MrSphay/Modrinth-plus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Daedalus doesn't instantly fail when upstream loses versions (#5518)

Browse Source 
* Daedalus doesn't instantly fail when upstream loses versions

* fix shear

* Revert DashMap changes for try_join_all
This commit is contained in:
aecsocket
2026-03-10 18:59:47 +00:00
committed by GitHub
parent 0c98f6bf45
commit 2b8175ad66
4 changed files with 119 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
apps/daedalus_client/src/fabric.rs
View File

@@ -1,17 +1,17 @@
use crate::util::{download_file, fetch_json, format_url};
use crate::{Error, MirrorArtifact, UploadFile, insert_mirrored_artifact};
use crate::{
Error, FetchResult, MirrorArtifact, UploadFile, insert_mirrored_artifact,
};
use daedalus::modded::{DUMMY_REPLACE_STRING, Manifest, PartialVersionInfo};
use dashmap::DashMap;
use serde::Deserialize;
use std::sync::Arc;
use tokio::sync::Semaphore;
#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))]
#[tracing::instrument(skip(semaphore))]
pub async fn fetch_fabric(
semaphore: Arc<Semaphore>,
upload_files: &DashMap<String, UploadFile>,
mirror_artifacts: &DashMap<String, MirrorArtifact>,
) -> Result<(), Error> {
) -> Result<FetchResult, Error> {
fetch(
daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
"fabric",
@@ -19,18 +19,14 @@ pub async fn fetch_fabric(
"https://maven.fabricmc.net/",
&[],
semaphore,
upload_files,
mirror_artifacts,
)
.await
}
#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))]
#[tracing::instrument(skip(semaphore))]
pub async fn fetch_quilt(
semaphore: Arc<Semaphore>,
upload_files: &DashMap<String, UploadFile>,
mirror_artifacts: &DashMap<String, MirrorArtifact>,
) -> Result<(), Error> {
) -> Result<FetchResult, Error> {
fetch(
daedalus::modded::CURRENT_QUILT_FORMAT_VERSION,
"quilt",
@@ -41,14 +37,12 @@ pub async fn fetch_quilt(
"0.17.5-beta.4",
],
semaphore,
upload_files,
mirror_artifacts,
)
.await
}
#[allow(clippy::too_many_arguments)]
#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))]
#[tracing::instrument(skip(semaphore))]
async fn fetch(
format_version: usize,
mod_loader: &str,
@@ -56,9 +50,9 @@ async fn fetch(
maven_url: &str,
skip_versions: &[&str],
semaphore: Arc<Semaphore>,
upload_files: &DashMap<String, UploadFile>,
mirror_artifacts: &DashMap<String, MirrorArtifact>,
) -> Result<(), Error> {
) -> Result<FetchResult, Error> {
let upload_files = DashMap::new();
let mirror_artifacts = DashMap::<String, MirrorArtifact>::new();
let modrinth_manifest = fetch_json::<Manifest>(
&format_url(&format!("{mod_loader}/v{format_version}/manifest.json",)),
&semaphore,
@@ -124,7 +118,7 @@ async fn fetch(
None,
vec![maven_url.to_string()],
false,
mirror_artifacts,
&mirror_artifacts,
)?;
}
}
@@ -175,7 +169,7 @@ async fn fetch(
.unwrap_or_else(|| maven_url.to_string()),
],
false,
mirror_artifacts,
&mirror_artifacts,
)?;
} else {
lib.name = new_name;
@@ -268,7 +262,10 @@ async fn fetch(
);
}
Ok(())
Ok(FetchResult {
upload_files,
mirror_artifacts,
})
}
#[derive(Deserialize, Debug, Clone)]

42
apps/daedalus_client/src/forge.rs
View File

@@ -1,5 +1,7 @@
use crate::util::{download_file, fetch_json, fetch_xml, format_url};
use crate::{Error, MirrorArtifact, UploadFile, insert_mirrored_artifact};
use crate::{
Error, FetchResult, MirrorArtifact, UploadFile, insert_mirrored_artifact,
};
use chrono::{DateTime, Utc};
use daedalus::get_path_from_artifact;
use daedalus::modded::PartialVersionInfo;
@@ -13,12 +15,10 @@ use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::Semaphore;
#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))]
#[tracing::instrument(skip(semaphore))]
pub async fn fetch_forge(
semaphore: Arc<Semaphore>,
upload_files: &DashMap<String, UploadFile>,
mirror_artifacts: &DashMap<String, MirrorArtifact>,
) -> Result<(), Error> {
) -> Result<FetchResult, Error> {
let forge_manifest = fetch_json::<IndexMap<String, Vec<String>>>(
"https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json",
&semaphore,
@@ -90,18 +90,14 @@ pub async fn fetch_forge(
"https://maven.minecraftforge.net/",
forge_versions,
semaphore,
upload_files,
mirror_artifacts,
)
.await
}
#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))]
#[tracing::instrument(skip(semaphore))]
pub async fn fetch_neo(
semaphore: Arc<Semaphore>,
upload_files: &DashMap<String, UploadFile>,
mirror_artifacts: &DashMap<String, MirrorArtifact>,
) -> Result<(), Error> {
) -> Result<FetchResult, Error> {
#[derive(Debug, Deserialize)]
struct Metadata {
versioning: Versioning,
@@ -188,27 +184,20 @@ pub async fn fetch_neo(
"https://maven.neoforged.net/",
parsed_versions,
semaphore,
upload_files,
mirror_artifacts,
)
.await
}
#[tracing::instrument(skip(
forge_versions,
semaphore,
upload_files,
mirror_artifacts
))]
#[tracing::instrument(skip(forge_versions, semaphore))]
async fn fetch(
format_version: usize,
mod_loader: &str,
maven_url: &str,
forge_versions: Vec<ForgeVersion>,
semaphore: Arc<Semaphore>,
upload_files: &DashMap<String, UploadFile>,
mirror_artifacts: &DashMap<String, MirrorArtifact>,
) -> Result<(), Error> {
) -> Result<FetchResult, Error> {
let upload_files = DashMap::new();
let mirror_artifacts = DashMap::<String, MirrorArtifact>::new();
let modrinth_manifest = fetch_json::<daedalus::modded::Manifest>(
&format_url(&format!("{mod_loader}/v{format_version}/manifest.json",)),
&semaphore,
@@ -708,8 +697,8 @@ async fn fetch(
loader,
maven_url,
mod_loader,
upload_files,
mirror_artifacts,
&upload_files,
&mirror_artifacts,
)
}),
)
@@ -778,7 +767,10 @@ async fn fetch(
);
}
Ok(())
Ok(FetchResult {
upload_files,
mirror_artifacts,
})
}
#[derive(Debug)]

103
apps/daedalus_client/src/main.rs
View File

@@ -43,47 +43,64 @@ async fn main() -> Result<()> {
.unwrap_or(10),
));
// path, upload file
let upload_files: DashMap<String, UploadFile> = DashMap::new();
// path, mirror artifact
let mirror_artifacts: DashMap<String, MirrorArtifact> = DashMap::new();
let mut fetch_result = FetchResult::default();
minecraft::fetch(semaphore.clone(), &upload_files, &mirror_artifacts)
.await?;
fabric::fetch_fabric(semaphore.clone(), &upload_files, &mirror_artifacts)
.await?;
fabric::fetch_quilt(semaphore.clone(), &upload_files, &mirror_artifacts)
.await?;
forge::fetch_neo(semaphore.clone(), &upload_files, &mirror_artifacts)
.await?;
forge::fetch_forge(semaphore.clone(), &upload_files, &mirror_artifacts)
.await?;
match minecraft::fetch(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"),
}
futures::future::try_join_all(upload_files.iter().map(|x| {
match fabric::fetch_fabric(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"),
}
match fabric::fetch_quilt(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"),
}
match forge::fetch_neo(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"),
}
match forge::fetch_forge(semaphore.clone()).await {
Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
Err(err) => tracing::warn!(error = %err, "Forge fetch failed"),
}
let FetchResult {
upload_files,
mirror_artifacts,
} = fetch_result;
futures::future::try_join_all(upload_files.iter().map(|entry| {
upload_file_to_bucket(
x.key().clone(),
x.value().file.clone(),
x.value().content_type.clone(),
entry.key().clone(),
entry.value().file.clone(),
entry.value().content_type.clone(),
&semaphore,
)
}))
.await?;
futures::future::try_join_all(mirror_artifacts.iter().map(|x| {
futures::future::try_join_all(mirror_artifacts.iter().map(|entry| {
upload_url_to_bucket_mirrors(
format!("maven/{}", x.key()),
x.value()
format!("maven/{}", entry.key()),
entry
.value()
.mirrors
.iter()
.map(|mirror| {
if mirror.entire_url {
mirror.path.clone()
} else {
format!("{}{}", mirror.path, x.key())
format!("{}{}", mirror.path, entry.key())
}
})
.collect(),
x.sha1.clone(),
entry.value().sha1.clone(),
&semaphore,
)
}))
@@ -139,17 +156,47 @@ pub struct UploadFile {
content_type: Option<String>,
}
#[derive(Default)]
pub struct FetchResult {
pub upload_files: DashMap<String, UploadFile>,
pub mirror_artifacts: DashMap<String, MirrorArtifact>,
}
pub struct MirrorArtifact {
pub sha1: Option<String>,
pub mirrors: DashSet<Mirror>,
}
#[derive(Eq, PartialEq, Hash)]
#[derive(Clone, Eq, PartialEq, Hash)]
pub struct Mirror {
path: String,
entire_url: bool,
}
fn merge_fetch_result(fetch_result: &mut FetchResult, fetched: FetchResult) {
for (path, upload_file) in fetched.upload_files {
fetch_result.upload_files.insert(path, upload_file);
}
for (artifact_path, fetched_mirror_artifact) in fetched.mirror_artifacts {
let mut val = fetch_result
.mirror_artifacts
.entry(artifact_path)
.or_insert(MirrorArtifact {
sha1: fetched_mirror_artifact.sha1.clone(),
mirrors: DashSet::new(),
});
if val.sha1.is_none() {
val.sha1 = fetched_mirror_artifact.sha1;
}
for mirror in fetched_mirror_artifact.mirrors {
val.mirrors.insert(mirror);
}
}
}
#[tracing::instrument(skip(mirror_artifacts))]
pub fn insert_mirrored_artifact(
artifact: &str,
@@ -158,13 +205,17 @@ pub fn insert_mirrored_artifact(
entire_url: bool,
mirror_artifacts: &DashMap<String, MirrorArtifact>,
) -> Result<()> {
let val = mirror_artifacts
let mut val = mirror_artifacts
.entry(get_path_from_artifact(artifact)?)
.or_insert(MirrorArtifact {
sha1,
sha1: sha1.clone(),
mirrors: DashSet::new(),
});
if val.sha1.is_none() {
val.sha1 = sha1;
}
for mirror in mirrors {
val.mirrors.insert(Mirror {
path: mirror,

16
apps/daedalus_client/src/minecraft.rs
View File

@@ -1,6 +1,6 @@
use crate::util::fetch_json;
use crate::{
Error, MirrorArtifact, UploadFile, util::download_file, util::format_url,
Error, FetchResult, UploadFile, util::download_file, util::format_url,
util::sha1_async,
};
use daedalus::minecraft::{
@@ -12,12 +12,9 @@ use serde::Deserialize;
use std::sync::Arc;
use tokio::sync::Semaphore;
#[tracing::instrument(skip(semaphore, upload_files, _mirror_artifacts))]
pub async fn fetch(
semaphore: Arc<Semaphore>,
upload_files: &DashMap<String, UploadFile>,
_mirror_artifacts: &DashMap<String, MirrorArtifact>,
) -> Result<(), Error> {
#[tracing::instrument(skip(semaphore))]
pub async fn fetch(semaphore: Arc<Semaphore>) -> Result<FetchResult, Error> {
let upload_files = DashMap::new();
let modrinth_manifest = fetch_json::<VersionManifest>(
&format_url(&format!(
"minecraft/v{}/manifest.json",
@@ -169,7 +166,10 @@ pub async fn fetch(
);
}
Ok(())
Ok(FetchResult {
upload_files,
mirror_artifacts: DashMap::new(),
})
}
#[derive(Deserialize, Debug)]