From 2b8175ad666cab86888a29a20374f374593310c7 Mon Sep 17 00:00:00 2001 From: aecsocket Date: Tue, 10 Mar 2026 18:59:47 +0000 Subject: [PATCH] Daedalus doesn't instantly fail when upstream loses versions (#5518) * Daedalus doesn't instantly fail when upstream loses versions * fix shear * Revert DashMap changes for try_join_all --- apps/daedalus_client/src/fabric.rs | 37 +++++---- apps/daedalus_client/src/forge.rs | 42 +++++------ apps/daedalus_client/src/main.rs | 103 +++++++++++++++++++------- apps/daedalus_client/src/minecraft.rs | 16 ++-- 4 files changed, 119 insertions(+), 79 deletions(-) diff --git a/apps/daedalus_client/src/fabric.rs b/apps/daedalus_client/src/fabric.rs index 805d0e288..8c2a1c4b2 100644 --- a/apps/daedalus_client/src/fabric.rs +++ b/apps/daedalus_client/src/fabric.rs @@ -1,17 +1,17 @@ use crate::util::{download_file, fetch_json, format_url}; -use crate::{Error, MirrorArtifact, UploadFile, insert_mirrored_artifact}; +use crate::{ + Error, FetchResult, MirrorArtifact, UploadFile, insert_mirrored_artifact, +}; use daedalus::modded::{DUMMY_REPLACE_STRING, Manifest, PartialVersionInfo}; use dashmap::DashMap; use serde::Deserialize; use std::sync::Arc; use tokio::sync::Semaphore; -#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +#[tracing::instrument(skip(semaphore))] pub async fn fetch_fabric( semaphore: Arc, - upload_files: &DashMap, - mirror_artifacts: &DashMap, -) -> Result<(), Error> { +) -> Result { fetch( daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, "fabric", @@ -19,18 +19,14 @@ pub async fn fetch_fabric( "https://maven.fabricmc.net/", &[], semaphore, - upload_files, - mirror_artifacts, ) .await } -#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +#[tracing::instrument(skip(semaphore))] pub async fn fetch_quilt( semaphore: Arc, - upload_files: &DashMap, - mirror_artifacts: &DashMap, -) -> Result<(), Error> { +) -> Result { fetch( daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, "quilt", @@ -41,14 +37,12 @@ pub async fn fetch_quilt( "0.17.5-beta.4", ], semaphore, - upload_files, - mirror_artifacts, ) .await } #[allow(clippy::too_many_arguments)] -#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +#[tracing::instrument(skip(semaphore))] async fn fetch( format_version: usize, mod_loader: &str, @@ -56,9 +50,9 @@ async fn fetch( maven_url: &str, skip_versions: &[&str], semaphore: Arc, - upload_files: &DashMap, - mirror_artifacts: &DashMap, -) -> Result<(), Error> { +) -> Result { + let upload_files = DashMap::new(); + let mirror_artifacts = DashMap::::new(); let modrinth_manifest = fetch_json::( &format_url(&format!("{mod_loader}/v{format_version}/manifest.json",)), &semaphore, @@ -124,7 +118,7 @@ async fn fetch( None, vec![maven_url.to_string()], false, - mirror_artifacts, + &mirror_artifacts, )?; } } @@ -175,7 +169,7 @@ async fn fetch( .unwrap_or_else(|| maven_url.to_string()), ], false, - mirror_artifacts, + &mirror_artifacts, )?; } else { lib.name = new_name; @@ -268,7 +262,10 @@ async fn fetch( ); } - Ok(()) + Ok(FetchResult { + upload_files, + mirror_artifacts, + }) } #[derive(Deserialize, Debug, Clone)] diff --git a/apps/daedalus_client/src/forge.rs b/apps/daedalus_client/src/forge.rs index c1bf60564..8bbd7cc94 100644 --- a/apps/daedalus_client/src/forge.rs +++ b/apps/daedalus_client/src/forge.rs @@ -1,5 +1,7 @@ use crate::util::{download_file, fetch_json, fetch_xml, format_url}; -use crate::{Error, MirrorArtifact, UploadFile, insert_mirrored_artifact}; +use crate::{ + Error, FetchResult, MirrorArtifact, UploadFile, insert_mirrored_artifact, +}; use chrono::{DateTime, Utc}; use daedalus::get_path_from_artifact; use daedalus::modded::PartialVersionInfo; @@ -13,12 +15,10 @@ use std::collections::HashMap; use std::sync::Arc; use tokio::sync::Semaphore; -#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +#[tracing::instrument(skip(semaphore))] pub async fn fetch_forge( semaphore: Arc, - upload_files: &DashMap, - mirror_artifacts: &DashMap, -) -> Result<(), Error> { +) -> Result { let forge_manifest = fetch_json::>>( "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json", &semaphore, @@ -90,18 +90,14 @@ pub async fn fetch_forge( "https://maven.minecraftforge.net/", forge_versions, semaphore, - upload_files, - mirror_artifacts, ) .await } -#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +#[tracing::instrument(skip(semaphore))] pub async fn fetch_neo( semaphore: Arc, - upload_files: &DashMap, - mirror_artifacts: &DashMap, -) -> Result<(), Error> { +) -> Result { #[derive(Debug, Deserialize)] struct Metadata { versioning: Versioning, @@ -188,27 +184,20 @@ pub async fn fetch_neo( "https://maven.neoforged.net/", parsed_versions, semaphore, - upload_files, - mirror_artifacts, ) .await } -#[tracing::instrument(skip( - forge_versions, - semaphore, - upload_files, - mirror_artifacts -))] +#[tracing::instrument(skip(forge_versions, semaphore))] async fn fetch( format_version: usize, mod_loader: &str, maven_url: &str, forge_versions: Vec, semaphore: Arc, - upload_files: &DashMap, - mirror_artifacts: &DashMap, -) -> Result<(), Error> { +) -> Result { + let upload_files = DashMap::new(); + let mirror_artifacts = DashMap::::new(); let modrinth_manifest = fetch_json::( &format_url(&format!("{mod_loader}/v{format_version}/manifest.json",)), &semaphore, @@ -708,8 +697,8 @@ async fn fetch( loader, maven_url, mod_loader, - upload_files, - mirror_artifacts, + &upload_files, + &mirror_artifacts, ) }), ) @@ -778,7 +767,10 @@ async fn fetch( ); } - Ok(()) + Ok(FetchResult { + upload_files, + mirror_artifacts, + }) } #[derive(Debug)] diff --git a/apps/daedalus_client/src/main.rs b/apps/daedalus_client/src/main.rs index a09454715..8b132a0c1 100644 --- a/apps/daedalus_client/src/main.rs +++ b/apps/daedalus_client/src/main.rs @@ -43,47 +43,64 @@ async fn main() -> Result<()> { .unwrap_or(10), )); - // path, upload file - let upload_files: DashMap = DashMap::new(); - // path, mirror artifact - let mirror_artifacts: DashMap = DashMap::new(); + let mut fetch_result = FetchResult::default(); - minecraft::fetch(semaphore.clone(), &upload_files, &mirror_artifacts) - .await?; - fabric::fetch_fabric(semaphore.clone(), &upload_files, &mirror_artifacts) - .await?; - fabric::fetch_quilt(semaphore.clone(), &upload_files, &mirror_artifacts) - .await?; - forge::fetch_neo(semaphore.clone(), &upload_files, &mirror_artifacts) - .await?; - forge::fetch_forge(semaphore.clone(), &upload_files, &mirror_artifacts) - .await?; + match minecraft::fetch(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"), + } - futures::future::try_join_all(upload_files.iter().map(|x| { + match fabric::fetch_fabric(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"), + } + + match fabric::fetch_quilt(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"), + } + + match forge::fetch_neo(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"), + } + + match forge::fetch_forge(semaphore.clone()).await { + Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched), + Err(err) => tracing::warn!(error = %err, "Forge fetch failed"), + } + + let FetchResult { + upload_files, + mirror_artifacts, + } = fetch_result; + + futures::future::try_join_all(upload_files.iter().map(|entry| { upload_file_to_bucket( - x.key().clone(), - x.value().file.clone(), - x.value().content_type.clone(), + entry.key().clone(), + entry.value().file.clone(), + entry.value().content_type.clone(), &semaphore, ) })) .await?; - futures::future::try_join_all(mirror_artifacts.iter().map(|x| { + futures::future::try_join_all(mirror_artifacts.iter().map(|entry| { upload_url_to_bucket_mirrors( - format!("maven/{}", x.key()), - x.value() + format!("maven/{}", entry.key()), + entry + .value() .mirrors .iter() .map(|mirror| { if mirror.entire_url { mirror.path.clone() } else { - format!("{}{}", mirror.path, x.key()) + format!("{}{}", mirror.path, entry.key()) } }) .collect(), - x.sha1.clone(), + entry.value().sha1.clone(), &semaphore, ) })) @@ -139,17 +156,47 @@ pub struct UploadFile { content_type: Option, } +#[derive(Default)] +pub struct FetchResult { + pub upload_files: DashMap, + pub mirror_artifacts: DashMap, +} + pub struct MirrorArtifact { pub sha1: Option, pub mirrors: DashSet, } -#[derive(Eq, PartialEq, Hash)] +#[derive(Clone, Eq, PartialEq, Hash)] pub struct Mirror { path: String, entire_url: bool, } +fn merge_fetch_result(fetch_result: &mut FetchResult, fetched: FetchResult) { + for (path, upload_file) in fetched.upload_files { + fetch_result.upload_files.insert(path, upload_file); + } + + for (artifact_path, fetched_mirror_artifact) in fetched.mirror_artifacts { + let mut val = fetch_result + .mirror_artifacts + .entry(artifact_path) + .or_insert(MirrorArtifact { + sha1: fetched_mirror_artifact.sha1.clone(), + mirrors: DashSet::new(), + }); + + if val.sha1.is_none() { + val.sha1 = fetched_mirror_artifact.sha1; + } + + for mirror in fetched_mirror_artifact.mirrors { + val.mirrors.insert(mirror); + } + } +} + #[tracing::instrument(skip(mirror_artifacts))] pub fn insert_mirrored_artifact( artifact: &str, @@ -158,13 +205,17 @@ pub fn insert_mirrored_artifact( entire_url: bool, mirror_artifacts: &DashMap, ) -> Result<()> { - let val = mirror_artifacts + let mut val = mirror_artifacts .entry(get_path_from_artifact(artifact)?) .or_insert(MirrorArtifact { - sha1, + sha1: sha1.clone(), mirrors: DashSet::new(), }); + if val.sha1.is_none() { + val.sha1 = sha1; + } + for mirror in mirrors { val.mirrors.insert(Mirror { path: mirror, diff --git a/apps/daedalus_client/src/minecraft.rs b/apps/daedalus_client/src/minecraft.rs index bdeed603f..a3af06e62 100644 --- a/apps/daedalus_client/src/minecraft.rs +++ b/apps/daedalus_client/src/minecraft.rs @@ -1,6 +1,6 @@ use crate::util::fetch_json; use crate::{ - Error, MirrorArtifact, UploadFile, util::download_file, util::format_url, + Error, FetchResult, UploadFile, util::download_file, util::format_url, util::sha1_async, }; use daedalus::minecraft::{ @@ -12,12 +12,9 @@ use serde::Deserialize; use std::sync::Arc; use tokio::sync::Semaphore; -#[tracing::instrument(skip(semaphore, upload_files, _mirror_artifacts))] -pub async fn fetch( - semaphore: Arc, - upload_files: &DashMap, - _mirror_artifacts: &DashMap, -) -> Result<(), Error> { +#[tracing::instrument(skip(semaphore))] +pub async fn fetch(semaphore: Arc) -> Result { + let upload_files = DashMap::new(); let modrinth_manifest = fetch_json::( &format_url(&format!( "minecraft/v{}/manifest.json", @@ -169,7 +166,10 @@ pub async fn fetch( ); } - Ok(()) + Ok(FetchResult { + upload_files, + mirror_artifacts: DashMap::new(), + }) } #[derive(Deserialize, Debug)]