Daedalus doesn't instantly fail when upstream loses versions (#5518)

* Daedalus doesn't instantly fail when upstream loses versions

* fix shear

* Revert DashMap changes for try_join_all

apps/daedalus_client/src/fabric.rs

@@ -1,17 +1,17 @@
 use crate::util::{download_file, fetch_json, format_url};
-use crate::{Error, MirrorArtifact, UploadFile, insert_mirrored_artifact};
+use crate::{
+    Error, FetchResult, MirrorArtifact, UploadFile, insert_mirrored_artifact,
+};
 use daedalus::modded::{DUMMY_REPLACE_STRING, Manifest, PartialVersionInfo};
 use dashmap::DashMap;
 use serde::Deserialize;
 use std::sync::Arc;
 use tokio::sync::Semaphore;
 
-#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))]
+#[tracing::instrument(skip(semaphore))]
 pub async fn fetch_fabric(
     semaphore: Arc<Semaphore>,
-    upload_files: &DashMap<String, UploadFile>,
+) -> Result<FetchResult, Error> {
-    mirror_artifacts: &DashMap<String, MirrorArtifact>,
-) -> Result<(), Error> {
     fetch(
         daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
         "fabric",
@@ -19,18 +19,14 @@ pub async fn fetch_fabric(
         "https://maven.fabricmc.net/",
         &[],
         semaphore,
-        upload_files,
-        mirror_artifacts,
     )
     .await
 }
 
-#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))]
+#[tracing::instrument(skip(semaphore))]
 pub async fn fetch_quilt(
     semaphore: Arc<Semaphore>,
-    upload_files: &DashMap<String, UploadFile>,
+) -> Result<FetchResult, Error> {
-    mirror_artifacts: &DashMap<String, MirrorArtifact>,
-) -> Result<(), Error> {
     fetch(
         daedalus::modded::CURRENT_QUILT_FORMAT_VERSION,
         "quilt",
@@ -41,14 +37,12 @@ pub async fn fetch_quilt(
             "0.17.5-beta.4",
         ],
         semaphore,
-        upload_files,
-        mirror_artifacts,
     )
     .await
 }
 
 #[allow(clippy::too_many_arguments)]
-#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))]
+#[tracing::instrument(skip(semaphore))]
 async fn fetch(
     format_version: usize,
     mod_loader: &str,
@@ -56,9 +50,9 @@ async fn fetch(
     maven_url: &str,
     skip_versions: &[&str],
     semaphore: Arc<Semaphore>,
-    upload_files: &DashMap<String, UploadFile>,
+) -> Result<FetchResult, Error> {
-    mirror_artifacts: &DashMap<String, MirrorArtifact>,
+    let upload_files = DashMap::new();
-) -> Result<(), Error> {
+    let mirror_artifacts = DashMap::<String, MirrorArtifact>::new();
     let modrinth_manifest = fetch_json::<Manifest>(
         &format_url(&format!("{mod_loader}/v{format_version}/manifest.json",)),
         &semaphore,
@@ -124,7 +118,7 @@ async fn fetch(
                 None,
                 vec![maven_url.to_string()],
                 false,
-                mirror_artifacts,
+                &mirror_artifacts,
             )?;
         }
     }
@@ -175,7 +169,7 @@ async fn fetch(
                                     .unwrap_or_else(|| maven_url.to_string()),
                             ],
                             false,
-                            mirror_artifacts,
+                            &mirror_artifacts,
                         )?;
                     } else {
                         lib.name = new_name;
@@ -268,7 +262,10 @@ async fn fetch(
         );
     }
 
-    Ok(())
+    Ok(FetchResult {
+        upload_files,
+        mirror_artifacts,
+    })
 }
 
 #[derive(Deserialize, Debug, Clone)]

apps/daedalus_client/src/forge.rs

@@ -1,5 +1,7 @@
 use crate::util::{download_file, fetch_json, fetch_xml, format_url};
-use crate::{Error, MirrorArtifact, UploadFile, insert_mirrored_artifact};
+use crate::{
+    Error, FetchResult, MirrorArtifact, UploadFile, insert_mirrored_artifact,
+};
 use chrono::{DateTime, Utc};
 use daedalus::get_path_from_artifact;
 use daedalus::modded::PartialVersionInfo;
@@ -13,12 +15,10 @@ use std::collections::HashMap;
 use std::sync::Arc;
 use tokio::sync::Semaphore;
 
-#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))]
+#[tracing::instrument(skip(semaphore))]
 pub async fn fetch_forge(
     semaphore: Arc<Semaphore>,
-    upload_files: &DashMap<String, UploadFile>,
+) -> Result<FetchResult, Error> {
-    mirror_artifacts: &DashMap<String, MirrorArtifact>,
-) -> Result<(), Error> {
     let forge_manifest = fetch_json::<IndexMap<String, Vec<String>>>(
         "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json",
         &semaphore,
@@ -90,18 +90,14 @@ pub async fn fetch_forge(
         "https://maven.minecraftforge.net/",
         forge_versions,
         semaphore,
-        upload_files,
-        mirror_artifacts,
     )
     .await
 }
 
-#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))]
+#[tracing::instrument(skip(semaphore))]
 pub async fn fetch_neo(
     semaphore: Arc<Semaphore>,
-    upload_files: &DashMap<String, UploadFile>,
+) -> Result<FetchResult, Error> {
-    mirror_artifacts: &DashMap<String, MirrorArtifact>,
-) -> Result<(), Error> {
     #[derive(Debug, Deserialize)]
     struct Metadata {
         versioning: Versioning,
@@ -188,27 +184,20 @@ pub async fn fetch_neo(
         "https://maven.neoforged.net/",
         parsed_versions,
         semaphore,
-        upload_files,
-        mirror_artifacts,
     )
     .await
 }
 
-#[tracing::instrument(skip(
+#[tracing::instrument(skip(forge_versions, semaphore))]
-    forge_versions,
-    semaphore,
-    upload_files,
-    mirror_artifacts
-))]
 async fn fetch(
     format_version: usize,
     mod_loader: &str,
     maven_url: &str,
     forge_versions: Vec<ForgeVersion>,
     semaphore: Arc<Semaphore>,
-    upload_files: &DashMap<String, UploadFile>,
+) -> Result<FetchResult, Error> {
-    mirror_artifacts: &DashMap<String, MirrorArtifact>,
+    let upload_files = DashMap::new();
-) -> Result<(), Error> {
+    let mirror_artifacts = DashMap::<String, MirrorArtifact>::new();
     let modrinth_manifest = fetch_json::<daedalus::modded::Manifest>(
         &format_url(&format!("{mod_loader}/v{format_version}/manifest.json",)),
         &semaphore,
@@ -708,8 +697,8 @@ async fn fetch(
                         loader,
                         maven_url,
                         mod_loader,
-                        upload_files,
+                        &upload_files,
-                        mirror_artifacts,
+                        &mirror_artifacts,
                     )
                 }),
         )
@@ -778,7 +767,10 @@ async fn fetch(
         );
     }
 
-    Ok(())
+    Ok(FetchResult {
+        upload_files,
+        mirror_artifacts,
+    })
 }
 
 #[derive(Debug)]

apps/daedalus_client/src/main.rs

@@ -43,47 +43,64 @@ async fn main() -> Result<()> {
             .unwrap_or(10),
     ));
 
-    // path, upload file
+    let mut fetch_result = FetchResult::default();
-    let upload_files: DashMap<String, UploadFile> = DashMap::new();
-    // path, mirror artifact
-    let mirror_artifacts: DashMap<String, MirrorArtifact> = DashMap::new();
 
-    minecraft::fetch(semaphore.clone(), &upload_files, &mirror_artifacts)
+    match minecraft::fetch(semaphore.clone()).await {
-        .await?;
+        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
-    fabric::fetch_fabric(semaphore.clone(), &upload_files, &mirror_artifacts)
+        Err(err) => tracing::warn!(error = %err, "Minecraft fetch failed"),
-        .await?;
+    }
-    fabric::fetch_quilt(semaphore.clone(), &upload_files, &mirror_artifacts)
-        .await?;
-    forge::fetch_neo(semaphore.clone(), &upload_files, &mirror_artifacts)
-        .await?;
-    forge::fetch_forge(semaphore.clone(), &upload_files, &mirror_artifacts)
-        .await?;
 
-    futures::future::try_join_all(upload_files.iter().map(|x| {
+    match fabric::fetch_fabric(semaphore.clone()).await {
+        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+        Err(err) => tracing::warn!(error = %err, "Fabric fetch failed"),
+    }
+
+    match fabric::fetch_quilt(semaphore.clone()).await {
+        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+        Err(err) => tracing::warn!(error = %err, "Quilt fetch failed"),
+    }
+
+    match forge::fetch_neo(semaphore.clone()).await {
+        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+        Err(err) => tracing::warn!(error = %err, "NeoForge fetch failed"),
+    }
+
+    match forge::fetch_forge(semaphore.clone()).await {
+        Ok(fetched) => merge_fetch_result(&mut fetch_result, fetched),
+        Err(err) => tracing::warn!(error = %err, "Forge fetch failed"),
+    }
+
+    let FetchResult {
+        upload_files,
+        mirror_artifacts,
+    } = fetch_result;
+
+    futures::future::try_join_all(upload_files.iter().map(|entry| {
         upload_file_to_bucket(
-            x.key().clone(),
+            entry.key().clone(),
-            x.value().file.clone(),
+            entry.value().file.clone(),
-            x.value().content_type.clone(),
+            entry.value().content_type.clone(),
             &semaphore,
         )
     }))
     .await?;
 
-    futures::future::try_join_all(mirror_artifacts.iter().map(|x| {
+    futures::future::try_join_all(mirror_artifacts.iter().map(|entry| {
         upload_url_to_bucket_mirrors(
-            format!("maven/{}", x.key()),
+            format!("maven/{}", entry.key()),
-            x.value()
+            entry
+                .value()
                 .mirrors
                 .iter()
                 .map(|mirror| {
                     if mirror.entire_url {
                         mirror.path.clone()
                     } else {
-                        format!("{}{}", mirror.path, x.key())
+                        format!("{}{}", mirror.path, entry.key())
                     }
                 })
                 .collect(),
-            x.sha1.clone(),
+            entry.value().sha1.clone(),
             &semaphore,
         )
     }))
@@ -139,17 +156,47 @@ pub struct UploadFile {
     content_type: Option<String>,
 }
 
+#[derive(Default)]
+pub struct FetchResult {
+    pub upload_files: DashMap<String, UploadFile>,
+    pub mirror_artifacts: DashMap<String, MirrorArtifact>,
+}
+
 pub struct MirrorArtifact {
     pub sha1: Option<String>,
     pub mirrors: DashSet<Mirror>,
 }
 
-#[derive(Eq, PartialEq, Hash)]
+#[derive(Clone, Eq, PartialEq, Hash)]
 pub struct Mirror {
     path: String,
     entire_url: bool,
 }
 
+fn merge_fetch_result(fetch_result: &mut FetchResult, fetched: FetchResult) {
+    for (path, upload_file) in fetched.upload_files {
+        fetch_result.upload_files.insert(path, upload_file);
+    }
+
+    for (artifact_path, fetched_mirror_artifact) in fetched.mirror_artifacts {
+        let mut val = fetch_result
+            .mirror_artifacts
+            .entry(artifact_path)
+            .or_insert(MirrorArtifact {
+                sha1: fetched_mirror_artifact.sha1.clone(),
+                mirrors: DashSet::new(),
+            });
+
+        if val.sha1.is_none() {
+            val.sha1 = fetched_mirror_artifact.sha1;
+        }
+
+        for mirror in fetched_mirror_artifact.mirrors {
+            val.mirrors.insert(mirror);
+        }
+    }
+}
+
 #[tracing::instrument(skip(mirror_artifacts))]
 pub fn insert_mirrored_artifact(
     artifact: &str,
@@ -158,13 +205,17 @@ pub fn insert_mirrored_artifact(
     entire_url: bool,
     mirror_artifacts: &DashMap<String, MirrorArtifact>,
 ) -> Result<()> {
-    let val = mirror_artifacts
+    let mut val = mirror_artifacts
         .entry(get_path_from_artifact(artifact)?)
         .or_insert(MirrorArtifact {
-            sha1,
+            sha1: sha1.clone(),
             mirrors: DashSet::new(),
         });
 
+    if val.sha1.is_none() {
+        val.sha1 = sha1;
+    }
+
     for mirror in mirrors {
         val.mirrors.insert(Mirror {
             path: mirror,

apps/daedalus_client/src/minecraft.rs

@@ -1,6 +1,6 @@
 use crate::util::fetch_json;
 use crate::{
-    Error, MirrorArtifact, UploadFile, util::download_file, util::format_url,
+    Error, FetchResult, UploadFile, util::download_file, util::format_url,
     util::sha1_async,
 };
 use daedalus::minecraft::{
@@ -12,12 +12,9 @@ use serde::Deserialize;
 use std::sync::Arc;
 use tokio::sync::Semaphore;
 
-#[tracing::instrument(skip(semaphore, upload_files, _mirror_artifacts))]
+#[tracing::instrument(skip(semaphore))]
-pub async fn fetch(
+pub async fn fetch(semaphore: Arc<Semaphore>) -> Result<FetchResult, Error> {
-    semaphore: Arc<Semaphore>,
+    let upload_files = DashMap::new();
-    upload_files: &DashMap<String, UploadFile>,
-    _mirror_artifacts: &DashMap<String, MirrorArtifact>,
-) -> Result<(), Error> {
     let modrinth_manifest = fetch_json::<VersionManifest>(
         &format_url(&format!(
             "minecraft/v{}/manifest.json",
@@ -169,7 +166,10 @@ pub async fn fetch(
         );
     }
 
-    Ok(())
+    Ok(FetchResult {
+        upload_files,
+        mirror_artifacts: DashMap::new(),
+    })
 }
 
 #[derive(Deserialize, Debug)]