Fix how analytics writes are serialized (#5926)

apps/labrinth/src/models/v3/analytics.rs

@@ -1,4 +1,5 @@
 use clickhouse::Row;
+use derive_more::Display;
 use serde::{Deserialize, Serialize};
 use std::hash::Hash;
 use std::net::Ipv6Addr;
@@ -24,14 +25,18 @@ pub struct Download {
     pub user_agent: String,
     pub headers: Vec<(String, String)>,
 
-    // added retroactively - may be missing
-    pub reason: Option<DownloadReason>,
-    pub game_version: Option<String>,
-    pub loader: Option<String>,
+    // added retroactively - may be empty
+    pub reason: String,
+    pub game_version: String,
+    pub loader: String,
 }
 
 /// Why a project was downloaded.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[derive(
+    Debug, Display, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize,
+)]
+#[serde(rename_all = "snake_case")]
+#[display(rename_all = "snake_case")]
 pub enum DownloadReason {
     /// Project was downloaded directly by the user.
     Standalone,
@@ -95,8 +100,8 @@ pub struct Playtime {
     /// Parent modpack this playtime was recorded in
     pub parent: u64,
 
-    // added retroactively - may be missing
-    pub country: Option<String>,
+    // added retroactively - may be empty
+    pub country: String,
 }
 
 #[derive(Row, Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Hash)]

apps/labrinth/src/queue/analytics/mod.rs

@@ -9,6 +9,7 @@ use crate::routes::analytics::MINECRAFT_SERVER_PLAYS;
 use dashmap::{DashMap, DashSet};
 use redis::cmd;
 use std::collections::HashMap;
+use tracing::trace;
 
 pub mod cache;
 
@@ -262,6 +263,7 @@ impl AnalyticsQueue {
         }
 
         if !downloads_queue.is_empty() {
+            let downloads_count = downloads_queue.len();
             let mut downloads_keys = Vec::new();
             let raw_downloads = DashMap::new();
 
@@ -319,6 +321,11 @@ impl AnalyticsQueue {
             let mut version_downloads: HashMap<i64, i32> = HashMap::new();
             let mut project_downloads: HashMap<i64, i32> = HashMap::new();
 
+            trace!(
+                "inserting {} raw downloads out of {downloads_count} downloads",
+                raw_downloads.len()
+            );
+
             for (_, download) in raw_downloads {
                 *version_downloads
                     .entry(download.version_id as i64)
@@ -327,6 +334,8 @@ impl AnalyticsQueue {
                     .entry(download.project_id as i64)
                     .or_default() += 1;
 
+                trace!("writing download {download:?}");
+
                 downloads.write(&download).await?;
             }
 

apps/labrinth/src/routes/analytics.rs

@@ -234,7 +234,8 @@ async fn playtime_ingest(
                 parent: playtime.parent.map_or(0, |x| x.0),
                 country: headers
                     .get("cf-ipcountry")
-                    .and_then(|c| c.to_str().map(|s| s.to_string()).ok()),
+                    .and_then(|c| c.to_str().map(|s| s.to_string()).ok())
+                    .unwrap_or_default(),
             });
         }
     }

apps/labrinth/src/routes/internal/admin.rs

@@ -16,6 +16,7 @@ use serde::Deserialize;
 use std::collections::HashMap;
 use std::net::Ipv4Addr;
 use std::sync::Arc;
+use tracing::trace;
 
 pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
     cfg.service(
@@ -129,12 +130,16 @@ pub async fn count_download(
     let ip = crate::util::ip::convert_to_ip_v6(&download_body.ip)
         .unwrap_or_else(|_| Ipv4Addr::new(127, 0, 0, 1).to_ipv6_mapped());
 
-    let meta = download_body
-        .headers
-        .get(DOWNLOAD_META_HEADER)
-        .and_then(|v| serde_json::from_str::<DownloadMeta>(v).ok());
+    let meta =
+        if let Some(meta) = download_body.headers.get(DOWNLOAD_META_HEADER) {
+            serde_json::from_str::<DownloadMeta>(meta)
+                .map(Some)
+                .wrap_request_err("invalid download meta")?
+        } else {
+            None
+        };
 
-    analytics_queue.add_download(Download {
+    let download = Download {
         recorded: get_current_tenths_of_ms(),
         domain: url.host_str().unwrap_or_default().to_string(),
         site_path: url.path().to_string(),
@@ -169,10 +174,19 @@ pub async fn count_download(
                     .contains(&&*x.0.to_lowercase())
             })
             .collect(),
-        reason: meta.as_ref().map(|m| m.reason),
-        game_version: meta.as_ref().map(|m| m.game_version.clone()),
-        loader: meta.as_ref().map(|m| m.loader.clone()),
-    });
+        reason: meta
+            .as_ref()
+            .map(|m| m.reason.to_string())
+            .unwrap_or_default(),
+        game_version: meta
+            .as_ref()
+            .map(|m| m.game_version.clone())
+            .unwrap_or_default(),
+        loader: meta.as_ref().map(|m| m.loader.clone()).unwrap_or_default(),
+    };
+    trace!("added download {download:#?}");
+
+    analytics_queue.add_download(download);
 
     Ok(HttpResponse::NoContent().body(""))
 }

apps/labrinth/src/routes/v3/analytics_get.rs

@@ -54,10 +54,12 @@ pub struct GetRequest {
     /// What time range to return statistics for.
     pub time_range: TimeRange,
     /// What analytics metrics to return data for.
+    #[serde(default)]
     pub return_metrics: ReturnMetrics,
     /// What project IDs to return data for.
     ///
     /// If this is empty, all of the user's projects will be included.
+    #[serde(default)]
     pub project_ids: Vec<ProjectId>,
 }