Track new analytics metrics in backend (#5895)

* Allow filtering by project IDs in analytics route

* Download meta info in header

* add recursion limit

* Track playtime country

* fix clickhouse migrations

apps/labrinth/src/clickhouse/mod.rs

@@ -9,6 +9,9 @@ use crate::env::ENV;
 use crate::queue::server_ping;
 use crate::routes::analytics::MINECRAFT_SERVER_PLAYS;
 
+pub const DOWNLOADS: &str = "downloads";
+pub const PLAYTIME: &str = "playtime";
+
 pub async fn init_client() -> clickhouse::error::Result<clickhouse::Client> {
     init_client_with_database(&ENV.CLICKHOUSE_DATABASE).await
 }
@@ -90,7 +93,7 @@ pub async fn init_client_with_database(
     client
         .query(&format!(
             "
-            CREATE TABLE IF NOT EXISTS {database}.downloads {cluster_line}
+            CREATE TABLE IF NOT EXISTS {database}.{DOWNLOADS} {cluster_line}
             (
                 recorded DateTime64(4),
                 domain String,
@@ -117,7 +120,7 @@ pub async fn init_client_with_database(
     client
         .query(&format!(
             "
-            CREATE TABLE IF NOT EXISTS {database}.playtime {cluster_line}
+            CREATE TABLE IF NOT EXISTS {database}.{PLAYTIME} {cluster_line}
             (
                 recorded DateTime64(4),
                 seconds UInt64,
@@ -238,5 +241,27 @@ pub async fn init_client_with_database(
         .execute()
         .await?;
 
+    client
+        .query(&format!(
+            "
+            ALTER TABLE {database}.{DOWNLOADS} {cluster_line}
+            ADD COLUMN IF NOT EXISTS reason String,
+            ADD COLUMN IF NOT EXISTS game_version String,
+            ADD COLUMN IF NOT EXISTS loader String
+            "
+        ))
+        .execute()
+        .await?;
+
+    client
+        .query(&format!(
+            "
+            ALTER TABLE {database}.{PLAYTIME} {cluster_line}
+            ADD COLUMN IF NOT EXISTS country String
+            "
+        ))
+        .execute()
+        .await?;
+
     Ok(client.with_database(database))
 }

apps/labrinth/src/lib.rs

@@ -1,3 +1,5 @@
+#![recursion_limit = "256"]
+
 use std::sync::Arc;
 use std::time::Duration;
 

apps/labrinth/src/models/v3/analytics.rs

@@ -23,6 +23,23 @@ pub struct Download {
     pub country: String,
     pub user_agent: String,
     pub headers: Vec<(String, String)>,
+
+    // added retroactively - may be missing
+    pub reason: Option<DownloadReason>,
+    pub game_version: Option<String>,
+    pub loader: Option<String>,
+}
+
+/// Why a project was downloaded.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum DownloadReason {
+    /// Project was downloaded directly by the user.
+    Standalone,
+    /// Project was downloaded as a dependency, possibly transitive, of another
+    /// project.
+    Dependency,
+    /// Project was downloaded as part of a modpack.
+    Modpack,
 }
 
 #[derive(Debug, Row, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)]
@@ -77,6 +94,9 @@ pub struct Playtime {
     pub game_version: String,
     /// Parent modpack this playtime was recorded in
     pub parent: u64,
+
+    // added retroactively - may be missing
+    pub country: Option<String>,
 }
 
 #[derive(Row, Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Hash)]

apps/labrinth/src/routes/analytics.rs

@@ -214,6 +214,8 @@ async fn playtime_ingest(
     )
     .await?;
 
+    let headers = req.headers();
+
     for (id, playtime) in playtimes {
         if playtime.seconds > 300 {
             continue;
@@ -230,6 +232,9 @@ async fn playtime_ingest(
                 loader: playtime.loader,
                 game_version: playtime.game_version,
                 parent: playtime.parent.map_or(0, |x| x.0),
+                country: headers
+                    .get("cf-ipcountry")
+                    .and_then(|c| c.to_str().map(|s| s.to_string()).ok()),
             });
         }
     }

apps/labrinth/src/routes/internal/admin.rs

@@ -1,7 +1,7 @@
 use crate::auth::validate::get_user_record_from_bearer_token;
 use crate::database::PgPool;
 use crate::database::redis::RedisPool;
-use crate::models::analytics::Download;
+use crate::models::analytics::{Download, DownloadReason};
 use crate::models::ids::ProjectId;
 use crate::models::pats::Scopes;
 use crate::queue::analytics::AnalyticsQueue;
@@ -35,6 +35,17 @@ pub struct DownloadBody {
     pub headers: HashMap<String, String>,
 }
 
+/// Extra data attached to each download request, transmitted through the
+/// [`DOWNLOAD_META_HEADER`] header.
+#[derive(Debug, Clone, Deserialize)]
+pub struct DownloadMeta {
+    pub reason: DownloadReason,
+    pub game_version: String,
+    pub loader: String,
+}
+
+pub const DOWNLOAD_META_HEADER: &str = "modrinth-download-meta";
+
 // This is an internal route, cannot be used without key
 #[utoipa::path(
     patch,
@@ -118,6 +129,11 @@ pub async fn count_download(
     let ip = crate::util::ip::convert_to_ip_v6(&download_body.ip)
         .unwrap_or_else(|_| Ipv4Addr::new(127, 0, 0, 1).to_ipv6_mapped());
 
+    let meta = download_body
+        .headers
+        .get(DOWNLOAD_META_HEADER)
+        .and_then(|v| serde_json::from_str::<DownloadMeta>(v).ok());
+
     analytics_queue.add_download(Download {
         recorded: get_current_tenths_of_ms(),
         domain: url.host_str().unwrap_or_default().to_string(),
@@ -153,6 +169,9 @@ pub async fn count_download(
                     .contains(&&*x.0.to_lowercase())
             })
             .collect(),
+        reason: meta.as_ref().map(|m| m.reason),
+        game_version: meta.as_ref().map(|m| m.game_version.clone()),
+        loader: meta.as_ref().map(|m| m.loader.clone()),
     });
 
     Ok(HttpResponse::NoContent().body(""))

apps/labrinth/src/routes/internal/mod.rs

@@ -1,4 +1,4 @@
-pub(crate) mod admin;
+pub mod admin;
 pub mod affiliate;
 pub mod billing;
 pub mod delphi;

apps/labrinth/src/routes/v3/analytics_get.rs

@@ -55,6 +55,10 @@ pub struct GetRequest {
     pub time_range: TimeRange,
     /// What analytics metrics to return data for.
     pub return_metrics: ReturnMetrics,
+    /// What project IDs to return data for.
+    ///
+    /// If this is empty, all of the user's projects will be included.
+    pub project_ids: Vec<ProjectId>,
 }
 
 /// Time range for fetching analytics.
@@ -108,10 +112,6 @@ pub struct ReturnMetrics {
     pub affiliate_code_revenue: Option<Metrics<AffiliateCodeRevenueField>>,
 }
 
-/// Replacement for `()` because of a `utoipa` limitation.
-#[derive(Debug, Default, Serialize, Deserialize, utoipa::ToSchema)]
-pub struct Unit {}
-
 /// See [`ReturnMetrics`].
 #[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)]
 pub struct Metrics<F> {
@@ -612,9 +612,16 @@ pub async fn fetch_analytics(
 
     let mut time_slices = vec![TimeSlice::default(); num_time_slices];
 
-    // TODO fetch from req
-    let project_ids =
-        DBUser::get_projects(user.id.into(), &**pool, &redis).await?;
+    let project_ids = {
+        if req.project_ids.is_empty() {
+            DBUser::get_projects(user.id.into(), &**pool, &redis).await?
+        } else {
+            req.project_ids
+                .iter()
+                .map(|id| DBProjectId::from(*id))
+                .collect::<Vec<_>>()
+        }
+    };
 
     let project_ids =
         filter_allowed_project_ids(&project_ids, &user, &pool, &redis).await?;