diff --git a/apps/labrinth/src/clickhouse/mod.rs b/apps/labrinth/src/clickhouse/mod.rs index dc19d3c2d..78d95b59e 100644 --- a/apps/labrinth/src/clickhouse/mod.rs +++ b/apps/labrinth/src/clickhouse/mod.rs @@ -9,6 +9,9 @@ use crate::env::ENV; use crate::queue::server_ping; use crate::routes::analytics::MINECRAFT_SERVER_PLAYS; +pub const DOWNLOADS: &str = "downloads"; +pub const PLAYTIME: &str = "playtime"; + pub async fn init_client() -> clickhouse::error::Result { init_client_with_database(&ENV.CLICKHOUSE_DATABASE).await } @@ -90,7 +93,7 @@ pub async fn init_client_with_database( client .query(&format!( " - CREATE TABLE IF NOT EXISTS {database}.downloads {cluster_line} + CREATE TABLE IF NOT EXISTS {database}.{DOWNLOADS} {cluster_line} ( recorded DateTime64(4), domain String, @@ -117,7 +120,7 @@ pub async fn init_client_with_database( client .query(&format!( " - CREATE TABLE IF NOT EXISTS {database}.playtime {cluster_line} + CREATE TABLE IF NOT EXISTS {database}.{PLAYTIME} {cluster_line} ( recorded DateTime64(4), seconds UInt64, @@ -238,5 +241,27 @@ pub async fn init_client_with_database( .execute() .await?; + client + .query(&format!( + " + ALTER TABLE {database}.{DOWNLOADS} {cluster_line} + ADD COLUMN IF NOT EXISTS reason String, + ADD COLUMN IF NOT EXISTS game_version String, + ADD COLUMN IF NOT EXISTS loader String + " + )) + .execute() + .await?; + + client + .query(&format!( + " + ALTER TABLE {database}.{PLAYTIME} {cluster_line} + ADD COLUMN IF NOT EXISTS country String + " + )) + .execute() + .await?; + Ok(client.with_database(database)) } diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index 1293ccbce..15363ed09 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "256"] + use std::sync::Arc; use std::time::Duration; diff --git a/apps/labrinth/src/models/v3/analytics.rs b/apps/labrinth/src/models/v3/analytics.rs index ea3c22e37..eb028a03f 100644 --- a/apps/labrinth/src/models/v3/analytics.rs +++ b/apps/labrinth/src/models/v3/analytics.rs @@ -23,6 +23,23 @@ pub struct Download { pub country: String, pub user_agent: String, pub headers: Vec<(String, String)>, + + // added retroactively - may be missing + pub reason: Option, + pub game_version: Option, + pub loader: Option, +} + +/// Why a project was downloaded. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum DownloadReason { + /// Project was downloaded directly by the user. + Standalone, + /// Project was downloaded as a dependency, possibly transitive, of another + /// project. + Dependency, + /// Project was downloaded as part of a modpack. + Modpack, } #[derive(Debug, Row, Serialize, Deserialize, Clone, Eq, PartialEq, Hash)] @@ -77,6 +94,9 @@ pub struct Playtime { pub game_version: String, /// Parent modpack this playtime was recorded in pub parent: u64, + + // added retroactively - may be missing + pub country: Option, } #[derive(Row, Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Hash)] diff --git a/apps/labrinth/src/routes/analytics.rs b/apps/labrinth/src/routes/analytics.rs index 15dab8295..5f84fd330 100644 --- a/apps/labrinth/src/routes/analytics.rs +++ b/apps/labrinth/src/routes/analytics.rs @@ -214,6 +214,8 @@ async fn playtime_ingest( ) .await?; + let headers = req.headers(); + for (id, playtime) in playtimes { if playtime.seconds > 300 { continue; @@ -230,6 +232,9 @@ async fn playtime_ingest( loader: playtime.loader, game_version: playtime.game_version, parent: playtime.parent.map_or(0, |x| x.0), + country: headers + .get("cf-ipcountry") + .and_then(|c| c.to_str().map(|s| s.to_string()).ok()), }); } } diff --git a/apps/labrinth/src/routes/internal/admin.rs b/apps/labrinth/src/routes/internal/admin.rs index d7d3010a3..8d14ce4a6 100644 --- a/apps/labrinth/src/routes/internal/admin.rs +++ b/apps/labrinth/src/routes/internal/admin.rs @@ -1,7 +1,7 @@ use crate::auth::validate::get_user_record_from_bearer_token; use crate::database::PgPool; use crate::database::redis::RedisPool; -use crate::models::analytics::Download; +use crate::models::analytics::{Download, DownloadReason}; use crate::models::ids::ProjectId; use crate::models::pats::Scopes; use crate::queue::analytics::AnalyticsQueue; @@ -35,6 +35,17 @@ pub struct DownloadBody { pub headers: HashMap, } +/// Extra data attached to each download request, transmitted through the +/// [`DOWNLOAD_META_HEADER`] header. +#[derive(Debug, Clone, Deserialize)] +pub struct DownloadMeta { + pub reason: DownloadReason, + pub game_version: String, + pub loader: String, +} + +pub const DOWNLOAD_META_HEADER: &str = "modrinth-download-meta"; + // This is an internal route, cannot be used without key #[utoipa::path( patch, @@ -118,6 +129,11 @@ pub async fn count_download( let ip = crate::util::ip::convert_to_ip_v6(&download_body.ip) .unwrap_or_else(|_| Ipv4Addr::new(127, 0, 0, 1).to_ipv6_mapped()); + let meta = download_body + .headers + .get(DOWNLOAD_META_HEADER) + .and_then(|v| serde_json::from_str::(v).ok()); + analytics_queue.add_download(Download { recorded: get_current_tenths_of_ms(), domain: url.host_str().unwrap_or_default().to_string(), @@ -153,6 +169,9 @@ pub async fn count_download( .contains(&&*x.0.to_lowercase()) }) .collect(), + reason: meta.as_ref().map(|m| m.reason), + game_version: meta.as_ref().map(|m| m.game_version.clone()), + loader: meta.as_ref().map(|m| m.loader.clone()), }); Ok(HttpResponse::NoContent().body("")) diff --git a/apps/labrinth/src/routes/internal/mod.rs b/apps/labrinth/src/routes/internal/mod.rs index 0299b16df..eac882c33 100644 --- a/apps/labrinth/src/routes/internal/mod.rs +++ b/apps/labrinth/src/routes/internal/mod.rs @@ -1,4 +1,4 @@ -pub(crate) mod admin; +pub mod admin; pub mod affiliate; pub mod billing; pub mod delphi; diff --git a/apps/labrinth/src/routes/v3/analytics_get.rs b/apps/labrinth/src/routes/v3/analytics_get.rs index e35193d5d..43bd84b26 100644 --- a/apps/labrinth/src/routes/v3/analytics_get.rs +++ b/apps/labrinth/src/routes/v3/analytics_get.rs @@ -55,6 +55,10 @@ pub struct GetRequest { pub time_range: TimeRange, /// What analytics metrics to return data for. pub return_metrics: ReturnMetrics, + /// What project IDs to return data for. + /// + /// If this is empty, all of the user's projects will be included. + pub project_ids: Vec, } /// Time range for fetching analytics. @@ -108,10 +112,6 @@ pub struct ReturnMetrics { pub affiliate_code_revenue: Option>, } -/// Replacement for `()` because of a `utoipa` limitation. -#[derive(Debug, Default, Serialize, Deserialize, utoipa::ToSchema)] -pub struct Unit {} - /// See [`ReturnMetrics`]. #[derive(Debug, Serialize, Deserialize, utoipa::ToSchema)] pub struct Metrics { @@ -612,9 +612,16 @@ pub async fn fetch_analytics( let mut time_slices = vec![TimeSlice::default(); num_time_slices]; - // TODO fetch from req - let project_ids = - DBUser::get_projects(user.id.into(), &**pool, &redis).await?; + let project_ids = { + if req.project_ids.is_empty() { + DBUser::get_projects(user.id.into(), &**pool, &redis).await? + } else { + req.project_ids + .iter() + .map(|id| DBProjectId::from(*id)) + .collect::>() + } + }; let project_ids = filter_allowed_project_ids(&project_ids, &user, &pool, &redis).await?;