External projects moderator database (#5692)

* Begin external projects moderator database frontend

* add copy link button

* begin project page permissions settings

* MEL database backend routes

* include filename in external files

* Hook up frontend external license page to backend

* more work on user-facing external projects stuff

* put user-facing stuff behind feature flag

* prepr

* clippy

---------

Co-authored-by: aecsocket <aecsocket@tutanota.com>

apps/labrinth/src/database/models/mod.rs

@@ -11,6 +11,7 @@ pub mod ids;
 pub mod image_item;
 pub mod legacy_loader_fields;
 pub mod loader_fields;
+pub mod moderation_external_item;
 pub mod moderation_lock_item;
 pub mod notification_item;
 pub mod notifications_deliveries_item;

apps/labrinth/src/database/models/moderation_external_item.rs

@@ -0,0 +1,98 @@
+use chrono::{DateTime, Utc};
+
+use crate::database::models::DBUserId;
+
+pub struct ExternalLicense {
+    pub id: i64,
+    pub title: Option<String>,
+    pub status: String,
+    pub link: Option<String>,
+    pub proof: Option<String>,
+    pub flame_project_id: Option<i32>,
+}
+
+impl ExternalLicense {
+    pub async fn insert_many(
+        exec: impl sqlx::PgExecutor<'_>,
+        licenses: &[ExternalLicense],
+        user_id: DBUserId,
+    ) -> sqlx::Result<()> {
+        let now = Utc::now();
+
+        let ids: Vec<i64> = licenses.iter().map(|x| x.id).collect();
+        let titles: Vec<Option<String>> =
+            licenses.iter().map(|x| x.title.clone()).collect();
+        let statuses: Vec<String> =
+            licenses.iter().map(|x| x.status.clone()).collect();
+        let links: Vec<Option<String>> =
+            licenses.iter().map(|x| x.link.clone()).collect();
+        let proofs: Vec<Option<String>> =
+            licenses.iter().map(|x| x.proof.clone()).collect();
+        let flame_ids: Vec<Option<i32>> =
+            licenses.iter().map(|x| x.flame_project_id).collect();
+        let nows: Vec<DateTime<Utc>> = vec![now; licenses.len()];
+        let user_ids: Vec<i64> = vec![user_id.0; licenses.len()];
+
+        sqlx::query!(
+            r#"
+            INSERT INTO moderation_external_licenses (id, title, status, link, proof, flame_project_id, inserted_at, inserted_by, updated_at, updated_by)
+            SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::varchar[], $4::varchar[], $5::varchar[], $6::integer[], $7::timestamptz[], $8::bigint[], $7::timestamptz[], $8::bigint[])
+            ON CONFLICT (id) DO UPDATE SET
+                title = EXCLUDED.title,
+                status = EXCLUDED.status,
+                link = EXCLUDED.link,
+                proof = EXCLUDED.proof,
+                flame_project_id = EXCLUDED.flame_project_id,
+                updated_at = EXCLUDED.updated_at,
+                updated_by = EXCLUDED.updated_by
+            "#,
+            &ids,
+            &titles as _,
+            &statuses,
+            &links as _,
+            &proofs as _,
+            &flame_ids as _,
+            &nows,
+            &user_ids,
+        )
+        .execute(exec)
+        .await?;
+
+        Ok(())
+    }
+
+    pub async fn insert_files(
+        exec: impl sqlx::PgExecutor<'_>,
+        hashes: &[Vec<u8>],
+        filenames: &[Option<String>],
+        license_ids: &[i64],
+        user_id: DBUserId,
+    ) -> sqlx::Result<()> {
+        let now = Utc::now();
+        let nows: Vec<DateTime<Utc>> = vec![now; license_ids.len()];
+        let user_ids: Vec<i64> = vec![user_id.0; license_ids.len()];
+
+        let filenames: Vec<Option<String>> = filenames.to_vec();
+
+        sqlx::query!(
+            r#"
+            INSERT INTO moderation_external_files (sha1, filename, external_license_id, inserted_at, inserted_by, updated_at, updated_by)
+            SELECT * FROM UNNEST ($1::bytea[], $2::varchar[], $3::bigint[], $4::timestamptz[], $5::bigint[], $4::timestamptz[], $5::bigint[])
+            ON CONFLICT (sha1) DO UPDATE SET
+                filename = COALESCE(EXCLUDED.filename, moderation_external_files.filename),
+                external_license_id = EXCLUDED.external_license_id,
+                updated_at = EXCLUDED.updated_at,
+                updated_by = EXCLUDED.updated_by
+            "#,
+            hashes,
+            &filenames as _,
+            license_ids,
+            &nows,
+            &user_ids,
+        )
+        .execute(exec)
+        .await?;
+
+        Ok(())
+    }
+}

apps/labrinth/src/queue/moderation.rs

@@ -1,6 +1,7 @@
 use crate::auth::checks::filter_visible_versions;
 use crate::database;
 use crate::database::PgPool;
+use crate::database::models::DBUserId;
 use crate::database::models::notification_item::NotificationBuilder;
 use crate::database::models::thread_item::ThreadMessageBuilder;
 use crate::database::redis::RedisPool;
@@ -507,6 +508,7 @@ impl AutomatedModerationQueue {
                                         .fetch_all(&pool).await?;
 
                                     let mut insert_hashes = Vec::new();
+                                    let mut insert_filenames = Vec::new();
                                     let mut insert_ids = Vec::new();
 
                                     for row in rows {
@@ -518,6 +520,7 @@ impl AutomatedModerationQueue {
                                                 });
 
                                                 insert_hashes.push(hash.clone().as_bytes().to_vec());
+                                                insert_filenames.push(Some(file_name.clone()));
                                                 insert_ids.push(row.id);
 
                                                 hashes.remove(index);
@@ -526,16 +529,13 @@ impl AutomatedModerationQueue {
                                     }
 
                                     if !insert_ids.is_empty() && !insert_hashes.is_empty() {
-                                        sqlx::query!(
-                                            "
-                                            INSERT INTO moderation_external_files (sha1, external_license_id)
-                                            SELECT * FROM UNNEST ($1::bytea[], $2::bigint[])
-                                            ON CONFLICT (sha1) DO NOTHING
-                                            ",
-                                            &insert_hashes[..],
-                                            &insert_ids[..]
+                                        crate::database::models::moderation_external_item::ExternalLicense::insert_files(
+                                            &pool,
+                                            &insert_hashes,
+                                            &insert_filenames,
+                                            &insert_ids,
+                                            DBUserId(0),
                                         )
-                                            .execute(&pool)
                                             .await?;
                                     }
 

apps/labrinth/src/routes/internal/moderation/external_license.rs

@@ -0,0 +1,334 @@
+use std::collections::HashMap;
+
+use actix_web::{HttpRequest, get, patch, post, web};
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+
+use crate::database::PgPool;
+use crate::database::redis::RedisPool;
+use crate::models::pats::Scopes;
+use crate::queue::moderation::ApprovalType;
+use crate::routes::ApiError;
+use crate::{auth::check_is_moderator_from_headers, queue::session::AuthQueue};
+
+pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
+    cfg.service(search)
+        .service(get_by_sha1)
+        .service(update_license);
+}
+
+#[derive(Serialize, Deserialize, utoipa::ToSchema)]
+pub struct ExternalProject {
+    pub id: i64,
+    pub title: Option<String>,
+    pub status: ApprovalType,
+    pub link: Option<String>,
+    pub exceptions: Option<String>,
+    pub proof: Option<String>,
+    pub flame_project_id: Option<i32>,
+    pub inserted_at: Option<DateTime<Utc>>,
+    pub inserted_by: Option<i64>,
+    pub updated_at: Option<DateTime<Utc>>,
+    pub updated_by: Option<i64>,
+    pub linked_files: Vec<LinkedFile>,
+}
+
+#[derive(Serialize, Deserialize, Clone, utoipa::ToSchema)]
+pub struct LinkedFile {
+    pub name: Option<String>,
+    pub sha1: String,
+}
+
+#[derive(Deserialize, utoipa::ToSchema)]
+pub struct SearchRequest {
+    pub title: Option<String>,
+    pub flame_id: Option<i32>,
+}
+
+#[derive(Deserialize, utoipa::ToSchema)]
+pub struct UpdateLicenseRequest {
+    pub title: Option<String>,
+    pub status: ApprovalType,
+    pub link: Option<String>,
+    pub exceptions: Option<String>,
+    pub proof: Option<String>,
+    pub flame_project_id: Option<i32>,
+}
+
+struct LicenseRow {
+    id: i64,
+    title: Option<String>,
+    status: String,
+    link: Option<String>,
+    exceptions: Option<String>,
+    proof: Option<String>,
+    flame_project_id: Option<i32>,
+    inserted_at: Option<DateTime<Utc>>,
+    inserted_by: Option<i64>,
+    updated_at: Option<DateTime<Utc>>,
+    updated_by: Option<i64>,
+}
+
+impl LicenseRow {
+    fn into_external_project(
+        self,
+        linked_files: Vec<LinkedFile>,
+    ) -> ExternalProject {
+        ExternalProject {
+            id: self.id,
+            title: self.title,
+            status: ApprovalType::from_string(&self.status)
+                .unwrap_or(ApprovalType::Unidentified),
+            link: self.link,
+            exceptions: self.exceptions,
+            proof: self.proof,
+            flame_project_id: self.flame_project_id,
+            inserted_at: self.inserted_at,
+            inserted_by: self.inserted_by,
+            updated_at: self.updated_at,
+            updated_by: self.updated_by,
+            linked_files,
+        }
+    }
+}
+
+async fn fetch_linked_files(
+    pool: &PgPool,
+    license_ids: &[i64],
+) -> Result<HashMap<i64, Vec<LinkedFile>>, ApiError> {
+    if license_ids.is_empty() {
+        return Ok(HashMap::new());
+    }
+
+    let file_rows = sqlx::query!(
+        r#"
+        SELECT
+            mef.external_license_id,
+            mef.sha1,
+            mef.filename
+        FROM moderation_external_files mef
+        WHERE mef.external_license_id = ANY($1)
+        "#,
+        license_ids,
+    )
+    .fetch_all(pool)
+    .await?;
+
+    let mut map: HashMap<i64, Vec<LinkedFile>> = HashMap::new();
+    for row in file_rows {
+        map.entry(row.external_license_id)
+            .or_default()
+            .push(LinkedFile {
+                name: row.filename,
+                sha1: hex::encode(&row.sha1),
+            });
+    }
+    Ok(map)
+}
+
+#[utoipa::path]
+#[post("/search")]
+async fn search(
+    req: HttpRequest,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+    body: web::Json<SearchRequest>,
+) -> Result<web::Json<Vec<ExternalProject>>, ApiError> {
+    check_is_moderator_from_headers(
+        &req,
+        &**pool,
+        &redis,
+        &session_queue,
+        Scopes::PROJECT_READ,
+    )
+    .await?;
+
+    let rows = sqlx::query!(
+        r#"
+        SELECT
+            mel.id,
+            mel.title,
+            mel.status,
+            mel.link,
+            mel.exceptions,
+            mel.proof,
+            mel.flame_project_id,
+            mel.inserted_at,
+            mel.inserted_by,
+            mel.updated_at,
+            mel.updated_by
+        FROM moderation_external_licenses mel
+        WHERE ($1::text IS NULL OR mel.title ILIKE '%' || $1 || '%')
+          AND ($2::integer IS NULL OR mel.flame_project_id = $2)
+        ORDER BY mel.id
+        "#,
+        body.title,
+        body.flame_id,
+    )
+    .fetch_all(&**pool)
+    .await?;
+
+    let license_ids: Vec<i64> = rows.iter().map(|r| r.id).collect();
+    let files_map = fetch_linked_files(&pool, &license_ids).await?;
+
+    let results = rows
+        .into_iter()
+        .map(|row| {
+            let linked_files =
+                files_map.get(&row.id).cloned().unwrap_or_default();
+            LicenseRow {
+                id: row.id,
+                title: row.title,
+                status: row.status,
+                link: row.link,
+                exceptions: row.exceptions,
+                proof: row.proof,
+                flame_project_id: row.flame_project_id,
+                inserted_at: row.inserted_at,
+                inserted_by: row.inserted_by,
+                updated_at: row.updated_at,
+                updated_by: row.updated_by,
+            }
+            .into_external_project(linked_files)
+        })
+        .collect();
+
+    Ok(web::Json(results))
+}
+
+#[utoipa::path]
+#[get("/by-sha1/{sha1}")]
+async fn get_by_sha1(
+    req: HttpRequest,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+    path: web::Path<(String,)>,
+) -> Result<web::Json<ExternalProject>, ApiError> {
+    check_is_moderator_from_headers(
+        &req,
+        &**pool,
+        &redis,
+        &session_queue,
+        Scopes::PROJECT_READ,
+    )
+    .await?;
+
+    let sha1 = path.into_inner().0;
+
+    let row = sqlx::query!(
+        r#"
+        SELECT
+            mel.id,
+            mel.title,
+            mel.status,
+            mel.link,
+            mel.exceptions,
+            mel.proof,
+            mel.flame_project_id,
+            mel.inserted_at,
+            mel.inserted_by,
+            mel.updated_at,
+            mel.updated_by
+        FROM moderation_external_files mef
+        INNER JOIN moderation_external_licenses mel ON mel.id = mef.external_license_id
+        WHERE mef.sha1 = $1
+        "#,
+        sha1.as_bytes().to_vec(),
+    )
+    .fetch_optional(&**pool)
+    .await?
+    .ok_or(ApiError::NotFound)?;
+
+    let files_map = fetch_linked_files(&pool, &[row.id]).await?;
+    let linked_files = files_map.get(&row.id).cloned().unwrap_or_default();
+
+    Ok(web::Json(
+        LicenseRow {
+            id: row.id,
+            title: row.title,
+            status: row.status,
+            link: row.link,
+            exceptions: row.exceptions,
+            proof: row.proof,
+            flame_project_id: row.flame_project_id,
+            inserted_at: row.inserted_at,
+            inserted_by: row.inserted_by,
+            updated_at: row.updated_at,
+            updated_by: row.updated_by,
+        }
+        .into_external_project(linked_files),
+    ))
+}
+
+#[utoipa::path]
+#[patch("/{id}")]
+async fn update_license(
+    req: HttpRequest,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+    path: web::Path<(i64,)>,
+    body: web::Json<UpdateLicenseRequest>,
+) -> Result<web::Json<ExternalProject>, ApiError> {
+    let user = check_is_moderator_from_headers(
+        &req,
+        &**pool,
+        &redis,
+        &session_queue,
+        Scopes::PROJECT_READ,
+    )
+    .await?;
+
+    let id = path.into_inner().0;
+
+    let result = sqlx::query!(
+        r#"
+        UPDATE moderation_external_licenses
+        SET title = COALESCE($2, title),
+            status = $3,
+            link = COALESCE($4, link),
+            exceptions = COALESCE($5, exceptions),
+            proof = COALESCE($6, proof),
+            flame_project_id = COALESCE($7, flame_project_id),
+            updated_at = $8,
+            updated_by = $9
+        WHERE id = $1
+        RETURNING id, title, status, link, exceptions, proof, flame_project_id,
+                  inserted_at, inserted_by, updated_at, updated_by
+        "#,
+        id,
+        body.title,
+        body.status.as_str(),
+        body.link,
+        body.exceptions,
+        body.proof,
+        body.flame_project_id,
+        Utc::now(),
+        user.id.0 as i64,
+    )
+    .fetch_optional(&**pool)
+    .await?
+    .ok_or(ApiError::NotFound)?;
+
+    let files_map = fetch_linked_files(&pool, &[id]).await?;
+    let linked_files = files_map.get(&id).cloned().unwrap_or_default();
+
+    Ok(web::Json(
+        LicenseRow {
+            id: result.id,
+            title: result.title,
+            status: result.status,
+            link: result.link,
+            exceptions: result.exceptions,
+            proof: result.proof,
+            flame_project_id: result.flame_project_id,
+            inserted_at: result.inserted_at,
+            inserted_by: result.inserted_by,
+            updated_at: result.updated_at,
+            updated_by: result.updated_by,
+        }
+        .into_external_project(linked_files),
+    ))
+}

apps/labrinth/src/routes/internal/moderation/mod.rs

@@ -3,6 +3,7 @@ use crate::auth::get_user_from_headers;
 use crate::database;
 use crate::database::PgPool;
 use crate::database::models::DBModerationLock;
+use crate::database::models::moderation_external_item;
 use crate::database::redis::RedisPool;
 use crate::models::ids::OrganizationId;
 use crate::models::projects::{Project, ProjectStatus};
@@ -20,6 +21,7 @@ use ownership::get_projects_ownership;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
+mod external_license;
 mod ownership;
 mod tech_review;
 
@@ -36,6 +38,10 @@ pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
         .service(
             utoipa_actix_web::scope("/tech-review")
                 .configure(tech_review::config),
+        )
+        .service(
+            utoipa_actix_web::scope("/external-license")
+                .configure(external_license::config),
         );
 }
 
@@ -412,7 +418,7 @@ async fn set_project_meta(
     session_queue: web::Data<AuthQueue>,
     judgements: web::Json<HashMap<String, Judgement>>,
 ) -> Result<(), ApiError> {
-    check_is_moderator_from_headers(
+    let user = check_is_moderator_from_headers(
         &req,
         &**pool,
         &redis,
@@ -423,14 +429,10 @@ async fn set_project_meta(
 
     let mut transaction = pool.begin().await?;
 
-    let mut ids = Vec::new();
-    let mut titles = Vec::new();
-    let mut statuses = Vec::new();
-    let mut links = Vec::new();
-    let mut proofs = Vec::new();
-    let mut flame_ids = Vec::new();
-
+    let mut licenses = Vec::new();
     let mut file_hashes = Vec::new();
+    let mut file_filenames = Vec::new();
+    let mut file_license_ids = Vec::new();
 
     for (hash, judgement) in judgements.0 {
         let id = random_base62(8);
@@ -456,41 +458,38 @@ async fn set_project_meta(
             } => (title, status, link, proof, None),
         };
 
-        ids.push(id as i64);
-        titles.push(title);
-        statuses.push(status.as_str());
-        links.push(link);
-        proofs.push(proof);
-        flame_ids.push(flame_id);
+        licenses.push(moderation_external_item::ExternalLicense {
+            id: id as i64,
+            title,
+            status: status.as_str().to_string(),
+            link,
+            proof,
+            flame_project_id: flame_id,
+        });
         file_hashes.push(hash);
+        file_filenames.push(None);
+        file_license_ids.push(id as i64);
     }
 
-    sqlx::query(
-    "
-        INSERT INTO moderation_external_licenses (id, title, status, link, proof, flame_project_id)
-        SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::varchar[], $4::varchar[], $5::varchar[], $6::integer[])
-        "
-    )
-        .bind(&ids[..])
-        .bind(&titles[..])
-        .bind(&statuses[..])
-        .bind(&links[..])
-        .bind(&proofs[..])
-        .bind(&flame_ids[..])
-        .execute(&mut transaction)
-        .await?;
+    let user_id = database::models::ids::DBUserId::from(user.id);
 
-    sqlx::query(
-        "
-        INSERT INTO moderation_external_files (sha1, external_license_id)
-        SELECT * FROM UNNEST ($1::bytea[], $2::bigint[])
-        ON CONFLICT (sha1)
-        DO NOTHING
-        ",
+    moderation_external_item::ExternalLicense::insert_many(
+        &mut transaction,
+        &licenses,
+        user_id,
+    )
+    .await?;
+
+    moderation_external_item::ExternalLicense::insert_files(
+        &mut transaction,
+        &file_hashes
+            .iter()
+            .map(|x| x.as_bytes().to_vec())
+            .collect::<Vec<_>>(),
+        &file_filenames,
+        &file_license_ids,
+        user_id,
     )
-    .bind(&file_hashes[..])
-    .bind(&ids[..])
-    .execute(&mut transaction)
     .await?;
 
     transaction.commit().await?;