Search backend refactor with typesense impl (#5528)

* initial elasticsearch impl

* working elastic cluster

* replace SearchError with ApiError for preparation of search backend

* start factoring meili out to trait

* move meili to backend

* update routes to use search backend trait

* wip

* Update projects.rs

* search backend is only init'd once in config

* wip

* wip: backend agnostic

* change search internal routes to delegate to backend

* initial elasticsearch impl

* fix filtering

* elastic impl

* refactor indexing into its own module

* clean up elastic code

* fix ci

* fix tests

* fix elastic health check

* fix up env rebase

* fix compile

* dummy commit to update github pr

* Fix rebase

* Elastic basic https auth

* Fix duplicate projects showing up

* Fix up tests

* Replace search `ApiErrors` with `eyre::Reports`, propagate background task errors

* clean up agents files

* make index chunk size configurable

* make `match_phrase` in elastic case-insensitive

* use current/next indices and swap between them

* test case for error body

* Fix failing case

* da merge

* factor out common stuff from search backends

* allow fetching hit metadata from search results

* allow customising elasticsearch search config

* bit of docs

* add mappings to indices for elastic

* Implement Typesense

* wip

* fix up some sort fields stuff

* use different approach to filterable field sets

* remove a bunch of search fields which weren't used for filtering

* bucket text matches

* Bucketing by text_match for typesense

* fix tombi lint

* fix some sentry errors and dont prioritise 2+ term matches

* tweak ts query settings

* expose some more search settings

* query sort changes

* small fixes

* should fix pagination stuff

* fix healthcheck maybe

* ragebait ci

* tests

* tests

* revert environment

Cargo.lock

@@ -2714,6 +2714,29 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "elasticsearch"
+version = "9.1.0-alpha.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12bb303aa6e1d28c0c86b6fbfe484fd0fd3f512629aeed1ac4f6b85f81d9834a"
+dependencies = [
+ "base64 0.22.1",
+ "bytes",
+ "dyn-clone",
+ "flate2",
+ "lazy_static",
+ "parking_lot",
+ "percent-encoding",
+ "reqwest 0.12.24",
+ "rustc_version",
+ "serde",
+ "serde_json",
+ "serde_with",
+ "tokio",
+ "url",
+ "void",
+]
+
 [[package]]
 name = "elliptic-curve"
 version = "0.13.8"
@@ -4895,6 +4918,7 @@ dependencies = [
  "dotenv-build",
  "dotenvy",
  "either",
+ "elasticsearch",
  "eyre",
  "futures",
  "futures-util",

Cargo.toml

@@ -72,6 +72,7 @@ dotenv-build = "0.1.1"
 dotenvy = "0.15.7"
 dunce = "1.0.5"
 either = "1.15.0"
+elasticsearch = "9.1.0-alpha.1"
 encoding_rs = "0.8.35"
 enumset = "1.1.10"
 eyre = "0.6.12"

apps/labrinth/.env.docker-compose

@@ -16,9 +16,18 @@ DATABASE_URL=postgresql://labrinth:labrinth@labrinth-postgres/labrinth
 DATABASE_MIN_CONNECTIONS=0
 DATABASE_MAX_CONNECTIONS=16
 
+SEARCH_BACKEND=typesense
 MEILISEARCH_READ_ADDR=http://localhost:7700
 MEILISEARCH_WRITE_ADDRS=http://localhost:7700
 MEILISEARCH_KEY=modrinth
+ELASTICSEARCH_URL=http://localhost:9200
+ELASTICSEARCH_INDEX_PREFIX=labrinth
+ELASTICSEARCH_USERNAME=elastic
+ELASTICSEARCH_PASSWORD=elastic
+SEARCH_INDEX_CHUNK_SIZE=5000
+TYPESENSE_URL=http://localhost:8108
+TYPESENSE_API_KEY=modrinth
+TYPESENSE_INDEX_PREFIX=labrinth
 
 REDIS_URL=redis://labrinth-redis
 REDIS_MIN_CONNECTIONS=0

apps/labrinth/.env.local

@@ -16,16 +16,36 @@ DATABASE_URL=postgresql://labrinth:labrinth@localhost/labrinth
 DATABASE_MIN_CONNECTIONS=0
 DATABASE_MAX_CONNECTIONS=16
 
+SEARCH_BACKEND=meilisearch
+
+# Meilisearch configuration
 MEILISEARCH_READ_ADDR=http://localhost:7700
 MEILISEARCH_WRITE_ADDRS=http://localhost:7700
 # 5 minutes in milliseconds
 SEARCH_OPERATION_TIMEOUT=300000
 
+ELASTICSEARCH_URL=http://localhost:9200
+ELASTICSEARCH_INDEX_PREFIX=labrinth
+
 # # For a sharded Meilisearch setup (sharded-meilisearch docker compose profile)
 # MEILISEARCH_READ_ADDR=http://localhost:7710
 # MEILISEARCH_WRITE_ADDRS=http://localhost:7700,http://localhost:7701
 
+SEARCH_BACKEND=typesense
+
 MEILISEARCH_KEY=modrinth
+MEILISEARCH_META_NAMESPACE=
+
+# Elasticsearch configuration
+ELASTICSEARCH_URL=http://localhost:9200
+ELASTICSEARCH_INDEX_PREFIX=labrinth
+ELASTICSEARCH_USERNAME=
+ELASTICSEARCH_PASSWORD=
+
+SEARCH_INDEX_CHUNK_SIZE=5000
+TYPESENSE_URL=http://localhost:8108
+TYPESENSE_API_KEY=modrinth
+TYPESENSE_INDEX_PREFIX=labrinth
 
 REDIS_URL=redis://localhost
 REDIS_MIN_CONNECTIONS=0

apps/labrinth/Cargo.toml

@@ -26,7 +26,7 @@ async-stripe = { workspace = true, features = [
   "billing",
   "checkout",
   "connect",
-  "webhook-events",
+  "webhook-events"
 ] }
 async-trait = { workspace = true }
 base64 = { workspace = true }
@@ -44,6 +44,7 @@ deadpool-redis.workspace = true
 derive_more = { workspace = true, features = ["deref", "deref_mut"] }
 dotenvy = { workspace = true }
 either = { workspace = true }
+elasticsearch = { workspace = true, features = ["experimental-apis"] }
 eyre = { workspace = true }
 futures = { workspace = true }
 futures-util = { workspace = true }
@@ -86,11 +87,11 @@ reqwest = { workspace = true, features = [
   "http2",
   "json",
   "multipart",
-  "rustls-tls-webpki-roots",
+  "rustls-tls-webpki-roots"
 ] }
 rust_decimal = { workspace = true, features = [
   "serde-with-float",
-  "serde-with-str",
+  "serde-with-str"
 ] }
 rust_iso3166 = { workspace = true }
 rust-s3 = { workspace = true }
@@ -114,7 +115,7 @@ sqlx = { workspace = true, features = [
   "tls-rustls-aws-lc-rs",
 ] }
 sqlx-tracing = { workspace = true, features = ["postgres"] }
-strum = { workspace = true }
+strum = { workspace = true, features = ["derive"] }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "sync"] }
 tokio-stream = { workspace = true }

apps/labrinth/src/auth/mod.rs

@@ -20,8 +20,6 @@ use thiserror::Error;
 pub enum AuthenticationError {
     #[error(transparent)]
     Internal(#[from] eyre::Report),
-    #[error("Environment Error")]
-    Env(#[from] dotenvy::Error),
     #[error("An unknown database error occurred: {0}")]
     Sqlx(#[from] sqlx::Error),
     #[error("Database Error: {0}")]
@@ -58,7 +56,6 @@ impl actix_web::ResponseError for AuthenticationError {
             AuthenticationError::Internal(..) => {
                 StatusCode::INTERNAL_SERVER_ERROR
             }
-            AuthenticationError::Env(..) => StatusCode::INTERNAL_SERVER_ERROR,
             AuthenticationError::Sqlx(..) => StatusCode::INTERNAL_SERVER_ERROR,
             AuthenticationError::Database(..) => {
                 StatusCode::INTERNAL_SERVER_ERROR
@@ -94,7 +91,6 @@ impl AuthenticationError {
     pub fn error_name(&self) -> &'static str {
         match self {
             AuthenticationError::Internal(..) => "internal_error",
-            AuthenticationError::Env(..) => "environment_error",
             AuthenticationError::Sqlx(..) => "database_error",
             AuthenticationError::Database(..) => "database_error",
             AuthenticationError::SerDe(..) => "invalid_input",

apps/labrinth/src/background_task.rs

@@ -1,3 +1,4 @@
+use crate::database;
 use crate::database::PgPool;
 use crate::database::redis::RedisPool;
 use crate::queue::analytics::cache::cache_analytics;
@@ -8,9 +9,9 @@ use crate::queue::payouts::{
     insert_bank_balances_and_webhook, process_affiliate_payouts,
     process_payout, remove_payouts_for_refunded_charges,
 };
-use crate::search::indexing::index_projects;
+use crate::search::SearchBackend;
 use crate::util::anrok;
-use crate::{database, search};
+use actix_web::web;
 use clap::ValueEnum;
 use eyre::WrapErr;
 use tracing::info;
@@ -42,7 +43,7 @@ impl BackgroundTask {
         pool: PgPool,
         ro_pool: PgPool,
         redis_pool: RedisPool,
-        search_config: search::SearchConfig,
+        search_backend: web::Data<dyn SearchBackend>,
         clickhouse: clickhouse::Client,
         stripe_client: stripe::Client,
         anrok_client: anrok::Client,
@@ -53,7 +54,7 @@ impl BackgroundTask {
         match self {
             Migrations => run_migrations().await,
             IndexSearch => {
-                index_search(ro_pool, redis_pool, search_config).await
+                index_search(ro_pool, redis_pool, search_backend).await
             }
             ReleaseScheduled => release_scheduled(pool).await,
             UpdateVersions => update_versions(pool, redis_pool).await,
@@ -132,10 +133,10 @@ pub async fn run_migrations() -> eyre::Result<()> {
 pub async fn index_search(
     ro_pool: PgPool,
     redis_pool: RedisPool,
-    search_config: search::SearchConfig,
+    search_backend: web::Data<dyn SearchBackend>,
 ) -> eyre::Result<()> {
     info!("Indexing local database");
-    index_projects(ro_pool, redis_pool, &search_config).await
+    search_backend.index_projects(ro_pool, redis_pool).await
 }
 
 pub async fn release_scheduled(pool: PgPool) -> eyre::Result<()> {

apps/labrinth/src/env.rs

@@ -82,6 +82,7 @@ where
 }
 
 pub fn init() -> eyre::Result<()> {
+    dotenvy::dotenv().ok();
     EnvVars::from_env()?;
     LazyLock::force(&ENV);
     Ok(())
@@ -128,9 +129,6 @@ vars! {
     LABRINTH_EXTERNAL_NOTIFICATION_KEY: String;
     RATE_LIMIT_IGNORE_KEY: String;
     DATABASE_URL: String;
-    MEILISEARCH_READ_ADDR: String;
-    MEILISEARCH_WRITE_ADDRS: StringCsv;
-    MEILISEARCH_KEY: String;
     REDIS_URL: String;
     BIND_ADDR: String;
     SELF_ADDR: String;
@@ -142,6 +140,20 @@ vars! {
     ALLOWED_CALLBACK_URLS: Json<Vec<String>>;
     ANALYTICS_ALLOWED_ORIGINS: Json<Vec<String>>;
 
+    // search
+    SEARCH_BACKEND: crate::search::SearchBackendKind = crate::search::SearchBackendKind::Typesense;
+    MEILISEARCH_READ_ADDR: String;
+    MEILISEARCH_WRITE_ADDRS: StringCsv;
+    MEILISEARCH_KEY: String;
+    ELASTICSEARCH_URL: String;
+    ELASTICSEARCH_INDEX_PREFIX: String;
+    ELASTICSEARCH_USERNAME: String = "";
+    ELASTICSEARCH_PASSWORD: String = "";
+    SEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64;
+    TYPESENSE_URL: String = "http://localhost:8108";
+    TYPESENSE_API_KEY: String = "modrinth";
+    TYPESENSE_INDEX_PREFIX: String = "labrinth";
+
     // storage
     STORAGE_BACKEND: crate::file_hosting::FileHostKind;
 

apps/labrinth/src/lib.rs

@@ -58,7 +58,7 @@ pub struct LabrinthConfig {
     pub file_host: Arc<dyn file_hosting::FileHost + Send + Sync>,
     pub scheduler: Arc<scheduler::Scheduler>,
     pub ip_salt: Pepper,
-    pub search_config: search::SearchConfig,
+    pub search_backend: web::Data<dyn search::SearchBackend>,
     pub session_queue: web::Data<AuthQueue>,
     pub payouts_queue: web::Data<PayoutsQueue>,
     pub analytics_queue: Arc<AnalyticsQueue>,
@@ -78,7 +78,7 @@ pub fn app_setup(
     pool: PgPool,
     ro_pool: ReadOnlyPgPool,
     redis_pool: RedisPool,
-    search_config: search::SearchConfig,
+    search_backend: actix_web::web::Data<dyn search::SearchBackend>,
     clickhouse: &mut Client,
     file_host: Arc<dyn file_hosting::FileHost + Send + Sync>,
     stripe_client: stripe::Client,
@@ -129,21 +129,21 @@ pub fn app_setup(
         let local_index_interval =
             Duration::from_secs(ENV.LOCAL_INDEX_INTERVAL);
         let pool_ref = pool.clone();
-        let search_config_ref = search_config.clone();
         let redis_pool_ref = redis_pool.clone();
+        let search_backend_ref = search_backend.clone();
         scheduler.run(local_index_interval, move || {
             let pool_ref = pool_ref.clone();
             let redis_pool_ref = redis_pool_ref.clone();
-            let search_config_ref = search_config_ref.clone();
+            let search_backend = search_backend_ref.clone();
             async move {
-                if let Err(e) = background_task::index_search(
+                if let Err(err) = background_task::index_search(
                     pool_ref,
                     redis_pool_ref,
-                    search_config_ref,
+                    search_backend,
                 )
                 .await
                 {
-                    warn!("Local project indexing failed: {e:#}");
+                    warn!("Failed to index search: {err:?}");
                 }
             }
         });
@@ -299,7 +299,7 @@ pub fn app_setup(
         file_host,
         scheduler: Arc::new(scheduler),
         ip_salt,
-        search_config,
+        search_backend,
         session_queue,
         payouts_queue: web::Data::new(PayoutsQueue::new()),
         analytics_queue,
@@ -338,7 +338,7 @@ pub fn app_config(
     .app_data(web::Data::new(labrinth_config.pool.clone()))
     .app_data(web::Data::new(labrinth_config.ro_pool.clone()))
     .app_data(web::Data::new(labrinth_config.file_host.clone()))
-    .app_data(web::Data::new(labrinth_config.search_config.clone()))
+    .app_data(labrinth_config.search_backend.clone())
     .app_data(web::Data::new(labrinth_config.gotenberg_client.clone()))
     .app_data(labrinth_config.http_client.clone())
     .app_data(labrinth_config.session_queue.clone())

apps/labrinth/src/main.rs

@@ -56,7 +56,6 @@ struct Args {
 
 fn main() -> std::io::Result<()> {
     color_eyre::install().expect("failed to install `color-eyre`");
-    dotenvy::dotenv().ok();
     modrinth_util::log::init().expect("failed to initialize logging");
     env::init().expect("failed to initialize environment variables");
 
@@ -152,7 +151,8 @@ async fn app() -> std::io::Result<()> {
     info!("Initializing clickhouse connection");
     let mut clickhouse = clickhouse::init_client().await.unwrap();
 
-    let search_config = search::SearchConfig::new(None);
+    let search_backend =
+        actix_web::web::Data::from(Arc::from(search::backend(None)));
 
     let stripe_client = stripe::Client::new(ENV.STRIPE_API_KEY.clone());
 
@@ -171,7 +171,7 @@ async fn app() -> std::io::Result<()> {
             pool,
             ro_pool.into_inner(),
             redis_pool,
-            search_config,
+            search_backend,
             clickhouse,
             stripe_client,
             anrok_client.clone(),
@@ -207,7 +207,7 @@ async fn app() -> std::io::Result<()> {
         pool.clone(),
         ro_pool.clone(),
         redis_pool.clone(),
-        search_config.clone(),
+        search_backend.clone(),
         &mut clickhouse,
         file_host.clone(),
         stripe_client,

apps/labrinth/src/models/v2/search.rs

@@ -159,15 +159,20 @@ impl LegacySearchResults {
     pub fn from(search_results: crate::search::SearchResults) -> Self {
         let limit = search_results.hits_per_page;
         let offset = (search_results.page - 1) * limit;
+        let crate::search::SearchResults {
+            hits,
+            page: _,
+            hits_per_page: _,
+            total_hits,
+        } = search_results;
         Self {
-            hits: search_results
+            hits: hits
-                .hits
                 .into_iter()
                 .map(LegacyResultSearchProject::from)
                 .collect(),
             offset,
             limit,
-            total_hits: search_results.total_hits,
+            total_hits,
         }
     }
 }

apps/labrinth/src/models/v3/projects.rs

@@ -1020,20 +1020,3 @@ impl FileType {
 )]
 #[serde(transparent)]
 pub struct Loader(pub String);
-
-// These fields must always succeed parsing; deserialize errors aren't
-// processed correctly (don't return JSON errors)
-#[derive(Serialize, Deserialize, Debug)]
-pub struct SearchRequest {
-    pub query: Option<String>,
-    pub offset: Option<String>,
-    pub index: Option<String>,
-    pub limit: Option<String>,
-
-    pub new_filters: Option<String>,
-
-    // TODO: Deprecated values below. WILL BE REMOVED V3!
-    pub facets: Option<String>,
-    pub filters: Option<String>,
-    pub version: Option<String>,
-}

apps/labrinth/src/queue/email.rs

@@ -102,8 +102,6 @@ impl Mailer {
 
 #[derive(Error, Debug)]
 pub enum MailError {
-    #[error("Environment Error")]
-    Env(#[from] dotenvy::Error),
     #[error("Mail Error: {0}")]
     Mail(#[from] lettre::error::Error),
     #[error("Address Parse Error: {0}")]
@@ -136,7 +134,7 @@ impl EmailQueue {
             pg,
             redis,
             mailer: Arc::new(TokioMutex::new(Mailer::Uninitialized)),
-            identity: templates::MailingIdentity::from_env()?,
+            identity: templates::MailingIdentity::from_env(),
             client: Client::builder()
                 .user_agent("Modrinth")
                 .build()

apps/labrinth/src/queue/email/templates.rs

@@ -95,8 +95,8 @@ pub struct MailingIdentity {
 }
 
 impl MailingIdentity {
-    pub fn from_env() -> dotenvy::Result<Self> {
+    pub fn from_env() -> Self {
-        Ok(Self {
+        Self {
             from_name: ENV.SMTP_FROM_NAME.clone(),
             from_address: ENV.SMTP_FROM_ADDRESS.clone(),
             reply_name: if ENV.SMTP_REPLY_TO_NAME.is_empty() {
@@ -109,7 +109,7 @@ impl MailingIdentity {
             } else {
                 Some(ENV.SMTP_REPLY_TO_ADDRESS.clone())
             },
-        })
+        }
     }
 }
 

apps/labrinth/src/routes/internal/admin.rs

@@ -7,7 +7,7 @@ use crate::models::pats::Scopes;
 use crate::queue::analytics::AnalyticsQueue;
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
-use crate::search::SearchConfig;
+use crate::search::SearchBackend;
 use crate::util::date::get_current_tenths_of_ms;
 use crate::util::error::Context;
 use crate::util::guards::admin_key_guard;
@@ -154,11 +154,11 @@ pub async fn count_download(
 pub async fn force_reindex(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
-    config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
 ) -> Result<HttpResponse, ApiError> {
-    use crate::search::indexing::index_projects;
     let redis = redis.get_ref();
-    index_projects(pool.as_ref().clone(), redis.clone(), &config)
+    search_backend
+        .index_projects(pool.as_ref().clone(), redis.clone())
         .await
         .wrap_internal_err("failed to index projects")?;
     Ok(HttpResponse::NoContent().finish())

apps/labrinth/src/routes/internal/search.rs

@@ -1,12 +1,9 @@
-use crate::routes::ApiError;
-use crate::search::SearchConfig;
 use crate::util::guards::admin_key_guard;
-use actix_web::{HttpResponse, delete, get, web};
+use crate::{
-use meilisearch_sdk::tasks::{Task, TasksCancelQuery};
+    routes::ApiError,
-use serde::{Deserialize, Serialize};
+    search::{SearchBackend, TasksCancelFilter},
-use std::collections::HashMap;
+};
-use std::time::Duration;
+use actix_web::{delete, get, web};
-use utoipa::ToSchema;
 
 pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
     cfg.service(tasks).service(tasks_cancel);
@@ -15,107 +12,20 @@ pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
 #[utoipa::path]
 #[get("tasks", guard = "admin_key_guard")]
 pub async fn tasks(
-    config: web::Data<SearchConfig>,
+    search: web::Data<dyn SearchBackend>,
-) -> Result<HttpResponse, ApiError> {
+) -> Result<web::Json<serde_json::Value>, ApiError> {
-    let client = config.make_batch_client()?;
+    Ok(web::Json(search.tasks().await.map_err(ApiError::Internal)?))
-    let tasks = client
-        .with_all_clients("get_tasks", async |client| {
-            let tasks = client.get_tasks().await?;
-
-            Ok(tasks.results)
-        })
-        .await?;
-
-    #[derive(Serialize, ToSchema)]
-    struct MeiliTask<Time> {
-        uid: u32,
-        status: &'static str,
-        duration: Option<Duration>,
-        enqueued_at: Option<Time>,
-    }
-
-    #[derive(Serialize, ToSchema)]
-    struct TaskList<Time> {
-        by_instance: HashMap<String, Vec<MeiliTask<Time>>>,
-    }
-
-    let response = tasks
-        .into_iter()
-        .enumerate()
-        .map(|(idx, instance_tasks)| {
-            let tasks = instance_tasks
-                .into_iter()
-                .filter_map(|task| {
-                    Some(match task {
-                        Task::Enqueued { content } => MeiliTask {
-                            uid: content.uid,
-                            status: "enqueued",
-                            duration: None,
-                            enqueued_at: Some(content.enqueued_at),
-                        },
-                        Task::Processing { content } => MeiliTask {
-                            uid: content.uid,
-                            status: "processing",
-                            duration: None,
-                            enqueued_at: Some(content.enqueued_at),
-                        },
-                        Task::Failed { content } => MeiliTask {
-                            uid: content.task.uid,
-                            status: "failed",
-                            duration: Some(content.task.duration),
-                            enqueued_at: Some(content.task.enqueued_at),
-                        },
-                        Task::Succeeded { content: _ } => return None,
-                    })
-                })
-                .collect();
-
-            (idx.to_string(), tasks)
-        })
-        .collect::<HashMap<String, Vec<MeiliTask<_>>>>();
-
-    Ok(HttpResponse::Ok().json(TaskList {
-        by_instance: response,
-    }))
-}
-
-#[derive(Deserialize, Serialize, ToSchema)]
-#[serde(tag = "type", rename_all = "snake_case")]
-enum TasksCancelFilter {
-    All,
-    AllEnqueued,
-    Indexes { indexes: Vec<String> },
 }
 
 #[utoipa::path]
 #[delete("tasks", guard = "admin_key_guard")]
 pub async fn tasks_cancel(
-    config: web::Data<SearchConfig>,
+    search: web::Data<dyn SearchBackend>,
     body: web::Json<TasksCancelFilter>,
-) -> Result<HttpResponse, ApiError> {
+) -> Result<(), ApiError> {
-    let client = config.make_batch_client()?;
+    search
-    let all_results = client
+        .tasks_cancel(&body)
-        .with_all_clients("cancel_tasks", async |client| {
+        .await
-            let mut q = TasksCancelQuery::new(client);
+        .map_err(ApiError::Internal)?;
-            match &body.0 {
+    Ok(())
-                TasksCancelFilter::All => {}
-                TasksCancelFilter::Indexes { indexes } => {
-                    q.with_index_uids(indexes.iter().map(|s| s.as_str()));
-                }
-                TasksCancelFilter::AllEnqueued => {
-                    q.with_statuses(["enqueued"]);
-                }
-            };
-
-            let result = client.cancel_tasks_with(&q).await;
-
-            Ok(result)
-        })
-        .await?;
-
-    for r in all_results {
-        r?;
-    }
-
-    Ok(HttpResponse::Ok().finish())
 }

apps/labrinth/src/routes/mod.rs

@@ -93,8 +93,6 @@ pub enum ApiError {
     Auth(eyre::Report),
     #[error("Invalid input: {0}")]
     InvalidInput(String),
-    #[error("Environment error")]
-    Env(#[from] dotenvy::Error),
     #[error("Error while uploading file: {0}")]
     FileHosting(#[from] FileHostingError),
     #[error("database error")]
@@ -117,8 +115,6 @@ pub enum ApiError {
     Validation(String),
     #[error("Search error: {0}")]
     Search(#[from] meilisearch_sdk::errors::Error),
-    #[error("search indexing error")]
-    Indexing(#[from] crate::search::indexing::IndexingError),
     #[error("Payments error: {0}")]
     Payments(String),
     #[error("Discord error: {0}")]
@@ -176,7 +172,6 @@ impl ApiError {
                 Self::Internal(..) => "internal_error",
                 Self::Request(..) => "request_error",
                 Self::Auth(..) => "auth_error",
-                Self::Env(..) => "environment_error",
                 Self::Database(..) => "database_error",
                 Self::SqlxDatabase(..) => "database_error",
                 Self::RedisDatabase(..) => "database_error",
@@ -185,7 +180,6 @@ impl ApiError {
                 Self::Xml(..) => "xml_error",
                 Self::Json(..) => "json_error",
                 Self::Search(..) => "search_error",
-                Self::Indexing(..) => "indexing_error",
                 Self::FileHosting(..) => "file_hosting_error",
                 Self::InvalidInput(..) => "invalid_input",
                 Self::Validation(..) => "invalid_input",
@@ -241,7 +235,6 @@ impl actix_web::ResponseError for ApiError {
             Self::Request(..) => StatusCode::BAD_REQUEST,
             Self::Auth(..) => StatusCode::UNAUTHORIZED,
             Self::InvalidInput(..) => StatusCode::BAD_REQUEST,
-            Self::Env(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::Database(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::SqlxDatabase(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::RedisDatabase(..) => StatusCode::INTERNAL_SERVER_ERROR,
@@ -251,7 +244,6 @@ impl actix_web::ResponseError for ApiError {
             Self::Xml(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::Json(..) => StatusCode::BAD_REQUEST,
             Self::Search(..) => StatusCode::INTERNAL_SERVER_ERROR,
-            Self::Indexing(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::FileHosting(..) => StatusCode::INTERNAL_SERVER_ERROR,
             Self::Validation(..) => StatusCode::BAD_REQUEST,
             Self::Payments(..) => StatusCode::FAILED_DEPENDENCY,

apps/labrinth/src/routes/v2/projects.rs

@@ -4,7 +4,7 @@ use crate::database::models::{project_item, version_item};
 use crate::database::redis::RedisPool;
 use crate::file_hosting::FileHost;
 use crate::models::projects::{
-    Link, MonetizationStatus, Project, ProjectStatus, SearchRequest, Version,
+    Link, MonetizationStatus, Project, ProjectStatus, Version,
 };
 use crate::models::v2::projects::{
     DonationLink, LegacyProject, LegacySideType, LegacyVersion,
@@ -14,7 +14,7 @@ use crate::queue::moderation::AutomatedModerationQueue;
 use crate::queue::session::AuthQueue;
 use crate::routes::v3::projects::ProjectIds;
 use crate::routes::{ApiError, v2_reroute, v3};
-use crate::search::{SearchConfig, SearchError, search_for_project};
+use crate::search::{SearchBackend, SearchRequest};
 use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post, web};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -53,9 +53,9 @@ pub fn config(cfg: &mut web::ServiceConfig) {
 #[get("search")]
 pub async fn project_search(
     web::Query(info): web::Query<SearchRequest>,
-    config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
     redis: web::Data<RedisPool>,
-) -> Result<HttpResponse, SearchError> {
+) -> Result<HttpResponse, ApiError> {
     // Search now uses loader_fields instead of explicit 'client_side' and 'server_side' fields
     // While the backend for this has changed, it doesnt affect much
     // in the API calls except that 'versions:x' is now 'game_versions:x'
@@ -100,7 +100,7 @@ pub async fn project_search(
         ..info
     };
 
-    let results = search_for_project(&info, &config, &redis).await?;
+    let results = search_backend.search_for_project(&info, &redis).await?;
 
     let results = LegacySearchResults::from(results);
 
@@ -410,7 +410,7 @@ pub async fn project_edit(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
-    search_config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
     new_project: web::Json<EditProject>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
@@ -524,7 +524,7 @@ pub async fn project_edit(
         req.clone(),
         info,
         pool.clone(),
-        search_config,
+        search_backend,
         web::Json(new_project),
         redis.clone(),
         session_queue.clone(),
@@ -918,7 +918,7 @@ pub async fn project_delete(
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
-    search_config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so no need to convert
@@ -927,7 +927,7 @@ pub async fn project_delete(
         info,
         pool,
         redis,
-        search_config,
+        search_backend,
         session_queue,
     )
     .await

apps/labrinth/src/routes/v2/versions.rs

@@ -11,7 +11,7 @@ use crate::models::projects::{
 use crate::models::v2::projects::LegacyVersion;
 use crate::queue::session::AuthQueue;
 use crate::routes::{v2_reroute, v3};
-use crate::search::SearchConfig;
+use crate::search::SearchBackend;
 use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web};
 use serde::{Deserialize, Serialize};
 use validator::Validate;
@@ -357,7 +357,7 @@ pub async fn version_delete(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
-    search_config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
 ) -> Result<HttpResponse, ApiError> {
     // Returns NoContent, so we don't need to convert the response
     v3::versions::version_delete(
@@ -366,7 +366,7 @@ pub async fn version_delete(
         pool,
         redis,
         session_queue,
-        search_config,
+        search_backend,
     )
     .await
     .or_else(v2_reroute::flatten_404_error)

apps/labrinth/src/routes/v3/project_creation.rs

@@ -22,7 +22,6 @@ use crate::models::teams::{OrganizationPermissions, ProjectPermissions};
 use crate::models::threads::ThreadType;
 use crate::models::v3::user_limits::UserLimits;
 use crate::queue::session::AuthQueue;
-use crate::search::indexing::IndexingError;
 use crate::util::guards::admin_key_guard;
 use crate::util::http::HttpClient;
 use crate::util::img::upload_image_optimized;
@@ -55,14 +54,10 @@ pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
 
 #[derive(Error, Debug)]
 pub enum CreateError {
-    #[error("Environment Error")]
-    EnvError(#[from] dotenvy::Error),
     #[error("An unknown database error occurred")]
     SqlxDatabaseError(#[from] sqlx::Error),
     #[error("Database Error: {0}")]
     DatabaseError(#[from] models::DatabaseError),
-    #[error("Indexing Error: {0}")]
-    IndexingError(#[from] IndexingError),
     #[error("Error while parsing multipart payload: {0}")]
     MultipartError(#[from] actix_multipart::MultipartError),
     #[error("Error while parsing JSON: {0}")]
@@ -126,12 +121,10 @@ impl From<crate::routes::ApiError> for CreateError {
 impl actix_web::ResponseError for CreateError {
     fn status_code(&self) -> StatusCode {
         match self {
-            CreateError::EnvError(..) => StatusCode::INTERNAL_SERVER_ERROR,
             CreateError::SqlxDatabaseError(..) => {
                 StatusCode::INTERNAL_SERVER_ERROR
             }
             CreateError::DatabaseError(..) => StatusCode::INTERNAL_SERVER_ERROR,
-            CreateError::IndexingError(..) => StatusCode::INTERNAL_SERVER_ERROR,
             CreateError::FileHostingError(..) => {
                 StatusCode::INTERNAL_SERVER_ERROR
             }
@@ -159,10 +152,8 @@ impl actix_web::ResponseError for CreateError {
     fn error_response(&self) -> HttpResponse {
         HttpResponse::build(self.status_code()).json(ApiError {
             error: match self {
-                CreateError::EnvError(..) => "environment_error",
                 CreateError::SqlxDatabaseError(..) => "database_error",
                 CreateError::DatabaseError(..) => "database_error",
-                CreateError::IndexingError(..) => "indexing_error",
                 CreateError::FileHostingError(..) => "file_hosting_error",
                 CreateError::SerDeError(..) => "invalid_input",
                 CreateError::MultipartError(..) => "invalid_input",

apps/labrinth/src/routes/v3/projects.rs

@@ -20,8 +20,7 @@ use crate::models::images::ImageContext;
 use crate::models::notifications::NotificationBody;
 use crate::models::pats::Scopes;
 use crate::models::projects::{
-    MonetizationStatus, Project, ProjectStatus, SearchRequest,
+    MonetizationStatus, Project, ProjectStatus, SideTypesMigrationReviewStatus,
-    SideTypesMigrationReviewStatus,
 };
 use crate::models::teams::ProjectPermissions;
 use crate::models::threads::MessageBody;
@@ -30,8 +29,7 @@ use crate::queue::moderation::AutomatedModerationQueue;
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
 use crate::routes::internal::delphi;
-use crate::search::indexing::remove_documents;
+use crate::search::{SearchBackend, SearchQuery, SearchRequest, SearchResults};
-use crate::search::{SearchConfig, SearchError, search_for_project};
 use crate::util::error::Context;
 use crate::util::img;
 use crate::util::img::{delete_old_images, upload_image_optimized};
@@ -48,6 +46,7 @@ use validator::Validate;
 
 pub fn config(cfg: &mut web::ServiceConfig) {
     cfg.route("search", web::get().to(project_search));
+    cfg.service(project_search_post);
     cfg.route("projects", web::get().to(projects_get));
     cfg.route("projects", web::patch().to(projects_edit));
     cfg.route("projects_random", web::get().to(random_projects_get));
@@ -292,7 +291,7 @@ async fn project_edit(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
-    search_config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
     web::Json(new_project): web::Json<EditProject>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
@@ -302,7 +301,7 @@ async fn project_edit(
         req,
         info,
         pool,
-        search_config,
+        search_backend,
         web::Json(new_project),
         redis,
         session_queue,
@@ -315,7 +314,7 @@ pub async fn project_edit_internal(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
-    search_config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
     web::Json(new_project): web::Json<EditProject>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
@@ -1126,16 +1125,16 @@ pub async fn project_edit_internal(
         project_item.inner.status.is_searchable(),
         new_project.status.map(|status| status.is_searchable()),
     ) {
-        remove_documents(
+        search_backend
-            &project_item
+            .remove_documents(
-                .versions
+                &project_item
-                .into_iter()
+                    .versions
-                .map(|x| x.into())
+                    .into_iter()
-                .collect::<Vec<_>>(),
+                    .map(|x| x.into())
-            &search_config,
+                    .collect::<Vec<_>>(),
-        )
+            )
-        .await
+            .await
-        .wrap_internal_err("failed to remove documents")?;
+            .wrap_internal_err("failed to remove documents")?;
     }
 
     Ok(HttpResponse::NoContent().body(""))
@@ -1190,11 +1189,13 @@ pub async fn edit_project_categories(
 // }
 
 pub async fn project_search(
-    web::Query(info): web::Query<SearchRequest>,
+    web::Query(info): web::Query<SearchQuery>,
-    config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
     redis: web::Data<RedisPool>,
-) -> Result<HttpResponse, SearchError> {
+) -> Result<web::Json<SearchResults>, ApiError> {
-    let results = search_for_project(&info, &config, &redis).await?;
+    let results = search_backend
+        .search_for_project(&SearchRequest::from(info), &redis)
+        .await?;
 
     // TODO: add this back
     // let results = ReturnSearchResults {
@@ -1208,7 +1209,18 @@ pub async fn project_search(
     //     total_hits: results.total_hits,
     // };
 
-    Ok(HttpResponse::Ok().json(results))
+    Ok(web::Json(results))
+}
+
+// for more complicated search queries
+#[post("/search")]
+pub async fn project_search_post(
+    web::Json(info): web::Json<SearchRequest>,
+    search_backend: web::Data<dyn SearchBackend>,
+    redis: web::Data<RedisPool>,
+) -> Result<web::Json<SearchResults>, ApiError> {
+    let results = search_backend.search_for_project(&info, &redis).await?;
+    Ok(web::Json(results))
 }
 
 //checks the validity of a project id or slug
@@ -2452,7 +2464,7 @@ async fn project_delete(
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
-    search_config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
     session_queue: web::Data<AuthQueue>,
 ) -> Result<(), ApiError> {
     project_delete_internal(
@@ -2460,7 +2472,7 @@ async fn project_delete(
         info,
         pool,
         redis,
-        search_config,
+        search_backend,
         session_queue,
     )
     .await
@@ -2471,7 +2483,7 @@ pub async fn project_delete_internal(
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
-    search_config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
     session_queue: web::Data<AuthQueue>,
 ) -> Result<(), ApiError> {
     let (_, user) = get_user_from_headers(
@@ -2583,16 +2595,16 @@ pub async fn project_delete_internal(
         .await
         .wrap_internal_err("failed to commit transaction")?;
 
-    remove_documents(
+    search_backend
-        &project
+        .remove_documents(
-            .versions
+            &project
-            .into_iter()
+                .versions
-            .map(|x| x.into())
+                .into_iter()
-            .collect::<Vec<_>>(),
+                .map(|x| x.into())
-        &search_config,
+                .collect::<Vec<_>>(),
-    )
+        )
-    .await
+        .await
-    .wrap_internal_err("failed to remove project version documents")?;
+        .wrap_internal_err("failed to remove project version documents")?;
 
     if result.is_some() {
         Ok(())

apps/labrinth/src/routes/v3/versions.rs

@@ -26,8 +26,7 @@ use crate::models::projects::{Loader, skip_nulls};
 use crate::models::teams::ProjectPermissions;
 use crate::queue::session::AuthQueue;
 use crate::routes::internal::delphi;
-use crate::search::SearchConfig;
+use crate::search::SearchBackend;
-use crate::search::indexing::remove_documents;
 use crate::util::error::Context;
 use crate::util::img;
 use crate::util::validate::validation_errors_to_string;
@@ -915,7 +914,7 @@ pub async fn version_delete(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
-    search_config: web::Data<SearchConfig>,
+    search_backend: web::Data<dyn SearchBackend>,
 ) -> Result<HttpResponse, ApiError> {
     let user = get_user_from_headers(
         &req,
@@ -1022,10 +1021,10 @@ pub async fn version_delete(
         &redis,
     )
     .await?;
-    remove_documents(&[version.inner.id.into()], &search_config)
+    search_backend
+        .remove_documents(&[version.inner.id.into()])
         .await
         .wrap_internal_err("failed to remove documents")?;
-
     if result.is_some() {
         Ok(HttpResponse::NoContent().body(""))
     } else {

apps/labrinth/src/search/backend/common.rs

@@ -0,0 +1,123 @@
+use crate::routes::ApiError;
+use crate::search::SearchRequest;
+use crate::util::error::Context;
+use eyre::eyre;
+use std::borrow::Cow;
+
+pub struct ParsedSearchRequest<'a> {
+    pub offset: usize,
+    pub hits_per_page: usize,
+    pub page: usize,
+    pub index: &'a str,
+    pub query: &'a str,
+}
+
+pub fn parse_search_request(
+    info: &SearchRequest,
+) -> Result<ParsedSearchRequest<'_>, ApiError> {
+    let offset = info
+        .offset
+        .as_deref()
+        .unwrap_or("0")
+        .parse::<usize>()
+        .wrap_request_err("invalid offset")?;
+    let limit = info
+        .limit
+        .as_deref()
+        .unwrap_or("10")
+        .parse::<usize>()
+        .wrap_request_err("invalid limit")?
+        .min(100);
+    let hits_per_page = if limit == 0 { 1 } else { limit };
+
+    Ok(ParsedSearchRequest {
+        offset,
+        hits_per_page,
+        page: offset / hits_per_page + 1,
+        index: info.index.as_deref().unwrap_or("relevance"),
+        query: info.query.as_deref().unwrap_or_default(),
+    })
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SearchIndex {
+    Relevance,
+    Downloads,
+    Follows,
+    Updated,
+    Newest,
+    MinecraftJavaServerVerifiedPlays2w,
+    MinecraftJavaServerPlayersOnline,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SearchIndexName {
+    Projects,
+    ProjectsFiltered,
+}
+
+pub struct SearchSort {
+    pub index_name: SearchIndexName,
+    pub index: SearchIndex,
+}
+
+pub fn parse_search_index(
+    index: &str,
+    new_filters: Option<&str>,
+) -> Result<SearchSort, ApiError> {
+    let projects_name = SearchIndexName::Projects;
+    let projects_filtered_name = SearchIndexName::ProjectsFiltered;
+
+    // TODO: this is a dumb hack, the frontend should pass the project type it's filtering directly
+    let is_server = new_filters
+        .is_some_and(|f| f.contains("project_types = minecraft_java_server"));
+
+    Ok(match index {
+        "relevance" => SearchSort {
+            index_name: projects_name,
+            index: if is_server {
+                SearchIndex::MinecraftJavaServerVerifiedPlays2w
+            } else {
+                SearchIndex::Relevance
+            },
+        },
+        "downloads" => SearchSort {
+            index_name: projects_filtered_name,
+            index: SearchIndex::Downloads,
+        },
+        "follows" => SearchSort {
+            index_name: projects_name,
+            index: SearchIndex::Follows,
+        },
+        "updated" | "date_modified" => SearchSort {
+            index_name: projects_name,
+            index: SearchIndex::Updated,
+        },
+        "newest" | "date_created" => SearchSort {
+            index_name: projects_name,
+            index: SearchIndex::Newest,
+        },
+        "minecraft_java_server.verified_plays_2w" => SearchSort {
+            index_name: projects_name,
+            index: SearchIndex::MinecraftJavaServerVerifiedPlays2w,
+        },
+        "minecraft_java_server.ping.data.players_online" => SearchSort {
+            index_name: projects_name,
+            index: SearchIndex::MinecraftJavaServerPlayersOnline,
+        },
+        i => return Err(ApiError::Request(eyre!("invalid index '{i}'"))),
+    })
+}
+
+pub fn combined_search_filters(info: &SearchRequest) -> Option<Cow<'_, str>> {
+    if let Some(filters) = info.new_filters.as_deref() {
+        return Some(filters.into());
+    }
+
+    match (info.filters.as_deref(), info.version.as_deref()) {
+        (Some(f), Some(v)) => Some(format!("({f}) AND ({v})").into()),
+        (Some(f), None) => Some(f.into()),
+        (None, Some(v)) => Some(v.into()),
+        (None, None) => None,
+    }
+}

apps/labrinth/src/search/backend/meilisearch/indexing.rs

@@ -1,43 +1,23 @@
-/// This module is used for the indexing from any source.
+use std::sync::LazyLock;
-pub mod local_import;
-
 use std::time::Duration;
 
 use crate::database::PgPool;
 use crate::database::redis::RedisPool;
 use crate::env::ENV;
-use crate::search::{SearchConfig, UploadSearchProject};
+use crate::search::backend::meilisearch::MeilisearchConfig;
+use crate::search::indexing::index_local;
+use crate::search::{SearchField, UploadSearchProject};
 use crate::util::error::Context;
 use ariadne::ids::base62_impl::to_base62;
-use eyre::eyre;
+use eyre::{Result, eyre};
 use futures::StreamExt;
 use futures::stream::FuturesOrdered;
-use local_import::index_local;
 use meilisearch_sdk::client::{Client, SwapIndexes};
 use meilisearch_sdk::indexes::Index;
 use meilisearch_sdk::settings::{PaginationSetting, Settings};
 use meilisearch_sdk::task_info::TaskInfo;
-use thiserror::Error;
 use tracing::{Instrument, error, info, info_span, instrument};
 
-#[derive(Error, Debug)]
-pub enum IndexingError {
-    #[error(transparent)]
-    Internal(#[from] eyre::Report),
-    #[error("Error while connecting to the MeiliSearch database")]
-    Indexing(#[from] meilisearch_sdk::errors::Error),
-    #[error("Error while serializing or deserializing JSON: {0}")]
-    Serde(#[from] serde_json::Error),
-    #[error("Database Error: {0}")]
-    Sqlx(#[from] sqlx::error::Error),
-    #[error("Database Error: {0}")]
-    Database(#[from] crate::database::models::DatabaseError),
-    #[error("Environment Error")]
-    Env(#[from] dotenvy::Error),
-    #[error("Error while awaiting index creation task")]
-    Task,
-}
-
 // // The chunk size for adding projects to the indexing database. If the request size
 // // is too large (>10MiB) then the request fails with an error.  This chunk size
 // // assumes a max average size of 4KiB per project to avoid this cap.
@@ -51,8 +31,8 @@ fn search_operation_timeout() -> std::time::Duration {
 
 pub async fn remove_documents(
     ids: &[crate::models::ids::VersionId],
-    config: &SearchConfig,
+    config: &MeilisearchConfig,
-) -> eyre::Result<()> {
+) -> Result<()> {
     let mut indexes = get_indexes_for_indexing(config, false, false)
         .await
         .wrap_err("failed to get current indexes")?;
@@ -108,8 +88,8 @@ pub async fn remove_documents(
 pub async fn index_projects(
     ro_pool: PgPool,
     redis: RedisPool,
-    config: &SearchConfig,
+    config: &MeilisearchConfig,
-) -> eyre::Result<()> {
+) -> Result<()> {
     info!("Indexing projects.");
 
     info!("Ensuring current indexes exists");
@@ -197,9 +177,9 @@ pub async fn index_projects(
 }
 
 pub async fn swap_index(
-    config: &SearchConfig,
+    config: &MeilisearchConfig,
     index_name: &str,
-) -> Result<(), IndexingError> {
+) -> Result<()> {
     let client = config.make_batch_client()?;
     let index_name_next = config.get_index_name(index_name, true);
     let index_name = config.get_index_name(index_name, false);
@@ -210,12 +190,13 @@ pub async fn swap_index(
 
     let swap_indices_ref = &swap_indices;
 
+    // is it "indexes" or "indices"? who knows! roll a die!
     client
         .with_all_clients("swap_indexes", |client| async move {
             let task = client
                 .swap_indexes([swap_indices_ref])
                 .await
-                .map_err(IndexingError::Indexing)?;
+                .wrap_err("failed to swap indices")?;
 
             monitor_task(
                 client,
@@ -233,10 +214,10 @@ pub async fn swap_index(
 
 #[instrument(skip(config))]
 pub async fn get_indexes_for_indexing(
-    config: &SearchConfig,
+    config: &MeilisearchConfig,
     next: bool, // Get the 'next' one
     update_settings: bool,
-) -> Result<Vec<Vec<Index>>, IndexingError> {
+) -> Result<Vec<Vec<Index>>> {
     let client = config.make_batch_client()?;
     let project_name = config.get_index_name("projects", next);
     let project_filtered_name =
@@ -381,7 +362,7 @@ async fn add_to_index(
     client: &Client,
     index: &Index,
     mods: &[UploadSearchProject],
-) -> Result<(), IndexingError> {
+) -> Result<()> {
     for chunk in mods.chunks(MEILISEARCH_CHUNK_SIZE) {
         info!(
             "Adding chunk of {} versions starting with version id {}",
@@ -419,7 +400,7 @@ async fn monitor_task(
     task: TaskInfo,
     timeout: Duration,
     poll: Option<Duration>,
-) -> Result<(), IndexingError> {
+) -> Result<()> {
     let now = std::time::Instant::now();
 
     let id = task.get_task_uid();
@@ -465,7 +446,7 @@ async fn update_and_add_to_index(
     index: &Index,
     projects: &[UploadSearchProject],
     _additional_fields: &[String],
-) -> Result<(), IndexingError> {
+) -> Result<()> {
     // TODO: Uncomment this- hardcoding loader_fields is a band-aid fix, and will be fixed soon
     // let mut new_filterable_attributes: Vec<String> = index.get_filterable_attributes().await?;
     // let mut new_displayed_attributes = index.get_displayed_attributes().await?;
@@ -509,8 +490,8 @@ pub async fn add_projects_batch_client(
     indices: &[Vec<Index>],
     projects: Vec<UploadSearchProject>,
     additional_fields: Vec<String>,
-    config: &SearchConfig,
+    config: &MeilisearchConfig,
-) -> Result<(), IndexingError> {
+) -> Result<()> {
     let client = config.make_batch_client()?;
 
     let index_references = indices
@@ -558,12 +539,92 @@ fn default_settings() -> Settings {
         .with_displayed_attributes(DEFAULT_DISPLAYED_ATTRIBUTES)
         .with_searchable_attributes(DEFAULT_SEARCHABLE_ATTRIBUTES)
         .with_sortable_attributes(DEFAULT_SORTABLE_ATTRIBUTES)
-        .with_filterable_attributes(DEFAULT_ATTRIBUTES_FOR_FACETING)
+        .with_filterable_attributes(&*MEILI_FILTERABLE_ATTRIBUTES)
         .with_pagination(PaginationSetting {
             max_total_hits: 2147483647,
         })
 }
 
+pub struct MeilisearchFieldSpec {
+    pub path: &'static str,
+    pub filterable: bool,
+}
+
+impl SearchField {
+    pub const fn meilisearch_spec(self) -> MeilisearchFieldSpec {
+        match self {
+            SearchField::Categories => MeilisearchFieldSpec {
+                path: "categories",
+                filterable: true,
+            },
+            SearchField::ProjectTypes => MeilisearchFieldSpec {
+                path: "project_types",
+                filterable: true,
+            },
+            SearchField::ProjectId => MeilisearchFieldSpec {
+                path: "project_id",
+                filterable: true,
+            },
+            SearchField::OpenSource => MeilisearchFieldSpec {
+                path: "open_source",
+                filterable: true,
+            },
+            SearchField::Environment => MeilisearchFieldSpec {
+                path: "environment",
+                filterable: true,
+            },
+            SearchField::GameVersions => MeilisearchFieldSpec {
+                path: "game_versions",
+                filterable: true,
+            },
+            SearchField::ClientSide => MeilisearchFieldSpec {
+                path: "client_side",
+                filterable: true,
+            },
+            SearchField::ServerSide => MeilisearchFieldSpec {
+                path: "server_side",
+                filterable: true,
+            },
+            SearchField::MinecraftServerRegion => MeilisearchFieldSpec {
+                path: "minecraft_server.region",
+                filterable: true,
+            },
+            SearchField::MinecraftServerLanguages => MeilisearchFieldSpec {
+                path: "minecraft_server.languages",
+                filterable: true,
+            },
+            SearchField::MinecraftJavaServerContentKind => {
+                MeilisearchFieldSpec {
+                    path: "minecraft_java_server.content.kind",
+                    filterable: true,
+                }
+            }
+            SearchField::MinecraftJavaServerContentSupportedGameVersions => {
+                MeilisearchFieldSpec {
+                    path: "minecraft_java_server.content.supported_game_versions",
+                    filterable: true,
+                }
+            }
+            SearchField::MinecraftJavaServerPingData => MeilisearchFieldSpec {
+                path: "minecraft_java_server.ping.data",
+                filterable: true,
+            },
+        }
+    }
+}
+
+static MEILI_FILTERABLE_ATTRIBUTES: LazyLock<Vec<&'static str>> =
+    LazyLock::new(|| {
+        use strum::IntoEnumIterator;
+
+        SearchField::iter()
+            .filter_map(|field| {
+                let spec = field.meilisearch_spec();
+                spec.filterable.then_some(spec.path)
+            })
+            .collect()
+    });
+
 const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[
     "project_id",
     "version_id",
@@ -617,41 +678,6 @@ const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[
 const DEFAULT_SEARCHABLE_ATTRIBUTES: &[&str] =
     &["name", "summary", "author", "slug"];
 
-const DEFAULT_ATTRIBUTES_FOR_FACETING: &[&str] = &[
-    "categories",
-    "license",
-    "project_types",
-    "downloads",
-    "follows",
-    "author",
-    "name",
-    "date_created",
-    "created_timestamp",
-    "date_modified",
-    "modified_timestamp",
-    "version_published_timestamp",
-    "project_id",
-    "open_source",
-    "color",
-    // Note: loader fields are not here, but are added on as they are needed (so they can be dynamically added depending on which exist).
-    // TODO: remove these- as they should be automatically populated. This is a band-aid fix.
-    "environment",
-    "game_versions",
-    "mrpack_loaders",
-    // V2 legacy fields for logical consistency
-    "client_side",
-    "server_side",
-    "minecraft_server.country",
-    "minecraft_server.region",
-    "minecraft_server.languages",
-    "minecraft_java_server.content.kind",
-    "minecraft_java_server.content.supported_game_versions",
-    "minecraft_java_server.content.recommended_game_version",
-    "minecraft_java_server.verified_plays_2w",
-    "minecraft_java_server.ping.data",
-    "minecraft_java_server.ping.data.players_online",
-];
-
 const DEFAULT_SORTABLE_ATTRIBUTES: &[&str] = &[
     "downloads",
     "follows",

apps/labrinth/src/search/backend/meilisearch/mod.rs

@@ -0,0 +1,489 @@
+use crate::database::PgPool;
+use crate::database::redis::RedisPool;
+use crate::env::ENV;
+use crate::models::ids::VersionId;
+use crate::routes::ApiError;
+use crate::search::backend::{
+    SearchIndex, SearchIndexName, combined_search_filters, parse_search_index,
+    parse_search_request,
+};
+use crate::search::{
+    ResultSearchProject, SearchBackend, SearchRequest, SearchResults,
+    TasksCancelFilter,
+};
+use crate::util::error::Context;
+use async_trait::async_trait;
+use eyre::Result;
+use futures::TryStreamExt;
+use futures::stream::FuturesOrdered;
+use itertools::Itertools;
+use meilisearch_sdk::client::Client;
+use meilisearch_sdk::tasks::{Task, TasksCancelQuery};
+use serde::Serialize;
+use serde_json::Value;
+use std::collections::HashMap;
+use std::fmt::Write;
+use std::time::Duration;
+use tracing::{Instrument, info_span};
+
+pub mod indexing;
+
+#[derive(Debug, Clone)]
+pub struct MeilisearchReadClient {
+    pub client: Client,
+}
+
+impl std::ops::Deref for MeilisearchReadClient {
+    type Target = Client;
+
+    fn deref(&self) -> &Self::Target {
+        &self.client
+    }
+}
+
+pub struct BatchClient {
+    pub clients: Vec<Client>,
+}
+
+impl BatchClient {
+    pub fn new(clients: Vec<Client>) -> Self {
+        Self { clients }
+    }
+
+    pub async fn with_all_clients<'a, T, G, Fut>(
+        &'a self,
+        task_name: &str,
+        generator: G,
+    ) -> Result<Vec<T>>
+    where
+        G: Fn(&'a Client) -> Fut,
+        Fut: Future<Output = Result<T>> + 'a,
+    {
+        let mut tasks = FuturesOrdered::new();
+        for (idx, client) in self.clients.iter().enumerate() {
+            tasks.push_back(generator(client).instrument(info_span!(
+                "client_task",
+                task.name = task_name,
+                client.idx = idx,
+            )));
+        }
+
+        let results = tasks.try_collect::<Vec<T>>().await?;
+        Ok(results)
+    }
+
+    pub fn across_all<T, F, R>(&self, data: Vec<T>, mut predicate: F) -> Vec<R>
+    where
+        F: FnMut(T, &Client) -> R,
+    {
+        assert_eq!(
+            data.len(),
+            self.clients.len(),
+            "mismatch between data len and meilisearch client count"
+        );
+        self.clients
+            .iter()
+            .zip(data)
+            .map(|(client, item)| predicate(item, client))
+            .collect()
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct MeilisearchConfig {
+    pub addresses: Vec<String>,
+    pub read_lb_address: String,
+    pub key: String,
+    pub meta_namespace: String,
+}
+
+impl MeilisearchConfig {
+    pub fn new(meta_namespace: Option<String>) -> Self {
+        Self {
+            addresses: ENV.MEILISEARCH_WRITE_ADDRS.0.clone(),
+            key: ENV.MEILISEARCH_KEY.clone(),
+            meta_namespace: meta_namespace.unwrap_or_default(),
+            read_lb_address: ENV.MEILISEARCH_READ_ADDR.clone(),
+        }
+    }
+
+    pub fn make_loadbalanced_read_client(
+        &self,
+    ) -> Result<MeilisearchReadClient, meilisearch_sdk::errors::Error> {
+        Ok(MeilisearchReadClient {
+            client: Client::new(&self.read_lb_address, Some(&self.key))?,
+        })
+    }
+
+    pub fn make_batch_client(
+        &self,
+    ) -> Result<BatchClient, meilisearch_sdk::errors::Error> {
+        Ok(BatchClient::new(
+            self.addresses
+                .iter()
+                .map(|address| {
+                    Client::new(address.as_str(), Some(self.key.as_str()))
+                })
+                .collect::<Result<Vec<_>, _>>()?,
+        ))
+    }
+
+    pub fn get_index_name(&self, index: &str, next: bool) -> String {
+        let alt = if next { "_alt" } else { "" };
+        format!("{}_{}_{}", self.meta_namespace, index, alt)
+    }
+}
+
+pub struct Meilisearch {
+    pub config: MeilisearchConfig,
+}
+
+impl Meilisearch {
+    pub fn new(config: MeilisearchConfig) -> Self {
+        Self { config }
+    }
+
+    fn get_sort_index(
+        &self,
+        index: &str,
+        new_filters: Option<&str>,
+    ) -> Result<(String, &'static [&'static str]), ApiError> {
+        let sort = parse_search_index(index, new_filters)?;
+        let index_name = match sort.index_name {
+            SearchIndexName::Projects => {
+                self.config.get_index_name("projects", false)
+            }
+            SearchIndexName::ProjectsFiltered => {
+                self.config.get_index_name("projects_filtered", false)
+            }
+        };
+
+        Ok(match sort.index {
+            SearchIndex::Relevance => (
+                index_name,
+                &["downloads:desc", "version_published_timestamp:desc"],
+            ),
+            SearchIndex::Downloads => (
+                index_name,
+                &["downloads:desc", "version_published_timestamp:desc"],
+            ),
+            SearchIndex::Follows => (
+                index_name,
+                &["follows:desc", "version_published_timestamp:desc"],
+            ),
+            SearchIndex::Updated => (
+                index_name,
+                &["date_modified:desc", "version_published_timestamp:desc"],
+            ),
+            SearchIndex::Newest => (
+                index_name,
+                &["date_created:desc", "version_published_timestamp:desc"],
+            ),
+            SearchIndex::MinecraftJavaServerVerifiedPlays2w => (
+                index_name,
+                &[
+                    "minecraft_java_server.verified_plays_2w:desc",
+                    "minecraft_java_server.ping.data.players_online:desc",
+                    "version_published_timestamp:desc",
+                ],
+            ),
+            SearchIndex::MinecraftJavaServerPlayersOnline => (
+                index_name,
+                &[
+                    "minecraft_java_server.ping.data.players_online:desc",
+                    "version_published_timestamp:desc",
+                ],
+            ),
+        })
+    }
+}
+
+#[async_trait]
+impl SearchBackend for Meilisearch {
+    async fn search_for_project_raw(
+        &self,
+        info: &SearchRequest,
+    ) -> Result<SearchResults, ApiError> {
+        let parsed = parse_search_request(info)?;
+
+        let (index_name, sort_name) =
+            self.get_sort_index(parsed.index, info.new_filters.as_deref())?;
+        let client = self
+            .config
+            .make_loadbalanced_read_client()
+            .wrap_internal_err("failed to make load-balanced read client")?;
+        let meilisearch_index = client
+            .get_index(index_name)
+            .await
+            .wrap_internal_err("failed to get index")?;
+
+        let mut filter_string = String::new();
+
+        let results = {
+            let mut query = meilisearch_index.search();
+            query
+                .with_page(parsed.page)
+                .with_hits_per_page(parsed.hits_per_page)
+                .with_query(parsed.query)
+                .with_sort(sort_name);
+
+            if let Some(new_filters) = info.new_filters.as_deref() {
+                query.with_filter(new_filters);
+            } else {
+                let facets = if let Some(facets) = &info.facets {
+                    let facets =
+                        serde_json::from_str::<Vec<Vec<Value>>>(facets)
+                            .wrap_request_err("failed to parse facets")?;
+                    Some(facets)
+                } else {
+                    None
+                };
+
+                let filters =
+                    combined_search_filters(info).unwrap_or_else(|| "".into());
+
+                if let Some(facets) = facets {
+                    let facets: Vec<Vec<Vec<String>>> =
+                        facets
+                            .into_iter()
+                            .map(|facets| {
+                                facets
+								.into_iter()
+								.map(|facet| {
+									if facet.is_array() {
+										serde_json::from_value::<Vec<String>>(facet)
+											.unwrap_or_default()
+									} else {
+										vec![
+											serde_json::from_value::<String>(facet)
+												.unwrap_or_default(),
+										]
+									}
+								})
+								.collect_vec()
+                            })
+                            .collect_vec();
+
+                    filter_string.push('(');
+                    for (index, facet_outer_list) in facets.iter().enumerate() {
+                        filter_string.push('(');
+
+                        for (facet_outer_index, facet_inner_list) in
+                            facet_outer_list.iter().enumerate()
+                        {
+                            filter_string.push('(');
+                            for (facet_inner_index, facet) in
+                                facet_inner_list.iter().enumerate()
+                            {
+                                filter_string
+                                    .push_str(&facet.replace(':', " = "));
+                                if facet_inner_index
+                                    != (facet_inner_list.len() - 1)
+                                {
+                                    filter_string.push_str(" AND ")
+                                }
+                            }
+                            filter_string.push(')');
+
+                            if facet_outer_index != (facet_outer_list.len() - 1)
+                            {
+                                filter_string.push_str(" OR ")
+                            }
+                        }
+
+                        filter_string.push(')');
+
+                        if index != (facets.len() - 1) {
+                            filter_string.push_str(" AND ")
+                        }
+                    }
+                    filter_string.push(')');
+
+                    if !filters.is_empty() {
+                        write!(filter_string, " AND ({filters})")
+                            .expect("write should not fail");
+                    }
+                } else {
+                    filter_string.push_str(&filters);
+                }
+
+                if !filter_string.is_empty() {
+                    query.with_filter(&filter_string);
+                }
+            }
+
+            if info.show_metadata {
+                query.with_show_ranking_score(true);
+                query.with_show_ranking_score_details(true);
+                query.execute().await?
+            } else {
+                query.execute::<ResultSearchProject>().await?
+            }
+        };
+
+        if info.show_metadata {
+            let hits = results
+                .hits
+                .into_iter()
+                .map(|hit| {
+                    let metadata = serde_json::to_value(&hit)
+                        .ok()
+                        .and_then(|value| value.as_object().cloned())
+                        .map(|mut value| {
+                            value.remove("_formatted");
+                            value.remove("_matchesPosition");
+                            value.remove("_federation");
+                            let result = value.remove("result");
+                            let metadata = Value::Object(value);
+                            (result, metadata)
+                        });
+
+                    let (result, metadata) =
+                        metadata.unwrap_or((None, Value::Null));
+                    let mut result = result
+                        .and_then(|value| {
+                            serde_json::from_value::<ResultSearchProject>(value)
+                                .ok()
+                        })
+                        .unwrap_or(hit.result);
+
+                    if !metadata.is_null() {
+                        result.search_metadata = Some(metadata);
+                    }
+
+                    result
+                })
+                .collect();
+
+            Ok(SearchResults {
+                hits,
+                page: results.page.unwrap_or_default(),
+                hits_per_page: results.hits_per_page.unwrap_or_default(),
+                total_hits: results.total_hits.unwrap_or_default(),
+            })
+        } else {
+            Ok(SearchResults {
+                hits: results.hits.into_iter().map(|r| r.result).collect(),
+                page: results.page.unwrap_or_default(),
+                hits_per_page: results.hits_per_page.unwrap_or_default(),
+                total_hits: results.total_hits.unwrap_or_default(),
+            })
+        }
+    }
+
+    async fn index_projects(
+        &self,
+        ro_pool: PgPool,
+        redis: RedisPool,
+    ) -> eyre::Result<()> {
+        indexing::index_projects(ro_pool, redis, &self.config).await?;
+        Ok(())
+    }
+
+    async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()> {
+        indexing::remove_documents(ids, &self.config).await?;
+        Ok(())
+    }
+
+    async fn tasks(&self) -> eyre::Result<Value> {
+        let client = self
+            .config
+            .make_batch_client()
+            .wrap_internal_err("failed to make batch client")?;
+        let tasks = client
+            .with_all_clients("get_tasks", async |client| {
+                let tasks = client.get_tasks().await?;
+                Ok(tasks.results)
+            })
+            .await
+            .wrap_internal_err("failed to get tasks")?;
+
+        #[derive(Serialize)]
+        struct MeiliTask<Time> {
+            uid: u32,
+            status: &'static str,
+            duration: Option<Duration>,
+            enqueued_at: Option<Time>,
+        }
+
+        #[derive(Serialize)]
+        struct TaskList<Time> {
+            by_instance: HashMap<String, Vec<MeiliTask<Time>>>,
+        }
+
+        let response = tasks
+            .into_iter()
+            .enumerate()
+            .map(|(idx, instance_tasks)| {
+                let tasks = instance_tasks
+                    .into_iter()
+                    .filter_map(|task| {
+                        Some(match task {
+                            Task::Enqueued { content } => MeiliTask {
+                                uid: content.uid,
+                                status: "enqueued",
+                                duration: None,
+                                enqueued_at: Some(content.enqueued_at),
+                            },
+                            Task::Processing { content } => MeiliTask {
+                                uid: content.uid,
+                                status: "processing",
+                                duration: None,
+                                enqueued_at: Some(content.enqueued_at),
+                            },
+                            Task::Failed { content } => MeiliTask {
+                                uid: content.task.uid,
+                                status: "failed",
+                                duration: Some(content.task.duration),
+                                enqueued_at: Some(content.task.enqueued_at),
+                            },
+                            Task::Succeeded { .. } => return None,
+                        })
+                    })
+                    .collect();
+
+                (idx.to_string(), tasks)
+            })
+            .collect::<HashMap<String, Vec<MeiliTask<_>>>>();
+
+        let response = serde_json::to_value(TaskList {
+            by_instance: response,
+        })
+        .wrap_internal_err("failed to serialize tasks response")?;
+        Ok(response)
+    }
+
+    async fn tasks_cancel(
+        &self,
+        filter: &TasksCancelFilter,
+    ) -> eyre::Result<()> {
+        let client = self
+            .config
+            .make_batch_client()
+            .wrap_internal_err("failed to make batch client")?;
+        let all_results = client
+            .with_all_clients("cancel_tasks", async |client| {
+                let mut q = TasksCancelQuery::new(client);
+                match filter {
+                    TasksCancelFilter::All => {}
+                    TasksCancelFilter::Indexes { indexes } => {
+                        q.with_index_uids(indexes.iter().map(|s| s.as_str()));
+                    }
+                    TasksCancelFilter::AllEnqueued => {
+                        q.with_statuses(["enqueued"]);
+                    }
+                };
+
+                let result = client.cancel_tasks_with(&q).await;
+                Ok(result)
+            })
+            .await
+            .wrap_internal_err("failed to cancel tasks")?;
+
+        for r in all_results {
+            r.wrap_internal_err("failed to cancel tasks")?;
+        }
+
+        Ok(())
+    }
+}

apps/labrinth/src/search/backend/mod.rs

@@ -0,0 +1,12 @@
+mod common;
+pub mod elasticsearch;
+pub mod meilisearch;
+pub mod typesense;
+
+pub use common::{
+    ParsedSearchRequest, SearchIndex, SearchIndexName, SearchSort,
+    combined_search_filters, parse_search_index, parse_search_request,
+};
+pub use elasticsearch::Elasticsearch;
+pub use meilisearch::{Meilisearch, MeilisearchConfig};
+pub use typesense::{Typesense, TypesenseConfig};

apps/labrinth/src/search/indexing.rs

@@ -1,11 +1,11 @@
 use chrono::{DateTime, Utc};
 use dashmap::DashMap;
+use eyre::Result;
 use futures::TryStreamExt;
 use itertools::Itertools;
 use std::collections::HashMap;
 use tracing::info;
 
-use super::IndexingError;
 use crate::database::PgPool;
 use crate::database::models::loader_fields::{
     QueryLoaderField, QueryLoaderFieldEnumValue, QueryVersionField,
@@ -29,7 +29,7 @@ pub async fn index_local(
     redis: &RedisPool,
     cursor: i64,
     limit: i64,
-) -> Result<(Vec<UploadSearchProject>, i64), IndexingError> {
+) -> eyre::Result<(Vec<UploadSearchProject>, i64)> {
     info!("Indexing local projects!");
 
     // todo: loaders, project type, game versions
@@ -84,7 +84,8 @@ pub async fn index_local(
             }
         })
         .try_collect::<Vec<PartialProject>>()
-        .await?;
+        .await
+        .wrap_err("failed to fetch projects")?;
 
     let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
     let project_components = db_projects
@@ -430,6 +431,7 @@ pub async fn index_local(
                     display_categories: display_categories.clone(),
                     follows: project.follows,
                     downloads: project.downloads,
+                    log_downloads: (project.downloads.max(1) as f64).ln(),
                     icon_url: project.icon_url.clone(),
                     author: owner.clone(),
                     date_created: project.approved,
@@ -473,7 +475,7 @@ struct PartialVersion {
 async fn index_versions(
     pool: &PgPool,
     project_ids: Vec<i64>,
-) -> Result<HashMap<DBProjectId, Vec<PartialVersion>>, IndexingError> {
+) -> Result<HashMap<DBProjectId, Vec<PartialVersion>>> {
     let versions: HashMap<DBProjectId, Vec<(DBVersionId, DateTime<Utc>)>> =
         sqlx::query!(
             "
@@ -497,7 +499,8 @@ async fn index_versions(
                 async move { Ok(acc) }
             },
         )
-        .await?;
+        .await
+        .wrap_err("failed to fetch versions")?;
 
     // Get project types, loaders
     #[derive(Default)]
@@ -538,7 +541,8 @@ async fn index_versions(
         (version_id, version_loader_data)
     })
     .try_collect()
-    .await?;
+    .await
+    .wrap_err("failed to fetch loaders and project types")?;
 
     // Get version fields
     let version_fields: DashMap<DBVersionId, Vec<QueryVersionField>> =
@@ -570,7 +574,10 @@ async fn index_versions(
                 async move { Ok(acc) }
             },
         )
-        .await?;
+        .await
+        .wrap_err("failed to fetch version fields")?;
+
+    // Get version fields
 
     // Convert to partial versions
     let mut res_versions: HashMap<DBProjectId, Vec<PartialVersion>> =

apps/labrinth/src/search/mod.rs

@@ -1,188 +1,222 @@
-use crate::env::ENV;
+use crate::database::redis::RedisPool;
 use crate::models::exp;
 use crate::models::exp::minecraft::JavaServerPing;
-use crate::models::ids::ProjectId;
+use crate::models::ids::{ProjectId, VersionId};
-use crate::models::projects::SearchRequest;
 use crate::queue::server_ping;
-use crate::{database::models::DatabaseError, database::redis::RedisPool};
+use crate::routes::ApiError;
-use crate::{models::error::ApiError, search::indexing::IndexingError};
+use crate::{database::PgPool, env::ENV};
-use actix_web::HttpResponse;
-use actix_web::http::StatusCode;
 use ariadne::ids::base62_impl::parse_base62;
+use async_trait::async_trait;
 use chrono::{DateTime, Utc};
-use futures::TryStreamExt;
-use futures::stream::FuturesOrdered;
-use itertools::Itertools;
-use meilisearch_sdk::client::Client;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use std::borrow::Cow;
+use std::{collections::HashMap, str::FromStr};
-use std::collections::HashMap;
-use std::fmt::Write;
 use thiserror::Error;
-use tracing::{Instrument, info_span};
+use utoipa::ToSchema;
 
+pub mod backend;
 pub mod indexing;
 
-#[derive(Error, Debug)]
+/// Search parameters which can fit in a URL query string.
-pub enum SearchError {
+///
-    #[error("MeiliSearch Error: {0}")]
+/// Used with `GET /*/search` endpoints.
-    MeiliSearch(#[from] meilisearch_sdk::errors::Error),
+///
-    #[error("Error while serializing or deserializing JSON: {0}")]
+/// Can be converted into a [`SearchRequest`] using [`From`].
-    Serde(#[from] serde_json::Error),
+#[derive(Serialize, Deserialize, Debug)]
-    #[error("Error while parsing an integer: {0}")]
+pub struct SearchQuery {
-    IntParsing(#[from] std::num::ParseIntError),
+    pub query: Option<String>,
-    #[error("Error while formatting strings: {0}")]
+    pub offset: Option<String>,
-    FormatError(#[from] std::fmt::Error),
+    pub index: Option<String>,
-    #[error("Environment Error")]
+    pub limit: Option<String>,
-    Env(#[from] dotenvy::Error),
+
-    #[error("Invalid index to sort by: {0}")]
+    pub new_filters: Option<String>,
-    InvalidIndex(String),
+
-    #[error("Database error: {0}")]
+    // TODO: Deprecated values below. WILL BE REMOVED V3!
-    Database(#[from] DatabaseError),
+    pub facets: Option<String>,
+    pub filters: Option<String>,
+    pub version: Option<String>,
 }
 
-impl actix_web::ResponseError for SearchError {
+/// Search parameters which are more complicated and more suitable for a POST
-    fn status_code(&self) -> StatusCode {
+/// request body.
-        match self {
+///
-            SearchError::Env(..) => StatusCode::INTERNAL_SERVER_ERROR,
+/// Used with `POST /*/search` endpoints.
-            SearchError::MeiliSearch(..) => StatusCode::BAD_REQUEST,
+///
-            SearchError::Serde(..) => StatusCode::BAD_REQUEST,
+/// Can be converted from a [`SearchQuery`] using [`From`].
-            SearchError::IntParsing(..) => StatusCode::BAD_REQUEST,
+#[derive(Serialize, Deserialize, Debug)]
-            SearchError::InvalidIndex(..) => StatusCode::BAD_REQUEST,
+pub struct SearchRequest {
-            SearchError::FormatError(..) => StatusCode::BAD_REQUEST,
+    pub query: Option<String>,
-            SearchError::Database(..) => StatusCode::INTERNAL_SERVER_ERROR,
+    pub offset: Option<String>,
+    pub index: Option<String>,
+    pub limit: Option<String>,
+    #[serde(default)]
+    pub show_metadata: bool,
+    #[serde(default)]
+    pub elasticsearch_config: backend::elasticsearch::RequestConfig,
+    #[serde(default)]
+    pub typesense_config: backend::typesense::RequestConfig,
+
+    pub new_filters: Option<String>,
+
+    pub facets: Option<String>,
+    pub filters: Option<String>,
+    pub version: Option<String>,
+}
+
+impl From<SearchQuery> for SearchRequest {
+    fn from(query: SearchQuery) -> Self {
+        Self {
+            query: query.query,
+            offset: query.offset,
+            index: query.index,
+            limit: query.limit,
+            show_metadata: false,
+            elasticsearch_config:
+                backend::elasticsearch::RequestConfig::default(),
+            typesense_config: backend::typesense::RequestConfig::default(),
+            new_filters: query.new_filters,
+            facets: query.facets,
+            filters: query.filters,
+            version: query.version,
         }
     }
-
-    fn error_response(&self) -> HttpResponse {
-        HttpResponse::build(self.status_code()).json(ApiError {
-            error: match self {
-                SearchError::Env(..) => "environment_error",
-                SearchError::MeiliSearch(..) => "meilisearch_error",
-                SearchError::Serde(..) => "invalid_input",
-                SearchError::IntParsing(..) => "invalid_input",
-                SearchError::InvalidIndex(..) => "invalid_input",
-                SearchError::FormatError(..) => "invalid_input",
-                SearchError::Database(..) => "database_error",
-            },
-            description: self.to_string(),
-            details: None,
-        })
-    }
 }
 
-#[derive(Debug, Clone)]
+#[async_trait]
-pub struct MeilisearchReadClient {
+pub trait SearchBackend: Send + Sync {
-    pub client: Client,
+    async fn search_for_project(
-}
+        &self,
-
+        info: &SearchRequest,
-impl std::ops::Deref for MeilisearchReadClient {
+        redis: &RedisPool,
-    type Target = Client;
+    ) -> Result<SearchResults, ApiError> {
-
+        let mut results = self.search_for_project_raw(info).await?;
-    fn deref(&self) -> &Self::Target {
+        hydrate_search_results(&mut results.hits, redis)
-        &self.client
+            .await
-    }
+            .map_err(ApiError::Internal)?;
-}
-
-pub struct BatchClient {
-    pub clients: Vec<Client>,
-}
-
-impl BatchClient {
-    pub fn new(clients: Vec<Client>) -> Self {
-        Self { clients }
-    }
-
-    pub async fn with_all_clients<'a, T, G, Fut>(
-        &'a self,
-        task_name: &str,
-        generator: G,
-    ) -> Result<Vec<T>, IndexingError>
-    where
-        G: Fn(&'a Client) -> Fut,
-        Fut: Future<Output = Result<T, IndexingError>> + 'a,
-    {
-        let mut tasks = FuturesOrdered::new();
-        for (idx, client) in self.clients.iter().enumerate() {
-            tasks.push_back(generator(client).instrument(info_span!(
-                "client_task",
-                task.name = task_name,
-                client.idx = idx,
-            )));
-        }
-
-        let results = tasks.try_collect::<Vec<T>>().await?;
         Ok(results)
     }
 
-    pub fn across_all<T, F, R>(&self, data: Vec<T>, mut predicate: F) -> Vec<R>
+    async fn search_for_project_raw(
-    where
+        &self,
-        F: FnMut(T, &Client) -> R,
+        info: &SearchRequest,
-    {
+    ) -> Result<SearchResults, ApiError>;
-        assert_eq!(
+
-            data.len(),
+    async fn index_projects(
-            self.clients.len(),
+        &self,
-            "mismatch between data len and meilisearch client count"
+        ro_pool: PgPool,
-        );
+        redis: RedisPool,
-        self.clients
+    ) -> eyre::Result<()>;
-            .iter()
+
-            .zip(data)
+    async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()>;
-            .map(|(client, item)| predicate(item, client))
+
-            .collect()
+    async fn tasks(&self) -> eyre::Result<Value>;
-    }
+
+    async fn tasks_cancel(
+        &self,
+        filter: &TasksCancelFilter,
+    ) -> eyre::Result<()>;
 }
 
-#[derive(Debug, Clone)]
+async fn hydrate_search_results(
-pub struct SearchConfig {
+    hits: &mut [ResultSearchProject],
-    pub addresses: Vec<String>,
+    redis_pool: &RedisPool,
-    pub read_lb_address: String,
+) -> eyre::Result<()> {
-    pub key: String,
+    // Minecraft Java servers should fetch the latest player count that we have
-    pub meta_namespace: String,
+    // from Redis, rather than the (pretty stale) data from search backend
-}
+    // TODO: this block should be made generic over the component type,
+    // for now we can hardcode MC java servers tho
 
-impl SearchConfig {
+    let project_ids = hits
-    // Panics if the environment variables are not set,
+        .iter()
-    // but these are already checked for on startup.
+        .filter(|hit| hit.components.minecraft_java_server.is_some())
-    pub fn new(meta_namespace: Option<String>) -> Self {
+        .filter_map(|hit| parse_base62(&hit.project_id).ok().map(ProjectId))
-        Self {
+        .collect::<Vec<_>>();
-            addresses: ENV.MEILISEARCH_WRITE_ADDRS.0.clone(),
+
-            key: ENV.MEILISEARCH_KEY.clone(),
+    let pings_by_project_id = if project_ids.is_empty() {
-            meta_namespace: meta_namespace.unwrap_or_default(),
+        HashMap::new()
-            read_lb_address: ENV.MEILISEARCH_READ_ADDR.clone(),
+    } else {
+        let mut redis = redis_pool.connect().await?;
+        let ping_results = redis
+            .get_many_deserialized_from_json::<JavaServerPing>(
+                server_ping::REDIS_NAMESPACE,
+                &project_ids
+                    .iter()
+                    .map(ToString::to_string)
+                    .collect::<Vec<_>>(),
+            )
+            .await?;
+
+        ping_results
+            .into_iter()
+            .enumerate()
+            .filter_map(|(idx, ping)| ping.map(|ping| (project_ids[idx], ping)))
+            .collect::<HashMap<_, _>>()
+    };
+
+    for hit in hits {
+        let Some(java_server) = hit.components.minecraft_java_server.as_mut()
+        else {
+            continue;
+        };
+        if let Ok(project_id) = parse_base62(&hit.project_id).map(ProjectId) {
+            java_server.ping = pings_by_project_id.get(&project_id).cloned();
+        } else {
+            java_server.ping = None;
         }
     }
 
-    pub fn make_loadbalanced_read_client(
+    Ok(())
-        &self,
+}
-    ) -> Result<MeilisearchReadClient, meilisearch_sdk::errors::Error> {
+
-        Ok(MeilisearchReadClient {
+#[derive(Deserialize, Serialize, ToSchema)]
-            client: Client::new(&self.read_lb_address, Some(&self.key))?,
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum TasksCancelFilter {
+    All,
+    AllEnqueued,
+    Indexes { indexes: Vec<String> },
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum SearchBackendKind {
+    Meilisearch,
+    Elasticsearch,
+    Typesense,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum::EnumIter)]
+pub enum SearchField {
+    Categories,
+    ProjectTypes,
+    ProjectId,
+    OpenSource,
+    Environment,
+    GameVersions,
+    ClientSide,
+    ServerSide,
+    MinecraftServerRegion,
+    MinecraftServerLanguages,
+    MinecraftJavaServerContentKind,
+    MinecraftJavaServerContentSupportedGameVersions,
+    MinecraftJavaServerPingData,
+}
+
+#[derive(Debug, Error)]
+#[error("invalid search backend kind")]
+pub struct InvalidSearchBackendKind;
+
+impl FromStr for SearchBackendKind {
+    type Err = InvalidSearchBackendKind;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s {
+            "meilisearch" => SearchBackendKind::Meilisearch,
+            "elasticsearch" => SearchBackendKind::Elasticsearch,
+            "typesense" => SearchBackendKind::Typesense,
+            _ => return Err(InvalidSearchBackendKind),
         })
     }
-
-    pub fn make_batch_client(
-        &self,
-    ) -> Result<BatchClient, meilisearch_sdk::errors::Error> {
-        Ok(BatchClient::new(
-            self.addresses
-                .iter()
-                .map(|address| {
-                    Client::new(address.as_str(), Some(self.key.as_str()))
-                })
-                .collect::<Result<Vec<_>, _>>()?,
-        ))
-    }
-
-    // Next: true if we want the next index (we are preparing the next swap), false if we want the current index (searching)
-    pub fn get_index_name(&self, index: &str, next: bool) -> String {
-        let alt = if next { "_alt" } else { "" };
-        format!("{}_{}_{}", self.meta_namespace, index, alt)
-    }
 }
 
-/// A project document used for uploading projects to MeiliSearch's indices.
-/// This contains some extra data that is not returned by search results.
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct UploadSearchProject {
     pub version_id: String,
@@ -197,6 +231,7 @@ pub struct UploadSearchProject {
     pub display_categories: Vec<String>,
     pub follows: i32,
     pub downloads: i32,
+    pub log_downloads: f64,
     pub icon_url: Option<String>,
     pub license: String,
     pub gallery: Vec<String>,
@@ -263,281 +298,52 @@ pub struct ResultSearchProject {
     pub components: exp::ProjectQuery,
     #[serde(flatten)]
     pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub search_metadata: Option<Value>,
 }
 
-pub fn get_sort_index(
+impl From<UploadSearchProject> for ResultSearchProject {
-    config: &SearchConfig,
+    fn from(source: UploadSearchProject) -> Self {
-    index: &str,
+        Self {
-    new_filters: Option<&str>,
+            version_id: source.version_id,
-) -> Result<(String, &'static [&'static str]), SearchError> {
+            project_id: source.project_id,
-    let projects_name = config.get_index_name("projects", false);
+            project_types: source.project_types,
-    let projects_filtered_name =
+            slug: source.slug,
-        config.get_index_name("projects_filtered", false);
+            author: source.author,
-
+            name: source.name,
-    // TODO: this is a dumb hack, the frontend should pass the project type it's filtering directly
+            summary: source.summary,
-    let is_server = new_filters
+            categories: source.categories,
-        .is_some_and(|f| f.contains("project_types = minecraft_java_server"));
+            display_categories: source.display_categories,
-
+            downloads: source.downloads,
-    Ok(match index {
+            follows: source.follows,
-        "relevance" => (
+            icon_url: source.icon_url,
-            projects_name,
+            date_created: source.date_created.to_rfc3339(),
-            if is_server {
+            date_modified: source.date_modified.to_rfc3339(),
-                &[
+            license: source.license,
-                    "minecraft_java_server.verified_plays_2w:desc",
+            gallery: source.gallery,
-                    "minecraft_java_server.ping.data.players_online:desc",
+            featured_gallery: source.featured_gallery,
-                    "version_published_timestamp:desc",
+            color: source.color,
-                ]
+            loaders: source.loaders,
-            } else {
+            project_loader_fields: source.project_loader_fields,
-                &["downloads:desc", "version_published_timestamp:desc"]
+            components: source.components,
-            },
+            loader_fields: source.loader_fields,
-        ),
+            search_metadata: None,
-        "downloads" => (
-            projects_filtered_name,
-            &["downloads:desc", "version_published_timestamp:desc"],
-        ),
-        "follows" => (
-            projects_name,
-            &["follows:desc", "version_published_timestamp:desc"],
-        ),
-        "updated" | "date_modified" => (
-            projects_name,
-            &["date_modified:desc", "version_published_timestamp:desc"],
-        ),
-        "newest" | "date_created" => (
-            projects_name,
-            &["date_created:desc", "version_published_timestamp:desc"],
-        ),
-        "minecraft_java_server.verified_plays_2w" => (
-            projects_name,
-            &[
-                "minecraft_java_server.verified_plays_2w:desc",
-                "version_published_timestamp:desc",
-            ],
-        ),
-        "minecraft_java_server.ping.data.players_online" => (
-            projects_name,
-            &[
-                "minecraft_java_server.ping.data.players_online:desc",
-                "version_published_timestamp:desc",
-            ],
-        ),
-        i => return Err(SearchError::InvalidIndex(i.to_string())),
-    })
-}
-
-fn normalize_filter_aliases(filters: &str) -> String {
-    let mut filters = filters.replace("components.", "");
-    for (from, to) in [
-        (
-            "minecraft_java_server.content =",
-            "minecraft_java_server.content.kind =",
-        ),
-        (
-            "minecraft_java_server.content !=",
-            "minecraft_java_server.content.kind !=",
-        ),
-        (
-            "minecraft_java_server.content IN ",
-            "minecraft_java_server.content.kind IN ",
-        ),
-        (
-            "minecraft_java_server.content NOT IN ",
-            "minecraft_java_server.content.kind NOT IN ",
-        ),
-    ] {
-        filters = filters.replace(from, to);
-    }
-    filters
-}
-
-pub async fn search_for_project(
-    info: &SearchRequest,
-    config: &SearchConfig,
-    redis_pool: &RedisPool,
-) -> Result<SearchResults, SearchError> {
-    let offset: usize = info.offset.as_deref().unwrap_or("0").parse()?;
-    let index = info.index.as_deref().unwrap_or("relevance");
-    let limit = info
-        .limit
-        .as_deref()
-        .unwrap_or("10")
-        .parse::<usize>()?
-        .min(100);
-
-    let sort = get_sort_index(config, index, info.new_filters.as_deref())?;
-    let client = config.make_loadbalanced_read_client()?;
-    let meilisearch_index = client.get_index(sort.0).await?;
-
-    let mut filter_string = String::new();
-
-    // Convert offset and limit to page and hits_per_page
-    let hits_per_page = if limit == 0 { 1 } else { limit };
-
-    let page = offset / hits_per_page + 1;
-
-    let results = {
-        let mut query = meilisearch_index.search();
-        query
-            .with_page(page)
-            .with_hits_per_page(hits_per_page)
-            .with_query(info.query.as_deref().unwrap_or_default())
-            .with_sort(sort.1);
-
-        let normalized_new_filters =
-            info.new_filters.as_deref().map(normalize_filter_aliases);
-        if let Some(new_filters) = normalized_new_filters.as_deref() {
-            query.with_filter(new_filters);
-        } else {
-            let facets = if let Some(facets) = &info.facets {
-                Some(serde_json::from_str::<Vec<Vec<Value>>>(facets)?)
-            } else {
-                None
-            };
-
-            let filters: Cow<_> =
-                match (info.filters.as_deref(), info.version.as_deref()) {
-                    (Some(f), Some(v)) => format!("({f}) AND ({v})").into(),
-                    (Some(f), None) => f.into(),
-                    (None, Some(v)) => v.into(),
-                    (None, None) => "".into(),
-                };
-            let filters = normalize_filter_aliases(&filters);
-
-            if let Some(facets) = facets {
-                // Search can now *optionally* have a third inner array: So Vec(AND)<Vec(OR)<Vec(AND)< _ >>>
-                // For every inner facet, we will check if it can be deserialized into a Vec<&str>, and do so.
-                // If not, we will assume it is a single facet and wrap it in a Vec.
-                let facets: Vec<Vec<Vec<String>>> = facets
-                    .into_iter()
-                    .map(|facets| {
-                        facets
-                            .into_iter()
-                            .map(|facet| {
-                                if facet.is_array() {
-                                    serde_json::from_value::<Vec<String>>(facet)
-                                        .unwrap_or_default()
-                                } else {
-                                    vec![
-                                        serde_json::from_value::<String>(facet)
-                                            .unwrap_or_default(),
-                                    ]
-                                }
-                            })
-                            .collect_vec()
-                    })
-                    .collect_vec();
-
-                filter_string.push('(');
-                for (index, facet_outer_list) in facets.iter().enumerate() {
-                    filter_string.push('(');
-
-                    for (facet_outer_index, facet_inner_list) in
-                        facet_outer_list.iter().enumerate()
-                    {
-                        filter_string.push('(');
-                        for (facet_inner_index, facet) in
-                            facet_inner_list.iter().enumerate()
-                        {
-                            let facet = normalize_filter_aliases(
-                                &facet.replace(':', " = "),
-                            );
-                            filter_string.push_str(&facet);
-                            if facet_inner_index != (facet_inner_list.len() - 1)
-                            {
-                                filter_string.push_str(" AND ")
-                            }
-                        }
-                        filter_string.push(')');
-
-                        if facet_outer_index != (facet_outer_list.len() - 1) {
-                            filter_string.push_str(" OR ")
-                        }
-                    }
-
-                    filter_string.push(')');
-
-                    if index != (facets.len() - 1) {
-                        filter_string.push_str(" AND ")
-                    }
-                }
-                filter_string.push(')');
-
-                if !filters.is_empty() {
-                    write!(filter_string, " AND ({filters})")?;
-                }
-            } else {
-                filter_string.push_str(&filters);
-            }
-
-            if !filter_string.is_empty() {
-                query.with_filter(&filter_string);
-            }
-        }
-
-        query.execute::<ResultSearchProject>().await?
-    };
-
-    // Minecraft Java servers should fetch the latest player count that we have
-    // from Redis, rather than the (pretty stale) data from search backend
-    // TODO: this block should be made generic over the component type,
-    // for now we can hardcode MC java servers tho
-    let mut hits = results.hits.into_iter().map(|r| r.result).collect_vec();
-
-    let project_ids = hits
-        .iter()
-        .filter(|hit| hit.components.minecraft_java_server.is_some())
-        .filter_map(|hit| parse_base62(&hit.project_id).ok().map(ProjectId))
-        .collect_vec();
-
-    let pings_by_project_id = if project_ids.is_empty() {
-        HashMap::new()
-    } else {
-        let mut redis = redis_pool.connect().await?;
-        let ping_results = redis
-            .get_many_deserialized_from_json::<JavaServerPing>(
-                server_ping::REDIS_NAMESPACE,
-                &project_ids.iter().map(ToString::to_string).collect_vec(),
-            )
-            .await?;
-
-        ping_results
-            .into_iter()
-            .enumerate()
-            .filter_map(|(idx, ping)| ping.map(|ping| (project_ids[idx], ping)))
-            .collect::<HashMap<_, _>>()
-    };
-
-    for hit in &mut hits {
-        let Some(java_server) = hit.components.minecraft_java_server.as_mut()
-        else {
-            continue;
-        };
-        if let Ok(project_id) = parse_base62(&hit.project_id).map(ProjectId) {
-            java_server.ping = pings_by_project_id.get(&project_id).cloned();
-        } else {
-            java_server.ping = None;
         }
     }
-
-    Ok(SearchResults {
-        hits,
-        page: results.page.unwrap_or_default(),
-        hits_per_page: results.hits_per_page.unwrap_or_default(),
-        total_hits: results.total_hits.unwrap_or_default(),
-    })
 }
 
-#[cfg(test)]
+pub fn backend(meta_namespace: Option<String>) -> Box<dyn SearchBackend> {
-mod tests {
+    match ENV.SEARCH_BACKEND {
-    use super::normalize_filter_aliases;
+        SearchBackendKind::Meilisearch => {
-
+            let config = backend::MeilisearchConfig::new(meta_namespace);
-    #[test]
+            Box::new(backend::Meilisearch::new(config))
-    fn normalizes_component_filter_aliases() {
+        }
-        assert_eq!(
+        SearchBackendKind::Elasticsearch => {
-            normalize_filter_aliases(
+            Box::new(backend::Elasticsearch::new(meta_namespace).unwrap())
-                "components.minecraft_java_server.content = vanilla AND components.minecraft_server.country = US"
+        }
-            ),
+        SearchBackendKind::Typesense => {
-            "minecraft_java_server.content.kind = vanilla AND minecraft_server.country = US"
+            let config = backend::TypesenseConfig::new(meta_namespace);
-        );
+            Box::new(backend::Typesense::new(config))
+        }
     }
 }

apps/labrinth/src/test/database.rs

@@ -2,8 +2,9 @@ use crate::database::PgPool;
 use crate::database::redis::RedisPool;
 use crate::database::{MIGRATOR, ReadOnlyPgPool};
 use crate::env::ENV;
-use crate::search;
+use crate::search::{self, SearchBackend};
 use sqlx::postgres::PgPoolOptions;
+use std::sync::Arc;
 use std::time::Duration;
 use url::Url;
 
@@ -42,8 +43,8 @@ pub struct TemporaryDatabase {
     pub pool: PgPool,
     pub ro_pool: ReadOnlyPgPool,
     pub redis_pool: RedisPool,
-    pub search_config: crate::search::SearchConfig,
     pub database_name: String,
+    pub search_backend: Arc<dyn SearchBackend>,
 }
 
 impl TemporaryDatabase {
@@ -91,15 +92,14 @@ impl TemporaryDatabase {
         // Gets new Redis pool
         let redis_pool = RedisPool::new(temp_database_name.clone());
 
-        // Create new meilisearch config
+        // Create search backend
-        let search_config =
+        let search_backend = search::backend(Some(temp_database_name.clone()));
-            search::SearchConfig::new(Some(temp_database_name.clone()));
         Self {
             pool,
             ro_pool,
             database_name: temp_database_name,
             redis_pool,
-            search_config,
+            search_backend: Arc::from(search_backend),
         }
     }
 
@@ -193,7 +193,9 @@ impl TemporaryDatabase {
                         ro_pool: ReadOnlyPgPool::from(pool.clone()),
                         database_name: TEMPLATE_DATABASE_NAME.to_string(),
                         redis_pool: RedisPool::new(name.clone()),
-                        search_config: search::SearchConfig::new(Some(name)),
+                        search_backend: Arc::from(search::backend(Some(
+                            name.clone(),
+                        ))),
                     };
                     let setup_api =
                         TestEnvironment::<ApiV3>::build_setup_api(&db).await;

apps/labrinth/src/test/mod.rs

@@ -23,7 +23,6 @@ pub mod search;
 // If making a test, you should probably use environment::TestEnvironment::build() (which calls this)
 pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
     println!("Setting up labrinth config");
-    dotenvy::dotenv().ok();
     env::init().expect("failed to initialize environment variables");
 
     let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
@@ -31,7 +30,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
     let pool = db.pool.clone();
     let ro_pool = db.ro_pool.clone();
     let redis_pool = db.redis_pool.clone();
-    let search_config = db.search_config.clone();
+    let search_backend = db.search_backend.clone();
     let file_host: Arc<dyn file_hosting::FileHost + Send + Sync> =
         Arc::new(file_hosting::MockHost::new());
     let mut clickhouse = clickhouse::init_client().await.unwrap();
@@ -48,7 +47,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
         pool.clone(),
         ro_pool.clone(),
         redis_pool.clone(),
-        search_config,
+        search_backend.into(),
         &mut clickhouse,
         file_host.clone(),
         stripe_client,

apps/labrinth/tests/search.rs

@@ -101,7 +101,7 @@ async fn search_projects() {
                     async move {
                         let projects = api
                             .search_deserialized(
-                                Some(&format!("\"&{test_name}\"")),
+                                Some(&format!("&{test_name}")),
                                 Some(facets.clone()),
                                 USER_USER_PAT,
                             )

apps/labrinth/tests/v2/search.rs

@@ -316,7 +316,7 @@ async fn search_projects() {
                 async move {
                     let projects = api
                         .search_deserialized(
-                            Some(&format!("\"&{test_name}\"")),
+                            Some(&format!("&{test_name}")),
                             Some(facets.clone()),
                             USER_USER_PAT,
                         )
@@ -337,7 +337,7 @@ async fn search_projects() {
         // A couple additional tests for the search type returned, making sure it is properly translated back
         let client_side_required = api
             .search_deserialized(
-                Some(&format!("\"&{test_name}\"")),
+                Some(&format!("&{test_name}")),
                 Some(json!([["client_side:required"]])),
                 USER_USER_PAT,
             )
@@ -348,7 +348,7 @@ async fn search_projects() {
 
         let server_side_required = api
             .search_deserialized(
-                Some(&format!("\"&{test_name}\"")),
+                Some(&format!("&{test_name}")),
                 Some(json!([["server_side:required"]])),
                 USER_USER_PAT,
             )
@@ -359,7 +359,7 @@ async fn search_projects() {
 
         let client_side_unsupported = api
             .search_deserialized(
-                Some(&format!("\"&{test_name}\"")),
+                Some(&format!("&{test_name}")),
                 Some(json!([["client_side:unsupported"]])),
                 USER_USER_PAT,
             )
@@ -370,7 +370,7 @@ async fn search_projects() {
 
         let client_side_optional_server_side_optional = api
             .search_deserialized(
-                Some(&format!("\"&{test_name}\"")),
+                Some(&format!("&{test_name}")),
                 Some(json!([["client_side:optional"], ["server_side:optional"]])),
                 USER_USER_PAT,
             )
@@ -384,7 +384,7 @@ async fn search_projects() {
         // over all versions of a project
         let game_versions = api
             .search_deserialized(
-                Some(&format!("\"&{test_name}\"")),
+                Some(&format!("&{test_name}")),
                 Some(json!([["categories:forge"], ["versions:1.20.2"]])),
                 USER_USER_PAT,
             )

docker-compose.yml

@@ -18,6 +18,20 @@ services:
       interval: 3s
       timeout: 5s
       retries: 3
+  typesense0:
+    image: typesense/typesense:30.1
+    container_name: labrinth-typesense0
+    restart: on-failure
+    ports:
+      - '127.0.0.1:8108:8108'
+    volumes:
+      - typesense-data:/data
+    command: --data-dir=/data --api-key=modrinth --enable-cors
+    healthcheck:
+      test: ['CMD-SHELL', "bash -lc '</dev/tcp/127.0.0.1/8108'"]
+      interval: 3s
+      timeout: 5s
+      retries: 3
   meilisearch0:
     image: getmeili/meilisearch:v1.12.0
     container_name: labrinth-meilisearch0
@@ -37,6 +51,167 @@ services:
       interval: 3s
       timeout: 5s
       retries: 3
+  elasticsearch-certs:
+    image: elasticsearch:9.3.0
+    container_name: labrinth-elasticsearch-certs
+    user: '0'
+    networks:
+      - elasticsearch-mesh
+    restart: 'no'
+    volumes:
+      - elasticsearch-certs:/usr/share/elasticsearch/config/certs
+    command: |
+      bash -c '
+        set -euo pipefail
+        if [ ! -s config/certs/ca/ca.crt ] || [ ! -s config/certs/elasticsearch0/elasticsearch0.crt ] || [ ! -s config/certs/elasticsearch1/elasticsearch1.crt ] || [ ! -s config/certs/elasticsearch2/elasticsearch2.crt ]; then
+          rm -rf config/certs/*
+          printf "%s\n" \
+            "instances:" \
+            "  - name: elasticsearch0" \
+            "    dns:" \
+            "      - elasticsearch0" \
+            "      - localhost" \
+            "    ip:" \
+            "      - 127.0.0.1" \
+            "  - name: elasticsearch1" \
+            "    dns:" \
+            "      - elasticsearch1" \
+            "      - localhost" \
+            "    ip:" \
+            "      - 127.0.0.1" \
+            "  - name: elasticsearch2" \
+            "    dns:" \
+            "      - elasticsearch2" \
+            "      - localhost" \
+            "    ip:" \
+            "      - 127.0.0.1" \
+            > config/certs/instances.yml
+          bin/elasticsearch-certutil ca --silent --pem --out config/certs/ca.zip
+          unzip config/certs/ca.zip -d config/certs
+          bin/elasticsearch-certutil cert --silent --pem --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key --out config/certs/certs.zip
+          unzip config/certs/certs.zip -d config/certs
+        fi
+        chown -R 1000:0 config/certs
+        find config/certs -type d -exec chmod 750 {} \;
+        find config/certs -type f -exec chmod 640 {} \;
+        echo "Set up certificates"
+      '
+  elasticsearch0:
+    image: elasticsearch:9.3.0
+    container_name: labrinth-elasticsearch0
+    networks:
+      - elasticsearch-mesh
+    restart: on-failure
+    depends_on:
+      elasticsearch-certs:
+        condition: service_completed_successfully
+    ports:
+      - '127.0.0.1:9200:9200'
+    volumes:
+      - elasticsearch0-data:/usr/share/elasticsearch/data
+      - elasticsearch-certs:/usr/share/elasticsearch/config/certs:ro
+    environment:
+      - logger.level=WARN
+      - node.name=elasticsearch0
+      - cluster.name=labrinth
+      - cluster.initial_master_nodes=elasticsearch0,elasticsearch1,elasticsearch2
+      - discovery.seed_hosts=elasticsearch1,elasticsearch2
+      - bootstrap.memory_lock=false
+      # auth
+      - xpack.security.enabled=true
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.security.transport.ssl.key=certs/elasticsearch0/elasticsearch0.key
+      - xpack.security.transport.ssl.certificate=certs/elasticsearch0/elasticsearch0.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - ELASTIC_USERNAME=elastic
+      - ELASTIC_PASSWORD=elastic
+    mem_limit: 1g
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          'curl -s -u elastic:elastic http://localhost:9200/_cluster/health | grep -qE "\"status\":\"(yellow|green)\""',
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+  elasticsearch1:
+    image: elasticsearch:9.3.0
+    container_name: labrinth-elasticsearch1
+    networks:
+      - elasticsearch-mesh
+    restart: on-failure
+    depends_on:
+      elasticsearch-certs:
+        condition: service_completed_successfully
+    volumes:
+      - elasticsearch1-data:/usr/share/elasticsearch/data
+      - elasticsearch-certs:/usr/share/elasticsearch/config/certs:ro
+    environment:
+      - logger.level=WARN
+      - node.name=elasticsearch1
+      - cluster.name=labrinth
+      - cluster.initial_master_nodes=elasticsearch0,elasticsearch1,elasticsearch2
+      - discovery.seed_hosts=elasticsearch0,elasticsearch2
+      - bootstrap.memory_lock=false
+      # auth
+      - xpack.security.enabled=true
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.security.transport.ssl.key=certs/elasticsearch1/elasticsearch1.key
+      - xpack.security.transport.ssl.certificate=certs/elasticsearch1/elasticsearch1.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - ELASTIC_USERNAME=elastic
+      - ELASTIC_PASSWORD=elastic
+    mem_limit: 1g
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          'curl -s -u elastic:elastic http://localhost:9200/_cluster/health | grep -qE "\"status\":\"(yellow|green)\""',
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+  elasticsearch2:
+    image: elasticsearch:9.3.0
+    container_name: labrinth-elasticsearch2
+    networks:
+      - elasticsearch-mesh
+    restart: on-failure
+    depends_on:
+      elasticsearch-certs:
+        condition: service_completed_successfully
+    volumes:
+      - elasticsearch2-data:/usr/share/elasticsearch/data
+      - elasticsearch-certs:/usr/share/elasticsearch/config/certs:ro
+    environment:
+      - logger.level=WARN
+      - node.name=elasticsearch2
+      - cluster.name=labrinth
+      - cluster.initial_master_nodes=elasticsearch0,elasticsearch1,elasticsearch2
+      - discovery.seed_hosts=elasticsearch0,elasticsearch1
+      - bootstrap.memory_lock=false
+      # auth
+      - xpack.security.enabled=true
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.security.transport.ssl.key=certs/elasticsearch2/elasticsearch2.key
+      - xpack.security.transport.ssl.certificate=certs/elasticsearch2/elasticsearch2.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - ELASTIC_USERNAME=elastic
+      - ELASTIC_PASSWORD=elastic
+    mem_limit: 1g
+    healthcheck:
+      test:
+        [
+          'CMD-SHELL',
+          'curl -s -u elastic:elastic http://localhost:9200/_cluster/health | grep -qE "\"status\":\"(yellow|green)\""',
+        ]
+      interval: 10s
+      timeout: 5s
+      retries: 10
   redis:
     image: redis:alpine
     container_name: labrinth-redis
@@ -109,6 +284,12 @@ services:
         condition: service_healthy
       meilisearch:
         condition: service_healthy
+      elasticsearch0:
+        condition: service_healthy
+      elasticsearch1:
+        condition: service_healthy
+      elasticsearch2:
+        condition: service_healthy
       redis:
         condition: service_healthy
       clickhouse:
@@ -143,7 +324,6 @@ services:
       # Delphi must send a message on a webhook to our backend,
       # so it must have access to our local network
       - 'host.docker.internal:host-gateway'
-
   # Sharded Meilisearch
   meilisearch1:
     profiles:
@@ -166,7 +346,6 @@ services:
       interval: 3s
       timeout: 5s
       retries: 3
-
   nginx-meilisearch-lb:
     profiles:
       - sharded-meilisearch
@@ -186,9 +365,16 @@ services:
 networks:
   meilisearch-mesh:
     driver: bridge
+  elasticsearch-mesh:
+    driver: bridge
 volumes:
+  typesense-data:
   meilisearch-data:
   meilisearch1-data:
+  elasticsearch0-data:
+  elasticsearch1-data:
+  elasticsearch2-data:
+  elasticsearch-certs:
   db-data:
   redis-data:
   labrinth-cdn-data:

packages/async-minecraft-ping/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "async-minecraft-ping"
 version = "0.8.0"
-authors = ["Jay Vana <jaysvana@gmail.com>"]
+# authors = ["Jay Vana <jaysvana@gmail.com>"] # deprecated
 edition = "2021"
-license = "MIT OR Apache-2.0"
 description = "An async Rust client for the Minecraft ServerListPing protocol"
 readme = "README.md"
 repository = "https://github.com/jsvana/async-minecraft-ping/"
+license = "MIT OR Apache-2.0"
 keywords = ["mc", "minecraft", "serverlistping"]
 categories = ["api-bindings", "asynchronous"]