Search backend refactor with typesense impl (#5528)

* initial elasticsearch impl

* working elastic cluster

* replace SearchError with ApiError for preparation of search backend

* start factoring meili out to trait

* move meili to backend

* update routes to use search backend trait

* wip

* Update projects.rs

* search backend is only init'd once in config

* wip

* wip: backend agnostic

* change search internal routes to delegate to backend

* initial elasticsearch impl

* fix filtering

* elastic impl

* refactor indexing into its own module

* clean up elastic code

* fix ci

* fix tests

* fix elastic health check

* fix up env rebase

* fix compile

* dummy commit to update github pr

* Fix rebase

* Elastic basic https auth

* Fix duplicate projects showing up

* Fix up tests

* Replace search `ApiErrors` with `eyre::Reports`, propagate background task errors

* clean up agents files

* make index chunk size configurable

* make `match_phrase` in elastic case-insensitive

* use current/next indices and swap between them

* test case for error body

* Fix failing case

* da merge

* factor out common stuff from search backends

* allow fetching hit metadata from search results

* allow customising elasticsearch search config

* bit of docs

* add mappings to indices for elastic

* Implement Typesense

* wip

* fix up some sort fields stuff

* use different approach to filterable field sets

* remove a bunch of search fields which weren't used for filtering

* bucket text matches

* Bucketing by text_match for typesense

* fix tombi lint

* fix some sentry errors and dont prioritise 2+ term matches

* tweak ts query settings

* expose some more search settings

* query sort changes

* small fixes

* should fix pagination stuff

* fix healthcheck maybe

* ragebait ci

* tests

* tests

* revert environment

apps/labrinth/src/test/database.rs

@@ -2,8 +2,9 @@ use crate::database::PgPool;
 use crate::database::redis::RedisPool;
 use crate::database::{MIGRATOR, ReadOnlyPgPool};
 use crate::env::ENV;
-use crate::search;
+use crate::search::{self, SearchBackend};
 use sqlx::postgres::PgPoolOptions;
+use std::sync::Arc;
 use std::time::Duration;
 use url::Url;
 
@@ -42,8 +43,8 @@ pub struct TemporaryDatabase {
     pub pool: PgPool,
     pub ro_pool: ReadOnlyPgPool,
     pub redis_pool: RedisPool,
-    pub search_config: crate::search::SearchConfig,
     pub database_name: String,
+    pub search_backend: Arc<dyn SearchBackend>,
 }
 
 impl TemporaryDatabase {
@@ -91,15 +92,14 @@ impl TemporaryDatabase {
         // Gets new Redis pool
         let redis_pool = RedisPool::new(temp_database_name.clone());
 
-        // Create new meilisearch config
-        let search_config =
-            search::SearchConfig::new(Some(temp_database_name.clone()));
+        // Create search backend
+        let search_backend = search::backend(Some(temp_database_name.clone()));
         Self {
             pool,
             ro_pool,
             database_name: temp_database_name,
             redis_pool,
-            search_config,
+            search_backend: Arc::from(search_backend),
         }
     }
 
@@ -193,7 +193,9 @@ impl TemporaryDatabase {
                         ro_pool: ReadOnlyPgPool::from(pool.clone()),
                         database_name: TEMPLATE_DATABASE_NAME.to_string(),
                         redis_pool: RedisPool::new(name.clone()),
-                        search_config: search::SearchConfig::new(Some(name)),
+                        search_backend: Arc::from(search::backend(Some(
+                            name.clone(),
+                        ))),
                     };
                     let setup_api =
                         TestEnvironment::<ApiV3>::build_setup_api(&db).await;

apps/labrinth/src/test/mod.rs

@@ -23,7 +23,6 @@ pub mod search;
 // If making a test, you should probably use environment::TestEnvironment::build() (which calls this)
 pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
     println!("Setting up labrinth config");
-    dotenvy::dotenv().ok();
     env::init().expect("failed to initialize environment variables");
 
     let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
@@ -31,7 +30,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
     let pool = db.pool.clone();
     let ro_pool = db.ro_pool.clone();
     let redis_pool = db.redis_pool.clone();
-    let search_config = db.search_config.clone();
+    let search_backend = db.search_backend.clone();
     let file_host: Arc<dyn file_hosting::FileHost + Send + Sync> =
         Arc::new(file_hosting::MockHost::new());
     let mut clickhouse = clickhouse::init_client().await.unwrap();
@@ -48,7 +47,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
         pool.clone(),
         ro_pool.clone(),
         redis_pool.clone(),
-        search_config,
+        search_backend.into(),
         &mut clickhouse,
         file_host.clone(),
         stripe_client,