MrSphay/Modrinth-plus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Search backend refactor with typesense impl (#5528)

Browse Source 
* initial elasticsearch impl

* working elastic cluster

* replace SearchError with ApiError for preparation of search backend

* start factoring meili out to trait

* move meili to backend

* update routes to use search backend trait

* wip

* Update projects.rs

* search backend is only init'd once in config

* wip

* wip: backend agnostic

* change search internal routes to delegate to backend

* initial elasticsearch impl

* fix filtering

* elastic impl

* refactor indexing into its own module

* clean up elastic code

* fix ci

* fix tests

* fix elastic health check

* fix up env rebase

* fix compile

* dummy commit to update github pr

* Fix rebase

* Elastic basic https auth

* Fix duplicate projects showing up

* Fix up tests

* Replace search `ApiErrors` with `eyre::Reports`, propagate background task errors

* clean up agents files

* make index chunk size configurable

* make `match_phrase` in elastic case-insensitive

* use current/next indices and swap between them

* test case for error body

* Fix failing case

* da merge

* factor out common stuff from search backends

* allow fetching hit metadata from search results

* allow customising elasticsearch search config

* bit of docs

* add mappings to indices for elastic

* Implement Typesense

* wip

* fix up some sort fields stuff

* use different approach to filterable field sets

* remove a bunch of search fields which weren't used for filtering

* bucket text matches

* Bucketing by text_match for typesense

* fix tombi lint

* fix some sentry errors and dont prioritise 2+ term matches

* tweak ts query settings

* expose some more search settings

* query sort changes

* small fixes

* should fix pagination stuff

* fix healthcheck maybe

* ragebait ci

* tests

* tests

* revert environment
This commit is contained in:
aecsocket
2026-03-12 17:58:55 +00:00
committed by GitHub
parent 1c1683adb6
commit f0224dfff7
36 changed files with 3848 additions and 762 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
Cargo.lock generated
View File

@@ -2714,6 +2714,29 @@ dependencies = [
"serde",
]
[[package]]
name = "elasticsearch"
version = "9.1.0-alpha.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12bb303aa6e1d28c0c86b6fbfe484fd0fd3f512629aeed1ac4f6b85f81d9834a"
dependencies = [
"base64 0.22.1",
"bytes",
"dyn-clone",
"flate2",
"lazy_static",
"parking_lot",
"percent-encoding",
"reqwest 0.12.24",
"rustc_version",
"serde",
"serde_json",
"serde_with",
"tokio",
"url",
"void",
]
[[package]]
name = "elliptic-curve"
version = "0.13.8"
@@ -4895,6 +4918,7 @@ dependencies = [
"dotenv-build",
"dotenvy",
"either",
"elasticsearch",
"eyre",
"futures",
"futures-util",

1
Cargo.toml
View File

@@ -72,6 +72,7 @@ dotenv-build = "0.1.1"
dotenvy = "0.15.7"
dunce = "1.0.5"
either = "1.15.0"
elasticsearch = "9.1.0-alpha.1"
encoding_rs = "0.8.35"
enumset = "1.1.10"
eyre = "0.6.12"

9
apps/labrinth/.env.docker-compose
View File

@@ -16,9 +16,18 @@ DATABASE_URL=postgresql://labrinth:labrinth@labrinth-postgres/labrinth
DATABASE_MIN_CONNECTIONS=0
DATABASE_MAX_CONNECTIONS=16
SEARCH_BACKEND=typesense
MEILISEARCH_READ_ADDR=http://localhost:7700
MEILISEARCH_WRITE_ADDRS=http://localhost:7700
MEILISEARCH_KEY=modrinth
ELASTICSEARCH_URL=http://localhost:9200
ELASTICSEARCH_INDEX_PREFIX=labrinth
ELASTICSEARCH_USERNAME=elastic
ELASTICSEARCH_PASSWORD=elastic
SEARCH_INDEX_CHUNK_SIZE=5000
TYPESENSE_URL=http://localhost:8108
TYPESENSE_API_KEY=modrinth
TYPESENSE_INDEX_PREFIX=labrinth
REDIS_URL=redis://labrinth-redis
REDIS_MIN_CONNECTIONS=0

20
apps/labrinth/.env.local
View File

@@ -16,16 +16,36 @@ DATABASE_URL=postgresql://labrinth:labrinth@localhost/labrinth
DATABASE_MIN_CONNECTIONS=0
DATABASE_MAX_CONNECTIONS=16
SEARCH_BACKEND=meilisearch
# Meilisearch configuration
MEILISEARCH_READ_ADDR=http://localhost:7700
MEILISEARCH_WRITE_ADDRS=http://localhost:7700
# 5 minutes in milliseconds
SEARCH_OPERATION_TIMEOUT=300000
ELASTICSEARCH_URL=http://localhost:9200
ELASTICSEARCH_INDEX_PREFIX=labrinth
# # For a sharded Meilisearch setup (sharded-meilisearch docker compose profile)
# MEILISEARCH_READ_ADDR=http://localhost:7710
# MEILISEARCH_WRITE_ADDRS=http://localhost:7700,http://localhost:7701
SEARCH_BACKEND=typesense
MEILISEARCH_KEY=modrinth
MEILISEARCH_META_NAMESPACE=
# Elasticsearch configuration
ELASTICSEARCH_URL=http://localhost:9200
ELASTICSEARCH_INDEX_PREFIX=labrinth
ELASTICSEARCH_USERNAME=
ELASTICSEARCH_PASSWORD=
SEARCH_INDEX_CHUNK_SIZE=5000
TYPESENSE_URL=http://localhost:8108
TYPESENSE_API_KEY=modrinth
TYPESENSE_INDEX_PREFIX=labrinth
REDIS_URL=redis://localhost
REDIS_MIN_CONNECTIONS=0

9
apps/labrinth/Cargo.toml
View File

@@ -26,7 +26,7 @@ async-stripe = { workspace = true, features = [
"billing",
"checkout",
"connect",
"webhook-events",
"webhook-events"
] }
async-trait = { workspace = true }
base64 = { workspace = true }
@@ -44,6 +44,7 @@ deadpool-redis.workspace = true
derive_more = { workspace = true, features = ["deref", "deref_mut"] }
dotenvy = { workspace = true }
either = { workspace = true }
elasticsearch = { workspace = true, features = ["experimental-apis"] }
eyre = { workspace = true }
futures = { workspace = true }
futures-util = { workspace = true }
@@ -86,11 +87,11 @@ reqwest = { workspace = true, features = [
"http2",
"json",
"multipart",
"rustls-tls-webpki-roots",
"rustls-tls-webpki-roots"
] }
rust_decimal = { workspace = true, features = [
"serde-with-float",
"serde-with-str",
"serde-with-str"
] }
rust_iso3166 = { workspace = true }
rust-s3 = { workspace = true }
@@ -114,7 +115,7 @@ sqlx = { workspace = true, features = [
"tls-rustls-aws-lc-rs",
] }
sqlx-tracing = { workspace = true, features = ["postgres"] }
strum = { workspace = true }
strum = { workspace = true, features = ["derive"] }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "sync"] }
tokio-stream = { workspace = true }

4
apps/labrinth/src/auth/mod.rs
View File

@@ -20,8 +20,6 @@ use thiserror::Error;
pub enum AuthenticationError {
#[error(transparent)]
Internal(#[from] eyre::Report),
#[error("Environment Error")]
Env(#[from] dotenvy::Error),
#[error("An unknown database error occurred: {0}")]
Sqlx(#[from] sqlx::Error),
#[error("Database Error: {0}")]
@@ -58,7 +56,6 @@ impl actix_web::ResponseError for AuthenticationError {
AuthenticationError::Internal(..) => {
StatusCode::INTERNAL_SERVER_ERROR
}
AuthenticationError::Env(..) => StatusCode::INTERNAL_SERVER_ERROR,
AuthenticationError::Sqlx(..) => StatusCode::INTERNAL_SERVER_ERROR,
AuthenticationError::Database(..) => {
StatusCode::INTERNAL_SERVER_ERROR
@@ -94,7 +91,6 @@ impl AuthenticationError {
pub fn error_name(&self) -> &'static str {
match self {
AuthenticationError::Internal(..) => "internal_error",
AuthenticationError::Env(..) => "environment_error",
AuthenticationError::Sqlx(..) => "database_error",
AuthenticationError::Database(..) => "database_error",
AuthenticationError::SerDe(..) => "invalid_input",

13
apps/labrinth/src/background_task.rs
View File

@@ -1,3 +1,4 @@
use crate::database;
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::queue::analytics::cache::cache_analytics;
@@ -8,9 +9,9 @@ use crate::queue::payouts::{
insert_bank_balances_and_webhook, process_affiliate_payouts,
process_payout, remove_payouts_for_refunded_charges,
};
use crate::search::indexing::index_projects;
use crate::search::SearchBackend;
use crate::util::anrok;
use crate::{database, search};
use actix_web::web;
use clap::ValueEnum;
use eyre::WrapErr;
use tracing::info;
@@ -42,7 +43,7 @@ impl BackgroundTask {
pool: PgPool,
ro_pool: PgPool,
redis_pool: RedisPool,
search_config: search::SearchConfig,
search_backend: web::Data<dyn SearchBackend>,
clickhouse: clickhouse::Client,
stripe_client: stripe::Client,
anrok_client: anrok::Client,
@@ -53,7 +54,7 @@ impl BackgroundTask {
match self {
Migrations => run_migrations().await,
IndexSearch => {
index_search(ro_pool, redis_pool, search_config).await
index_search(ro_pool, redis_pool, search_backend).await
}
ReleaseScheduled => release_scheduled(pool).await,
UpdateVersions => update_versions(pool, redis_pool).await,
@@ -132,10 +133,10 @@ pub async fn run_migrations() -> eyre::Result<()> {
pub async fn index_search(
ro_pool: PgPool,
redis_pool: RedisPool,
search_config: search::SearchConfig,
search_backend: web::Data<dyn SearchBackend>,
) -> eyre::Result<()> {
info!("Indexing local database");
index_projects(ro_pool, redis_pool, &search_config).await
search_backend.index_projects(ro_pool, redis_pool).await
}
pub async fn release_scheduled(pool: PgPool) -> eyre::Result<()> {

18
apps/labrinth/src/env.rs
View File

@@ -82,6 +82,7 @@ where
}
pub fn init() -> eyre::Result<()> {
dotenvy::dotenv().ok();
EnvVars::from_env()?;
LazyLock::force(&ENV);
Ok(())
@@ -128,9 +129,6 @@ vars! {
LABRINTH_EXTERNAL_NOTIFICATION_KEY: String;
RATE_LIMIT_IGNORE_KEY: String;
DATABASE_URL: String;
MEILISEARCH_READ_ADDR: String;
MEILISEARCH_WRITE_ADDRS: StringCsv;
MEILISEARCH_KEY: String;
REDIS_URL: String;
BIND_ADDR: String;
SELF_ADDR: String;
@@ -142,6 +140,20 @@ vars! {
ALLOWED_CALLBACK_URLS: Json<Vec<String>>;
ANALYTICS_ALLOWED_ORIGINS: Json<Vec<String>>;
// search
SEARCH_BACKEND: crate::search::SearchBackendKind = crate::search::SearchBackendKind::Typesense;
MEILISEARCH_READ_ADDR: String;
MEILISEARCH_WRITE_ADDRS: StringCsv;
MEILISEARCH_KEY: String;
ELASTICSEARCH_URL: String;
ELASTICSEARCH_INDEX_PREFIX: String;
ELASTICSEARCH_USERNAME: String = "";
ELASTICSEARCH_PASSWORD: String = "";
SEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64;
TYPESENSE_URL: String = "http://localhost:8108";
TYPESENSE_API_KEY: String = "modrinth";
TYPESENSE_INDEX_PREFIX: String = "labrinth";
// storage
STORAGE_BACKEND: crate::file_hosting::FileHostKind;

18
apps/labrinth/src/lib.rs
View File

@@ -58,7 +58,7 @@ pub struct LabrinthConfig {
pub file_host: Arc<dyn file_hosting::FileHost + Send + Sync>,
pub scheduler: Arc<scheduler::Scheduler>,
pub ip_salt: Pepper,
pub search_config: search::SearchConfig,
pub search_backend: web::Data<dyn search::SearchBackend>,
pub session_queue: web::Data<AuthQueue>,
pub payouts_queue: web::Data<PayoutsQueue>,
pub analytics_queue: Arc<AnalyticsQueue>,
@@ -78,7 +78,7 @@ pub fn app_setup(
pool: PgPool,
ro_pool: ReadOnlyPgPool,
redis_pool: RedisPool,
search_config: search::SearchConfig,
search_backend: actix_web::web::Data<dyn search::SearchBackend>,
clickhouse: &mut Client,
file_host: Arc<dyn file_hosting::FileHost + Send + Sync>,
stripe_client: stripe::Client,
@@ -129,21 +129,21 @@ pub fn app_setup(
let local_index_interval =
Duration::from_secs(ENV.LOCAL_INDEX_INTERVAL);
let pool_ref = pool.clone();
let search_config_ref = search_config.clone();
let redis_pool_ref = redis_pool.clone();
let search_backend_ref = search_backend.clone();
scheduler.run(local_index_interval, move || {
let pool_ref = pool_ref.clone();
let redis_pool_ref = redis_pool_ref.clone();
let search_config_ref = search_config_ref.clone();
let search_backend = search_backend_ref.clone();
async move {
if let Err(e) = background_task::index_search(
if let Err(err) = background_task::index_search(
pool_ref,
redis_pool_ref,
search_config_ref,
search_backend,
)
.await
{
warn!("Local project indexing failed: {e:#}");
warn!("Failed to index search: {err:?}");
}
}
});
@@ -299,7 +299,7 @@ pub fn app_setup(
file_host,
scheduler: Arc::new(scheduler),
ip_salt,
search_config,
search_backend,
session_queue,
payouts_queue: web::Data::new(PayoutsQueue::new()),
analytics_queue,
@@ -338,7 +338,7 @@ pub fn app_config(
.app_data(web::Data::new(labrinth_config.pool.clone()))
.app_data(web::Data::new(labrinth_config.ro_pool.clone()))
.app_data(web::Data::new(labrinth_config.file_host.clone()))
.app_data(web::Data::new(labrinth_config.search_config.clone()))
.app_data(labrinth_config.search_backend.clone())
.app_data(web::Data::new(labrinth_config.gotenberg_client.clone()))
.app_data(labrinth_config.http_client.clone())
.app_data(labrinth_config.session_queue.clone())

8
apps/labrinth/src/main.rs
View File

@@ -56,7 +56,6 @@ struct Args {
fn main() -> std::io::Result<()> {
color_eyre::install().expect("failed to install `color-eyre`");
dotenvy::dotenv().ok();
modrinth_util::log::init().expect("failed to initialize logging");
env::init().expect("failed to initialize environment variables");
@@ -152,7 +151,8 @@ async fn app() -> std::io::Result<()> {
info!("Initializing clickhouse connection");
let mut clickhouse = clickhouse::init_client().await.unwrap();
let search_config = search::SearchConfig::new(None);
let search_backend =
actix_web::web::Data::from(Arc::from(search::backend(None)));
let stripe_client = stripe::Client::new(ENV.STRIPE_API_KEY.clone());
@@ -171,7 +171,7 @@ async fn app() -> std::io::Result<()> {
pool,
ro_pool.into_inner(),
redis_pool,
search_config,
search_backend,
clickhouse,
stripe_client,
anrok_client.clone(),
@@ -207,7 +207,7 @@ async fn app() -> std::io::Result<()> {
pool.clone(),
ro_pool.clone(),
redis_pool.clone(),
search_config.clone(),
search_backend.clone(),
&mut clickhouse,
file_host.clone(),
stripe_client,

11
apps/labrinth/src/models/v2/search.rs
View File

@@ -159,15 +159,20 @@ impl LegacySearchResults {
pub fn from(search_results: crate::search::SearchResults) -> Self {
let limit = search_results.hits_per_page;
let offset = (search_results.page - 1) * limit;
let crate::search::SearchResults {
hits,
page: _,
hits_per_page: _,
total_hits,
} = search_results;
Self {
hits: search_results
.hits
hits: hits
.into_iter()
.map(LegacyResultSearchProject::from)
.collect(),
offset,
limit,
total_hits: search_results.total_hits,
total_hits,
}
}
}

17
apps/labrinth/src/models/v3/projects.rs
View File

@@ -1020,20 +1020,3 @@ impl FileType {
)]
#[serde(transparent)]
pub struct Loader(pub String);
// These fields must always succeed parsing; deserialize errors aren't
// processed correctly (don't return JSON errors)
#[derive(Serialize, Deserialize, Debug)]
pub struct SearchRequest {
pub query: Option<String>,
pub offset: Option<String>,
pub index: Option<String>,
pub limit: Option<String>,
pub new_filters: Option<String>,
// TODO: Deprecated values below. WILL BE REMOVED V3!
pub facets: Option<String>,
pub filters: Option<String>,
pub version: Option<String>,
}

4
apps/labrinth/src/queue/email.rs
View File

@@ -102,8 +102,6 @@ impl Mailer {
#[derive(Error, Debug)]
pub enum MailError {
#[error("Environment Error")]
Env(#[from] dotenvy::Error),
#[error("Mail Error: {0}")]
Mail(#[from] lettre::error::Error),
#[error("Address Parse Error: {0}")]
@@ -136,7 +134,7 @@ impl EmailQueue {
pg,
redis,
mailer: Arc::new(TokioMutex::new(Mailer::Uninitialized)),
identity: templates::MailingIdentity::from_env()?,
identity: templates::MailingIdentity::from_env(),
client: Client::builder()
.user_agent("Modrinth")
.build()

6
apps/labrinth/src/queue/email/templates.rs
View File

@@ -95,8 +95,8 @@ pub struct MailingIdentity {
}
impl MailingIdentity {
pub fn from_env() -> dotenvy::Result<Self> {
Ok(Self {
pub fn from_env() -> Self {
Self {
from_name: ENV.SMTP_FROM_NAME.clone(),
from_address: ENV.SMTP_FROM_ADDRESS.clone(),
reply_name: if ENV.SMTP_REPLY_TO_NAME.is_empty() {
@@ -109,7 +109,7 @@ impl MailingIdentity {
} else {
Some(ENV.SMTP_REPLY_TO_ADDRESS.clone())
},
})
}
}
}

8
apps/labrinth/src/routes/internal/admin.rs
View File

@@ -7,7 +7,7 @@ use crate::models::pats::Scopes;
use crate::queue::analytics::AnalyticsQueue;
use crate::queue::session::AuthQueue;
use crate::routes::ApiError;
use crate::search::SearchConfig;
use crate::search::SearchBackend;
use crate::util::date::get_current_tenths_of_ms;
use crate::util::error::Context;
use crate::util::guards::admin_key_guard;
@@ -154,11 +154,11 @@ pub async fn count_download(
pub async fn force_reindex(
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
config: web::Data<SearchConfig>,
search_backend: web::Data<dyn SearchBackend>,
) -> Result<HttpResponse, ApiError> {
use crate::search::indexing::index_projects;
let redis = redis.get_ref();
index_projects(pool.as_ref().clone(), redis.clone(), &config)
search_backend
.index_projects(pool.as_ref().clone(), redis.clone())
.await
.wrap_internal_err("failed to index projects")?;
Ok(HttpResponse::NoContent().finish())

120
apps/labrinth/src/routes/internal/search.rs
View File

@@ -1,12 +1,9 @@
use crate::routes::ApiError;
use crate::search::SearchConfig;
use crate::util::guards::admin_key_guard;
use actix_web::{HttpResponse, delete, get, web};
use meilisearch_sdk::tasks::{Task, TasksCancelQuery};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::Duration;
use utoipa::ToSchema;
use crate::{
routes::ApiError,
search::{SearchBackend, TasksCancelFilter},
};
use actix_web::{delete, get, web};
pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
cfg.service(tasks).service(tasks_cancel);
@@ -15,107 +12,20 @@ pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
#[utoipa::path]
#[get("tasks", guard = "admin_key_guard")]
pub async fn tasks(
config: web::Data<SearchConfig>,
) -> Result<HttpResponse, ApiError> {
let client = config.make_batch_client()?;
let tasks = client
.with_all_clients("get_tasks", async |client| {
let tasks = client.get_tasks().await?;
Ok(tasks.results)
})
.await?;
#[derive(Serialize, ToSchema)]
struct MeiliTask<Time> {
uid: u32,
status: &'static str,
duration: Option<Duration>,
enqueued_at: Option<Time>,
}
#[derive(Serialize, ToSchema)]
struct TaskList<Time> {
by_instance: HashMap<String, Vec<MeiliTask<Time>>>,
}
let response = tasks
.into_iter()
.enumerate()
.map(|(idx, instance_tasks)| {
let tasks = instance_tasks
.into_iter()
.filter_map(|task| {
Some(match task {
Task::Enqueued { content } => MeiliTask {
uid: content.uid,
status: "enqueued",
duration: None,
enqueued_at: Some(content.enqueued_at),
},
Task::Processing { content } => MeiliTask {
uid: content.uid,
status: "processing",
duration: None,
enqueued_at: Some(content.enqueued_at),
},
Task::Failed { content } => MeiliTask {
uid: content.task.uid,
status: "failed",
duration: Some(content.task.duration),
enqueued_at: Some(content.task.enqueued_at),
},
Task::Succeeded { content: _ } => return None,
})
})
.collect();
(idx.to_string(), tasks)
})
.collect::<HashMap<String, Vec<MeiliTask<_>>>>();
Ok(HttpResponse::Ok().json(TaskList {
by_instance: response,
}))
}
#[derive(Deserialize, Serialize, ToSchema)]
#[serde(tag = "type", rename_all = "snake_case")]
enum TasksCancelFilter {
All,
AllEnqueued,
Indexes { indexes: Vec<String> },
search: web::Data<dyn SearchBackend>,
) -> Result<web::Json<serde_json::Value>, ApiError> {
Ok(web::Json(search.tasks().await.map_err(ApiError::Internal)?))
}
#[utoipa::path]
#[delete("tasks", guard = "admin_key_guard")]
pub async fn tasks_cancel(
config: web::Data<SearchConfig>,
search: web::Data<dyn SearchBackend>,
body: web::Json<TasksCancelFilter>,
) -> Result<HttpResponse, ApiError> {
let client = config.make_batch_client()?;
let all_results = client
.with_all_clients("cancel_tasks", async |client| {
let mut q = TasksCancelQuery::new(client);
match &body.0 {
TasksCancelFilter::All => {}
TasksCancelFilter::Indexes { indexes } => {
q.with_index_uids(indexes.iter().map(|s| s.as_str()));
}
TasksCancelFilter::AllEnqueued => {
q.with_statuses(["enqueued"]);
}
};
let result = client.cancel_tasks_with(&q).await;
Ok(result)
})
.await?;
for r in all_results {
r?;
}
Ok(HttpResponse::Ok().finish())
) -> Result<(), ApiError> {
search
.tasks_cancel(&body)
.await
.map_err(ApiError::Internal)?;
Ok(())
}

8
apps/labrinth/src/routes/mod.rs
View File

@@ -93,8 +93,6 @@ pub enum ApiError {
Auth(eyre::Report),
#[error("Invalid input: {0}")]
InvalidInput(String),
#[error("Environment error")]
Env(#[from] dotenvy::Error),
#[error("Error while uploading file: {0}")]
FileHosting(#[from] FileHostingError),
#[error("database error")]
@@ -117,8 +115,6 @@ pub enum ApiError {
Validation(String),
#[error("Search error: {0}")]
Search(#[from] meilisearch_sdk::errors::Error),
#[error("search indexing error")]
Indexing(#[from] crate::search::indexing::IndexingError),
#[error("Payments error: {0}")]
Payments(String),
#[error("Discord error: {0}")]
@@ -176,7 +172,6 @@ impl ApiError {
Self::Internal(..) => "internal_error",
Self::Request(..) => "request_error",
Self::Auth(..) => "auth_error",
Self::Env(..) => "environment_error",
Self::Database(..) => "database_error",
Self::SqlxDatabase(..) => "database_error",
Self::RedisDatabase(..) => "database_error",
@@ -185,7 +180,6 @@ impl ApiError {
Self::Xml(..) => "xml_error",
Self::Json(..) => "json_error",
Self::Search(..) => "search_error",
Self::Indexing(..) => "indexing_error",
Self::FileHosting(..) => "file_hosting_error",
Self::InvalidInput(..) => "invalid_input",
Self::Validation(..) => "invalid_input",
@@ -241,7 +235,6 @@ impl actix_web::ResponseError for ApiError {
Self::Request(..) => StatusCode::BAD_REQUEST,
Self::Auth(..) => StatusCode::UNAUTHORIZED,
Self::InvalidInput(..) => StatusCode::BAD_REQUEST,
Self::Env(..) => StatusCode::INTERNAL_SERVER_ERROR,
Self::Database(..) => StatusCode::INTERNAL_SERVER_ERROR,
Self::SqlxDatabase(..) => StatusCode::INTERNAL_SERVER_ERROR,
Self::RedisDatabase(..) => StatusCode::INTERNAL_SERVER_ERROR,
@@ -251,7 +244,6 @@ impl actix_web::ResponseError for ApiError {
Self::Xml(..) => StatusCode::INTERNAL_SERVER_ERROR,
Self::Json(..) => StatusCode::BAD_REQUEST,
Self::Search(..) => StatusCode::INTERNAL_SERVER_ERROR,
Self::Indexing(..) => StatusCode::INTERNAL_SERVER_ERROR,
Self::FileHosting(..) => StatusCode::INTERNAL_SERVER_ERROR,
Self::Validation(..) => StatusCode::BAD_REQUEST,
Self::Payments(..) => StatusCode::FAILED_DEPENDENCY,

18
apps/labrinth/src/routes/v2/projects.rs
View File

@@ -4,7 +4,7 @@ use crate::database::models::{project_item, version_item};
use crate::database::redis::RedisPool;
use crate::file_hosting::FileHost;
use crate::models::projects::{
Link, MonetizationStatus, Project, ProjectStatus, SearchRequest, Version,
Link, MonetizationStatus, Project, ProjectStatus, Version,
};
use crate::models::v2::projects::{
DonationLink, LegacyProject, LegacySideType, LegacyVersion,
@@ -14,7 +14,7 @@ use crate::queue::moderation::AutomatedModerationQueue;
use crate::queue::session::AuthQueue;
use crate::routes::v3::projects::ProjectIds;
use crate::routes::{ApiError, v2_reroute, v3};
use crate::search::{SearchConfig, SearchError, search_for_project};
use crate::search::{SearchBackend, SearchRequest};
use actix_web::{HttpRequest, HttpResponse, delete, get, patch, post, web};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
@@ -53,9 +53,9 @@ pub fn config(cfg: &mut web::ServiceConfig) {
#[get("search")]
pub async fn project_search(
web::Query(info): web::Query<SearchRequest>,
config: web::Data<SearchConfig>,
search_backend: web::Data<dyn SearchBackend>,
redis: web::Data<RedisPool>,
) -> Result<HttpResponse, SearchError> {
) -> Result<HttpResponse, ApiError> {
// Search now uses loader_fields instead of explicit 'client_side' and 'server_side' fields
// While the backend for this has changed, it doesnt affect much
// in the API calls except that 'versions:x' is now 'game_versions:x'
@@ -100,7 +100,7 @@ pub async fn project_search(
..info
};
let results = search_for_project(&info, &config, &redis).await?;
let results = search_backend.search_for_project(&info, &redis).await?;
let results = LegacySearchResults::from(results);
@@ -410,7 +410,7 @@ pub async fn project_edit(
req: HttpRequest,
info: web::Path<(String,)>,
pool: web::Data<PgPool>,
search_config: web::Data<SearchConfig>,
search_backend: web::Data<dyn SearchBackend>,
new_project: web::Json<EditProject>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
@@ -524,7 +524,7 @@ pub async fn project_edit(
req.clone(),
info,
pool.clone(),
search_config,
search_backend,
web::Json(new_project),
redis.clone(),
session_queue.clone(),
@@ -918,7 +918,7 @@ pub async fn project_delete(
info: web::Path<(String,)>,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
search_config: web::Data<SearchConfig>,
search_backend: web::Data<dyn SearchBackend>,
session_queue: web::Data<AuthQueue>,
) -> Result<HttpResponse, ApiError> {
// Returns NoContent, so no need to convert
@@ -927,7 +927,7 @@ pub async fn project_delete(
info,
pool,
redis,
search_config,
search_backend,
session_queue,
)
.await

6
apps/labrinth/src/routes/v2/versions.rs
View File

@@ -11,7 +11,7 @@ use crate::models::projects::{
use crate::models::v2::projects::LegacyVersion;
use crate::queue::session::AuthQueue;
use crate::routes::{v2_reroute, v3};
use crate::search::SearchConfig;
use crate::search::SearchBackend;
use actix_web::{HttpRequest, HttpResponse, delete, get, patch, web};
use serde::{Deserialize, Serialize};
use validator::Validate;
@@ -357,7 +357,7 @@ pub async fn version_delete(
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
search_config: web::Data<SearchConfig>,
search_backend: web::Data<dyn SearchBackend>,
) -> Result<HttpResponse, ApiError> {
// Returns NoContent, so we don't need to convert the response
v3::versions::version_delete(
@@ -366,7 +366,7 @@ pub async fn version_delete(
pool,
redis,
session_queue,
search_config,
search_backend,
)
.await
.or_else(v2_reroute::flatten_404_error)

9
apps/labrinth/src/routes/v3/project_creation.rs
View File

@@ -22,7 +22,6 @@ use crate::models::teams::{OrganizationPermissions, ProjectPermissions};
use crate::models::threads::ThreadType;
use crate::models::v3::user_limits::UserLimits;
use crate::queue::session::AuthQueue;
use crate::search::indexing::IndexingError;
use crate::util::guards::admin_key_guard;
use crate::util::http::HttpClient;
use crate::util::img::upload_image_optimized;
@@ -55,14 +54,10 @@ pub fn config(cfg: &mut utoipa_actix_web::service_config::ServiceConfig) {
#[derive(Error, Debug)]
pub enum CreateError {
#[error("Environment Error")]
EnvError(#[from] dotenvy::Error),
#[error("An unknown database error occurred")]
SqlxDatabaseError(#[from] sqlx::Error),
#[error("Database Error: {0}")]
DatabaseError(#[from] models::DatabaseError),
#[error("Indexing Error: {0}")]
IndexingError(#[from] IndexingError),
#[error("Error while parsing multipart payload: {0}")]
MultipartError(#[from] actix_multipart::MultipartError),
#[error("Error while parsing JSON: {0}")]
@@ -126,12 +121,10 @@ impl From<crate::routes::ApiError> for CreateError {
impl actix_web::ResponseError for CreateError {
fn status_code(&self) -> StatusCode {
match self {
CreateError::EnvError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::SqlxDatabaseError(..) => {
StatusCode::INTERNAL_SERVER_ERROR
}
CreateError::DatabaseError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::IndexingError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::FileHostingError(..) => {
StatusCode::INTERNAL_SERVER_ERROR
}
@@ -159,10 +152,8 @@ impl actix_web::ResponseError for CreateError {
fn error_response(&self) -> HttpResponse {
HttpResponse::build(self.status_code()).json(ApiError {
error: match self {
CreateError::EnvError(..) => "environment_error",
CreateError::SqlxDatabaseError(..) => "database_error",
CreateError::DatabaseError(..) => "database_error",
CreateError::IndexingError(..) => "indexing_error",
CreateError::FileHostingError(..) => "file_hosting_error",
CreateError::SerDeError(..) => "invalid_input",
CreateError::MultipartError(..) => "invalid_input",

82
apps/labrinth/src/routes/v3/projects.rs
View File

@@ -20,8 +20,7 @@ use crate::models::images::ImageContext;
use crate::models::notifications::NotificationBody;
use crate::models::pats::Scopes;
use crate::models::projects::{
MonetizationStatus, Project, ProjectStatus, SearchRequest,
SideTypesMigrationReviewStatus,
MonetizationStatus, Project, ProjectStatus, SideTypesMigrationReviewStatus,
};
use crate::models::teams::ProjectPermissions;
use crate::models::threads::MessageBody;
@@ -30,8 +29,7 @@ use crate::queue::moderation::AutomatedModerationQueue;
use crate::queue::session::AuthQueue;
use crate::routes::ApiError;
use crate::routes::internal::delphi;
use crate::search::indexing::remove_documents;
use crate::search::{SearchConfig, SearchError, search_for_project};
use crate::search::{SearchBackend, SearchQuery, SearchRequest, SearchResults};
use crate::util::error::Context;
use crate::util::img;
use crate::util::img::{delete_old_images, upload_image_optimized};
@@ -48,6 +46,7 @@ use validator::Validate;
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.route("search", web::get().to(project_search));
cfg.service(project_search_post);
cfg.route("projects", web::get().to(projects_get));
cfg.route("projects", web::patch().to(projects_edit));
cfg.route("projects_random", web::get().to(random_projects_get));
@@ -292,7 +291,7 @@ async fn project_edit(
req: HttpRequest,
info: web::Path<(String,)>,
pool: web::Data<PgPool>,
search_config: web::Data<SearchConfig>,
search_backend: web::Data<dyn SearchBackend>,
web::Json(new_project): web::Json<EditProject>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
@@ -302,7 +301,7 @@ async fn project_edit(
req,
info,
pool,
search_config,
search_backend,
web::Json(new_project),
redis,
session_queue,
@@ -315,7 +314,7 @@ pub async fn project_edit_internal(
req: HttpRequest,
info: web::Path<(String,)>,
pool: web::Data<PgPool>,
search_config: web::Data<SearchConfig>,
search_backend: web::Data<dyn SearchBackend>,
web::Json(new_project): web::Json<EditProject>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
@@ -1126,16 +1125,16 @@ pub async fn project_edit_internal(
project_item.inner.status.is_searchable(),
new_project.status.map(|status| status.is_searchable()),
) {
remove_documents(
&project_item
.versions
.into_iter()
.map(|x| x.into())
.collect::<Vec<_>>(),
&search_config,
)
.await
.wrap_internal_err("failed to remove documents")?;
search_backend
.remove_documents(
&project_item
.versions
.into_iter()
.map(|x| x.into())
.collect::<Vec<_>>(),
)
.await
.wrap_internal_err("failed to remove documents")?;
}
Ok(HttpResponse::NoContent().body(""))
@@ -1190,11 +1189,13 @@ pub async fn edit_project_categories(
// }
pub async fn project_search(
web::Query(info): web::Query<SearchRequest>,
config: web::Data<SearchConfig>,
web::Query(info): web::Query<SearchQuery>,
search_backend: web::Data<dyn SearchBackend>,
redis: web::Data<RedisPool>,
) -> Result<HttpResponse, SearchError> {
let results = search_for_project(&info, &config, &redis).await?;
) -> Result<web::Json<SearchResults>, ApiError> {
let results = search_backend
.search_for_project(&SearchRequest::from(info), &redis)
.await?;
// TODO: add this back
// let results = ReturnSearchResults {
@@ -1208,7 +1209,18 @@ pub async fn project_search(
// total_hits: results.total_hits,
// };
Ok(HttpResponse::Ok().json(results))
Ok(web::Json(results))
}
// for more complicated search queries
#[post("/search")]
pub async fn project_search_post(
web::Json(info): web::Json<SearchRequest>,
search_backend: web::Data<dyn SearchBackend>,
redis: web::Data<RedisPool>,
) -> Result<web::Json<SearchResults>, ApiError> {
let results = search_backend.search_for_project(&info, &redis).await?;
Ok(web::Json(results))
}
//checks the validity of a project id or slug
@@ -2452,7 +2464,7 @@ async fn project_delete(
info: web::Path<(String,)>,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
search_config: web::Data<SearchConfig>,
search_backend: web::Data<dyn SearchBackend>,
session_queue: web::Data<AuthQueue>,
) -> Result<(), ApiError> {
project_delete_internal(
@@ -2460,7 +2472,7 @@ async fn project_delete(
info,
pool,
redis,
search_config,
search_backend,
session_queue,
)
.await
@@ -2471,7 +2483,7 @@ pub async fn project_delete_internal(
info: web::Path<(String,)>,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
search_config: web::Data<SearchConfig>,
search_backend: web::Data<dyn SearchBackend>,
session_queue: web::Data<AuthQueue>,
) -> Result<(), ApiError> {
let (_, user) = get_user_from_headers(
@@ -2583,16 +2595,16 @@ pub async fn project_delete_internal(
.await
.wrap_internal_err("failed to commit transaction")?;
remove_documents(
&project
.versions
.into_iter()
.map(|x| x.into())
.collect::<Vec<_>>(),
&search_config,
)
.await
.wrap_internal_err("failed to remove project version documents")?;
search_backend
.remove_documents(
&project
.versions
.into_iter()
.map(|x| x.into())
.collect::<Vec<_>>(),
)
.await
.wrap_internal_err("failed to remove project version documents")?;
if result.is_some() {
Ok(())

9
apps/labrinth/src/routes/v3/versions.rs
View File

@@ -26,8 +26,7 @@ use crate::models::projects::{Loader, skip_nulls};
use crate::models::teams::ProjectPermissions;
use crate::queue::session::AuthQueue;
use crate::routes::internal::delphi;
use crate::search::SearchConfig;
use crate::search::indexing::remove_documents;
use crate::search::SearchBackend;
use crate::util::error::Context;
use crate::util::img;
use crate::util::validate::validation_errors_to_string;
@@ -915,7 +914,7 @@ pub async fn version_delete(
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
search_config: web::Data<SearchConfig>,
search_backend: web::Data<dyn SearchBackend>,
) -> Result<HttpResponse, ApiError> {
let user = get_user_from_headers(
&req,
@@ -1022,10 +1021,10 @@ pub async fn version_delete(
&redis,
)
.await?;
remove_documents(&[version.inner.id.into()], &search_config)
search_backend
.remove_documents(&[version.inner.id.into()])
.await
.wrap_internal_err("failed to remove documents")?;
if result.is_some() {
Ok(HttpResponse::NoContent().body(""))
} else {

123
apps/labrinth/src/search/backend/common.rs Normal file
View File

@@ -0,0 +1,123 @@
use crate::routes::ApiError;
use crate::search::SearchRequest;
use crate::util::error::Context;
use eyre::eyre;
use std::borrow::Cow;
pub struct ParsedSearchRequest<'a> {
pub offset: usize,
pub hits_per_page: usize,
pub page: usize,
pub index: &'a str,
pub query: &'a str,
}
pub fn parse_search_request(
info: &SearchRequest,
) -> Result<ParsedSearchRequest<'_>, ApiError> {
let offset = info
.offset
.as_deref()
.unwrap_or("0")
.parse::<usize>()
.wrap_request_err("invalid offset")?;
let limit = info
.limit
.as_deref()
.unwrap_or("10")
.parse::<usize>()
.wrap_request_err("invalid limit")?
.min(100);
let hits_per_page = if limit == 0 { 1 } else { limit };
Ok(ParsedSearchRequest {
offset,
hits_per_page,
page: offset / hits_per_page + 1,
index: info.index.as_deref().unwrap_or("relevance"),
query: info.query.as_deref().unwrap_or_default(),
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SearchIndex {
Relevance,
Downloads,
Follows,
Updated,
Newest,
MinecraftJavaServerVerifiedPlays2w,
MinecraftJavaServerPlayersOnline,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SearchIndexName {
Projects,
ProjectsFiltered,
}
pub struct SearchSort {
pub index_name: SearchIndexName,
pub index: SearchIndex,
}
pub fn parse_search_index(
index: &str,
new_filters: Option<&str>,
) -> Result<SearchSort, ApiError> {
let projects_name = SearchIndexName::Projects;
let projects_filtered_name = SearchIndexName::ProjectsFiltered;
// TODO: this is a dumb hack, the frontend should pass the project type it's filtering directly
let is_server = new_filters
.is_some_and(|f| f.contains("project_types = minecraft_java_server"));
Ok(match index {
"relevance" => SearchSort {
index_name: projects_name,
index: if is_server {
SearchIndex::MinecraftJavaServerVerifiedPlays2w
} else {
SearchIndex::Relevance
},
},
"downloads" => SearchSort {
index_name: projects_filtered_name,
index: SearchIndex::Downloads,
},
"follows" => SearchSort {
index_name: projects_name,
index: SearchIndex::Follows,
},
"updated" | "date_modified" => SearchSort {
index_name: projects_name,
index: SearchIndex::Updated,
},
"newest" | "date_created" => SearchSort {
index_name: projects_name,
index: SearchIndex::Newest,
},
"minecraft_java_server.verified_plays_2w" => SearchSort {
index_name: projects_name,
index: SearchIndex::MinecraftJavaServerVerifiedPlays2w,
},
"minecraft_java_server.ping.data.players_online" => SearchSort {
index_name: projects_name,
index: SearchIndex::MinecraftJavaServerPlayersOnline,
},
i => return Err(ApiError::Request(eyre!("invalid index '{i}'"))),
})
}
pub fn combined_search_filters(info: &SearchRequest) -> Option<Cow<'_, str>> {
if let Some(filters) = info.new_filters.as_deref() {
return Some(filters.into());
}
match (info.filters.as_deref(), info.version.as_deref()) {
(Some(f), Some(v)) => Some(format!("({f}) AND ({v})").into()),
(Some(f), None) => Some(f.into()),
(None, Some(v)) => Some(v.into()),
(None, None) => None,
}
}

1388
apps/labrinth/src/search/backend/elasticsearch/mod.rs Normal file
View File

File diff suppressed because it is too large Load Diff

176
apps/labrinth/src/search/indexing/mod.rs → apps/labrinth/src/search/backend/meilisearch/indexing.rs
View File

@@ -1,43 +1,23 @@
/// This module is used for the indexing from any source.
pub mod local_import;
use std::sync::LazyLock;
use std::time::Duration;
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::env::ENV;
use crate::search::{SearchConfig, UploadSearchProject};
use crate::search::backend::meilisearch::MeilisearchConfig;
use crate::search::indexing::index_local;
use crate::search::{SearchField, UploadSearchProject};
use crate::util::error::Context;
use ariadne::ids::base62_impl::to_base62;
use eyre::eyre;
use eyre::{Result, eyre};
use futures::StreamExt;
use futures::stream::FuturesOrdered;
use local_import::index_local;
use meilisearch_sdk::client::{Client, SwapIndexes};
use meilisearch_sdk::indexes::Index;
use meilisearch_sdk::settings::{PaginationSetting, Settings};
use meilisearch_sdk::task_info::TaskInfo;
use thiserror::Error;
use tracing::{Instrument, error, info, info_span, instrument};
#[derive(Error, Debug)]
pub enum IndexingError {
#[error(transparent)]
Internal(#[from] eyre::Report),
#[error("Error while connecting to the MeiliSearch database")]
Indexing(#[from] meilisearch_sdk::errors::Error),
#[error("Error while serializing or deserializing JSON: {0}")]
Serde(#[from] serde_json::Error),
#[error("Database Error: {0}")]
Sqlx(#[from] sqlx::error::Error),
#[error("Database Error: {0}")]
Database(#[from] crate::database::models::DatabaseError),
#[error("Environment Error")]
Env(#[from] dotenvy::Error),
#[error("Error while awaiting index creation task")]
Task,
}
// // The chunk size for adding projects to the indexing database. If the request size
// // is too large (>10MiB) then the request fails with an error. This chunk size
// // assumes a max average size of 4KiB per project to avoid this cap.
@@ -51,8 +31,8 @@ fn search_operation_timeout() -> std::time::Duration {
pub async fn remove_documents(
ids: &[crate::models::ids::VersionId],
config: &SearchConfig,
) -> eyre::Result<()> {
config: &MeilisearchConfig,
) -> Result<()> {
let mut indexes = get_indexes_for_indexing(config, false, false)
.await
.wrap_err("failed to get current indexes")?;
@@ -108,8 +88,8 @@ pub async fn remove_documents(
pub async fn index_projects(
ro_pool: PgPool,
redis: RedisPool,
config: &SearchConfig,
) -> eyre::Result<()> {
config: &MeilisearchConfig,
) -> Result<()> {
info!("Indexing projects.");
info!("Ensuring current indexes exists");
@@ -197,9 +177,9 @@ pub async fn index_projects(
}
pub async fn swap_index(
config: &SearchConfig,
config: &MeilisearchConfig,
index_name: &str,
) -> Result<(), IndexingError> {
) -> Result<()> {
let client = config.make_batch_client()?;
let index_name_next = config.get_index_name(index_name, true);
let index_name = config.get_index_name(index_name, false);
@@ -210,12 +190,13 @@ pub async fn swap_index(
let swap_indices_ref = &swap_indices;
// is it "indexes" or "indices"? who knows! roll a die!
client
.with_all_clients("swap_indexes", |client| async move {
let task = client
.swap_indexes([swap_indices_ref])
.await
.map_err(IndexingError::Indexing)?;
.wrap_err("failed to swap indices")?;
monitor_task(
client,
@@ -233,10 +214,10 @@ pub async fn swap_index(
#[instrument(skip(config))]
pub async fn get_indexes_for_indexing(
config: &SearchConfig,
config: &MeilisearchConfig,
next: bool, // Get the 'next' one
update_settings: bool,
) -> Result<Vec<Vec<Index>>, IndexingError> {
) -> Result<Vec<Vec<Index>>> {
let client = config.make_batch_client()?;
let project_name = config.get_index_name("projects", next);
let project_filtered_name =
@@ -381,7 +362,7 @@ async fn add_to_index(
client: &Client,
index: &Index,
mods: &[UploadSearchProject],
) -> Result<(), IndexingError> {
) -> Result<()> {
for chunk in mods.chunks(MEILISEARCH_CHUNK_SIZE) {
info!(
"Adding chunk of {} versions starting with version id {}",
@@ -419,7 +400,7 @@ async fn monitor_task(
task: TaskInfo,
timeout: Duration,
poll: Option<Duration>,
) -> Result<(), IndexingError> {
) -> Result<()> {
let now = std::time::Instant::now();
let id = task.get_task_uid();
@@ -465,7 +446,7 @@ async fn update_and_add_to_index(
index: &Index,
projects: &[UploadSearchProject],
_additional_fields: &[String],
) -> Result<(), IndexingError> {
) -> Result<()> {
// TODO: Uncomment this- hardcoding loader_fields is a band-aid fix, and will be fixed soon
// let mut new_filterable_attributes: Vec<String> = index.get_filterable_attributes().await?;
// let mut new_displayed_attributes = index.get_displayed_attributes().await?;
@@ -509,8 +490,8 @@ pub async fn add_projects_batch_client(
indices: &[Vec<Index>],
projects: Vec<UploadSearchProject>,
additional_fields: Vec<String>,
config: &SearchConfig,
) -> Result<(), IndexingError> {
config: &MeilisearchConfig,
) -> Result<()> {
let client = config.make_batch_client()?;
let index_references = indices
@@ -558,12 +539,92 @@ fn default_settings() -> Settings {
.with_displayed_attributes(DEFAULT_DISPLAYED_ATTRIBUTES)
.with_searchable_attributes(DEFAULT_SEARCHABLE_ATTRIBUTES)
.with_sortable_attributes(DEFAULT_SORTABLE_ATTRIBUTES)
.with_filterable_attributes(DEFAULT_ATTRIBUTES_FOR_FACETING)
.with_filterable_attributes(&*MEILI_FILTERABLE_ATTRIBUTES)
.with_pagination(PaginationSetting {
max_total_hits: 2147483647,
})
}
pub struct MeilisearchFieldSpec {
pub path: &'static str,
pub filterable: bool,
}
impl SearchField {
pub const fn meilisearch_spec(self) -> MeilisearchFieldSpec {
match self {
SearchField::Categories => MeilisearchFieldSpec {
path: "categories",
filterable: true,
},
SearchField::ProjectTypes => MeilisearchFieldSpec {
path: "project_types",
filterable: true,
},
SearchField::ProjectId => MeilisearchFieldSpec {
path: "project_id",
filterable: true,
},
SearchField::OpenSource => MeilisearchFieldSpec {
path: "open_source",
filterable: true,
},
SearchField::Environment => MeilisearchFieldSpec {
path: "environment",
filterable: true,
},
SearchField::GameVersions => MeilisearchFieldSpec {
path: "game_versions",
filterable: true,
},
SearchField::ClientSide => MeilisearchFieldSpec {
path: "client_side",
filterable: true,
},
SearchField::ServerSide => MeilisearchFieldSpec {
path: "server_side",
filterable: true,
},
SearchField::MinecraftServerRegion => MeilisearchFieldSpec {
path: "minecraft_server.region",
filterable: true,
},
SearchField::MinecraftServerLanguages => MeilisearchFieldSpec {
path: "minecraft_server.languages",
filterable: true,
},
SearchField::MinecraftJavaServerContentKind => {
MeilisearchFieldSpec {
path: "minecraft_java_server.content.kind",
filterable: true,
}
}
SearchField::MinecraftJavaServerContentSupportedGameVersions => {
MeilisearchFieldSpec {
path: "minecraft_java_server.content.supported_game_versions",
filterable: true,
}
}
SearchField::MinecraftJavaServerPingData => MeilisearchFieldSpec {
path: "minecraft_java_server.ping.data",
filterable: true,
},
}
}
}
static MEILI_FILTERABLE_ATTRIBUTES: LazyLock<Vec<&'static str>> =
LazyLock::new(|| {
use strum::IntoEnumIterator;
SearchField::iter()
.filter_map(|field| {
let spec = field.meilisearch_spec();
spec.filterable.then_some(spec.path)
})
.collect()
});
const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[
"project_id",
"version_id",
@@ -617,41 +678,6 @@ const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[
const DEFAULT_SEARCHABLE_ATTRIBUTES: &[&str] =
&["name", "summary", "author", "slug"];
const DEFAULT_ATTRIBUTES_FOR_FACETING: &[&str] = &[
"categories",
"license",
"project_types",
"downloads",
"follows",
"author",
"name",
"date_created",
"created_timestamp",
"date_modified",
"modified_timestamp",
"version_published_timestamp",
"project_id",
"open_source",
"color",
// Note: loader fields are not here, but are added on as they are needed (so they can be dynamically added depending on which exist).
// TODO: remove these- as they should be automatically populated. This is a band-aid fix.
"environment",
"game_versions",
"mrpack_loaders",
// V2 legacy fields for logical consistency
"client_side",
"server_side",
"minecraft_server.country",
"minecraft_server.region",
"minecraft_server.languages",
"minecraft_java_server.content.kind",
"minecraft_java_server.content.supported_game_versions",
"minecraft_java_server.content.recommended_game_version",
"minecraft_java_server.verified_plays_2w",
"minecraft_java_server.ping.data",
"minecraft_java_server.ping.data.players_online",
];
const DEFAULT_SORTABLE_ATTRIBUTES: &[&str] = &[
"downloads",
"follows",

489
apps/labrinth/src/search/backend/meilisearch/mod.rs Normal file
View File

@@ -0,0 +1,489 @@
use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::env::ENV;
use crate::models::ids::VersionId;
use crate::routes::ApiError;
use crate::search::backend::{
SearchIndex, SearchIndexName, combined_search_filters, parse_search_index,
parse_search_request,
};
use crate::search::{
ResultSearchProject, SearchBackend, SearchRequest, SearchResults,
TasksCancelFilter,
};
use crate::util::error::Context;
use async_trait::async_trait;
use eyre::Result;
use futures::TryStreamExt;
use futures::stream::FuturesOrdered;
use itertools::Itertools;
use meilisearch_sdk::client::Client;
use meilisearch_sdk::tasks::{Task, TasksCancelQuery};
use serde::Serialize;
use serde_json::Value;
use std::collections::HashMap;
use std::fmt::Write;
use std::time::Duration;
use tracing::{Instrument, info_span};
pub mod indexing;
#[derive(Debug, Clone)]
pub struct MeilisearchReadClient {
pub client: Client,
}
impl std::ops::Deref for MeilisearchReadClient {
type Target = Client;
fn deref(&self) -> &Self::Target {
&self.client
}
}
pub struct BatchClient {
pub clients: Vec<Client>,
}
impl BatchClient {
pub fn new(clients: Vec<Client>) -> Self {
Self { clients }
}
pub async fn with_all_clients<'a, T, G, Fut>(
&'a self,
task_name: &str,
generator: G,
) -> Result<Vec<T>>
where
G: Fn(&'a Client) -> Fut,
Fut: Future<Output = Result<T>> + 'a,
{
let mut tasks = FuturesOrdered::new();
for (idx, client) in self.clients.iter().enumerate() {
tasks.push_back(generator(client).instrument(info_span!(
"client_task",
task.name = task_name,
client.idx = idx,
)));
}
let results = tasks.try_collect::<Vec<T>>().await?;
Ok(results)
}
pub fn across_all<T, F, R>(&self, data: Vec<T>, mut predicate: F) -> Vec<R>
where
F: FnMut(T, &Client) -> R,
{
assert_eq!(
data.len(),
self.clients.len(),
"mismatch between data len and meilisearch client count"
);
self.clients
.iter()
.zip(data)
.map(|(client, item)| predicate(item, client))
.collect()
}
}
#[derive(Debug, Clone)]
pub struct MeilisearchConfig {
pub addresses: Vec<String>,
pub read_lb_address: String,
pub key: String,
pub meta_namespace: String,
}
impl MeilisearchConfig {
pub fn new(meta_namespace: Option<String>) -> Self {
Self {
addresses: ENV.MEILISEARCH_WRITE_ADDRS.0.clone(),
key: ENV.MEILISEARCH_KEY.clone(),
meta_namespace: meta_namespace.unwrap_or_default(),
read_lb_address: ENV.MEILISEARCH_READ_ADDR.clone(),
}
}
pub fn make_loadbalanced_read_client(
&self,
) -> Result<MeilisearchReadClient, meilisearch_sdk::errors::Error> {
Ok(MeilisearchReadClient {
client: Client::new(&self.read_lb_address, Some(&self.key))?,
})
}
pub fn make_batch_client(
&self,
) -> Result<BatchClient, meilisearch_sdk::errors::Error> {
Ok(BatchClient::new(
self.addresses
.iter()
.map(|address| {
Client::new(address.as_str(), Some(self.key.as_str()))
})
.collect::<Result<Vec<_>, _>>()?,
))
}
pub fn get_index_name(&self, index: &str, next: bool) -> String {
let alt = if next { "_alt" } else { "" };
format!("{}_{}_{}", self.meta_namespace, index, alt)
}
}
pub struct Meilisearch {
pub config: MeilisearchConfig,
}
impl Meilisearch {
pub fn new(config: MeilisearchConfig) -> Self {
Self { config }
}
fn get_sort_index(
&self,
index: &str,
new_filters: Option<&str>,
) -> Result<(String, &'static [&'static str]), ApiError> {
let sort = parse_search_index(index, new_filters)?;
let index_name = match sort.index_name {
SearchIndexName::Projects => {
self.config.get_index_name("projects", false)
}
SearchIndexName::ProjectsFiltered => {
self.config.get_index_name("projects_filtered", false)
}
};
Ok(match sort.index {
SearchIndex::Relevance => (
index_name,
&["downloads:desc", "version_published_timestamp:desc"],
),
SearchIndex::Downloads => (
index_name,
&["downloads:desc", "version_published_timestamp:desc"],
),
SearchIndex::Follows => (
index_name,
&["follows:desc", "version_published_timestamp:desc"],
),
SearchIndex::Updated => (
index_name,
&["date_modified:desc", "version_published_timestamp:desc"],
),
SearchIndex::Newest => (
index_name,
&["date_created:desc", "version_published_timestamp:desc"],
),
SearchIndex::MinecraftJavaServerVerifiedPlays2w => (
index_name,
&[
"minecraft_java_server.verified_plays_2w:desc",
"minecraft_java_server.ping.data.players_online:desc",
"version_published_timestamp:desc",
],
),
SearchIndex::MinecraftJavaServerPlayersOnline => (
index_name,
&[
"minecraft_java_server.ping.data.players_online:desc",
"version_published_timestamp:desc",
],
),
})
}
}
#[async_trait]
impl SearchBackend for Meilisearch {
async fn search_for_project_raw(
&self,
info: &SearchRequest,
) -> Result<SearchResults, ApiError> {
let parsed = parse_search_request(info)?;
let (index_name, sort_name) =
self.get_sort_index(parsed.index, info.new_filters.as_deref())?;
let client = self
.config
.make_loadbalanced_read_client()
.wrap_internal_err("failed to make load-balanced read client")?;
let meilisearch_index = client
.get_index(index_name)
.await
.wrap_internal_err("failed to get index")?;
let mut filter_string = String::new();
let results = {
let mut query = meilisearch_index.search();
query
.with_page(parsed.page)
.with_hits_per_page(parsed.hits_per_page)
.with_query(parsed.query)
.with_sort(sort_name);
if let Some(new_filters) = info.new_filters.as_deref() {
query.with_filter(new_filters);
} else {
let facets = if let Some(facets) = &info.facets {
let facets =
serde_json::from_str::<Vec<Vec<Value>>>(facets)
.wrap_request_err("failed to parse facets")?;
Some(facets)
} else {
None
};
let filters =
combined_search_filters(info).unwrap_or_else(|| "".into());
if let Some(facets) = facets {
let facets: Vec<Vec<Vec<String>>> =
facets
.into_iter()
.map(|facets| {
facets
.into_iter()
.map(|facet| {
if facet.is_array() {
serde_json::from_value::<Vec<String>>(facet)
.unwrap_or_default()
} else {
vec![
serde_json::from_value::<String>(facet)
.unwrap_or_default(),
]
}
})
.collect_vec()
})
.collect_vec();
filter_string.push('(');
for (index, facet_outer_list) in facets.iter().enumerate() {
filter_string.push('(');
for (facet_outer_index, facet_inner_list) in
facet_outer_list.iter().enumerate()
{
filter_string.push('(');
for (facet_inner_index, facet) in
facet_inner_list.iter().enumerate()
{
filter_string
.push_str(&facet.replace(':', " = "));
if facet_inner_index
!= (facet_inner_list.len() - 1)
{
filter_string.push_str(" AND ")
}
}
filter_string.push(')');
if facet_outer_index != (facet_outer_list.len() - 1)
{
filter_string.push_str(" OR ")
}
}
filter_string.push(')');
if index != (facets.len() - 1) {
filter_string.push_str(" AND ")
}
}
filter_string.push(')');
if !filters.is_empty() {
write!(filter_string, " AND ({filters})")
.expect("write should not fail");
}
} else {
filter_string.push_str(&filters);
}
if !filter_string.is_empty() {
query.with_filter(&filter_string);
}
}
if info.show_metadata {
query.with_show_ranking_score(true);
query.with_show_ranking_score_details(true);
query.execute().await?
} else {
query.execute::<ResultSearchProject>().await?
}
};
if info.show_metadata {
let hits = results
.hits
.into_iter()
.map(|hit| {
let metadata = serde_json::to_value(&hit)
.ok()
.and_then(|value| value.as_object().cloned())
.map(|mut value| {
value.remove("_formatted");
value.remove("_matchesPosition");
value.remove("_federation");
let result = value.remove("result");
let metadata = Value::Object(value);
(result, metadata)
});
let (result, metadata) =
metadata.unwrap_or((None, Value::Null));
let mut result = result
.and_then(|value| {
serde_json::from_value::<ResultSearchProject>(value)
.ok()
})
.unwrap_or(hit.result);
if !metadata.is_null() {
result.search_metadata = Some(metadata);
}
result
})
.collect();
Ok(SearchResults {
hits,
page: results.page.unwrap_or_default(),
hits_per_page: results.hits_per_page.unwrap_or_default(),
total_hits: results.total_hits.unwrap_or_default(),
})
} else {
Ok(SearchResults {
hits: results.hits.into_iter().map(|r| r.result).collect(),
page: results.page.unwrap_or_default(),
hits_per_page: results.hits_per_page.unwrap_or_default(),
total_hits: results.total_hits.unwrap_or_default(),
})
}
}
async fn index_projects(
&self,
ro_pool: PgPool,
redis: RedisPool,
) -> eyre::Result<()> {
indexing::index_projects(ro_pool, redis, &self.config).await?;
Ok(())
}
async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()> {
indexing::remove_documents(ids, &self.config).await?;
Ok(())
}
async fn tasks(&self) -> eyre::Result<Value> {
let client = self
.config
.make_batch_client()
.wrap_internal_err("failed to make batch client")?;
let tasks = client
.with_all_clients("get_tasks", async |client| {
let tasks = client.get_tasks().await?;
Ok(tasks.results)
})
.await
.wrap_internal_err("failed to get tasks")?;
#[derive(Serialize)]
struct MeiliTask<Time> {
uid: u32,
status: &'static str,
duration: Option<Duration>,
enqueued_at: Option<Time>,
}
#[derive(Serialize)]
struct TaskList<Time> {
by_instance: HashMap<String, Vec<MeiliTask<Time>>>,
}
let response = tasks
.into_iter()
.enumerate()
.map(|(idx, instance_tasks)| {
let tasks = instance_tasks
.into_iter()
.filter_map(|task| {
Some(match task {
Task::Enqueued { content } => MeiliTask {
uid: content.uid,
status: "enqueued",
duration: None,
enqueued_at: Some(content.enqueued_at),
},
Task::Processing { content } => MeiliTask {
uid: content.uid,
status: "processing",
duration: None,
enqueued_at: Some(content.enqueued_at),
},
Task::Failed { content } => MeiliTask {
uid: content.task.uid,
status: "failed",
duration: Some(content.task.duration),
enqueued_at: Some(content.task.enqueued_at),
},
Task::Succeeded { .. } => return None,
})
})
.collect();
(idx.to_string(), tasks)
})
.collect::<HashMap<String, Vec<MeiliTask<_>>>>();
let response = serde_json::to_value(TaskList {
by_instance: response,
})
.wrap_internal_err("failed to serialize tasks response")?;
Ok(response)
}
async fn tasks_cancel(
&self,
filter: &TasksCancelFilter,
) -> eyre::Result<()> {
let client = self
.config
.make_batch_client()
.wrap_internal_err("failed to make batch client")?;
let all_results = client
.with_all_clients("cancel_tasks", async |client| {
let mut q = TasksCancelQuery::new(client);
match filter {
TasksCancelFilter::All => {}
TasksCancelFilter::Indexes { indexes } => {
q.with_index_uids(indexes.iter().map(|s| s.as_str()));
}
TasksCancelFilter::AllEnqueued => {
q.with_statuses(["enqueued"]);
}
};
let result = client.cancel_tasks_with(&q).await;
Ok(result)
})
.await
.wrap_internal_err("failed to cancel tasks")?;
for r in all_results {
r.wrap_internal_err("failed to cancel tasks")?;
}
Ok(())
}
}

12
apps/labrinth/src/search/backend/mod.rs Normal file
View File

@@ -0,0 +1,12 @@
mod common;
pub mod elasticsearch;
pub mod meilisearch;
pub mod typesense;
pub use common::{
ParsedSearchRequest, SearchIndex, SearchIndexName, SearchSort,
combined_search_filters, parse_search_index, parse_search_request,
};
pub use elasticsearch::Elasticsearch;
pub use meilisearch::{Meilisearch, MeilisearchConfig};
pub use typesense::{Typesense, TypesenseConfig};

1094
apps/labrinth/src/search/backend/typesense/mod.rs Normal file
View File

File diff suppressed because it is too large Load Diff

21
apps/labrinth/src/search/indexing/local_import.rs → apps/labrinth/src/search/indexing.rs
View File

@@ -1,11 +1,11 @@
use chrono::{DateTime, Utc};
use dashmap::DashMap;
use eyre::Result;
use futures::TryStreamExt;
use itertools::Itertools;
use std::collections::HashMap;
use tracing::info;
use super::IndexingError;
use crate::database::PgPool;
use crate::database::models::loader_fields::{
QueryLoaderField, QueryLoaderFieldEnumValue, QueryVersionField,
@@ -29,7 +29,7 @@ pub async fn index_local(
redis: &RedisPool,
cursor: i64,
limit: i64,
) -> Result<(Vec<UploadSearchProject>, i64), IndexingError> {
) -> eyre::Result<(Vec<UploadSearchProject>, i64)> {
info!("Indexing local projects!");
// todo: loaders, project type, game versions
@@ -84,7 +84,8 @@ pub async fn index_local(
}
})
.try_collect::<Vec<PartialProject>>()
.await?;
.await
.wrap_err("failed to fetch projects")?;
let project_ids = db_projects.iter().map(|x| x.id.0).collect::<Vec<i64>>();
let project_components = db_projects
@@ -430,6 +431,7 @@ pub async fn index_local(
display_categories: display_categories.clone(),
follows: project.follows,
downloads: project.downloads,
log_downloads: (project.downloads.max(1) as f64).ln(),
icon_url: project.icon_url.clone(),
author: owner.clone(),
date_created: project.approved,
@@ -473,7 +475,7 @@ struct PartialVersion {
async fn index_versions(
pool: &PgPool,
project_ids: Vec<i64>,
) -> Result<HashMap<DBProjectId, Vec<PartialVersion>>, IndexingError> {
) -> Result<HashMap<DBProjectId, Vec<PartialVersion>>> {
let versions: HashMap<DBProjectId, Vec<(DBVersionId, DateTime<Utc>)>> =
sqlx::query!(
"
@@ -497,7 +499,8 @@ async fn index_versions(
async move { Ok(acc) }
},
)
.await?;
.await
.wrap_err("failed to fetch versions")?;
// Get project types, loaders
#[derive(Default)]
@@ -538,7 +541,8 @@ async fn index_versions(
(version_id, version_loader_data)
})
.try_collect()
.await?;
.await
.wrap_err("failed to fetch loaders and project types")?;
// Get version fields
let version_fields: DashMap<DBVersionId, Vec<QueryVersionField>> =
@@ -570,7 +574,10 @@ async fn index_versions(
async move { Ok(acc) }
},
)
.await?;
.await
.wrap_err("failed to fetch version fields")?;
// Get version fields
// Convert to partial versions
let mut res_versions: HashMap<DBProjectId, Vec<PartialVersion>> =

656
apps/labrinth/src/search/mod.rs
View File

@@ -1,188 +1,222 @@
use crate::env::ENV;
use crate::database::redis::RedisPool;
use crate::models::exp;
use crate::models::exp::minecraft::JavaServerPing;
use crate::models::ids::ProjectId;
use crate::models::projects::SearchRequest;
use crate::models::ids::{ProjectId, VersionId};
use crate::queue::server_ping;
use crate::{database::models::DatabaseError, database::redis::RedisPool};
use crate::{models::error::ApiError, search::indexing::IndexingError};
use actix_web::HttpResponse;
use actix_web::http::StatusCode;
use crate::routes::ApiError;
use crate::{database::PgPool, env::ENV};
use ariadne::ids::base62_impl::parse_base62;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use futures::TryStreamExt;
use futures::stream::FuturesOrdered;
use itertools::Itertools;
use meilisearch_sdk::client::Client;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::borrow::Cow;
use std::collections::HashMap;
use std::fmt::Write;
use std::{collections::HashMap, str::FromStr};
use thiserror::Error;
use tracing::{Instrument, info_span};
use utoipa::ToSchema;
pub mod backend;
pub mod indexing;
#[derive(Error, Debug)]
pub enum SearchError {
#[error("MeiliSearch Error: {0}")]
MeiliSearch(#[from] meilisearch_sdk::errors::Error),
#[error("Error while serializing or deserializing JSON: {0}")]
Serde(#[from] serde_json::Error),
#[error("Error while parsing an integer: {0}")]
IntParsing(#[from] std::num::ParseIntError),
#[error("Error while formatting strings: {0}")]
FormatError(#[from] std::fmt::Error),
#[error("Environment Error")]
Env(#[from] dotenvy::Error),
#[error("Invalid index to sort by: {0}")]
InvalidIndex(String),
#[error("Database error: {0}")]
Database(#[from] DatabaseError),
/// Search parameters which can fit in a URL query string.
///
/// Used with `GET /*/search` endpoints.
///
/// Can be converted into a [`SearchRequest`] using [`From`].
#[derive(Serialize, Deserialize, Debug)]
pub struct SearchQuery {
pub query: Option<String>,
pub offset: Option<String>,
pub index: Option<String>,
pub limit: Option<String>,
pub new_filters: Option<String>,
// TODO: Deprecated values below. WILL BE REMOVED V3!
pub facets: Option<String>,
pub filters: Option<String>,
pub version: Option<String>,
}
impl actix_web::ResponseError for SearchError {
fn status_code(&self) -> StatusCode {
match self {
SearchError::Env(..) => StatusCode::INTERNAL_SERVER_ERROR,
SearchError::MeiliSearch(..) => StatusCode::BAD_REQUEST,
SearchError::Serde(..) => StatusCode::BAD_REQUEST,
SearchError::IntParsing(..) => StatusCode::BAD_REQUEST,
SearchError::InvalidIndex(..) => StatusCode::BAD_REQUEST,
SearchError::FormatError(..) => StatusCode::BAD_REQUEST,
SearchError::Database(..) => StatusCode::INTERNAL_SERVER_ERROR,
/// Search parameters which are more complicated and more suitable for a POST
/// request body.
///
/// Used with `POST /*/search` endpoints.
///
/// Can be converted from a [`SearchQuery`] using [`From`].
#[derive(Serialize, Deserialize, Debug)]
pub struct SearchRequest {
pub query: Option<String>,
pub offset: Option<String>,
pub index: Option<String>,
pub limit: Option<String>,
#[serde(default)]
pub show_metadata: bool,
#[serde(default)]
pub elasticsearch_config: backend::elasticsearch::RequestConfig,
#[serde(default)]
pub typesense_config: backend::typesense::RequestConfig,
pub new_filters: Option<String>,
pub facets: Option<String>,
pub filters: Option<String>,
pub version: Option<String>,
}
impl From<SearchQuery> for SearchRequest {
fn from(query: SearchQuery) -> Self {
Self {
query: query.query,
offset: query.offset,
index: query.index,
limit: query.limit,
show_metadata: false,
elasticsearch_config:
backend::elasticsearch::RequestConfig::default(),
typesense_config: backend::typesense::RequestConfig::default(),
new_filters: query.new_filters,
facets: query.facets,
filters: query.filters,
version: query.version,
}
}
fn error_response(&self) -> HttpResponse {
HttpResponse::build(self.status_code()).json(ApiError {
error: match self {
SearchError::Env(..) => "environment_error",
SearchError::MeiliSearch(..) => "meilisearch_error",
SearchError::Serde(..) => "invalid_input",
SearchError::IntParsing(..) => "invalid_input",
SearchError::InvalidIndex(..) => "invalid_input",
SearchError::FormatError(..) => "invalid_input",
SearchError::Database(..) => "database_error",
},
description: self.to_string(),
details: None,
})
}
}
#[derive(Debug, Clone)]
pub struct MeilisearchReadClient {
pub client: Client,
}
impl std::ops::Deref for MeilisearchReadClient {
type Target = Client;
fn deref(&self) -> &Self::Target {
&self.client
}
}
pub struct BatchClient {
pub clients: Vec<Client>,
}
impl BatchClient {
pub fn new(clients: Vec<Client>) -> Self {
Self { clients }
}
pub async fn with_all_clients<'a, T, G, Fut>(
&'a self,
task_name: &str,
generator: G,
) -> Result<Vec<T>, IndexingError>
where
G: Fn(&'a Client) -> Fut,
Fut: Future<Output = Result<T, IndexingError>> + 'a,
{
let mut tasks = FuturesOrdered::new();
for (idx, client) in self.clients.iter().enumerate() {
tasks.push_back(generator(client).instrument(info_span!(
"client_task",
task.name = task_name,
client.idx = idx,
)));
}
let results = tasks.try_collect::<Vec<T>>().await?;
#[async_trait]
pub trait SearchBackend: Send + Sync {
async fn search_for_project(
&self,
info: &SearchRequest,
redis: &RedisPool,
) -> Result<SearchResults, ApiError> {
let mut results = self.search_for_project_raw(info).await?;
hydrate_search_results(&mut results.hits, redis)
.await
.map_err(ApiError::Internal)?;
Ok(results)
}
pub fn across_all<T, F, R>(&self, data: Vec<T>, mut predicate: F) -> Vec<R>
where
F: FnMut(T, &Client) -> R,
{
assert_eq!(
data.len(),
self.clients.len(),
"mismatch between data len and meilisearch client count"
);
self.clients
.iter()
.zip(data)
.map(|(client, item)| predicate(item, client))
.collect()
}
async fn search_for_project_raw(
&self,
info: &SearchRequest,
) -> Result<SearchResults, ApiError>;
async fn index_projects(
&self,
ro_pool: PgPool,
redis: RedisPool,
) -> eyre::Result<()>;
async fn remove_documents(&self, ids: &[VersionId]) -> eyre::Result<()>;
async fn tasks(&self) -> eyre::Result<Value>;
async fn tasks_cancel(
&self,
filter: &TasksCancelFilter,
) -> eyre::Result<()>;
}
#[derive(Debug, Clone)]
pub struct SearchConfig {
pub addresses: Vec<String>,
pub read_lb_address: String,
pub key: String,
pub meta_namespace: String,
}
async fn hydrate_search_results(
hits: &mut [ResultSearchProject],
redis_pool: &RedisPool,
) -> eyre::Result<()> {
// Minecraft Java servers should fetch the latest player count that we have
// from Redis, rather than the (pretty stale) data from search backend
// TODO: this block should be made generic over the component type,
// for now we can hardcode MC java servers tho
impl SearchConfig {
// Panics if the environment variables are not set,
// but these are already checked for on startup.
pub fn new(meta_namespace: Option<String>) -> Self {
Self {
addresses: ENV.MEILISEARCH_WRITE_ADDRS.0.clone(),
key: ENV.MEILISEARCH_KEY.clone(),
meta_namespace: meta_namespace.unwrap_or_default(),
read_lb_address: ENV.MEILISEARCH_READ_ADDR.clone(),
let project_ids = hits
.iter()
.filter(|hit| hit.components.minecraft_java_server.is_some())
.filter_map(|hit| parse_base62(&hit.project_id).ok().map(ProjectId))
.collect::<Vec<_>>();
let pings_by_project_id = if project_ids.is_empty() {
HashMap::new()
} else {
let mut redis = redis_pool.connect().await?;
let ping_results = redis
.get_many_deserialized_from_json::<JavaServerPing>(
server_ping::REDIS_NAMESPACE,
&project_ids
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>(),
)
.await?;
ping_results
.into_iter()
.enumerate()
.filter_map(|(idx, ping)| ping.map(|ping| (project_ids[idx], ping)))
.collect::<HashMap<_, _>>()
};
for hit in hits {
let Some(java_server) = hit.components.minecraft_java_server.as_mut()
else {
continue;
};
if let Ok(project_id) = parse_base62(&hit.project_id).map(ProjectId) {
java_server.ping = pings_by_project_id.get(&project_id).cloned();
} else {
java_server.ping = None;
}
}
pub fn make_loadbalanced_read_client(
&self,
) -> Result<MeilisearchReadClient, meilisearch_sdk::errors::Error> {
Ok(MeilisearchReadClient {
client: Client::new(&self.read_lb_address, Some(&self.key))?,
Ok(())
}
#[derive(Deserialize, Serialize, ToSchema)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum TasksCancelFilter {
All,
AllEnqueued,
Indexes { indexes: Vec<String> },
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SearchBackendKind {
Meilisearch,
Elasticsearch,
Typesense,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, strum::EnumIter)]
pub enum SearchField {
Categories,
ProjectTypes,
ProjectId,
OpenSource,
Environment,
GameVersions,
ClientSide,
ServerSide,
MinecraftServerRegion,
MinecraftServerLanguages,
MinecraftJavaServerContentKind,
MinecraftJavaServerContentSupportedGameVersions,
MinecraftJavaServerPingData,
}
#[derive(Debug, Error)]
#[error("invalid search backend kind")]
pub struct InvalidSearchBackendKind;
impl FromStr for SearchBackendKind {
type Err = InvalidSearchBackendKind;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(match s {
"meilisearch" => SearchBackendKind::Meilisearch,
"elasticsearch" => SearchBackendKind::Elasticsearch,
"typesense" => SearchBackendKind::Typesense,
_ => return Err(InvalidSearchBackendKind),
})
}
pub fn make_batch_client(
&self,
) -> Result<BatchClient, meilisearch_sdk::errors::Error> {
Ok(BatchClient::new(
self.addresses
.iter()
.map(|address| {
Client::new(address.as_str(), Some(self.key.as_str()))
})
.collect::<Result<Vec<_>, _>>()?,
))
}
// Next: true if we want the next index (we are preparing the next swap), false if we want the current index (searching)
pub fn get_index_name(&self, index: &str, next: bool) -> String {
let alt = if next { "_alt" } else { "" };
format!("{}_{}_{}", self.meta_namespace, index, alt)
}
}
/// A project document used for uploading projects to MeiliSearch's indices.
/// This contains some extra data that is not returned by search results.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct UploadSearchProject {
pub version_id: String,
@@ -197,6 +231,7 @@ pub struct UploadSearchProject {
pub display_categories: Vec<String>,
pub follows: i32,
pub downloads: i32,
pub log_downloads: f64,
pub icon_url: Option<String>,
pub license: String,
pub gallery: Vec<String>,
@@ -263,281 +298,52 @@ pub struct ResultSearchProject {
pub components: exp::ProjectQuery,
#[serde(flatten)]
pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub search_metadata: Option<Value>,
}
pub fn get_sort_index(
config: &SearchConfig,
index: &str,
new_filters: Option<&str>,
) -> Result<(String, &'static [&'static str]), SearchError> {
let projects_name = config.get_index_name("projects", false);
let projects_filtered_name =
config.get_index_name("projects_filtered", false);
// TODO: this is a dumb hack, the frontend should pass the project type it's filtering directly
let is_server = new_filters
.is_some_and(|f| f.contains("project_types = minecraft_java_server"));
Ok(match index {
"relevance" => (
projects_name,
if is_server {
&[
"minecraft_java_server.verified_plays_2w:desc",
"minecraft_java_server.ping.data.players_online:desc",
"version_published_timestamp:desc",
]
} else {
&["downloads:desc", "version_published_timestamp:desc"]
},
),
"downloads" => (
projects_filtered_name,
&["downloads:desc", "version_published_timestamp:desc"],
),
"follows" => (
projects_name,
&["follows:desc", "version_published_timestamp:desc"],
),
"updated" | "date_modified" => (
projects_name,
&["date_modified:desc", "version_published_timestamp:desc"],
),
"newest" | "date_created" => (
projects_name,
&["date_created:desc", "version_published_timestamp:desc"],
),
"minecraft_java_server.verified_plays_2w" => (
projects_name,
&[
"minecraft_java_server.verified_plays_2w:desc",
"version_published_timestamp:desc",
],
),
"minecraft_java_server.ping.data.players_online" => (
projects_name,
&[
"minecraft_java_server.ping.data.players_online:desc",
"version_published_timestamp:desc",
],
),
i => return Err(SearchError::InvalidIndex(i.to_string())),
})
}
fn normalize_filter_aliases(filters: &str) -> String {
let mut filters = filters.replace("components.", "");
for (from, to) in [
(
"minecraft_java_server.content =",
"minecraft_java_server.content.kind =",
),
(
"minecraft_java_server.content !=",
"minecraft_java_server.content.kind !=",
),
(
"minecraft_java_server.content IN ",
"minecraft_java_server.content.kind IN ",
),
(
"minecraft_java_server.content NOT IN ",
"minecraft_java_server.content.kind NOT IN ",
),
] {
filters = filters.replace(from, to);
}
filters
}
pub async fn search_for_project(
info: &SearchRequest,
config: &SearchConfig,
redis_pool: &RedisPool,
) -> Result<SearchResults, SearchError> {
let offset: usize = info.offset.as_deref().unwrap_or("0").parse()?;
let index = info.index.as_deref().unwrap_or("relevance");
let limit = info
.limit
.as_deref()
.unwrap_or("10")
.parse::<usize>()?
.min(100);
let sort = get_sort_index(config, index, info.new_filters.as_deref())?;
let client = config.make_loadbalanced_read_client()?;
let meilisearch_index = client.get_index(sort.0).await?;
let mut filter_string = String::new();
// Convert offset and limit to page and hits_per_page
let hits_per_page = if limit == 0 { 1 } else { limit };
let page = offset / hits_per_page + 1;
let results = {
let mut query = meilisearch_index.search();
query
.with_page(page)
.with_hits_per_page(hits_per_page)
.with_query(info.query.as_deref().unwrap_or_default())
.with_sort(sort.1);
let normalized_new_filters =
info.new_filters.as_deref().map(normalize_filter_aliases);
if let Some(new_filters) = normalized_new_filters.as_deref() {
query.with_filter(new_filters);
} else {
let facets = if let Some(facets) = &info.facets {
Some(serde_json::from_str::<Vec<Vec<Value>>>(facets)?)
} else {
None
};
let filters: Cow<_> =
match (info.filters.as_deref(), info.version.as_deref()) {
(Some(f), Some(v)) => format!("({f}) AND ({v})").into(),
(Some(f), None) => f.into(),
(None, Some(v)) => v.into(),
(None, None) => "".into(),
};
let filters = normalize_filter_aliases(&filters);
if let Some(facets) = facets {
// Search can now *optionally* have a third inner array: So Vec(AND)<Vec(OR)<Vec(AND)< _ >>>
// For every inner facet, we will check if it can be deserialized into a Vec<&str>, and do so.
// If not, we will assume it is a single facet and wrap it in a Vec.
let facets: Vec<Vec<Vec<String>>> = facets
.into_iter()
.map(|facets| {
facets
.into_iter()
.map(|facet| {
if facet.is_array() {
serde_json::from_value::<Vec<String>>(facet)
.unwrap_or_default()
} else {
vec![
serde_json::from_value::<String>(facet)
.unwrap_or_default(),
]
}
})
.collect_vec()
})
.collect_vec();
filter_string.push('(');
for (index, facet_outer_list) in facets.iter().enumerate() {
filter_string.push('(');
for (facet_outer_index, facet_inner_list) in
facet_outer_list.iter().enumerate()
{
filter_string.push('(');
for (facet_inner_index, facet) in
facet_inner_list.iter().enumerate()
{
let facet = normalize_filter_aliases(
&facet.replace(':', " = "),
);
filter_string.push_str(&facet);
if facet_inner_index != (facet_inner_list.len() - 1)
{
filter_string.push_str(" AND ")
}
}
filter_string.push(')');
if facet_outer_index != (facet_outer_list.len() - 1) {
filter_string.push_str(" OR ")
}
}
filter_string.push(')');
if index != (facets.len() - 1) {
filter_string.push_str(" AND ")
}
}
filter_string.push(')');
if !filters.is_empty() {
write!(filter_string, " AND ({filters})")?;
}
} else {
filter_string.push_str(&filters);
}
if !filter_string.is_empty() {
query.with_filter(&filter_string);
}
}
query.execute::<ResultSearchProject>().await?
};
// Minecraft Java servers should fetch the latest player count that we have
// from Redis, rather than the (pretty stale) data from search backend
// TODO: this block should be made generic over the component type,
// for now we can hardcode MC java servers tho
let mut hits = results.hits.into_iter().map(|r| r.result).collect_vec();
let project_ids = hits
.iter()
.filter(|hit| hit.components.minecraft_java_server.is_some())
.filter_map(|hit| parse_base62(&hit.project_id).ok().map(ProjectId))
.collect_vec();
let pings_by_project_id = if project_ids.is_empty() {
HashMap::new()
} else {
let mut redis = redis_pool.connect().await?;
let ping_results = redis
.get_many_deserialized_from_json::<JavaServerPing>(
server_ping::REDIS_NAMESPACE,
&project_ids.iter().map(ToString::to_string).collect_vec(),
)
.await?;
ping_results
.into_iter()
.enumerate()
.filter_map(|(idx, ping)| ping.map(|ping| (project_ids[idx], ping)))
.collect::<HashMap<_, _>>()
};
for hit in &mut hits {
let Some(java_server) = hit.components.minecraft_java_server.as_mut()
else {
continue;
};
if let Ok(project_id) = parse_base62(&hit.project_id).map(ProjectId) {
java_server.ping = pings_by_project_id.get(&project_id).cloned();
} else {
java_server.ping = None;
impl From<UploadSearchProject> for ResultSearchProject {
fn from(source: UploadSearchProject) -> Self {
Self {
version_id: source.version_id,
project_id: source.project_id,
project_types: source.project_types,
slug: source.slug,
author: source.author,
name: source.name,
summary: source.summary,
categories: source.categories,
display_categories: source.display_categories,
downloads: source.downloads,
follows: source.follows,
icon_url: source.icon_url,
date_created: source.date_created.to_rfc3339(),
date_modified: source.date_modified.to_rfc3339(),
license: source.license,
gallery: source.gallery,
featured_gallery: source.featured_gallery,
color: source.color,
loaders: source.loaders,
project_loader_fields: source.project_loader_fields,
components: source.components,
loader_fields: source.loader_fields,
search_metadata: None,
}
}
Ok(SearchResults {
hits,
page: results.page.unwrap_or_default(),
hits_per_page: results.hits_per_page.unwrap_or_default(),
total_hits: results.total_hits.unwrap_or_default(),
})
}
#[cfg(test)]
mod tests {
use super::normalize_filter_aliases;
#[test]
fn normalizes_component_filter_aliases() {
assert_eq!(
normalize_filter_aliases(
"components.minecraft_java_server.content = vanilla AND components.minecraft_server.country = US"
),
"minecraft_java_server.content.kind = vanilla AND minecraft_server.country = US"
);
pub fn backend(meta_namespace: Option<String>) -> Box<dyn SearchBackend> {
match ENV.SEARCH_BACKEND {
SearchBackendKind::Meilisearch => {
let config = backend::MeilisearchConfig::new(meta_namespace);
Box::new(backend::Meilisearch::new(config))
}
SearchBackendKind::Elasticsearch => {
Box::new(backend::Elasticsearch::new(meta_namespace).unwrap())
}
SearchBackendKind::Typesense => {
let config = backend::TypesenseConfig::new(meta_namespace);
Box::new(backend::Typesense::new(config))
}
}
}

16
apps/labrinth/src/test/database.rs
View File

@@ -2,8 +2,9 @@ use crate::database::PgPool;
use crate::database::redis::RedisPool;
use crate::database::{MIGRATOR, ReadOnlyPgPool};
use crate::env::ENV;
use crate::search;
use crate::search::{self, SearchBackend};
use sqlx::postgres::PgPoolOptions;
use std::sync::Arc;
use std::time::Duration;
use url::Url;
@@ -42,8 +43,8 @@ pub struct TemporaryDatabase {
pub pool: PgPool,
pub ro_pool: ReadOnlyPgPool,
pub redis_pool: RedisPool,
pub search_config: crate::search::SearchConfig,
pub database_name: String,
pub search_backend: Arc<dyn SearchBackend>,
}
impl TemporaryDatabase {
@@ -91,15 +92,14 @@ impl TemporaryDatabase {
// Gets new Redis pool
let redis_pool = RedisPool::new(temp_database_name.clone());
// Create new meilisearch config
let search_config =
search::SearchConfig::new(Some(temp_database_name.clone()));
// Create search backend
let search_backend = search::backend(Some(temp_database_name.clone()));
Self {
pool,
ro_pool,
database_name: temp_database_name,
redis_pool,
search_config,
search_backend: Arc::from(search_backend),
}
}
@@ -193,7 +193,9 @@ impl TemporaryDatabase {
ro_pool: ReadOnlyPgPool::from(pool.clone()),
database_name: TEMPLATE_DATABASE_NAME.to_string(),
redis_pool: RedisPool::new(name.clone()),
search_config: search::SearchConfig::new(Some(name)),
search_backend: Arc::from(search::backend(Some(
name.clone(),
))),
};
let setup_api =
TestEnvironment::<ApiV3>::build_setup_api(&db).await;

5
apps/labrinth/src/test/mod.rs
View File

@@ -23,7 +23,6 @@ pub mod search;
// If making a test, you should probably use environment::TestEnvironment::build() (which calls this)
pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
println!("Setting up labrinth config");
dotenvy::dotenv().ok();
env::init().expect("failed to initialize environment variables");
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
@@ -31,7 +30,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
let pool = db.pool.clone();
let ro_pool = db.ro_pool.clone();
let redis_pool = db.redis_pool.clone();
let search_config = db.search_config.clone();
let search_backend = db.search_backend.clone();
let file_host: Arc<dyn file_hosting::FileHost + Send + Sync> =
Arc::new(file_hosting::MockHost::new());
let mut clickhouse = clickhouse::init_client().await.unwrap();
@@ -48,7 +47,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
pool.clone(),
ro_pool.clone(),
redis_pool.clone(),
search_config,
search_backend.into(),
&mut clickhouse,
file_host.clone(),
stripe_client,

2
apps/labrinth/tests/search.rs
View File

@@ -101,7 +101,7 @@ async fn search_projects() {
async move {
let projects = api
.search_deserialized(
Some(&format!("\"&{test_name}\"")),
Some(&format!("&{test_name}")),
Some(facets.clone()),
USER_USER_PAT,
)

12
apps/labrinth/tests/v2/search.rs
View File

@@ -316,7 +316,7 @@ async fn search_projects() {
async move {
let projects = api
.search_deserialized(
Some(&format!("\"&{test_name}\"")),
Some(&format!("&{test_name}")),
Some(facets.clone()),
USER_USER_PAT,
)
@@ -337,7 +337,7 @@ async fn search_projects() {
// A couple additional tests for the search type returned, making sure it is properly translated back
let client_side_required = api
.search_deserialized(
Some(&format!("\"&{test_name}\"")),
Some(&format!("&{test_name}")),
Some(json!([["client_side:required"]])),
USER_USER_PAT,
)
@@ -348,7 +348,7 @@ async fn search_projects() {
let server_side_required = api
.search_deserialized(
Some(&format!("\"&{test_name}\"")),
Some(&format!("&{test_name}")),
Some(json!([["server_side:required"]])),
USER_USER_PAT,
)
@@ -359,7 +359,7 @@ async fn search_projects() {
let client_side_unsupported = api
.search_deserialized(
Some(&format!("\"&{test_name}\"")),
Some(&format!("&{test_name}")),
Some(json!([["client_side:unsupported"]])),
USER_USER_PAT,
)
@@ -370,7 +370,7 @@ async fn search_projects() {
let client_side_optional_server_side_optional = api
.search_deserialized(
Some(&format!("\"&{test_name}\"")),
Some(&format!("&{test_name}")),
Some(json!([["client_side:optional"], ["server_side:optional"]])),
USER_USER_PAT,
)
@@ -384,7 +384,7 @@ async fn search_projects() {
// over all versions of a project
let game_versions = api
.search_deserialized(
Some(&format!("\"&{test_name}\"")),
Some(&format!("&{test_name}")),
Some(json!([["categories:forge"], ["versions:1.20.2"]])),
USER_USER_PAT,
)

190
docker-compose.yml
View File

@@ -18,6 +18,20 @@ services:
interval: 3s
timeout: 5s
retries: 3
typesense0:
image: typesense/typesense:30.1
container_name: labrinth-typesense0
restart: on-failure
ports:
- '127.0.0.1:8108:8108'
volumes:
- typesense-data:/data
command: --data-dir=/data --api-key=modrinth --enable-cors
healthcheck:
test: ['CMD-SHELL', "bash -lc '</dev/tcp/127.0.0.1/8108'"]
interval: 3s
timeout: 5s
retries: 3
meilisearch0:
image: getmeili/meilisearch:v1.12.0
container_name: labrinth-meilisearch0
@@ -37,6 +51,167 @@ services:
interval: 3s
timeout: 5s
retries: 3
elasticsearch-certs:
image: elasticsearch:9.3.0
container_name: labrinth-elasticsearch-certs
user: '0'
networks:
- elasticsearch-mesh
restart: 'no'
volumes:
- elasticsearch-certs:/usr/share/elasticsearch/config/certs
command: |
bash -c '
set -euo pipefail
if [ ! -s config/certs/ca/ca.crt ] || [ ! -s config/certs/elasticsearch0/elasticsearch0.crt ] || [ ! -s config/certs/elasticsearch1/elasticsearch1.crt ] || [ ! -s config/certs/elasticsearch2/elasticsearch2.crt ]; then
rm -rf config/certs/*
printf "%s\n" \
"instances:" \
" - name: elasticsearch0" \
" dns:" \
" - elasticsearch0" \
" - localhost" \
" ip:" \
" - 127.0.0.1" \
" - name: elasticsearch1" \
" dns:" \
" - elasticsearch1" \
" - localhost" \
" ip:" \
" - 127.0.0.1" \
" - name: elasticsearch2" \
" dns:" \
" - elasticsearch2" \
" - localhost" \
" ip:" \
" - 127.0.0.1" \
> config/certs/instances.yml
bin/elasticsearch-certutil ca --silent --pem --out config/certs/ca.zip
unzip config/certs/ca.zip -d config/certs
bin/elasticsearch-certutil cert --silent --pem --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key --out config/certs/certs.zip
unzip config/certs/certs.zip -d config/certs
fi
chown -R 1000:0 config/certs
find config/certs -type d -exec chmod 750 {} \;
find config/certs -type f -exec chmod 640 {} \;
echo "Set up certificates"
'
elasticsearch0:
image: elasticsearch:9.3.0
container_name: labrinth-elasticsearch0
networks:
- elasticsearch-mesh
restart: on-failure
depends_on:
elasticsearch-certs:
condition: service_completed_successfully
ports:
- '127.0.0.1:9200:9200'
volumes:
- elasticsearch0-data:/usr/share/elasticsearch/data
- elasticsearch-certs:/usr/share/elasticsearch/config/certs:ro
environment:
- logger.level=WARN
- node.name=elasticsearch0
- cluster.name=labrinth
- cluster.initial_master_nodes=elasticsearch0,elasticsearch1,elasticsearch2
- discovery.seed_hosts=elasticsearch1,elasticsearch2
- bootstrap.memory_lock=false
# auth
- xpack.security.enabled=true
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.security.transport.ssl.key=certs/elasticsearch0/elasticsearch0.key
- xpack.security.transport.ssl.certificate=certs/elasticsearch0/elasticsearch0.crt
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
- ELASTIC_USERNAME=elastic
- ELASTIC_PASSWORD=elastic
mem_limit: 1g
healthcheck:
test:
[
'CMD-SHELL',
'curl -s -u elastic:elastic http://localhost:9200/_cluster/health | grep -qE "\"status\":\"(yellow|green)\""',
]
interval: 10s
timeout: 5s
retries: 10
elasticsearch1:
image: elasticsearch:9.3.0
container_name: labrinth-elasticsearch1
networks:
- elasticsearch-mesh
restart: on-failure
depends_on:
elasticsearch-certs:
condition: service_completed_successfully
volumes:
- elasticsearch1-data:/usr/share/elasticsearch/data
- elasticsearch-certs:/usr/share/elasticsearch/config/certs:ro
environment:
- logger.level=WARN
- node.name=elasticsearch1
- cluster.name=labrinth
- cluster.initial_master_nodes=elasticsearch0,elasticsearch1,elasticsearch2
- discovery.seed_hosts=elasticsearch0,elasticsearch2
- bootstrap.memory_lock=false
# auth
- xpack.security.enabled=true
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.security.transport.ssl.key=certs/elasticsearch1/elasticsearch1.key
- xpack.security.transport.ssl.certificate=certs/elasticsearch1/elasticsearch1.crt
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
- ELASTIC_USERNAME=elastic
- ELASTIC_PASSWORD=elastic
mem_limit: 1g
healthcheck:
test:
[
'CMD-SHELL',
'curl -s -u elastic:elastic http://localhost:9200/_cluster/health | grep -qE "\"status\":\"(yellow|green)\""',
]
interval: 10s
timeout: 5s
retries: 10
elasticsearch2:
image: elasticsearch:9.3.0
container_name: labrinth-elasticsearch2
networks:
- elasticsearch-mesh
restart: on-failure
depends_on:
elasticsearch-certs:
condition: service_completed_successfully
volumes:
- elasticsearch2-data:/usr/share/elasticsearch/data
- elasticsearch-certs:/usr/share/elasticsearch/config/certs:ro
environment:
- logger.level=WARN
- node.name=elasticsearch2
- cluster.name=labrinth
- cluster.initial_master_nodes=elasticsearch0,elasticsearch1,elasticsearch2
- discovery.seed_hosts=elasticsearch0,elasticsearch1
- bootstrap.memory_lock=false
# auth
- xpack.security.enabled=true
- xpack.security.transport.ssl.enabled=true
- xpack.security.transport.ssl.verification_mode=certificate
- xpack.security.transport.ssl.key=certs/elasticsearch2/elasticsearch2.key
- xpack.security.transport.ssl.certificate=certs/elasticsearch2/elasticsearch2.crt
- xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
- ELASTIC_USERNAME=elastic
- ELASTIC_PASSWORD=elastic
mem_limit: 1g
healthcheck:
test:
[
'CMD-SHELL',
'curl -s -u elastic:elastic http://localhost:9200/_cluster/health | grep -qE "\"status\":\"(yellow|green)\""',
]
interval: 10s
timeout: 5s
retries: 10
redis:
image: redis:alpine
container_name: labrinth-redis
@@ -109,6 +284,12 @@ services:
condition: service_healthy
meilisearch:
condition: service_healthy
elasticsearch0:
condition: service_healthy
elasticsearch1:
condition: service_healthy
elasticsearch2:
condition: service_healthy
redis:
condition: service_healthy
clickhouse:
@@ -143,7 +324,6 @@ services:
# Delphi must send a message on a webhook to our backend,
# so it must have access to our local network
- 'host.docker.internal:host-gateway'
# Sharded Meilisearch
meilisearch1:
profiles:
@@ -166,7 +346,6 @@ services:
interval: 3s
timeout: 5s
retries: 3
nginx-meilisearch-lb:
profiles:
- sharded-meilisearch
@@ -186,9 +365,16 @@ services:
networks:
meilisearch-mesh:
driver: bridge
elasticsearch-mesh:
driver: bridge
volumes:
typesense-data:
meilisearch-data:
meilisearch1-data:
elasticsearch0-data:
elasticsearch1-data:
elasticsearch2-data:
elasticsearch-certs:
db-data:
redis-data:
labrinth-cdn-data:

4
packages/async-minecraft-ping/Cargo.toml
View File

@@ -1,12 +1,12 @@
[package]
name = "async-minecraft-ping"
version = "0.8.0"
authors = ["Jay Vana <jaysvana@gmail.com>"]
# authors = ["Jay Vana <jaysvana@gmail.com>"] # deprecated
edition = "2021"
license = "MIT OR Apache-2.0"
description = "An async Rust client for the Minecraft ServerListPing protocol"
readme = "README.md"
repository = "https://github.com/jsvana/async-minecraft-ping/"
license = "MIT OR Apache-2.0"
keywords = ["mc", "minecraft", "serverlistping"]
categories = ["api-bindings", "asynchronous"]