Files
Modrinth-plus/apps/labrinth/src/env.rs
aecsocket f0224dfff7 Search backend refactor with typesense impl (#5528)
* initial elasticsearch impl

* working elastic cluster

* replace SearchError with ApiError for preparation of search backend

* start factoring meili out to trait

* move meili to backend

* update routes to use search backend trait

* wip

* Update projects.rs

* search backend is only init'd once in config

* wip

* wip: backend agnostic

* change search internal routes to delegate to backend

* initial elasticsearch impl

* fix filtering

* elastic impl

* refactor indexing into its own module

* clean up elastic code

* fix ci

* fix tests

* fix elastic health check

* fix up env rebase

* fix compile

* dummy commit to update github pr

* Fix rebase

* Elastic basic https auth

* Fix duplicate projects showing up

* Fix up tests

* Replace search `ApiErrors` with `eyre::Reports`, propagate background task errors

* clean up agents files

* make index chunk size configurable

* make `match_phrase` in elastic case-insensitive

* use current/next indices and swap between them

* test case for error body

* Fix failing case

* da merge

* factor out common stuff from search backends

* allow fetching hit metadata from search results

* allow customising elasticsearch search config

* bit of docs

* add mappings to indices for elastic

* Implement Typesense

* wip

* fix up some sort fields stuff

* use different approach to filterable field sets

* remove a bunch of search fields which weren't used for filtering

* bucket text matches

* Bucketing by text_match for typesense

* fix tombi lint

* fix some sentry errors and dont prioritise 2+ term matches

* tweak ts query settings

* expose some more search settings

* query sort changes

* small fixes

* should fix pagination stuff

* fix healthcheck maybe

* ragebait ci

* tests

* tests

* revert environment
2026-03-12 18:58:55 +01:00

297 lines
8.2 KiB
Rust

use std::{any::type_name, convert::Infallible, str::FromStr, sync::LazyLock};
use derive_more::{Deref, DerefMut};
use eyre::{Context, eyre};
use rust_decimal::Decimal;
use serde::de::DeserializeOwned;
macro_rules! vars {
(
$(
$field:ident: $ty:ty $(= $default:expr)?;
)*
) => {
#[derive(Debug)]
#[allow(
non_snake_case,
reason = "environment variables are UPPER_SNAKE_CASE",
)]
pub struct EnvVars {
$(
pub $field: $ty,
)*
}
impl EnvVars {
pub fn from_env() -> eyre::Result<Self> {
let mut err = eyre!("failed to read environment variables");
$(
#[expect(
non_snake_case,
reason = "environment variables are UPPER_SNAKE_CASE",
)]
#[allow(
unused_assignments,
unused_mut,
reason = "`default` is not used if there is no default",
)]
let $field: Option<$ty> = {
let mut default = None::<$ty>;
$( default = Some({ $default }.into()); )?
match parse_value::<$ty>(stringify!($field), default) {
Ok(value) => Some(value),
Err(source) => {
err = err.wrap_err(eyre!("{source:#}"));
None
}
}
};
)*
Ok(EnvVars {
$(
$field: match $field {
Some(value) => value,
None => return Err(err),
},
)*
})
}
}
};
}
pub static ENV: LazyLock<EnvVars> = LazyLock::new(|| {
EnvVars::from_env().unwrap_or_else(|err| panic!("{err:?}"))
});
fn parse_value<T>(key: &str, default: Option<T>) -> eyre::Result<T>
where
T: FromStr,
T::Err: std::error::Error + Send + Sync + 'static,
{
match (dotenvy::var(key), default) {
(Ok(value), _) => value.parse::<T>().wrap_err_with(|| {
eyre!("`{key}` is not a valid `{}`", type_name::<T>())
}),
(Err(_), Some(default)) => Ok(default),
(Err(_), None) => Err(eyre!("`{key}` missing")),
}
}
pub fn init() -> eyre::Result<()> {
dotenvy::dotenv().ok();
EnvVars::from_env()?;
LazyLock::force(&ENV);
Ok(())
}
#[derive(
Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Deref, DerefMut,
)]
pub struct Json<T: DeserializeOwned>(pub T);
impl<T: DeserializeOwned> FromStr for Json<T> {
type Err = serde_json::Error;
fn from_str(s: &str) -> Result<Self, Self::Err> {
serde_json::from_str(s).map(Self)
}
}
#[derive(
Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Deref, DerefMut,
)]
pub struct StringCsv(pub Vec<String>);
impl FromStr for StringCsv {
type Err = Infallible;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let v = s
.split(',')
.filter(|s| !s.trim().is_empty())
.map(|s| s.to_string())
.collect::<Vec<_>>();
Ok(Self(v))
}
}
vars! {
SENTRY_ENVIRONMENT: String;
SENTRY_TRACES_SAMPLE_RATE: f32;
SITE_URL: String;
CDN_URL: String;
LABRINTH_ADMIN_KEY: String;
LABRINTH_MEDAL_KEY: String;
LABRINTH_EXTERNAL_NOTIFICATION_KEY: String;
RATE_LIMIT_IGNORE_KEY: String;
DATABASE_URL: String;
REDIS_URL: String;
BIND_ADDR: String;
SELF_ADDR: String;
LOCAL_INDEX_INTERVAL: u64;
VERSION_INDEX_INTERVAL: u64;
WHITELISTED_MODPACK_DOMAINS: Json<Vec<String>>;
ALLOWED_CALLBACK_URLS: Json<Vec<String>>;
ANALYTICS_ALLOWED_ORIGINS: Json<Vec<String>>;
// search
SEARCH_BACKEND: crate::search::SearchBackendKind = crate::search::SearchBackendKind::Typesense;
MEILISEARCH_READ_ADDR: String;
MEILISEARCH_WRITE_ADDRS: StringCsv;
MEILISEARCH_KEY: String;
ELASTICSEARCH_URL: String;
ELASTICSEARCH_INDEX_PREFIX: String;
ELASTICSEARCH_USERNAME: String = "";
ELASTICSEARCH_PASSWORD: String = "";
SEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64;
TYPESENSE_URL: String = "http://localhost:8108";
TYPESENSE_API_KEY: String = "modrinth";
TYPESENSE_INDEX_PREFIX: String = "labrinth";
// storage
STORAGE_BACKEND: crate::file_hosting::FileHostKind;
// s3
S3_PUBLIC_BUCKET_NAME: String = "";
S3_PUBLIC_USES_PATH_STYLE_BUCKET: bool = false;
S3_PUBLIC_REGION: String = "";
S3_PUBLIC_URL: String = "";
S3_PUBLIC_ACCESS_TOKEN: String = "";
S3_PUBLIC_SECRET: String = "";
S3_PRIVATE_BUCKET_NAME: String = "";
S3_PRIVATE_USES_PATH_STYLE_BUCKET: bool = false;
S3_PRIVATE_REGION: String = "";
S3_PRIVATE_URL: String = "";
S3_PRIVATE_ACCESS_TOKEN: String = "";
S3_PRIVATE_SECRET: String = "";
// local
MOCK_FILE_PATH: String = "";
GITHUB_CLIENT_ID: String;
GITHUB_CLIENT_SECRET: String;
GITLAB_CLIENT_ID: String;
GITLAB_CLIENT_SECRET: String;
DISCORD_CLIENT_ID: String;
DISCORD_CLIENT_SECRET: String;
MICROSOFT_CLIENT_ID: String;
MICROSOFT_CLIENT_SECRET: String;
GOOGLE_CLIENT_ID: String;
GOOGLE_CLIENT_SECRET: String;
STEAM_API_KEY: String;
TREMENDOUS_API_URL: String;
TREMENDOUS_API_KEY: String;
TREMENDOUS_PRIVATE_KEY: String;
PAYPAL_API_URL: String;
PAYPAL_WEBHOOK_ID: String;
PAYPAL_CLIENT_ID: String;
PAYPAL_CLIENT_SECRET: String;
PAYPAL_NVP_USERNAME: String;
PAYPAL_NVP_PASSWORD: String;
PAYPAL_NVP_SIGNATURE: String;
PAYPAL_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
BREX_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
TREMENDOUS_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
MURAL_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
HCAPTCHA_SECRET: String;
SMTP_USERNAME: String;
SMTP_PASSWORD: String;
SMTP_HOST: String;
SMTP_PORT: u16;
SMTP_TLS: String;
SMTP_FROM_NAME: String;
SMTP_FROM_ADDRESS: String;
SITE_VERIFY_EMAIL_PATH: String;
SITE_RESET_PASSWORD_PATH: String;
SITE_BILLING_PATH: String;
SENDY_URL: String;
SENDY_LIST_ID: String;
SENDY_API_KEY: String;
CLICKHOUSE_REPLICATED: bool;
CLICKHOUSE_URL: String;
CLICKHOUSE_USER: String;
CLICKHOUSE_PASSWORD: String;
CLICKHOUSE_DATABASE: String;
FLAME_ANVIL_URL: String;
GOTENBERG_URL: String;
GOTENBERG_CALLBACK_BASE: String;
GOTENBERG_TIMEOUT: u64;
STRIPE_API_KEY: String;
STRIPE_WEBHOOK_SECRET: String;
ADITUDE_API_KEY: String;
PYRO_API_KEY: String;
BREX_API_URL: String;
BREX_API_KEY: String;
DELPHI_URL: String;
AVALARA_1099_API_URL: String;
AVALARA_1099_API_KEY: String;
AVALARA_1099_API_TEAM_ID: String;
AVALARA_1099_COMPANY_ID: String;
ANROK_API_URL: String;
ANROK_API_KEY: String;
PAYOUT_ALERT_SLACK_WEBHOOK: String;
CLOUDFLARE_INTEGRATION: bool = false;
ARCHON_URL: String;
MURALPAY_API_URL: String;
MURALPAY_API_KEY: String;
MURALPAY_TRANSFER_API_KEY: String;
MURALPAY_SOURCE_ACCOUNT_ID: muralpay::AccountId = muralpay::AccountId(uuid::Uuid::nil());
DEFAULT_AFFILIATE_REVENUE_SPLIT: Decimal;
DATABASE_ACQUIRE_TIMEOUT_MS: u64 = 30000u64;
DATABASE_MIN_CONNECTIONS: u32 = 0u32;
DATABASE_MAX_CONNECTIONS: u32 = 16u32;
READONLY_DATABASE_URL: String = "";
READONLY_DATABASE_MIN_CONNECTIONS: u32 = 0u32;
READONLY_DATABASE_MAX_CONNECTIONS: u32 = 1u32;
REDIS_WAIT_TIMEOUT_MS: u64 = 15000u64;
REDIS_MAX_CONNECTIONS: u32 = 10000u32;
REDIS_MIN_CONNECTIONS: usize = 0usize;
SEARCH_OPERATION_TIMEOUT: u64 = 300000u64;
SMTP_REPLY_TO_NAME: String = "";
SMTP_REPLY_TO_ADDRESS: String = "";
PUBLIC_DISCORD_WEBHOOK: String = "";
MODERATION_SLACK_WEBHOOK: String = "";
DELPHI_SLACK_WEBHOOK: String = "";
TREMENDOUS_CAMPAIGN_ID: String = "";
// server pinging
SERVER_PING_MAX_CONCURRENT: usize = 16usize;
SERVER_PING_RETRIES: usize = 3usize;
SERVER_PING_MIN_INTERVAL_SEC: u64 = 30u64 * 60;
SERVER_PING_TIMEOUT_MS: u64 = 3u64 * 1000;
}