* initial elasticsearch impl * working elastic cluster * replace SearchError with ApiError for preparation of search backend * start factoring meili out to trait * move meili to backend * update routes to use search backend trait * wip * Update projects.rs * search backend is only init'd once in config * wip * wip: backend agnostic * change search internal routes to delegate to backend * initial elasticsearch impl * fix filtering * elastic impl * refactor indexing into its own module * clean up elastic code * fix ci * fix tests * fix elastic health check * fix up env rebase * fix compile * dummy commit to update github pr * Fix rebase * Elastic basic https auth * Fix duplicate projects showing up * Fix up tests * Replace search `ApiErrors` with `eyre::Reports`, propagate background task errors * clean up agents files * make index chunk size configurable * make `match_phrase` in elastic case-insensitive * use current/next indices and swap between them * test case for error body * Fix failing case * da merge * factor out common stuff from search backends * allow fetching hit metadata from search results * allow customising elasticsearch search config * bit of docs * add mappings to indices for elastic * Implement Typesense * wip * fix up some sort fields stuff * use different approach to filterable field sets * remove a bunch of search fields which weren't used for filtering * bucket text matches * Bucketing by text_match for typesense * fix tombi lint * fix some sentry errors and dont prioritise 2+ term matches * tweak ts query settings * expose some more search settings * query sort changes * small fixes * should fix pagination stuff * fix healthcheck maybe * ragebait ci * tests * tests * revert environment
297 lines
8.2 KiB
Rust
297 lines
8.2 KiB
Rust
use std::{any::type_name, convert::Infallible, str::FromStr, sync::LazyLock};
|
|
|
|
use derive_more::{Deref, DerefMut};
|
|
use eyre::{Context, eyre};
|
|
use rust_decimal::Decimal;
|
|
use serde::de::DeserializeOwned;
|
|
|
|
macro_rules! vars {
|
|
(
|
|
$(
|
|
$field:ident: $ty:ty $(= $default:expr)?;
|
|
)*
|
|
) => {
|
|
#[derive(Debug)]
|
|
#[allow(
|
|
non_snake_case,
|
|
reason = "environment variables are UPPER_SNAKE_CASE",
|
|
)]
|
|
pub struct EnvVars {
|
|
$(
|
|
pub $field: $ty,
|
|
)*
|
|
}
|
|
|
|
impl EnvVars {
|
|
pub fn from_env() -> eyre::Result<Self> {
|
|
let mut err = eyre!("failed to read environment variables");
|
|
|
|
$(
|
|
#[expect(
|
|
non_snake_case,
|
|
reason = "environment variables are UPPER_SNAKE_CASE",
|
|
)]
|
|
#[allow(
|
|
unused_assignments,
|
|
unused_mut,
|
|
reason = "`default` is not used if there is no default",
|
|
)]
|
|
let $field: Option<$ty> = {
|
|
let mut default = None::<$ty>;
|
|
$( default = Some({ $default }.into()); )?
|
|
|
|
match parse_value::<$ty>(stringify!($field), default) {
|
|
Ok(value) => Some(value),
|
|
Err(source) => {
|
|
err = err.wrap_err(eyre!("{source:#}"));
|
|
None
|
|
}
|
|
}
|
|
};
|
|
)*
|
|
|
|
Ok(EnvVars {
|
|
$(
|
|
$field: match $field {
|
|
Some(value) => value,
|
|
None => return Err(err),
|
|
},
|
|
)*
|
|
})
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
pub static ENV: LazyLock<EnvVars> = LazyLock::new(|| {
|
|
EnvVars::from_env().unwrap_or_else(|err| panic!("{err:?}"))
|
|
});
|
|
|
|
fn parse_value<T>(key: &str, default: Option<T>) -> eyre::Result<T>
|
|
where
|
|
T: FromStr,
|
|
T::Err: std::error::Error + Send + Sync + 'static,
|
|
{
|
|
match (dotenvy::var(key), default) {
|
|
(Ok(value), _) => value.parse::<T>().wrap_err_with(|| {
|
|
eyre!("`{key}` is not a valid `{}`", type_name::<T>())
|
|
}),
|
|
(Err(_), Some(default)) => Ok(default),
|
|
(Err(_), None) => Err(eyre!("`{key}` missing")),
|
|
}
|
|
}
|
|
|
|
pub fn init() -> eyre::Result<()> {
|
|
dotenvy::dotenv().ok();
|
|
EnvVars::from_env()?;
|
|
LazyLock::force(&ENV);
|
|
Ok(())
|
|
}
|
|
|
|
#[derive(
|
|
Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Deref, DerefMut,
|
|
)]
|
|
pub struct Json<T: DeserializeOwned>(pub T);
|
|
|
|
impl<T: DeserializeOwned> FromStr for Json<T> {
|
|
type Err = serde_json::Error;
|
|
|
|
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
|
serde_json::from_str(s).map(Self)
|
|
}
|
|
}
|
|
|
|
#[derive(
|
|
Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Deref, DerefMut,
|
|
)]
|
|
pub struct StringCsv(pub Vec<String>);
|
|
|
|
impl FromStr for StringCsv {
|
|
type Err = Infallible;
|
|
|
|
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
|
let v = s
|
|
.split(',')
|
|
.filter(|s| !s.trim().is_empty())
|
|
.map(|s| s.to_string())
|
|
.collect::<Vec<_>>();
|
|
Ok(Self(v))
|
|
}
|
|
}
|
|
|
|
vars! {
|
|
SENTRY_ENVIRONMENT: String;
|
|
SENTRY_TRACES_SAMPLE_RATE: f32;
|
|
SITE_URL: String;
|
|
CDN_URL: String;
|
|
LABRINTH_ADMIN_KEY: String;
|
|
LABRINTH_MEDAL_KEY: String;
|
|
LABRINTH_EXTERNAL_NOTIFICATION_KEY: String;
|
|
RATE_LIMIT_IGNORE_KEY: String;
|
|
DATABASE_URL: String;
|
|
REDIS_URL: String;
|
|
BIND_ADDR: String;
|
|
SELF_ADDR: String;
|
|
|
|
LOCAL_INDEX_INTERVAL: u64;
|
|
VERSION_INDEX_INTERVAL: u64;
|
|
|
|
WHITELISTED_MODPACK_DOMAINS: Json<Vec<String>>;
|
|
ALLOWED_CALLBACK_URLS: Json<Vec<String>>;
|
|
ANALYTICS_ALLOWED_ORIGINS: Json<Vec<String>>;
|
|
|
|
// search
|
|
SEARCH_BACKEND: crate::search::SearchBackendKind = crate::search::SearchBackendKind::Typesense;
|
|
MEILISEARCH_READ_ADDR: String;
|
|
MEILISEARCH_WRITE_ADDRS: StringCsv;
|
|
MEILISEARCH_KEY: String;
|
|
ELASTICSEARCH_URL: String;
|
|
ELASTICSEARCH_INDEX_PREFIX: String;
|
|
ELASTICSEARCH_USERNAME: String = "";
|
|
ELASTICSEARCH_PASSWORD: String = "";
|
|
SEARCH_INDEX_CHUNK_SIZE: i64 = 5000i64;
|
|
TYPESENSE_URL: String = "http://localhost:8108";
|
|
TYPESENSE_API_KEY: String = "modrinth";
|
|
TYPESENSE_INDEX_PREFIX: String = "labrinth";
|
|
|
|
// storage
|
|
STORAGE_BACKEND: crate::file_hosting::FileHostKind;
|
|
|
|
// s3
|
|
S3_PUBLIC_BUCKET_NAME: String = "";
|
|
S3_PUBLIC_USES_PATH_STYLE_BUCKET: bool = false;
|
|
S3_PUBLIC_REGION: String = "";
|
|
S3_PUBLIC_URL: String = "";
|
|
S3_PUBLIC_ACCESS_TOKEN: String = "";
|
|
S3_PUBLIC_SECRET: String = "";
|
|
|
|
S3_PRIVATE_BUCKET_NAME: String = "";
|
|
S3_PRIVATE_USES_PATH_STYLE_BUCKET: bool = false;
|
|
S3_PRIVATE_REGION: String = "";
|
|
S3_PRIVATE_URL: String = "";
|
|
S3_PRIVATE_ACCESS_TOKEN: String = "";
|
|
S3_PRIVATE_SECRET: String = "";
|
|
|
|
// local
|
|
MOCK_FILE_PATH: String = "";
|
|
|
|
GITHUB_CLIENT_ID: String;
|
|
GITHUB_CLIENT_SECRET: String;
|
|
GITLAB_CLIENT_ID: String;
|
|
GITLAB_CLIENT_SECRET: String;
|
|
DISCORD_CLIENT_ID: String;
|
|
DISCORD_CLIENT_SECRET: String;
|
|
MICROSOFT_CLIENT_ID: String;
|
|
MICROSOFT_CLIENT_SECRET: String;
|
|
GOOGLE_CLIENT_ID: String;
|
|
GOOGLE_CLIENT_SECRET: String;
|
|
STEAM_API_KEY: String;
|
|
|
|
TREMENDOUS_API_URL: String;
|
|
TREMENDOUS_API_KEY: String;
|
|
TREMENDOUS_PRIVATE_KEY: String;
|
|
|
|
PAYPAL_API_URL: String;
|
|
PAYPAL_WEBHOOK_ID: String;
|
|
PAYPAL_CLIENT_ID: String;
|
|
PAYPAL_CLIENT_SECRET: String;
|
|
PAYPAL_NVP_USERNAME: String;
|
|
PAYPAL_NVP_PASSWORD: String;
|
|
PAYPAL_NVP_SIGNATURE: String;
|
|
|
|
PAYPAL_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
|
|
BREX_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
|
|
TREMENDOUS_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
|
|
MURAL_BALANCE_ALERT_THRESHOLD: u64 = 0u64;
|
|
|
|
HCAPTCHA_SECRET: String;
|
|
|
|
SMTP_USERNAME: String;
|
|
SMTP_PASSWORD: String;
|
|
SMTP_HOST: String;
|
|
SMTP_PORT: u16;
|
|
SMTP_TLS: String;
|
|
SMTP_FROM_NAME: String;
|
|
SMTP_FROM_ADDRESS: String;
|
|
|
|
SITE_VERIFY_EMAIL_PATH: String;
|
|
SITE_RESET_PASSWORD_PATH: String;
|
|
SITE_BILLING_PATH: String;
|
|
|
|
SENDY_URL: String;
|
|
SENDY_LIST_ID: String;
|
|
SENDY_API_KEY: String;
|
|
|
|
CLICKHOUSE_REPLICATED: bool;
|
|
CLICKHOUSE_URL: String;
|
|
CLICKHOUSE_USER: String;
|
|
CLICKHOUSE_PASSWORD: String;
|
|
CLICKHOUSE_DATABASE: String;
|
|
|
|
FLAME_ANVIL_URL: String;
|
|
|
|
GOTENBERG_URL: String;
|
|
GOTENBERG_CALLBACK_BASE: String;
|
|
GOTENBERG_TIMEOUT: u64;
|
|
|
|
STRIPE_API_KEY: String;
|
|
STRIPE_WEBHOOK_SECRET: String;
|
|
|
|
ADITUDE_API_KEY: String;
|
|
|
|
PYRO_API_KEY: String;
|
|
|
|
BREX_API_URL: String;
|
|
BREX_API_KEY: String;
|
|
|
|
DELPHI_URL: String;
|
|
|
|
AVALARA_1099_API_URL: String;
|
|
AVALARA_1099_API_KEY: String;
|
|
AVALARA_1099_API_TEAM_ID: String;
|
|
AVALARA_1099_COMPANY_ID: String;
|
|
|
|
ANROK_API_URL: String;
|
|
ANROK_API_KEY: String;
|
|
|
|
PAYOUT_ALERT_SLACK_WEBHOOK: String;
|
|
CLOUDFLARE_INTEGRATION: bool = false;
|
|
|
|
ARCHON_URL: String;
|
|
|
|
MURALPAY_API_URL: String;
|
|
MURALPAY_API_KEY: String;
|
|
MURALPAY_TRANSFER_API_KEY: String;
|
|
MURALPAY_SOURCE_ACCOUNT_ID: muralpay::AccountId = muralpay::AccountId(uuid::Uuid::nil());
|
|
|
|
DEFAULT_AFFILIATE_REVENUE_SPLIT: Decimal;
|
|
|
|
DATABASE_ACQUIRE_TIMEOUT_MS: u64 = 30000u64;
|
|
DATABASE_MIN_CONNECTIONS: u32 = 0u32;
|
|
DATABASE_MAX_CONNECTIONS: u32 = 16u32;
|
|
READONLY_DATABASE_URL: String = "";
|
|
READONLY_DATABASE_MIN_CONNECTIONS: u32 = 0u32;
|
|
READONLY_DATABASE_MAX_CONNECTIONS: u32 = 1u32;
|
|
|
|
REDIS_WAIT_TIMEOUT_MS: u64 = 15000u64;
|
|
REDIS_MAX_CONNECTIONS: u32 = 10000u32;
|
|
REDIS_MIN_CONNECTIONS: usize = 0usize;
|
|
|
|
SEARCH_OPERATION_TIMEOUT: u64 = 300000u64;
|
|
|
|
SMTP_REPLY_TO_NAME: String = "";
|
|
SMTP_REPLY_TO_ADDRESS: String = "";
|
|
|
|
PUBLIC_DISCORD_WEBHOOK: String = "";
|
|
MODERATION_SLACK_WEBHOOK: String = "";
|
|
DELPHI_SLACK_WEBHOOK: String = "";
|
|
|
|
TREMENDOUS_CAMPAIGN_ID: String = "";
|
|
|
|
// server pinging
|
|
SERVER_PING_MAX_CONCURRENT: usize = 16usize;
|
|
SERVER_PING_RETRIES: usize = 3usize;
|
|
SERVER_PING_MIN_INTERVAL_SEC: u64 = 30u64 * 60;
|
|
SERVER_PING_TIMEOUT_MS: u64 = 3u64 * 1000;
|
|
}
|