fix: infer source fetch metrics
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 4s
Build / test-and-image (pull_request) Successful in 53s

This commit is contained in:
MrSphay
2026-05-17 14:44:21 +02:00
parent 8605d0baab
commit 2025ae09db
4 changed files with 90 additions and 4 deletions

View File

@@ -10,6 +10,44 @@ const fetchMetrics = {
recent: [],
};
const SOURCE_BY_HOST = [
[/api\.bls\.gov$/i, 'BLS'],
[/api\.fred\.stlouisfed\.org$/i, 'FRED'],
[/api\.eia\.gov$/i, 'EIA'],
[/api\.gdeltproject\.org$/i, 'GDELT'],
[/api\.weather\.gov$/i, 'NOAA'],
[/api\.open-notify\.org$/i, 'OpenNotify'],
[/opensky-network\.org$/i, 'OpenSky'],
[/firms\.modaps\.eosdis\.nasa\.gov$/i, 'FIRMS'],
[/api\.acleddata\.com$/i, 'ACLED'],
[/api\.reliefweb\.int$/i, 'ReliefWeb'],
[/receiverbook\.de$/i, 'KiwiSDR'],
[/safecast\.org$/i, 'Safecast'],
[/api\.patentsview\.org$/i, 'PatentsView'],
[/api\.trade\.gov$/i, 'Comtrade'],
[/api\.usaspending\.gov$/i, 'USASpending'],
[/api\.telegram\.org$/i, 'Telegram'],
[/oauth\.reddit\.com$/i, 'Reddit'],
[/reddit\.com$/i, 'Reddit'],
[/api\.bsky\.app$/i, 'Bluesky'],
[/api\.yahoo\.com$/i, 'YahooFinance'],
[/query\d?\.finance\.yahoo\.com$/i, 'YahooFinance'],
[/api\.cloudflare\.com$/i, 'CloudflareRadar'],
[/api\.opensanctions\.org$/i, 'OpenSanctions'],
[/home\.treasury\.gov$/i, 'Treasury'],
[/fiscaldata\.treasury\.gov$/i, 'Treasury'],
[/who\.int$/i, 'WHO'],
];
export function inferFetchSource(url) {
let host = 'unknown';
try { host = new URL(url).host.toLowerCase(); } catch { return 'unknown'; }
for (const [pattern, source] of SOURCE_BY_HOST) {
if (pattern.test(host)) return source;
}
return host;
}
function metricBucket(map, key) {
if (!map[key]) map[key] = { requests: 0, ok: 0, failed: 0, bytes: 0, lastStatus: null, lastError: null, lastMs: 0 };
return map[key];
@@ -38,7 +76,7 @@ export function getFetchMetrics() {
}
export async function safeFetch(url, opts = {}) {
const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts;
const { timeout = 15000, retries = 1, headers = {}, source = inferFetchSource(url) } = opts;
let lastError;
for (let i = 0; i <= retries; i++) {
const started = Date.now();
@@ -71,11 +109,11 @@ export async function safeFetch(url, opts = {}) {
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
}
}
return { error: lastError?.message || 'Unknown error', source: url };
return { error: lastError?.message || 'Unknown error', source };
}
export async function safeFetchText(url, opts = {}) {
const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts;
const { timeout = 15000, retries = 1, headers = {}, source = inferFetchSource(url) } = opts;
let lastError;
for (let i = 0; i <= retries; i++) {
const started = Date.now();