fix: infer source fetch metrics
This commit is contained in:
@@ -10,6 +10,44 @@ const fetchMetrics = {
|
||||
recent: [],
|
||||
};
|
||||
|
||||
const SOURCE_BY_HOST = [
|
||||
[/api\.bls\.gov$/i, 'BLS'],
|
||||
[/api\.fred\.stlouisfed\.org$/i, 'FRED'],
|
||||
[/api\.eia\.gov$/i, 'EIA'],
|
||||
[/api\.gdeltproject\.org$/i, 'GDELT'],
|
||||
[/api\.weather\.gov$/i, 'NOAA'],
|
||||
[/api\.open-notify\.org$/i, 'OpenNotify'],
|
||||
[/opensky-network\.org$/i, 'OpenSky'],
|
||||
[/firms\.modaps\.eosdis\.nasa\.gov$/i, 'FIRMS'],
|
||||
[/api\.acleddata\.com$/i, 'ACLED'],
|
||||
[/api\.reliefweb\.int$/i, 'ReliefWeb'],
|
||||
[/receiverbook\.de$/i, 'KiwiSDR'],
|
||||
[/safecast\.org$/i, 'Safecast'],
|
||||
[/api\.patentsview\.org$/i, 'PatentsView'],
|
||||
[/api\.trade\.gov$/i, 'Comtrade'],
|
||||
[/api\.usaspending\.gov$/i, 'USASpending'],
|
||||
[/api\.telegram\.org$/i, 'Telegram'],
|
||||
[/oauth\.reddit\.com$/i, 'Reddit'],
|
||||
[/reddit\.com$/i, 'Reddit'],
|
||||
[/api\.bsky\.app$/i, 'Bluesky'],
|
||||
[/api\.yahoo\.com$/i, 'YahooFinance'],
|
||||
[/query\d?\.finance\.yahoo\.com$/i, 'YahooFinance'],
|
||||
[/api\.cloudflare\.com$/i, 'CloudflareRadar'],
|
||||
[/api\.opensanctions\.org$/i, 'OpenSanctions'],
|
||||
[/home\.treasury\.gov$/i, 'Treasury'],
|
||||
[/fiscaldata\.treasury\.gov$/i, 'Treasury'],
|
||||
[/who\.int$/i, 'WHO'],
|
||||
];
|
||||
|
||||
export function inferFetchSource(url) {
|
||||
let host = 'unknown';
|
||||
try { host = new URL(url).host.toLowerCase(); } catch { return 'unknown'; }
|
||||
for (const [pattern, source] of SOURCE_BY_HOST) {
|
||||
if (pattern.test(host)) return source;
|
||||
}
|
||||
return host;
|
||||
}
|
||||
|
||||
function metricBucket(map, key) {
|
||||
if (!map[key]) map[key] = { requests: 0, ok: 0, failed: 0, bytes: 0, lastStatus: null, lastError: null, lastMs: 0 };
|
||||
return map[key];
|
||||
@@ -38,7 +76,7 @@ export function getFetchMetrics() {
|
||||
}
|
||||
|
||||
export async function safeFetch(url, opts = {}) {
|
||||
const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts;
|
||||
const { timeout = 15000, retries = 1, headers = {}, source = inferFetchSource(url) } = opts;
|
||||
let lastError;
|
||||
for (let i = 0; i <= retries; i++) {
|
||||
const started = Date.now();
|
||||
@@ -71,11 +109,11 @@ export async function safeFetch(url, opts = {}) {
|
||||
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
|
||||
}
|
||||
}
|
||||
return { error: lastError?.message || 'Unknown error', source: url };
|
||||
return { error: lastError?.message || 'Unknown error', source };
|
||||
}
|
||||
|
||||
export async function safeFetchText(url, opts = {}) {
|
||||
const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts;
|
||||
const { timeout = 15000, retries = 1, headers = {}, source = inferFetchSource(url) } = opts;
|
||||
let lastError;
|
||||
for (let i = 0; i <= retries; i++) {
|
||||
const started = Date.now();
|
||||
|
||||
Reference in New Issue
Block a user