// Shared fetch utility with timeout, retries, metrics, and error handling const fetchMetrics = { requests: 0, ok: 0, failed: 0, bytes: 0, byHost: {}, bySource: {}, recent: [], }; function metricBucket(map, key) { if (!map[key]) map[key] = { requests: 0, ok: 0, failed: 0, bytes: 0, lastStatus: null, lastError: null, lastMs: 0 }; return map[key]; } function recordFetchMetric({ url, source = 'unknown', ok, status, bytes, durationMs, error }) { let host = 'unknown'; try { host = new URL(url).host; } catch { } fetchMetrics.requests++; fetchMetrics.bytes += bytes || 0; if (ok) fetchMetrics.ok++; else fetchMetrics.failed++; for (const bucket of [metricBucket(fetchMetrics.byHost, host), metricBucket(fetchMetrics.bySource, source)]) { bucket.requests++; bucket.bytes += bytes || 0; bucket.lastStatus = status || null; bucket.lastMs = durationMs || 0; bucket.lastError = error || null; if (ok) bucket.ok++; else bucket.failed++; } fetchMetrics.recent.unshift({ at: new Date().toISOString(), source, host, ok, status, bytes: bytes || 0, durationMs, error: error || null }); fetchMetrics.recent = fetchMetrics.recent.slice(0, 100); } export function getFetchMetrics() { return JSON.parse(JSON.stringify(fetchMetrics)); } export async function safeFetch(url, opts = {}) { const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts; let lastError; for (let i = 0; i <= retries; i++) { const started = Date.now(); let metricRecorded = false; try { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeout); const res = await fetch(url, { signal: controller.signal, headers: { 'User-Agent': 'Crucix/1.0', ...headers }, }); clearTimeout(timer); const status = res.status; const text = await res.text(); if (!res.ok) { const error = `HTTP ${res.status}`; recordFetchMetric({ url, source, ok: false, status, bytes: text.length, durationMs: Date.now() - started, error }); metricRecorded = true; throw new Error(`${error}: ${text.slice(0, 200)}`); } const trimmed = text.trim(); const contentType = res.headers.get('content-type') || ''; if (contentType.includes('text/html') || trimmed.startsWith(' setTimeout(r, 2000 * (i + 1))); } } return { error: lastError?.message || 'Unknown error', source: url }; } export async function safeFetchText(url, opts = {}) { const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts; let lastError; for (let i = 0; i <= retries; i++) { const started = Date.now(); let metricRecorded = false; try { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeout); const res = await fetch(url, { signal: controller.signal, headers: { 'User-Agent': 'Crucix/1.0', ...headers }, }); clearTimeout(timer); const text = await res.text(); recordFetchMetric({ url, source, ok: res.ok, status: res.status, bytes: text.length, durationMs: Date.now() - started, error: res.ok ? null : `HTTP ${res.status}` }); metricRecorded = true; if (!res.ok) throw new Error(`HTTP ${res.status}: ${text.slice(0, 200)}`); return { text, status: res.status, bytes: text.length }; } catch (e) { lastError = e; if (!metricRecorded) { recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message }); } if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1))); } } return { error: lastError?.message || 'Unknown error' }; } export function ago(hours) { return new Date(Date.now() - hours * 3600000).toISOString(); } export function today() { return new Date().toISOString().split('T')[0]; } export function daysAgo(n) { const d = new Date(); d.setDate(d.getDate() - n); return d.toISOString().split('T')[0]; }