MrSphay/intelligence-terminal
1
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases Wiki Activity

merge: update adsb degraded branch
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 6s
Details
Build / test-and-image (pull_request) Successful in 1m13s
Details

Browse Source
This commit is contained in:
MrSphay
2026-05-17 19:03:49 +02:00
parent b2f604b120 2f2a10609f
commit d7f10bf545
27 changed files with 1561 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files

244
apis/sources/acled.mjs
View File

@@ -1,9 +1,9 @@
// ACLED — Armed Conflict Location & Event Data
// ACLED - Armed Conflict Location & Event Data.
// Auth strategy (tries in order):
// 1. Cookie-based session: POST /user/login?_format=json → session cookie
// 2. OAuth Bearer token: POST /oauth/token → Authorization header
// Set ACLED_EMAIL and ACLED_PASSWORD in .env (your myACLED login credentials).
// Data endpoint: GET https://acleddata.com/api/acled/read
// 1. OAuth Bearer token: POST /oauth/token -> Authorization header
// 2. Cookie-based session: POST /user/login?_format=json -> session cookie
// Set ACLED_EMAIL and ACLED_PASSWORD in .env. ACLED_USER or ACLED_USERNAME are
// accepted as aliases for ACLED_EMAIL.
import { daysAgo } from '../utils/fetch.mjs';
import '../utils/env.mjs';
@@ -12,124 +12,135 @@ const LOGIN_URL = 'https://acleddata.com/user/login?_format=json';
const TOKEN_URL = 'https://acleddata.com/oauth/token';
const API_BASE = 'https://acleddata.com/api/acled/read';
// Session cache
let sessionCache = { cookies: null, token: null, method: null, expires: 0 };
// Strategy 1: Cookie-based session login (mirrors browser login)
async function loginCookie(email, password) {
export function resetAcledSessionCache() {
sessionCache = { cookies: null, token: null, method: null, expires: 0 };
}
export function getAcledConfig(env = process.env) {
const email = env.ACLED_EMAIL || env.ACLED_USER || env.ACLED_USERNAME || '';
const password = env.ACLED_PASSWORD || '';
const missing = [];
if (!email) missing.push('ACLED_EMAIL');
if (!password) missing.push('ACLED_PASSWORD');
return { email, password, configured: missing.length === 0, missing };
}
function acledError(status, error, message, extra = {}) {
return { status, error, message, ...extra };
}
function safeText(value, max = 200) {
return String(value || '').replace(/Bearer\s+[A-Za-z0-9._-]+/gi, 'Bearer [redacted]').slice(0, max);
}
async function fetchWithTimeout(fetchImpl, url, init, timeoutMs) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 15000);
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const res = await fetch(LOGIN_URL, {
return await fetchImpl(url, { ...init, signal: controller.signal });
} finally {
clearTimeout(timer);
}
}
export async function loginCookie(email, password, opts = {}) {
const fetchImpl = opts.fetchImpl || globalThis.fetch;
try {
const res = await fetchWithTimeout(fetchImpl, LOGIN_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: email, pass: password }),
redirect: 'manual',
signal: controller.signal,
});
clearTimeout(timer);
}, 15000);
// Collect Set-Cookie headers
const setCookies = res.headers.getSetCookie?.() || [];
const cookieStr = setCookies.map(c => c.split(';')[0]).join('; ');
if (res.ok && cookieStr) {
return { cookies: cookieStr };
}
// Some Drupal sites return 303 redirect on successful login — cookies still set
if (res.status >= 300 && res.status < 400 && cookieStr) {
return { cookies: cookieStr };
if ((res.ok || (res.status >= 300 && res.status < 400)) && cookieStr) {
return { ok: true, cookies: cookieStr };
}
const errText = await res.text().catch(() => '');
return { error: `Cookie login failed (HTTP ${res.status}): ${errText.slice(0, 200)}` };
return acledError('auth_failed', `acled_cookie_http_${res.status}`, `ACLED cookie login failed with HTTP ${res.status}`, {
detail: safeText(errText),
});
} catch (e) {
clearTimeout(timer);
const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : '';
return { error: `Cookie login error: ${e.message}${cause}` };
return acledError('auth_failed', 'acled_cookie_request_failed', `ACLED cookie login error: ${e.message}`);
}
}
// Strategy 2: OAuth2 password grant
async function loginOAuth(email, password) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 15000);
export async function loginOAuth(email, password, opts = {}) {
const fetchImpl = opts.fetchImpl || globalThis.fetch;
try {
const body = new URLSearchParams({
username: email,
password: password,
password,
grant_type: 'password',
client_id: 'acled',
});
const res = await fetch(TOKEN_URL, {
const res = await fetchWithTimeout(fetchImpl, TOKEN_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: body.toString(),
signal: controller.signal,
});
clearTimeout(timer);
}, 15000);
if (!res.ok) {
const errText = await res.text().catch(() => '');
return { error: `OAuth failed (HTTP ${res.status}): ${errText.slice(0, 200)}` };
return acledError('auth_failed', `acled_oauth_http_${res.status}`, `ACLED OAuth failed with HTTP ${res.status}`, {
detail: safeText(errText),
});
}
const data = await res.json();
if (!data.access_token) {
return { error: `OAuth response missing access_token: ${JSON.stringify(data).slice(0, 200)}` };
return acledError('auth_failed', 'acled_oauth_missing_access_token', 'ACLED OAuth response did not include access_token');
}
return { token: data.access_token };
return { ok: true, token: data.access_token };
} catch (e) {
clearTimeout(timer);
const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : '';
return { error: `OAuth error: ${e.message}${cause}` };
return acledError('auth_failed', 'acled_oauth_request_failed', `ACLED OAuth error: ${e.message}`);
}
}
// Try both auth strategies
async function authenticate() {
const email = process.env.ACLED_EMAIL;
const password = process.env.ACLED_PASSWORD;
if (!email || !password) {
return { error: 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.' };
export async function authenticate(opts = {}) {
const env = opts.env || process.env;
const fetchImpl = opts.fetchImpl || globalThis.fetch;
const config = getAcledConfig(env);
if (!config.configured) {
return acledError('no_credentials', 'missing_acled_credentials', 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.', {
missing: config.missing,
});
}
// Return cached session if still valid
if (sessionCache.method && Date.now() < sessionCache.expires) {
return sessionCache;
}
const errors = [];
const debug = process.argv.includes('--debug');
// Try OAuth first (official programmatic method per ACLED docs)
const oauthResult = await loginOAuth(email, password);
if (oauthResult.token) {
if (debug) console.error(`[ACLED DEBUG] OAuth OK — token: ${oauthResult.token.slice(0, 20)}...`);
const diagnostics = [];
const oauthResult = await loginOAuth(config.email, config.password, { fetchImpl });
if (oauthResult.ok) {
sessionCache = { cookies: null, token: oauthResult.token, method: 'oauth', expires: Date.now() + 23 * 60 * 60 * 1000 };
return sessionCache;
}
errors.push(`OAuth: ${oauthResult.error}`);
if (debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
diagnostics.push({ method: 'oauth', status: oauthResult.status, error: oauthResult.error, message: oauthResult.message });
if (opts.debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
// Fall back to cookie-based session
const cookieResult = await loginCookie(email, password);
if (cookieResult.cookies) {
if (debug) console.error(`[ACLED DEBUG] Cookie OK — cookies: ${cookieResult.cookies.slice(0, 80)}...`);
const cookieResult = await loginCookie(config.email, config.password, { fetchImpl });
if (cookieResult.ok) {
sessionCache = { cookies: cookieResult.cookies, token: null, method: 'cookie', expires: Date.now() + 12 * 60 * 60 * 1000 };
return sessionCache;
}
errors.push(`Cookie: ${cookieResult.error}`);
diagnostics.push({ method: 'cookie', status: cookieResult.status, error: cookieResult.error, message: cookieResult.message });
if (opts.debug) console.error(`[ACLED DEBUG] Cookie login failed: ${cookieResult.error}`);
return { error: `All ACLED auth methods failed.\n${errors.join('\n')}` };
return acledError('auth_failed', 'acled_auth_failed', 'All ACLED auth methods failed.', { diagnostics });
}
// Build headers based on auth method
function authHeaders(session) {
const headers = { 'User-Agent': 'Crucix/1.0', 'Content-Type': 'application/json' };
const headers = { 'User-Agent': 'Crucix/2.0', 'Content-Type': 'application/json' };
if (session.method === 'cookie' && session.cookies) {
headers['Cookie'] = session.cookies;
} else if (session.method === 'oauth' && session.token) {
@@ -138,7 +149,6 @@ function authHeaders(session) {
return headers;
}
// Event type constants
export const EVENT_TYPES = [
'Battles',
'Explosions/Remote violence',
@@ -148,7 +158,6 @@ export const EVENT_TYPES = [
'Strategic developments',
];
// Query conflict events with flexible filters
export async function getEvents(opts = {}) {
const {
limit = 500,
@@ -157,10 +166,13 @@ export async function getEvents(opts = {}) {
eventType,
country,
region,
env = process.env,
fetchImpl = globalThis.fetch,
debug = process.argv.includes('--debug'),
} = opts;
const session = await authenticate();
if (session.error) return { error: session.error };
const session = await authenticate({ env, fetchImpl, debug });
if (session.error) return session;
const params = new URLSearchParams({ _format: 'json', limit: String(limit) });
if (eventDateStart && eventDateEnd) {
@@ -171,59 +183,43 @@ export async function getEvents(opts = {}) {
if (country) params.set('country', country);
if (region) params.set('region', String(region));
const debug = process.argv.includes('--debug');
try {
const url = `${API_BASE}?${params}`;
const hdrs = authHeaders(session);
if (debug) {
console.error(`[ACLED DEBUG] Data request: GET ${url}`);
console.error(`[ACLED DEBUG] Headers: ${JSON.stringify(hdrs)}`);
}
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 25000);
const res = await fetch(url, {
headers: hdrs,
signal: controller.signal,
});
clearTimeout(timer);
if (debug) console.error(`[ACLED DEBUG] Data request: GET ${url}`);
const res = await fetchWithTimeout(fetchImpl, url, { headers: authHeaders(session) }, 25000);
if (debug) console.error(`[ACLED DEBUG] Data response: HTTP ${res.status}`);
if (!res.ok) {
const errText = await res.text().catch(() => '');
if (debug) console.error(`[ACLED DEBUG] Error body: ${errText.slice(0, 500)}`);
if (res.status === 401 || res.status === 403) {
// Clear cache and report
sessionCache = { cookies: null, token: null, method: null, expires: 0 };
const hint = res.status === 403
? '\n→ Fix: Log in at https://acleddata.com/user/login, then:\n'
+ ' 1. Accept Terms of Use (profile → Terms of Use button → check the box)\n'
+ ' 2. Complete all required profile fields\n'
+ ' 3. Ensure your account has the "API" access group\n'
+ ' Contact access@acleddata.com if issues persist.'
: '';
return { error: `ACLED data access denied (HTTP ${res.status}, auth method: ${session.method}). Response: ${errText.slice(0, 300)}${hint}` };
return acledError('access_denied', `acled_data_http_${res.status}`, `ACLED data access denied with HTTP ${res.status}`, {
authMethod: session.method,
detail: safeText(errText, 300),
hint: 'Accept ACLED terms, complete profile fields, and confirm API access for the account.',
});
}
return { error: `HTTP ${res.status}: ${errText.slice(0, 200)}` };
return acledError('api_failed', `acled_data_http_${res.status}`, `ACLED data request failed with HTTP ${res.status}`, {
detail: safeText(errText),
});
}
const data = await res.json();
// ACLED may return a 200 with an error status in the body
if (data?.status && data.status !== 200) {
return { error: `ACLED API error: status ${data.status} — ${data.message || 'Unknown error'}` };
return acledError('api_failed', `acled_api_status_${data.status}`, `ACLED API returned status ${data.status}`, {
detail: safeText(data.message),
});
}
return data;
} catch (e) {
if (e.name === 'AbortError') {
return { error: 'ACLED data request timed out (25s)' };
return acledError('api_failed', 'acled_data_timeout', 'ACLED data request timed out after 25s');
}
const rootCause = e.cause ? `${e.cause.message || e.cause.code || e.cause}` : '';
return { error: `ACLED data error: ${e.message}${rootCause ? ' → ' + rootCause : ''}` };
return acledError('api_failed', 'acled_data_request_failed', `ACLED data error: ${e.message}`);
}
}
// Summarize events by a given field
function groupBy(events, field) {
const map = {};
for (const e of events) {
@@ -235,33 +231,47 @@ function groupBy(events, field) {
return map;
}
// Briefing — last 7 days of global conflict events
export async function briefing() {
if (!process.env.ACLED_EMAIL || !process.env.ACLED_PASSWORD) {
export async function briefing(opts = {}) {
const env = opts.env || process.env;
const fetchImpl = opts.fetchImpl || globalThis.fetch;
const config = getAcledConfig(env);
if (!config.configured) {
return {
source: 'ACLED',
timestamp: new Date().toISOString(),
status: 'no_credentials',
error: 'missing_acled_credentials',
missing: config.missing,
message: 'Set ACLED_EMAIL and ACLED_PASSWORD in .env. Register at https://acleddata.com/user/register',
};
}
const start = daysAgo(7);
const end = daysAgo(0);
const end = daysAgo(0);
const data = await getEvents({
eventDateStart: start,
eventDateEnd: end,
limit: 2000,
env,
fetchImpl,
debug: opts.debug,
});
if (data?.error) {
return { source: 'ACLED', timestamp: new Date().toISOString(), error: data.error };
return {
source: 'ACLED',
timestamp: new Date().toISOString(),
status: data.status || 'api_failed',
error: data.error,
message: data.message,
detail: data.detail,
hint: data.hint,
diagnostics: data.diagnostics,
};
}
let events = data?.data || [];
// Enrich all events with numeric lat/lon
events = events.map(e => ({
...e,
lat: parseFloat(e.latitude) || null,
@@ -272,10 +282,9 @@ export async function briefing() {
(sum, e) => sum + (parseInt(e.fatalities, 10) || 0), 0
);
const byRegion = groupBy(events, 'region');
const byType = groupBy(events, 'event_type');
const byRegion = groupBy(events, 'region');
const byType = groupBy(events, 'event_type');
const byCountry = groupBy(events, 'country');
const topCountries = Object.entries(byCountry)
.sort((a, b) => b[1].count - a[1].count)
.slice(0, 10)
@@ -286,20 +295,21 @@ export async function briefing() {
.sort((a, b) => (parseInt(b.fatalities, 10) || 0) - (parseInt(a.fatalities, 10) || 0))
.slice(0, 15)
.map(e => ({
date: e.event_date,
type: e.event_type,
subType: e.sub_event_type,
country: e.country,
location: e.location,
date: e.event_date,
type: e.event_type,
subType: e.sub_event_type,
country: e.country,
location: e.location,
fatalities: parseInt(e.fatalities, 10) || 0,
lat: parseFloat(e.latitude) || null,
lon: parseFloat(e.longitude) || null,
notes: e.notes?.slice(0, 200),
lat: parseFloat(e.latitude) || null,
lon: parseFloat(e.longitude) || null,
notes: e.notes?.slice(0, 200),
}));
return {
source: 'ACLED',
timestamp: new Date().toISOString(),
status: 'ok',
period: { start, end },
totalEvents: events.length,
totalFatalities,

139
apis/sources/reddit.mjs
View File

@@ -1,14 +1,15 @@
// Reddit — social sentiment intelligence
// Reddit now requires OAuth for API access (public JSON API returns 403).
// Gracefully degrades when not authenticated.
// To enable: register an app at https://www.reddit.com/prefs/apps/ and set
// REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET in .env
// Reddit social sentiment intelligence.
// Reddit API access requires OAuth. Runtime sweeps intentionally do not use
// unauthenticated reddit.com .json scraping because it is unreliable and not
// acceptable for production operation.
import { safeFetch } from '../utils/fetch.mjs';
import '../utils/env.mjs';
function delay(ms) { return new Promise(r => setTimeout(r, ms)); }
const USER_AGENT = 'Crucix/2.0 intelligence-engine';
const SUBREDDITS = [
'worldnews',
'geopolitics',
@@ -17,48 +18,95 @@ const SUBREDDITS = [
'commodities',
];
// Get OAuth token using client credentials flow (application-only)
async function getToken() {
const clientId = process.env.REDDIT_CLIENT_ID;
const clientSecret = process.env.REDDIT_CLIENT_SECRET;
if (!clientId || !clientSecret) return null;
export function getRedditConfig(env = process.env) {
const clientId = env.REDDIT_CLIENT_ID || '';
const clientSecret = env.REDDIT_CLIENT_SECRET || '';
const missing = [];
if (!clientId) missing.push('REDDIT_CLIENT_ID');
if (!clientSecret) missing.push('REDDIT_CLIENT_SECRET');
return {
clientId,
clientSecret,
configured: missing.length === 0,
missing,
};
}
function credentialsMessage(missing) {
return `Reddit requires OAuth. Register a script app at https://www.reddit.com/prefs/apps/ and set ${missing.join(' and ')} in .env`;
}
export async function getToken({ env = process.env, fetchImpl = globalThis.fetch } = {}) {
const config = getRedditConfig(env);
if (!config.configured) {
return {
ok: false,
status: 'no_credentials',
missing: config.missing,
error: 'missing_reddit_oauth_credentials',
message: credentialsMessage(config.missing),
};
}
try {
const auth = Buffer.from(`${clientId}:${clientSecret}`).toString('base64');
const res = await fetch('https://www.reddit.com/api/v1/access_token', {
const auth = Buffer.from(`${config.clientId}:${config.clientSecret}`).toString('base64');
const res = await fetchImpl('https://www.reddit.com/api/v1/access_token', {
method: 'POST',
headers: {
'Authorization': `Basic ${auth}`,
'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent': 'Crucix/1.0 intelligence-engine',
'User-Agent': USER_AGENT,
},
body: 'grant_type=client_credentials',
});
if (!res.ok) return null;
if (!res.ok) {
const body = await res.text().catch(() => '');
return {
ok: false,
status: 'auth_failed',
error: `reddit_oauth_http_${res.status}`,
message: `Reddit OAuth token request failed with HTTP ${res.status}`,
detail: body.slice(0, 200),
};
}
const data = await res.json();
return data.access_token || null;
} catch {
return null;
if (!data.access_token) {
return {
ok: false,
status: 'auth_failed',
error: 'reddit_oauth_missing_access_token',
message: 'Reddit OAuth token response did not include an access token',
};
}
return { ok: true, status: 'ok', token: data.access_token };
} catch (e) {
return {
ok: false,
status: 'auth_failed',
error: 'reddit_oauth_request_failed',
message: e.message,
};
}
}
// Fetch hot posts — tries OAuth first, then falls back to public endpoint
export async function getHot(subreddit, opts = {}) {
const { limit = 10, token = null } = opts;
if (token) {
// Use OAuth endpoint
return safeFetch(`https://oauth.reddit.com/r/${subreddit}/hot?limit=${limit}&raw_json=1`, {
headers: {
'Authorization': `Bearer ${token}`,
'User-Agent': 'Crucix/1.0 intelligence-engine',
},
});
if (!token) {
return {
status: 'no_credentials',
error: 'reddit_oauth_required',
message: 'Reddit source requires OAuth; unauthenticated reddit.com .json scraping is disabled',
};
}
// Try public endpoint (may 403)
return safeFetch(`https://www.reddit.com/r/${subreddit}/hot.json?limit=${limit}&raw_json=1`, {
headers: { 'User-Agent': 'Crucix/1.0 intelligence-engine' },
return safeFetch(`https://oauth.reddit.com/r/${subreddit}/hot?limit=${limit}&raw_json=1`, {
source: 'Reddit',
headers: {
'Authorization': `Bearer ${token}`,
'User-Agent': USER_AGENT,
},
});
}
@@ -74,29 +122,46 @@ function compactPost(child) {
};
}
export async function briefing() {
const token = await getToken();
export async function briefing(opts = {}) {
const {
env = process.env,
subreddits = SUBREDDITS,
delayMs = 1000,
fetchImpl = globalThis.fetch,
} = opts;
const tokenResult = await getToken({ env, fetchImpl });
if (!token && !process.env.REDDIT_CLIENT_ID) {
if (!tokenResult.ok) {
return {
source: 'Reddit',
timestamp: new Date().toISOString(),
status: 'no_key',
message: 'Reddit requires OAuth. Register at https://www.reddit.com/prefs/apps/ (script type), set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET in .env',
status: tokenResult.status,
error: tokenResult.error,
message: tokenResult.message,
missing: tokenResult.missing || [],
};
}
const subredditResults = {};
for (const sub of SUBREDDITS) {
const result = await getHot(sub, { limit: 10, token });
const errors = [];
for (const sub of subreddits) {
const result = await getHot(sub, { limit: 10, token: tokenResult.token });
if (result?.error) {
errors.push({ subreddit: sub, error: result.error });
subredditResults[sub] = [];
if (delayMs > 0) await delay(delayMs);
continue;
}
const children = result?.data?.children || [];
subredditResults[sub] = children.map(compactPost).filter(Boolean);
await delay(token ? 1000 : 2000);
if (delayMs > 0) await delay(delayMs);
}
return {
source: 'Reddit',
timestamp: new Date().toISOString(),
status: errors.length > 0 ? 'degraded' : 'ok',
...(errors.length > 0 ? { error: 'reddit_subreddit_fetch_failed', errors } : {}),
subreddits: subredditResults,
};
}

30
apis/utils/fetch.mjs
View File

@@ -42,6 +42,7 @@ export async function safeFetch(url, opts = {}) {
let lastError;
for (let i = 0; i <= retries; i++) {
const started = Date.now();
let metricRecorded = false;
try {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeout);
@@ -51,22 +52,29 @@ export async function safeFetch(url, opts = {}) {
});
clearTimeout(timer);
const status = res.status;
if (!res.ok) {
const body = await res.text().catch(() => '');
recordFetchMetric({ url, source, ok: false, status, bytes: body.length, durationMs: Date.now() - started, error: `HTTP ${res.status}` });
throw new Error(`HTTP ${res.status}: ${body.slice(0, 200)}`);
}
const text = await res.text();
recordFetchMetric({ url, source, ok: true, status, bytes: text.length, durationMs: Date.now() - started });
if (!res.ok) {
const error = `HTTP ${res.status}`;
recordFetchMetric({ url, source, ok: false, status, bytes: text.length, durationMs: Date.now() - started, error });
metricRecorded = true;
throw new Error(`${error}: ${text.slice(0, 200)}`);
}
const trimmed = text.trim();
const contentType = res.headers.get('content-type') || '';
if (contentType.includes('text/html') || trimmed.startsWith('<!DOCTYPE html') || trimmed.startsWith('<html')) {
throw new Error(`Expected JSON but received HTML from ${new URL(url).host}`);
const error = `Expected JSON but received HTML from ${new URL(url).host}`;
recordFetchMetric({ url, source, ok: false, status, bytes: text.length, durationMs: Date.now() - started, error });
metricRecorded = true;
throw new Error(error);
}
recordFetchMetric({ url, source, ok: true, status, bytes: text.length, durationMs: Date.now() - started });
metricRecorded = true;
try { return JSON.parse(text); } catch { return { rawText: text.slice(0, 500) }; }
} catch (e) {
lastError = e;
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
if (!metricRecorded) {
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
}
// GDELT needs 5s between requests, others are fine with shorter delays
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
}
@@ -79,6 +87,7 @@ export async function safeFetchText(url, opts = {}) {
let lastError;
for (let i = 0; i <= retries; i++) {
const started = Date.now();
let metricRecorded = false;
try {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeout);
@@ -89,11 +98,14 @@ export async function safeFetchText(url, opts = {}) {
clearTimeout(timer);
const text = await res.text();
recordFetchMetric({ url, source, ok: res.ok, status: res.status, bytes: text.length, durationMs: Date.now() - started, error: res.ok ? null : `HTTP ${res.status}` });
metricRecorded = true;
if (!res.ok) throw new Error(`HTTP ${res.status}: ${text.slice(0, 200)}`);
return { text, status: res.status, bytes: text.length };
} catch (e) {
lastError = e;
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
if (!metricRecorded) {
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
}
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
}
}