Code-Inc/intelligence-terminal
2
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-22-source-fetch-instrumentation
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Details
Build / test-and-image (pull_request) Successful in 55s
Details

Browse Source 
# Conflicts:
#	README.md
#	test/fetch-utils.test.mjs
This commit is contained in:
MrSphay
2026-05-17 20:37:21 +02:00
parent 2025ae09db d625bffd4a
commit c102017b16
27 changed files with 1196 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
apis/utils/fetch.mjs
View File

@@ -80,6 +80,7 @@ export async function safeFetch(url, opts = {}) {
let lastError;
for (let i = 0; i <= retries; i++) {
const started = Date.now();
let metricRecorded = false;
try {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeout);
@@ -89,22 +90,29 @@ export async function safeFetch(url, opts = {}) {
});
clearTimeout(timer);
const status = res.status;
if (!res.ok) {
const body = await res.text().catch(() => '');
recordFetchMetric({ url, source, ok: false, status, bytes: body.length, durationMs: Date.now() - started, error: `HTTP ${res.status}` });
throw new Error(`HTTP ${res.status}: ${body.slice(0, 200)}`);
}
const text = await res.text();
recordFetchMetric({ url, source, ok: true, status, bytes: text.length, durationMs: Date.now() - started });
if (!res.ok) {
const error = `HTTP ${res.status}`;
recordFetchMetric({ url, source, ok: false, status, bytes: text.length, durationMs: Date.now() - started, error });
metricRecorded = true;
throw new Error(`${error}: ${text.slice(0, 200)}`);
}
const trimmed = text.trim();
const contentType = res.headers.get('content-type') || '';
if (contentType.includes('text/html') || trimmed.startsWith('<!DOCTYPE html') || trimmed.startsWith('<html')) {
throw new Error(`Expected JSON but received HTML from ${new URL(url).host}`);
const error = `Expected JSON but received HTML from ${new URL(url).host}`;
recordFetchMetric({ url, source, ok: false, status, bytes: text.length, durationMs: Date.now() - started, error });
metricRecorded = true;
throw new Error(error);
}
recordFetchMetric({ url, source, ok: true, status, bytes: text.length, durationMs: Date.now() - started });
metricRecorded = true;
try { return JSON.parse(text); } catch { return { rawText: text.slice(0, 500) }; }
} catch (e) {
lastError = e;
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
if (!metricRecorded) {
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
}
// GDELT needs 5s between requests, others are fine with shorter delays
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
}
@@ -117,6 +125,7 @@ export async function safeFetchText(url, opts = {}) {
let lastError;
for (let i = 0; i <= retries; i++) {
const started = Date.now();
let metricRecorded = false;
try {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeout);
@@ -127,11 +136,14 @@ export async function safeFetchText(url, opts = {}) {
clearTimeout(timer);
const text = await res.text();
recordFetchMetric({ url, source, ok: res.ok, status: res.status, bytes: text.length, durationMs: Date.now() - started, error: res.ok ? null : `HTTP ${res.status}` });
metricRecorded = true;
if (!res.ok) throw new Error(`HTTP ${res.status}: ${text.slice(0, 200)}`);
return { text, status: res.status, bytes: text.length };
} catch (e) {
lastError = e;
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
if (!metricRecorded) {
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
}
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
}
}