fix: harden degraded source handling
This commit is contained in:
@@ -2,8 +2,6 @@
|
||||
// No auth required. Monitors the Specially Designated Nationals (SDN) list
|
||||
// and consolidated sanctions list for changes.
|
||||
|
||||
import { safeFetch } from '../utils/fetch.mjs';
|
||||
|
||||
const EXPORTS_BASE = 'https://sanctionslistservice.ofac.treas.gov/api/PublicationPreview/exports';
|
||||
|
||||
// SDN list endpoints
|
||||
@@ -42,21 +40,19 @@ function parseSDNMetadata(xml) {
|
||||
|
||||
// Fetch SDN list metadata (smaller initial chunk via timeout)
|
||||
export async function getSDNMetadata() {
|
||||
// The full SDN XML is large; safeFetch will get the first 500 chars
|
||||
// which should include the header/publish date
|
||||
const data = await safeFetch(SDN_XML_URL, { timeout: 20000 });
|
||||
const data = await fetchXmlSnippet(SDN_XML_URL);
|
||||
return parseSDNMetadata(data);
|
||||
}
|
||||
|
||||
// Fetch advanced SDN data (includes more structured info)
|
||||
export async function getSDNAdvanced() {
|
||||
const data = await safeFetch(SDN_ADVANCED_URL, { timeout: 20000 });
|
||||
const data = await fetchXmlSnippet(SDN_ADVANCED_URL);
|
||||
return parseSDNMetadata(data);
|
||||
}
|
||||
|
||||
// Fetch consolidated list metadata
|
||||
export async function getConsolidatedMetadata() {
|
||||
const data = await safeFetch(CONS_ADVANCED_URL, { timeout: 20000 });
|
||||
const data = await fetchXmlSnippet(CONS_ADVANCED_URL);
|
||||
return parseSDNMetadata(data);
|
||||
}
|
||||
|
||||
@@ -101,15 +97,13 @@ function parseRecentEntries(xml) {
|
||||
|
||||
// Briefing — report on sanctions list status and metadata
|
||||
export async function briefing() {
|
||||
const [sdnMeta, advancedMeta] = await Promise.all([
|
||||
const [sdnMeta, advancedMeta, advancedSnippet] = await Promise.all([
|
||||
getSDNMetadata(),
|
||||
getSDNAdvanced(),
|
||||
fetchXmlSnippet(SDN_ADVANCED_URL, { maxBytes: 256_000 }),
|
||||
]);
|
||||
|
||||
// Try to extract any entries visible in the advanced data
|
||||
const sampleEntries = parseRecentEntries(
|
||||
await safeFetch(SDN_ADVANCED_URL, { timeout: 25000 })
|
||||
);
|
||||
const sampleEntries = parseRecentEntries(advancedSnippet);
|
||||
|
||||
return {
|
||||
source: 'OFAC Sanctions',
|
||||
@@ -136,6 +130,46 @@ export async function briefing() {
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchXmlSnippet(url, opts = {}) {
|
||||
const timeout = opts.timeout || 9000;
|
||||
const maxBytes = opts.maxBytes || 128_000;
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeout);
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Crucix/2.0',
|
||||
'Range': `bytes=0-${maxBytes - 1}`,
|
||||
},
|
||||
signal: controller.signal,
|
||||
});
|
||||
if (!res.ok && res.status !== 206) {
|
||||
const body = await res.text().catch(() => '');
|
||||
return { error: `HTTP ${res.status}: ${body.slice(0, 160)}` };
|
||||
}
|
||||
if (!res.body?.getReader) {
|
||||
const text = await res.text();
|
||||
return { rawText: text.slice(0, maxBytes) };
|
||||
}
|
||||
const reader = res.body.getReader();
|
||||
const chunks = [];
|
||||
let bytes = 0;
|
||||
while (bytes < maxBytes) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
chunks.push(value);
|
||||
bytes += value.byteLength;
|
||||
}
|
||||
await reader.cancel().catch(() => {});
|
||||
const buffer = Buffer.concat(chunks.map(chunk => Buffer.from(chunk)), bytes);
|
||||
return { rawText: buffer.toString('utf8', 0, Math.min(bytes, maxBytes)) };
|
||||
} catch (error) {
|
||||
return { error: error.name === 'AbortError' ? `OFAC snippet request timed out after ${timeout}ms` : error.message };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
// Run standalone
|
||||
if (process.argv[1]?.endsWith('ofac.mjs')) {
|
||||
const data = await briefing();
|
||||
|
||||
Reference in New Issue
Block a user