merge: update adsb degraded branch

2026-05-17 19:03:49 +02:00
parent b2f604b120 2f2a10609f
commit d7f10bf545
27 changed files with 1561 additions and 320 deletions
--- a/test/acled-source.test.mjs
+++ b/test/acled-source.test.mjs
@@ -0,0 +1,95 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { authenticate, briefing, resetAcledSessionCache } from '../apis/sources/acled.mjs';
+
+function jsonResponse(status, body, ok = status >= 200 && status < 300) {
+  return {
+    ok,
+    status,
+    headers: { getSetCookie: () => [] },
+    json: async () => body,
+    text: async () => JSON.stringify(body),
+  };
+}
+
+test('ACLED reports missing credentials without network access', async () => {
+  resetAcledSessionCache();
+  let calls = 0;
+  const data = await briefing({
+    env: {},
+    fetchImpl: async () => {
+      calls++;
+      throw new Error('unexpected network access');
+    },
+  });
+
+  assert.equal(calls, 0);
+  assert.equal(data.status, 'no_credentials');
+  assert.equal(data.error, 'missing_acled_credentials');
+  assert.deepEqual(data.missing, ['ACLED_EMAIL', 'ACLED_PASSWORD']);
+});
+
+test('ACLED accepts ACLED_USER as email alias and returns empty valid result', async () => {
+  resetAcledSessionCache();
+  const urls = [];
+  const data = await briefing({
+    env: { ACLED_USER: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
+    fetchImpl: async url => {
+      urls.push(String(url));
+      if (String(url).includes('/oauth/token')) {
+        return jsonResponse(200, { access_token: 'token' });
+      }
+      return jsonResponse(200, { status: 200, data: [] });
+    },
+  });
+
+  assert.equal(data.status, 'ok');
+  assert.equal(data.totalEvents, 0);
+  assert.ok(urls.some(url => url.includes('/oauth/token')));
+  assert.ok(urls.some(url => url.includes('/api/acled/read')));
+});
+
+test('ACLED classifies auth failure without exposing credentials', async () => {
+  resetAcledSessionCache();
+  const result = await authenticate({
+    env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'super-secret' },
+    fetchImpl: async url => {
+      if (String(url).includes('/oauth/token')) {
+        return jsonResponse(401, { error: 'invalid_grant' }, false);
+      }
+      return {
+        ok: false,
+        status: 403,
+        headers: { getSetCookie: () => [] },
+        text: async () => 'forbidden',
+      };
+    },
+  });
+
+  assert.equal(result.status, 'auth_failed');
+  assert.equal(result.error, 'acled_auth_failed');
+  assert.equal(result.diagnostics.length, 2);
+  assert.doesNotMatch(JSON.stringify(result), /super-secret/);
+});
+
+test('ACLED classifies data access denied distinctly from auth failure', async () => {
+  resetAcledSessionCache();
+  const data = await briefing({
+    env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
+    fetchImpl: async url => {
+      if (String(url).includes('/oauth/token')) {
+        return jsonResponse(200, { access_token: 'token' });
+      }
+      return {
+        ok: false,
+        status: 403,
+        headers: { getSetCookie: () => [] },
+        text: async () => 'terms not accepted',
+      };
+    },
+  });
+
+  assert.equal(data.status, 'access_denied');
+  assert.equal(data.error, 'acled_data_http_403');
+  assert.match(data.hint, /Accept ACLED terms/);
+});
--- a/test/fetch-utils.test.mjs
+++ b/test/fetch-utils.test.mjs
@@ -1,9 +1,11 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
+import { readFileSync } from 'node:fs';
 import { safeFetch, safeFetchText, getFetchMetrics } from '../apis/utils/fetch.mjs';

 test('safeFetch reports HTML as degraded JSON response', async () => {
  const originalFetch = globalThis.fetch;
+  const source = 'unit-html-once';
  globalThis.fetch = async () => ({
    ok: true,
    status: 200,
@@ -11,9 +13,72 @@ test('safeFetch reports HTML as degraded JSON response', async () => {
    text: async () => '<html>not json</html>',
  });
  try {
-    const data = await safeFetch('https://example.test/json', { retries: 0, source: 'unit' });
+    const data = await safeFetch('https://example.test/json', { retries: 0, source });
    assert.match(data.error, /Expected JSON/);
-    assert.ok(getFetchMetrics().bySource.unit.requests >= 1);
+    const bucket = getFetchMetrics().bySource[source];
+    assert.equal(bucket.requests, 1);
+    assert.equal(bucket.ok, 0);
+    assert.equal(bucket.failed, 1);
+    assert.equal(bucket.lastStatus, 200);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test('safeFetch records HTTP failure once with status and bytes', async () => {
+  const originalFetch = globalThis.fetch;
+  const source = 'unit-http-failure-once';
+  globalThis.fetch = async () => ({
+    ok: false,
+    status: 503,
+    headers: { get: () => 'application/json' },
+    text: async () => 'service unavailable',
+  });
+  try {
+    const data = await safeFetch('https://example.test/fail', { retries: 0, source });
+    assert.match(data.error, /HTTP 503/);
+    const bucket = getFetchMetrics().bySource[source];
+    assert.equal(bucket.requests, 1);
+    assert.equal(bucket.ok, 0);
+    assert.equal(bucket.failed, 1);
+    assert.equal(bucket.lastStatus, 503);
+    assert.equal(bucket.bytes, 'service unavailable'.length);
+    assert.match(bucket.lastError, /HTTP 503/);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test('safeFetch retry metrics count one record per attempt', async () => {
+  const originalFetch = globalThis.fetch;
+  const source = 'unit-retry-attempts';
+  let calls = 0;
+  globalThis.fetch = async () => {
+    calls += 1;
+    if (calls === 1) {
+      return {
+        ok: false,
+        status: 502,
+        headers: { get: () => 'application/json' },
+        text: async () => 'bad gateway',
+      };
+    }
+    return {
+      ok: true,
+      status: 200,
+      headers: { get: () => 'application/json' },
+      text: async () => '{"ok":true}',
+    };
+  };
+  try {
+    const data = await safeFetch('https://example.test/retry', { retries: 1, source });
+    assert.equal(data.ok, true);
+    assert.equal(calls, 2);
+    const bucket = getFetchMetrics().bySource[source];
+    assert.equal(bucket.requests, 2);
+    assert.equal(bucket.ok, 1);
+    assert.equal(bucket.failed, 1);
+    assert.equal(bucket.lastStatus, 200);
  } finally {
    globalThis.fetch = originalFetch;
  }
@@ -34,3 +99,18 @@ test('safeFetchText returns text and byte count', async () => {
    globalThis.fetch = originalFetch;
  }
 });
+
+test('scenario watchlist feature is wired into sweep, briefing, and dashboard', () => {
+  const scenarios = readFileSync(new URL('../lib/scenarios.mjs', import.meta.url), 'utf8');
+  const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
+  const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
+  const readme = readFileSync(new URL('../README.md', import.meta.url), 'utf8');
+  assert.match(scenarios, /DEFAULT_SCENARIOS/);
+  assert.match(scenarios, /runsDir, 'scenarios\.json'/);
+  assert.match(scenarios, /scenario-state\.json/);
+  assert.match(scenarios, /watching.*building.*confirmed/s);
+  assert.match(server, /evaluateScenarios\(synthesized, delta, RUNS_DIR\)/);
+  assert.match(server, /\*Scenario Watchlist\*/);
+  assert.match(html, /Scenario Watchlist/);
+  assert.match(readme, /runs\/scenarios\.json/);
+});
--- a/test/mojibake-text.test.mjs
+++ b/test/mojibake-text.test.mjs
@@ -0,0 +1,65 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+
+const TEXT_ROOTS = ['locales'];
+
+const TEXT_FILES = [];
+
+const EXTENSIONS = new Set(['.json', '.html', '.mjs']);
+
+const MOJIBAKE_PATTERNS = [
+  { name: 'latin1-accent', pattern: /\u00c3./g },
+  { name: 'stray-cp1252-prefix', pattern: /\u00c2./g },
+  { name: 'emoji-mojibake', pattern: /\u00f0\u0178/g },
+  {
+    name: 'punctuation-mojibake',
+    pattern: /\u00e2[\u0080-\u009f\u20ac\u0153\u2018\u2019\u201c\u201d\u2013\u2014\u2022\u2026\u201e\u2021\u02c6\u2030\u2039\u203a\u0152\u017d]/g,
+  },
+  { name: 'variation-selector-mojibake', pattern: /\u00ef\u00b8/g },
+  { name: 'ligature-mojibake', pattern: /\u00c5[\u0080-\u017f]/g },
+  { name: 'replacement-character', pattern: /\ufffd/g },
+];
+
+function collectFiles(root) {
+  const out = [];
+  for (const entry of readdirSync(root, { withFileTypes: true })) {
+    const path = join(root, entry.name);
+    if (entry.isDirectory()) {
+      out.push(...collectFiles(path));
+    } else if (EXTENSIONS.has(path.slice(path.lastIndexOf('.')))) {
+      out.push(path);
+    }
+  }
+  return out;
+}
+
+function textFiles() {
+  const discovered = TEXT_ROOTS.flatMap(root => collectFiles(root));
+  const explicit = TEXT_FILES.filter(path => statSync(path, { throwIfNoEntry: false })?.isFile());
+  return [...new Set([...discovered, ...explicit])].sort();
+}
+
+test('locale JSON files are valid UTF-8 JSON', () => {
+  for (const file of collectFiles('locales')) {
+    assert.doesNotThrow(() => JSON.parse(readFileSync(file, 'utf8')), `${file} must parse as JSON`);
+  }
+});
+
+test('locale text does not contain known mojibake sequences', () => {
+  const failures = [];
+
+  for (const file of textFiles()) {
+    const text = readFileSync(file, 'utf8');
+    for (const { name, pattern } of MOJIBAKE_PATTERNS) {
+      for (const match of text.matchAll(pattern)) {
+        const start = Math.max(0, match.index - 30);
+        const end = Math.min(text.length, match.index + 50);
+        failures.push(`${file}: ${name}: ${JSON.stringify(text.slice(start, end))}`);
+      }
+    }
+  }
+
+  assert.deepEqual(failures, []);
+});
--- a/test/reddit-source.test.mjs
+++ b/test/reddit-source.test.mjs
@@ -0,0 +1,109 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { briefing, getHot, getRedditConfig, getToken } from '../apis/sources/reddit.mjs';
+
+test('Reddit reports missing OAuth credentials without network access', async () => {
+  let calls = 0;
+  const data = await briefing({
+    env: {},
+    delayMs: 0,
+    fetchImpl: async () => {
+      calls++;
+      throw new Error('unexpected network access');
+    },
+  });
+
+  assert.equal(calls, 0);
+  assert.equal(data.status, 'no_credentials');
+  assert.equal(data.error, 'missing_reddit_oauth_credentials');
+  assert.deepEqual(data.missing, ['REDDIT_CLIENT_ID', 'REDDIT_CLIENT_SECRET']);
+});
+
+test('Reddit hot posts require OAuth token and never use public JSON fallback', async () => {
+  const originalFetch = globalThis.fetch;
+  let calledUrl = null;
+  globalThis.fetch = async url => {
+    calledUrl = url;
+    throw new Error('unexpected public fallback');
+  };
+
+  try {
+    const data = await getHot('worldnews');
+    assert.equal(calledUrl, null);
+    assert.equal(data.status, 'no_credentials');
+    assert.equal(data.error, 'reddit_oauth_required');
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test('Reddit classifies OAuth HTTP failure without exposing secrets', async () => {
+  const result = await getToken({
+    env: { REDDIT_CLIENT_ID: 'client-id', REDDIT_CLIENT_SECRET: 'client-secret' },
+    fetchImpl: async () => ({
+      ok: false,
+      status: 401,
+      text: async () => 'invalid client',
+    }),
+  });
+
+  assert.equal(result.ok, false);
+  assert.equal(result.status, 'auth_failed');
+  assert.equal(result.error, 'reddit_oauth_http_401');
+  assert.doesNotMatch(JSON.stringify(result), /client-secret/);
+});
+
+test('Reddit fetches hot posts through oauth.reddit.com when configured', async () => {
+  const originalFetch = globalThis.fetch;
+  const urls = [];
+  globalThis.fetch = async url => {
+    urls.push(String(url));
+    if (String(url).includes('/api/v1/access_token')) {
+      return {
+        ok: true,
+        status: 200,
+        json: async () => ({ access_token: 'test-token' }),
+      };
+    }
+    return {
+      ok: true,
+      status: 200,
+      headers: { get: () => 'application/json' },
+      text: async () => JSON.stringify({
+        data: {
+          children: [
+            {
+              data: {
+                title: 'Market stress headline',
+                score: 42,
+                num_comments: 7,
+                url: 'https://example.test/post',
+                created_utc: 1700000000,
+              },
+            },
+          ],
+        },
+      }),
+    };
+  };
+
+  try {
+    const data = await briefing({
+      env: { REDDIT_CLIENT_ID: 'client-id', REDDIT_CLIENT_SECRET: 'client-secret' },
+      subreddits: ['worldnews'],
+      delayMs: 0,
+    });
+
+    assert.equal(data.status, 'ok');
+    assert.equal(data.subreddits.worldnews[0].title, 'Market stress headline');
+    assert.ok(urls.some(url => url === 'https://www.reddit.com/api/v1/access_token'));
+    assert.ok(urls.some(url => url.startsWith('https://oauth.reddit.com/r/worldnews/hot')));
+    assert.equal(urls.some(url => url.includes('hot.json')), false);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test('Reddit config reports partial credential state', () => {
+  assert.deepEqual(getRedditConfig({ REDDIT_CLIENT_ID: 'id' }).missing, ['REDDIT_CLIENT_SECRET']);
+});