Merge pull request 'Improve ACLED auth diagnostics and tests' (#14) from codex/issue-3-acled-diagnostics-auth-tests into codex/production-intelligence-terminal

Reviewed-on: #14
2026-05-17 15:48:03 +00:00
parent da180831bf 833b7dedd7
commit 7a562bc8cf
4 changed files with 230 additions and 122 deletions
--- a/README.md
+++ b/README.md
@@ -355,7 +355,7 @@ These three unlock the most valuable economic and satellite data. Each takes abo

 | Key | Source | How to Get |
 |-----|--------|------------|
-| `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2 |
+| `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2. `ACLED_USER` / `ACLED_USERNAME` are accepted as email aliases |
 | `AISSTREAM_API_KEY` | Maritime AIS vessel tracking | [aisstream.io](https://aisstream.io/) — free |
 | `ADSB_API_KEY` | Unfiltered flight tracking | [RapidAPI](https://rapidapi.com/adsbexchange/api/adsbexchange-com1) — ~$10/mo |
 | `REDDIT_CLIENT_ID` + `REDDIT_CLIENT_SECRET` | Reddit social sentiment | [reddit.com/prefs/apps](https://www.reddit.com/prefs/apps/) — create a script app |
--- a/apis/sources/acled.mjs
+++ b/apis/sources/acled.mjs
@@ -1,9 +1,9 @@
-// ACLED — Armed Conflict Location & Event Data
+// ACLED - Armed Conflict Location & Event Data.
 // Auth strategy (tries in order):
-//   1. Cookie-based session: POST /user/login?_format=json → session cookie
-//   2. OAuth Bearer token:   POST /oauth/token → Authorization header
-// Set ACLED_EMAIL and ACLED_PASSWORD in .env (your myACLED login credentials).
-// Data endpoint: GET https://acleddata.com/api/acled/read
+//   1. OAuth Bearer token:   POST /oauth/token -> Authorization header
+//   2. Cookie-based session: POST /user/login?_format=json -> session cookie
+// Set ACLED_EMAIL and ACLED_PASSWORD in .env. ACLED_USER or ACLED_USERNAME are
+// accepted as aliases for ACLED_EMAIL.

 import { daysAgo } from '../utils/fetch.mjs';
 import '../utils/env.mjs';
@@ -12,124 +12,135 @@ const LOGIN_URL = 'https://acleddata.com/user/login?_format=json';
 const TOKEN_URL = 'https://acleddata.com/oauth/token';
 const API_BASE  = 'https://acleddata.com/api/acled/read';

-// Session cache
 let sessionCache = { cookies: null, token: null, method: null, expires: 0 };

-// Strategy 1: Cookie-based session login (mirrors browser login)
-async function loginCookie(email, password) {
+export function resetAcledSessionCache() {
+  sessionCache = { cookies: null, token: null, method: null, expires: 0 };
+}
+
+export function getAcledConfig(env = process.env) {
+  const email = env.ACLED_EMAIL || env.ACLED_USER || env.ACLED_USERNAME || '';
+  const password = env.ACLED_PASSWORD || '';
+  const missing = [];
+  if (!email) missing.push('ACLED_EMAIL');
+  if (!password) missing.push('ACLED_PASSWORD');
+  return { email, password, configured: missing.length === 0, missing };
+}
+
+function acledError(status, error, message, extra = {}) {
+  return { status, error, message, ...extra };
+}
+
+function safeText(value, max = 200) {
+  return String(value || '').replace(/Bearer\s+[A-Za-z0-9._-]+/gi, 'Bearer [redacted]').slice(0, max);
+}
+
+async function fetchWithTimeout(fetchImpl, url, init, timeoutMs) {
  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), 15000);
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
-    const res = await fetch(LOGIN_URL, {
+    return await fetchImpl(url, { ...init, signal: controller.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export async function loginCookie(email, password, opts = {}) {
+  const fetchImpl = opts.fetchImpl || globalThis.fetch;
+  try {
+    const res = await fetchWithTimeout(fetchImpl, LOGIN_URL, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ name: email, pass: password }),
      redirect: 'manual',
-      signal: controller.signal,
-    });
-    clearTimeout(timer);
+    }, 15000);

-    // Collect Set-Cookie headers
    const setCookies = res.headers.getSetCookie?.() || [];
    const cookieStr = setCookies.map(c => c.split(';')[0]).join('; ');

-    if (res.ok && cookieStr) {
-      return { cookies: cookieStr };
-    }
-
-    // Some Drupal sites return 303 redirect on successful login — cookies still set
-    if (res.status >= 300 && res.status < 400 && cookieStr) {
-      return { cookies: cookieStr };
+    if ((res.ok || (res.status >= 300 && res.status < 400)) && cookieStr) {
+      return { ok: true, cookies: cookieStr };
    }

    const errText = await res.text().catch(() => '');
-    return { error: `Cookie login failed (HTTP ${res.status}): ${errText.slice(0, 200)}` };
+    return acledError('auth_failed', `acled_cookie_http_${res.status}`, `ACLED cookie login failed with HTTP ${res.status}`, {
+      detail: safeText(errText),
+    });
  } catch (e) {
-    clearTimeout(timer);
-    const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : '';
-    return { error: `Cookie login error: ${e.message}${cause}` };
+    return acledError('auth_failed', 'acled_cookie_request_failed', `ACLED cookie login error: ${e.message}`);
  }
 }

-// Strategy 2: OAuth2 password grant
-async function loginOAuth(email, password) {
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), 15000);
+export async function loginOAuth(email, password, opts = {}) {
+  const fetchImpl = opts.fetchImpl || globalThis.fetch;
  try {
    const body = new URLSearchParams({
      username: email,
-      password: password,
+      password,
      grant_type: 'password',
      client_id: 'acled',
    });

-    const res = await fetch(TOKEN_URL, {
+    const res = await fetchWithTimeout(fetchImpl, TOKEN_URL, {
      method: 'POST',
      headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
      body: body.toString(),
-      signal: controller.signal,
-    });
-    clearTimeout(timer);
+    }, 15000);

    if (!res.ok) {
      const errText = await res.text().catch(() => '');
-      return { error: `OAuth failed (HTTP ${res.status}): ${errText.slice(0, 200)}` };
+      return acledError('auth_failed', `acled_oauth_http_${res.status}`, `ACLED OAuth failed with HTTP ${res.status}`, {
+        detail: safeText(errText),
+      });
    }

    const data = await res.json();
    if (!data.access_token) {
-      return { error: `OAuth response missing access_token: ${JSON.stringify(data).slice(0, 200)}` };
+      return acledError('auth_failed', 'acled_oauth_missing_access_token', 'ACLED OAuth response did not include access_token');
    }

-    return { token: data.access_token };
+    return { ok: true, token: data.access_token };
  } catch (e) {
-    clearTimeout(timer);
-    const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : '';
-    return { error: `OAuth error: ${e.message}${cause}` };
+    return acledError('auth_failed', 'acled_oauth_request_failed', `ACLED OAuth error: ${e.message}`);
  }
 }

-// Try both auth strategies
-async function authenticate() {
-  const email    = process.env.ACLED_EMAIL;
-  const password = process.env.ACLED_PASSWORD;
-  if (!email || !password) {
-    return { error: 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.' };
+export async function authenticate(opts = {}) {
+  const env = opts.env || process.env;
+  const fetchImpl = opts.fetchImpl || globalThis.fetch;
+  const config = getAcledConfig(env);
+  if (!config.configured) {
+    return acledError('no_credentials', 'missing_acled_credentials', 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.', {
+      missing: config.missing,
+    });
  }

-  // Return cached session if still valid
  if (sessionCache.method && Date.now() < sessionCache.expires) {
    return sessionCache;
  }

-  const errors = [];
-  const debug = process.argv.includes('--debug');
-
-  // Try OAuth first (official programmatic method per ACLED docs)
-  const oauthResult = await loginOAuth(email, password);
-  if (oauthResult.token) {
-    if (debug) console.error(`[ACLED DEBUG] OAuth OK — token: ${oauthResult.token.slice(0, 20)}...`);
+  const diagnostics = [];
+  const oauthResult = await loginOAuth(config.email, config.password, { fetchImpl });
+  if (oauthResult.ok) {
    sessionCache = { cookies: null, token: oauthResult.token, method: 'oauth', expires: Date.now() + 23 * 60 * 60 * 1000 };
    return sessionCache;
  }
-  errors.push(`OAuth: ${oauthResult.error}`);
-  if (debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
+  diagnostics.push({ method: 'oauth', status: oauthResult.status, error: oauthResult.error, message: oauthResult.message });
+  if (opts.debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);

-  // Fall back to cookie-based session
-  const cookieResult = await loginCookie(email, password);
-  if (cookieResult.cookies) {
-    if (debug) console.error(`[ACLED DEBUG] Cookie OK — cookies: ${cookieResult.cookies.slice(0, 80)}...`);
+  const cookieResult = await loginCookie(config.email, config.password, { fetchImpl });
+  if (cookieResult.ok) {
    sessionCache = { cookies: cookieResult.cookies, token: null, method: 'cookie', expires: Date.now() + 12 * 60 * 60 * 1000 };
    return sessionCache;
  }
-  errors.push(`Cookie: ${cookieResult.error}`);
+  diagnostics.push({ method: 'cookie', status: cookieResult.status, error: cookieResult.error, message: cookieResult.message });
+  if (opts.debug) console.error(`[ACLED DEBUG] Cookie login failed: ${cookieResult.error}`);

-  return { error: `All ACLED auth methods failed.\n${errors.join('\n')}` };
+  return acledError('auth_failed', 'acled_auth_failed', 'All ACLED auth methods failed.', { diagnostics });
 }

-// Build headers based on auth method
 function authHeaders(session) {
-  const headers = { 'User-Agent': 'Crucix/1.0', 'Content-Type': 'application/json' };
+  const headers = { 'User-Agent': 'Crucix/2.0', 'Content-Type': 'application/json' };
  if (session.method === 'cookie' && session.cookies) {
    headers['Cookie'] = session.cookies;
  } else if (session.method === 'oauth' && session.token) {
@@ -138,7 +149,6 @@ function authHeaders(session) {
  return headers;
 }

-// Event type constants
 export const EVENT_TYPES = [
  'Battles',
  'Explosions/Remote violence',
@@ -148,7 +158,6 @@ export const EVENT_TYPES = [
  'Strategic developments',
 ];

-// Query conflict events with flexible filters
 export async function getEvents(opts = {}) {
  const {
    limit = 500,
@@ -157,10 +166,13 @@ export async function getEvents(opts = {}) {
    eventType,
    country,
    region,
+    env = process.env,
+    fetchImpl = globalThis.fetch,
+    debug = process.argv.includes('--debug'),
  } = opts;

-  const session = await authenticate();
-  if (session.error) return { error: session.error };
+  const session = await authenticate({ env, fetchImpl, debug });
+  if (session.error) return session;

  const params = new URLSearchParams({ _format: 'json', limit: String(limit) });
  if (eventDateStart && eventDateEnd) {
@@ -171,59 +183,43 @@ export async function getEvents(opts = {}) {
  if (country) params.set('country', country);
  if (region) params.set('region', String(region));

-  const debug = process.argv.includes('--debug');
  try {
    const url = `${API_BASE}?${params}`;
-    const hdrs = authHeaders(session);
-    if (debug) {
-      console.error(`[ACLED DEBUG] Data request: GET ${url}`);
-      console.error(`[ACLED DEBUG] Headers: ${JSON.stringify(hdrs)}`);
-    }
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), 25000);
-    const res = await fetch(url, {
-      headers: hdrs,
-      signal: controller.signal,
-    });
-    clearTimeout(timer);
+    if (debug) console.error(`[ACLED DEBUG] Data request: GET ${url}`);
+    const res = await fetchWithTimeout(fetchImpl, url, { headers: authHeaders(session) }, 25000);
    if (debug) console.error(`[ACLED DEBUG] Data response: HTTP ${res.status}`);

    if (!res.ok) {
      const errText = await res.text().catch(() => '');
-      if (debug) console.error(`[ACLED DEBUG] Error body: ${errText.slice(0, 500)}`);
      if (res.status === 401 || res.status === 403) {
-        // Clear cache and report
        sessionCache = { cookies: null, token: null, method: null, expires: 0 };
-        const hint = res.status === 403
-          ? '\n→ Fix: Log in at https://acleddata.com/user/login, then:\n'
-            + '  1. Accept Terms of Use (profile → Terms of Use button → check the box)\n'
-            + '  2. Complete all required profile fields\n'
-            + '  3. Ensure your account has the "API" access group\n'
-            + '  Contact access@acleddata.com if issues persist.'
-          : '';
-        return { error: `ACLED data access denied (HTTP ${res.status}, auth method: ${session.method}). Response: ${errText.slice(0, 300)}${hint}` };
+        return acledError('access_denied', `acled_data_http_${res.status}`, `ACLED data access denied with HTTP ${res.status}`, {
+          authMethod: session.method,
+          detail: safeText(errText, 300),
+          hint: 'Accept ACLED terms, complete profile fields, and confirm API access for the account.',
+        });
      }
-      return { error: `HTTP ${res.status}: ${errText.slice(0, 200)}` };
+      return acledError('api_failed', `acled_data_http_${res.status}`, `ACLED data request failed with HTTP ${res.status}`, {
+        detail: safeText(errText),
+      });
    }

    const data = await res.json();
-
-    // ACLED may return a 200 with an error status in the body
    if (data?.status && data.status !== 200) {
-      return { error: `ACLED API error: status ${data.status} — ${data.message || 'Unknown error'}` };
+      return acledError('api_failed', `acled_api_status_${data.status}`, `ACLED API returned status ${data.status}`, {
+        detail: safeText(data.message),
+      });
    }

    return data;
  } catch (e) {
    if (e.name === 'AbortError') {
-      return { error: 'ACLED data request timed out (25s)' };
+      return acledError('api_failed', 'acled_data_timeout', 'ACLED data request timed out after 25s');
    }
-    const rootCause = e.cause ? `${e.cause.message || e.cause.code || e.cause}` : '';
-    return { error: `ACLED data error: ${e.message}${rootCause ? ' → ' + rootCause : ''}` };
+    return acledError('api_failed', 'acled_data_request_failed', `ACLED data error: ${e.message}`);
  }
 }

-// Summarize events by a given field
 function groupBy(events, field) {
  const map = {};
  for (const e of events) {
@@ -235,33 +231,47 @@ function groupBy(events, field) {
  return map;
 }

-// Briefing — last 7 days of global conflict events
-export async function briefing() {
-  if (!process.env.ACLED_EMAIL || !process.env.ACLED_PASSWORD) {
+export async function briefing(opts = {}) {
+  const env = opts.env || process.env;
+  const fetchImpl = opts.fetchImpl || globalThis.fetch;
+  const config = getAcledConfig(env);
+  if (!config.configured) {
    return {
      source: 'ACLED',
      timestamp: new Date().toISOString(),
      status: 'no_credentials',
+      error: 'missing_acled_credentials',
+      missing: config.missing,
      message: 'Set ACLED_EMAIL and ACLED_PASSWORD in .env. Register at https://acleddata.com/user/register',
    };
  }

  const start = daysAgo(7);
-  const end   = daysAgo(0);
+  const end = daysAgo(0);

  const data = await getEvents({
    eventDateStart: start,
    eventDateEnd: end,
    limit: 2000,
+    env,
+    fetchImpl,
+    debug: opts.debug,
  });

  if (data?.error) {
-    return { source: 'ACLED', timestamp: new Date().toISOString(), error: data.error };
+    return {
+      source: 'ACLED',
+      timestamp: new Date().toISOString(),
+      status: data.status || 'api_failed',
+      error: data.error,
+      message: data.message,
+      detail: data.detail,
+      hint: data.hint,
+      diagnostics: data.diagnostics,
+    };
  }

  let events = data?.data || [];
-
-  // Enrich all events with numeric lat/lon
  events = events.map(e => ({
    ...e,
    lat: parseFloat(e.latitude) || null,
@@ -272,10 +282,9 @@ export async function briefing() {
    (sum, e) => sum + (parseInt(e.fatalities, 10) || 0), 0
  );

-  const byRegion  = groupBy(events, 'region');
-  const byType    = groupBy(events, 'event_type');
+  const byRegion = groupBy(events, 'region');
+  const byType = groupBy(events, 'event_type');
  const byCountry = groupBy(events, 'country');
-
  const topCountries = Object.entries(byCountry)
    .sort((a, b) => b[1].count - a[1].count)
    .slice(0, 10)
@@ -286,20 +295,21 @@ export async function briefing() {
    .sort((a, b) => (parseInt(b.fatalities, 10) || 0) - (parseInt(a.fatalities, 10) || 0))
    .slice(0, 15)
    .map(e => ({
-      date:       e.event_date,
-      type:       e.event_type,
-      subType:    e.sub_event_type,
-      country:    e.country,
-      location:   e.location,
+      date: e.event_date,
+      type: e.event_type,
+      subType: e.sub_event_type,
+      country: e.country,
+      location: e.location,
      fatalities: parseInt(e.fatalities, 10) || 0,
-      lat:        parseFloat(e.latitude) || null,
-      lon:        parseFloat(e.longitude) || null,
-      notes:      e.notes?.slice(0, 200),
+      lat: parseFloat(e.latitude) || null,
+      lon: parseFloat(e.longitude) || null,
+      notes: e.notes?.slice(0, 200),
    }));

  return {
    source: 'ACLED',
    timestamp: new Date().toISOString(),
+    status: 'ok',
    period: { start, end },
    totalEvents: events.length,
    totalFatalities,
--- a/docs/sources/acled.md
+++ b/docs/sources/acled.md
@@ -2,8 +2,11 @@

 Provides conflict events, fatalities, event types, and locations.

- Auth: `ACLED_EMAIL` and `ACLED_PASSWORD`.
+- Auth: `ACLED_EMAIL` and `ACLED_PASSWORD`. `ACLED_USER` or `ACLED_USERNAME` may be used as aliases for `ACLED_EMAIL`.
 - Flow: OAuth password grant is tried first, then cookie session fallback.
- Failure modes: missing credentials, terms/profile not completed, expired token, account missing API access.
- Behavior: missing or rejected credentials produce degraded source health with the ACLED error text.
- Test: set credentials, run `node apis/sources/acled.mjs`, then check `/api/health`.
+- Failure modes are classified as `no_credentials`, `auth_failed`, `access_denied`, or `api_failed`.
+- Behavior: missing, rejected, or unauthorized credentials produce degraded source health with a concise operator message.
+- Secret handling: debug output never prints bearer tokens, cookies, or the configured password.
+- Test: run `node --test test/acled-source.test.mjs`; with real credentials, run `node apis/sources/acled.mjs`, then check `/api/health`.
+
+`access_denied` normally means the login worked but the account cannot read API data. Check that ACLED terms are accepted, required profile fields are complete, and API access is enabled for the account.
--- a/test/acled-source.test.mjs
+++ b/test/acled-source.test.mjs
@@ -0,0 +1,95 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { authenticate, briefing, resetAcledSessionCache } from '../apis/sources/acled.mjs';
+
+function jsonResponse(status, body, ok = status >= 200 && status < 300) {
+  return {
+    ok,
+    status,
+    headers: { getSetCookie: () => [] },
+    json: async () => body,
+    text: async () => JSON.stringify(body),
+  };
+}
+
+test('ACLED reports missing credentials without network access', async () => {
+  resetAcledSessionCache();
+  let calls = 0;
+  const data = await briefing({
+    env: {},
+    fetchImpl: async () => {
+      calls++;
+      throw new Error('unexpected network access');
+    },
+  });
+
+  assert.equal(calls, 0);
+  assert.equal(data.status, 'no_credentials');
+  assert.equal(data.error, 'missing_acled_credentials');
+  assert.deepEqual(data.missing, ['ACLED_EMAIL', 'ACLED_PASSWORD']);
+});
+
+test('ACLED accepts ACLED_USER as email alias and returns empty valid result', async () => {
+  resetAcledSessionCache();
+  const urls = [];
+  const data = await briefing({
+    env: { ACLED_USER: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
+    fetchImpl: async url => {
+      urls.push(String(url));
+      if (String(url).includes('/oauth/token')) {
+        return jsonResponse(200, { access_token: 'token' });
+      }
+      return jsonResponse(200, { status: 200, data: [] });
+    },
+  });
+
+  assert.equal(data.status, 'ok');
+  assert.equal(data.totalEvents, 0);
+  assert.ok(urls.some(url => url.includes('/oauth/token')));
+  assert.ok(urls.some(url => url.includes('/api/acled/read')));
+});
+
+test('ACLED classifies auth failure without exposing credentials', async () => {
+  resetAcledSessionCache();
+  const result = await authenticate({
+    env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'super-secret' },
+    fetchImpl: async url => {
+      if (String(url).includes('/oauth/token')) {
+        return jsonResponse(401, { error: 'invalid_grant' }, false);
+      }
+      return {
+        ok: false,
+        status: 403,
+        headers: { getSetCookie: () => [] },
+        text: async () => 'forbidden',
+      };
+    },
+  });
+
+  assert.equal(result.status, 'auth_failed');
+  assert.equal(result.error, 'acled_auth_failed');
+  assert.equal(result.diagnostics.length, 2);
+  assert.doesNotMatch(JSON.stringify(result), /super-secret/);
+});
+
+test('ACLED classifies data access denied distinctly from auth failure', async () => {
+  resetAcledSessionCache();
+  const data = await briefing({
+    env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
+    fetchImpl: async url => {
+      if (String(url).includes('/oauth/token')) {
+        return jsonResponse(200, { access_token: 'token' });
+      }
+      return {
+        ok: false,
+        status: 403,
+        headers: { getSetCookie: () => [] },
+        text: async () => 'terms not accepted',
+      };
+    },
+  });
+
+  assert.equal(data.status, 'access_denied');
+  assert.equal(data.error, 'acled_data_http_403');
+  assert.match(data.hint, /Accept ACLED terms/);
+});