Merge pull request 'Improve ACLED auth diagnostics and tests' (#14) from codex/issue-3-acled-diagnostics-auth-tests into codex/production-intelligence-terminal

Reviewed-on: MrSphay/intelligence-terminal#14

README.md

@@ -355,7 +355,7 @@ These three unlock the most valuable economic and satellite data. Each takes abo
 
 | Key | Source | How to Get |
 |-----|--------|------------|
-| `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2 |
+| `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2. `ACLED_USER` / `ACLED_USERNAME` are accepted as email aliases |
 | `AISSTREAM_API_KEY` | Maritime AIS vessel tracking | [aisstream.io](https://aisstream.io/) — free |
 | `ADSB_API_KEY` | Unfiltered flight tracking | [RapidAPI](https://rapidapi.com/adsbexchange/api/adsbexchange-com1) — ~$10/mo |
 | `REDDIT_CLIENT_ID` + `REDDIT_CLIENT_SECRET` | Reddit social sentiment | [reddit.com/prefs/apps](https://www.reddit.com/prefs/apps/) — create a script app |

apis/sources/acled.mjs

@@ -1,9 +1,9 @@
-// ACLED — Armed Conflict Location & Event Data
+// ACLED - Armed Conflict Location & Event Data.
 // Auth strategy (tries in order):
-//   1. Cookie-based session: POST /user/login?_format=json → session cookie
+//   1. OAuth Bearer token:   POST /oauth/token -> Authorization header
-//   2. OAuth Bearer token:   POST /oauth/token → Authorization header
+//   2. Cookie-based session: POST /user/login?_format=json -> session cookie
-// Set ACLED_EMAIL and ACLED_PASSWORD in .env (your myACLED login credentials).
+// Set ACLED_EMAIL and ACLED_PASSWORD in .env. ACLED_USER or ACLED_USERNAME are
-// Data endpoint: GET https://acleddata.com/api/acled/read
+// accepted as aliases for ACLED_EMAIL.
 
 import { daysAgo } from '../utils/fetch.mjs';
 import '../utils/env.mjs';
@@ -12,124 +12,135 @@ const LOGIN_URL = 'https://acleddata.com/user/login?_format=json';
 const TOKEN_URL = 'https://acleddata.com/oauth/token';
 const API_BASE  = 'https://acleddata.com/api/acled/read';
 
-// Session cache
 let sessionCache = { cookies: null, token: null, method: null, expires: 0 };
 
-// Strategy 1: Cookie-based session login (mirrors browser login)
+export function resetAcledSessionCache() {
-async function loginCookie(email, password) {
+  sessionCache = { cookies: null, token: null, method: null, expires: 0 };
+}
+
+export function getAcledConfig(env = process.env) {
+  const email = env.ACLED_EMAIL || env.ACLED_USER || env.ACLED_USERNAME || '';
+  const password = env.ACLED_PASSWORD || '';
+  const missing = [];
+  if (!email) missing.push('ACLED_EMAIL');
+  if (!password) missing.push('ACLED_PASSWORD');
+  return { email, password, configured: missing.length === 0, missing };
+}
+
+function acledError(status, error, message, extra = {}) {
+  return { status, error, message, ...extra };
+}
+
+function safeText(value, max = 200) {
+  return String(value || '').replace(/Bearer\s+[A-Za-z0-9._-]+/gi, 'Bearer [redacted]').slice(0, max);
+}
+
+async function fetchWithTimeout(fetchImpl, url, init, timeoutMs) {
   const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), 15000);
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
   try {
-    const res = await fetch(LOGIN_URL, {
+    return await fetchImpl(url, { ...init, signal: controller.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export async function loginCookie(email, password, opts = {}) {
+  const fetchImpl = opts.fetchImpl || globalThis.fetch;
+  try {
+    const res = await fetchWithTimeout(fetchImpl, LOGIN_URL, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify({ name: email, pass: password }),
       redirect: 'manual',
-      signal: controller.signal,
+    }, 15000);
-    });
-    clearTimeout(timer);
 
-    // Collect Set-Cookie headers
     const setCookies = res.headers.getSetCookie?.() || [];
     const cookieStr = setCookies.map(c => c.split(';')[0]).join('; ');
 
-    if (res.ok && cookieStr) {
+    if ((res.ok || (res.status >= 300 && res.status < 400)) && cookieStr) {
-      return { cookies: cookieStr };
+      return { ok: true, cookies: cookieStr };
-    }
-
-    // Some Drupal sites return 303 redirect on successful login — cookies still set
-    if (res.status >= 300 && res.status < 400 && cookieStr) {
-      return { cookies: cookieStr };
     }
 
     const errText = await res.text().catch(() => '');
-    return { error: `Cookie login failed (HTTP ${res.status}): ${errText.slice(0, 200)}` };
+    return acledError('auth_failed', `acled_cookie_http_${res.status}`, `ACLED cookie login failed with HTTP ${res.status}`, {
+      detail: safeText(errText),
+    });
   } catch (e) {
-    clearTimeout(timer);
+    return acledError('auth_failed', 'acled_cookie_request_failed', `ACLED cookie login error: ${e.message}`);
-    const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : '';
-    return { error: `Cookie login error: ${e.message}${cause}` };
   }
 }
 
-// Strategy 2: OAuth2 password grant
+export async function loginOAuth(email, password, opts = {}) {
-async function loginOAuth(email, password) {
+  const fetchImpl = opts.fetchImpl || globalThis.fetch;
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), 15000);
   try {
     const body = new URLSearchParams({
       username: email,
-      password: password,
+      password,
       grant_type: 'password',
       client_id: 'acled',
     });
 
-    const res = await fetch(TOKEN_URL, {
+    const res = await fetchWithTimeout(fetchImpl, TOKEN_URL, {
       method: 'POST',
       headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
       body: body.toString(),
-      signal: controller.signal,
+    }, 15000);
-    });
-    clearTimeout(timer);
 
     if (!res.ok) {
       const errText = await res.text().catch(() => '');
-      return { error: `OAuth failed (HTTP ${res.status}): ${errText.slice(0, 200)}` };
+      return acledError('auth_failed', `acled_oauth_http_${res.status}`, `ACLED OAuth failed with HTTP ${res.status}`, {
+        detail: safeText(errText),
+      });
     }
 
     const data = await res.json();
     if (!data.access_token) {
-      return { error: `OAuth response missing access_token: ${JSON.stringify(data).slice(0, 200)}` };
+      return acledError('auth_failed', 'acled_oauth_missing_access_token', 'ACLED OAuth response did not include access_token');
     }
 
-    return { token: data.access_token };
+    return { ok: true, token: data.access_token };
   } catch (e) {
-    clearTimeout(timer);
+    return acledError('auth_failed', 'acled_oauth_request_failed', `ACLED OAuth error: ${e.message}`);
-    const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : '';
-    return { error: `OAuth error: ${e.message}${cause}` };
   }
 }
 
-// Try both auth strategies
+export async function authenticate(opts = {}) {
-async function authenticate() {
+  const env = opts.env || process.env;
-  const email    = process.env.ACLED_EMAIL;
+  const fetchImpl = opts.fetchImpl || globalThis.fetch;
-  const password = process.env.ACLED_PASSWORD;
+  const config = getAcledConfig(env);
-  if (!email || !password) {
+  if (!config.configured) {
-    return { error: 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.' };
+    return acledError('no_credentials', 'missing_acled_credentials', 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.', {
+      missing: config.missing,
+    });
   }
 
-  // Return cached session if still valid
   if (sessionCache.method && Date.now() < sessionCache.expires) {
     return sessionCache;
   }
 
-  const errors = [];
+  const diagnostics = [];
-  const debug = process.argv.includes('--debug');
+  const oauthResult = await loginOAuth(config.email, config.password, { fetchImpl });
-
+  if (oauthResult.ok) {
-  // Try OAuth first (official programmatic method per ACLED docs)
-  const oauthResult = await loginOAuth(email, password);
-  if (oauthResult.token) {
-    if (debug) console.error(`[ACLED DEBUG] OAuth OK — token: ${oauthResult.token.slice(0, 20)}...`);
     sessionCache = { cookies: null, token: oauthResult.token, method: 'oauth', expires: Date.now() + 23 * 60 * 60 * 1000 };
     return sessionCache;
   }
-  errors.push(`OAuth: ${oauthResult.error}`);
+  diagnostics.push({ method: 'oauth', status: oauthResult.status, error: oauthResult.error, message: oauthResult.message });
-  if (debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
+  if (opts.debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
 
-  // Fall back to cookie-based session
+  const cookieResult = await loginCookie(config.email, config.password, { fetchImpl });
-  const cookieResult = await loginCookie(email, password);
+  if (cookieResult.ok) {
-  if (cookieResult.cookies) {
-    if (debug) console.error(`[ACLED DEBUG] Cookie OK — cookies: ${cookieResult.cookies.slice(0, 80)}...`);
     sessionCache = { cookies: cookieResult.cookies, token: null, method: 'cookie', expires: Date.now() + 12 * 60 * 60 * 1000 };
     return sessionCache;
   }
-  errors.push(`Cookie: ${cookieResult.error}`);
+  diagnostics.push({ method: 'cookie', status: cookieResult.status, error: cookieResult.error, message: cookieResult.message });
+  if (opts.debug) console.error(`[ACLED DEBUG] Cookie login failed: ${cookieResult.error}`);
 
-  return { error: `All ACLED auth methods failed.\n${errors.join('\n')}` };
+  return acledError('auth_failed', 'acled_auth_failed', 'All ACLED auth methods failed.', { diagnostics });
 }
 
-// Build headers based on auth method
 function authHeaders(session) {
-  const headers = { 'User-Agent': 'Crucix/1.0', 'Content-Type': 'application/json' };
+  const headers = { 'User-Agent': 'Crucix/2.0', 'Content-Type': 'application/json' };
   if (session.method === 'cookie' && session.cookies) {
     headers['Cookie'] = session.cookies;
   } else if (session.method === 'oauth' && session.token) {
@@ -138,7 +149,6 @@ function authHeaders(session) {
   return headers;
 }
 
-// Event type constants
 export const EVENT_TYPES = [
   'Battles',
   'Explosions/Remote violence',
@@ -148,7 +158,6 @@ export const EVENT_TYPES = [
   'Strategic developments',
 ];
 
-// Query conflict events with flexible filters
 export async function getEvents(opts = {}) {
   const {
     limit = 500,
@@ -157,10 +166,13 @@ export async function getEvents(opts = {}) {
     eventType,
     country,
     region,
+    env = process.env,
+    fetchImpl = globalThis.fetch,
+    debug = process.argv.includes('--debug'),
   } = opts;
 
-  const session = await authenticate();
+  const session = await authenticate({ env, fetchImpl, debug });
-  if (session.error) return { error: session.error };
+  if (session.error) return session;
 
   const params = new URLSearchParams({ _format: 'json', limit: String(limit) });
   if (eventDateStart && eventDateEnd) {
@@ -171,59 +183,43 @@ export async function getEvents(opts = {}) {
   if (country) params.set('country', country);
   if (region) params.set('region', String(region));
 
-  const debug = process.argv.includes('--debug');
   try {
     const url = `${API_BASE}?${params}`;
-    const hdrs = authHeaders(session);
+    if (debug) console.error(`[ACLED DEBUG] Data request: GET ${url}`);
-    if (debug) {
+    const res = await fetchWithTimeout(fetchImpl, url, { headers: authHeaders(session) }, 25000);
-      console.error(`[ACLED DEBUG] Data request: GET ${url}`);
-      console.error(`[ACLED DEBUG] Headers: ${JSON.stringify(hdrs)}`);
-    }
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), 25000);
-    const res = await fetch(url, {
-      headers: hdrs,
-      signal: controller.signal,
-    });
-    clearTimeout(timer);
     if (debug) console.error(`[ACLED DEBUG] Data response: HTTP ${res.status}`);
 
     if (!res.ok) {
       const errText = await res.text().catch(() => '');
-      if (debug) console.error(`[ACLED DEBUG] Error body: ${errText.slice(0, 500)}`);
       if (res.status === 401 || res.status === 403) {
-        // Clear cache and report
         sessionCache = { cookies: null, token: null, method: null, expires: 0 };
-        const hint = res.status === 403
+        return acledError('access_denied', `acled_data_http_${res.status}`, `ACLED data access denied with HTTP ${res.status}`, {
-          ? '\n→ Fix: Log in at https://acleddata.com/user/login, then:\n'
+          authMethod: session.method,
-            + '  1. Accept Terms of Use (profile → Terms of Use button → check the box)\n'
+          detail: safeText(errText, 300),
-            + '  2. Complete all required profile fields\n'
+          hint: 'Accept ACLED terms, complete profile fields, and confirm API access for the account.',
-            + '  3. Ensure your account has the "API" access group\n'
+        });
-            + '  Contact access@acleddata.com if issues persist.'
-          : '';
-        return { error: `ACLED data access denied (HTTP ${res.status}, auth method: ${session.method}). Response: ${errText.slice(0, 300)}${hint}` };
       }
-      return { error: `HTTP ${res.status}: ${errText.slice(0, 200)}` };
+      return acledError('api_failed', `acled_data_http_${res.status}`, `ACLED data request failed with HTTP ${res.status}`, {
+        detail: safeText(errText),
+      });
     }
 
     const data = await res.json();
-
-    // ACLED may return a 200 with an error status in the body
     if (data?.status && data.status !== 200) {
-      return { error: `ACLED API error: status ${data.status} — ${data.message || 'Unknown error'}` };
+      return acledError('api_failed', `acled_api_status_${data.status}`, `ACLED API returned status ${data.status}`, {
+        detail: safeText(data.message),
+      });
     }
 
     return data;
   } catch (e) {
     if (e.name === 'AbortError') {
-      return { error: 'ACLED data request timed out (25s)' };
+      return acledError('api_failed', 'acled_data_timeout', 'ACLED data request timed out after 25s');
     }
-    const rootCause = e.cause ? `${e.cause.message || e.cause.code || e.cause}` : '';
+    return acledError('api_failed', 'acled_data_request_failed', `ACLED data error: ${e.message}`);
-    return { error: `ACLED data error: ${e.message}${rootCause ? ' → ' + rootCause : ''}` };
   }
 }
 
-// Summarize events by a given field
 function groupBy(events, field) {
   const map = {};
   for (const e of events) {
@@ -235,33 +231,47 @@ function groupBy(events, field) {
   return map;
 }
 
-// Briefing — last 7 days of global conflict events
+export async function briefing(opts = {}) {
-export async function briefing() {
+  const env = opts.env || process.env;
-  if (!process.env.ACLED_EMAIL || !process.env.ACLED_PASSWORD) {
+  const fetchImpl = opts.fetchImpl || globalThis.fetch;
+  const config = getAcledConfig(env);
+  if (!config.configured) {
     return {
       source: 'ACLED',
       timestamp: new Date().toISOString(),
       status: 'no_credentials',
+      error: 'missing_acled_credentials',
+      missing: config.missing,
       message: 'Set ACLED_EMAIL and ACLED_PASSWORD in .env. Register at https://acleddata.com/user/register',
     };
   }
 
   const start = daysAgo(7);
-  const end   = daysAgo(0);
+  const end = daysAgo(0);
 
   const data = await getEvents({
     eventDateStart: start,
     eventDateEnd: end,
     limit: 2000,
+    env,
+    fetchImpl,
+    debug: opts.debug,
   });
 
   if (data?.error) {
-    return { source: 'ACLED', timestamp: new Date().toISOString(), error: data.error };
+    return {
+      source: 'ACLED',
+      timestamp: new Date().toISOString(),
+      status: data.status || 'api_failed',
+      error: data.error,
+      message: data.message,
+      detail: data.detail,
+      hint: data.hint,
+      diagnostics: data.diagnostics,
+    };
   }
 
   let events = data?.data || [];
-
-  // Enrich all events with numeric lat/lon
   events = events.map(e => ({
     ...e,
     lat: parseFloat(e.latitude) || null,
@@ -272,10 +282,9 @@ export async function briefing() {
     (sum, e) => sum + (parseInt(e.fatalities, 10) || 0), 0
   );
 
-  const byRegion  = groupBy(events, 'region');
+  const byRegion = groupBy(events, 'region');
-  const byType    = groupBy(events, 'event_type');
+  const byType = groupBy(events, 'event_type');
   const byCountry = groupBy(events, 'country');
-
   const topCountries = Object.entries(byCountry)
     .sort((a, b) => b[1].count - a[1].count)
     .slice(0, 10)
@@ -286,20 +295,21 @@ export async function briefing() {
     .sort((a, b) => (parseInt(b.fatalities, 10) || 0) - (parseInt(a.fatalities, 10) || 0))
     .slice(0, 15)
     .map(e => ({
-      date:       e.event_date,
+      date: e.event_date,
-      type:       e.event_type,
+      type: e.event_type,
-      subType:    e.sub_event_type,
+      subType: e.sub_event_type,
-      country:    e.country,
+      country: e.country,
-      location:   e.location,
+      location: e.location,
       fatalities: parseInt(e.fatalities, 10) || 0,
-      lat:        parseFloat(e.latitude) || null,
+      lat: parseFloat(e.latitude) || null,
-      lon:        parseFloat(e.longitude) || null,
+      lon: parseFloat(e.longitude) || null,
-      notes:      e.notes?.slice(0, 200),
+      notes: e.notes?.slice(0, 200),
     }));
 
   return {
     source: 'ACLED',
     timestamp: new Date().toISOString(),
+    status: 'ok',
     period: { start, end },
     totalEvents: events.length,
     totalFatalities,

docs/sources/acled.md

@@ -2,8 +2,11 @@
 
 Provides conflict events, fatalities, event types, and locations.
 
-- Auth: `ACLED_EMAIL` and `ACLED_PASSWORD`.
+- Auth: `ACLED_EMAIL` and `ACLED_PASSWORD`. `ACLED_USER` or `ACLED_USERNAME` may be used as aliases for `ACLED_EMAIL`.
 - Flow: OAuth password grant is tried first, then cookie session fallback.
-- Failure modes: missing credentials, terms/profile not completed, expired token, account missing API access.
+- Failure modes are classified as `no_credentials`, `auth_failed`, `access_denied`, or `api_failed`.
-- Behavior: missing or rejected credentials produce degraded source health with the ACLED error text.
+- Behavior: missing, rejected, or unauthorized credentials produce degraded source health with a concise operator message.
-- Test: set credentials, run `node apis/sources/acled.mjs`, then check `/api/health`.
+- Secret handling: debug output never prints bearer tokens, cookies, or the configured password.
+- Test: run `node --test test/acled-source.test.mjs`; with real credentials, run `node apis/sources/acled.mjs`, then check `/api/health`.
+
+`access_denied` normally means the login worked but the account cannot read API data. Check that ACLED terms are accepted, required profile fields are complete, and API access is enabled for the account.

test/acled-source.test.mjs

@@ -0,0 +1,95 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { authenticate, briefing, resetAcledSessionCache } from '../apis/sources/acled.mjs';
+
+function jsonResponse(status, body, ok = status >= 200 && status < 300) {
+  return {
+    ok,
+    status,
+    headers: { getSetCookie: () => [] },
+    json: async () => body,
+    text: async () => JSON.stringify(body),
+  };
+}
+
+test('ACLED reports missing credentials without network access', async () => {
+  resetAcledSessionCache();
+  let calls = 0;
+  const data = await briefing({
+    env: {},
+    fetchImpl: async () => {
+      calls++;
+      throw new Error('unexpected network access');
+    },
+  });
+
+  assert.equal(calls, 0);
+  assert.equal(data.status, 'no_credentials');
+  assert.equal(data.error, 'missing_acled_credentials');
+  assert.deepEqual(data.missing, ['ACLED_EMAIL', 'ACLED_PASSWORD']);
+});
+
+test('ACLED accepts ACLED_USER as email alias and returns empty valid result', async () => {
+  resetAcledSessionCache();
+  const urls = [];
+  const data = await briefing({
+    env: { ACLED_USER: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
+    fetchImpl: async url => {
+      urls.push(String(url));
+      if (String(url).includes('/oauth/token')) {
+        return jsonResponse(200, { access_token: 'token' });
+      }
+      return jsonResponse(200, { status: 200, data: [] });
+    },
+  });
+
+  assert.equal(data.status, 'ok');
+  assert.equal(data.totalEvents, 0);
+  assert.ok(urls.some(url => url.includes('/oauth/token')));
+  assert.ok(urls.some(url => url.includes('/api/acled/read')));
+});
+
+test('ACLED classifies auth failure without exposing credentials', async () => {
+  resetAcledSessionCache();
+  const result = await authenticate({
+    env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'super-secret' },
+    fetchImpl: async url => {
+      if (String(url).includes('/oauth/token')) {
+        return jsonResponse(401, { error: 'invalid_grant' }, false);
+      }
+      return {
+        ok: false,
+        status: 403,
+        headers: { getSetCookie: () => [] },
+        text: async () => 'forbidden',
+      };
+    },
+  });
+
+  assert.equal(result.status, 'auth_failed');
+  assert.equal(result.error, 'acled_auth_failed');
+  assert.equal(result.diagnostics.length, 2);
+  assert.doesNotMatch(JSON.stringify(result), /super-secret/);
+});
+
+test('ACLED classifies data access denied distinctly from auth failure', async () => {
+  resetAcledSessionCache();
+  const data = await briefing({
+    env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
+    fetchImpl: async url => {
+      if (String(url).includes('/oauth/token')) {
+        return jsonResponse(200, { access_token: 'token' });
+      }
+      return {
+        ok: false,
+        status: 403,
+        headers: { getSetCookie: () => [] },
+        text: async () => 'terms not accepted',
+      };
+    },
+  });
+
+  assert.equal(data.status, 'access_denied');
+  assert.equal(data.error, 'acled_data_http_403');
+  assert.match(data.hint, /Accept ACLED terms/);
+});