Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-17-sse-heartbeat

# Conflicts:
#	README.md
#	test/fetch-utils.test.mjs

.env.example

@@ -10,8 +10,7 @@ STALE_ALERT_COOLDOWN_MINUTES=60
 DASHBOARD_URL=
 TERMINAL_ACTIONS_ENABLED=true
 SWEEP_TOKEN=
-TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
-TERMINAL_ACTION_RATE_LIMIT_MAX=10
+SSE_HEARTBEAT_INTERVAL_MS=25000
 BRIEF_VERBOSITY=standard
 
 # LLM layer

README.md

@@ -134,8 +134,7 @@ STALE_ALERT_COOLDOWN_MINUTES=60
 DASHBOARD_URL=https://intelligence.example.internal
 TERMINAL_ACTIONS_ENABLED=true
 SWEEP_TOKEN=
-TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
-TERMINAL_ACTION_RATE_LIMIT_MAX=10
+SSE_HEARTBEAT_INTERVAL_MS=25000
 BRIEF_VERBOSITY=standard
 
 LLM_PROVIDER=openrouter
@@ -187,52 +186,23 @@ LLM_MODEL=your-model
 
 For Pangolin or another reverse proxy, forward HTTP traffic to `intelligence-terminal:3117` (or the `PORT` you set). Missing API keys do not crash sweeps; affected sources are reported as degraded in `/api/health`.
 
-#### Terminal Action Exposure
-
-`POST /api/action` and `POST /api/sweep` can trigger operational actions such as manual sweeps. The dashboard has a **SET TOKEN** control that stores your `SWEEP_TOKEN` in browser local storage and sends it as the `x-crucix-token` header; do not put action tokens in URLs.
-
-Recommended settings:
-
-| Deployment | Settings |
-| --- | --- |
-| Private local machine | `NODE_ENV=development`, optional `SWEEP_TOKEN`, optional `TERMINAL_ACTIONS_ENABLED=true`. Localhost can run actions without a token for development. |
-| Private LAN / Dockge | Set a strong `SWEEP_TOKEN`, keep `TERMINAL_ACTIONS_ENABLED=true`, expose only to trusted clients. |
-| Pangolin-authenticated reverse proxy | Set a strong `SWEEP_TOKEN`, keep Pangolin auth in front, use the dashboard **SET TOKEN** flow once per browser. |
-| Public internet | Do not expose Terminal Actions directly. If exposure is unavoidable, require `SWEEP_TOKEN`, keep proxy authentication enabled, lower `TERMINAL_ACTION_RATE_LIMIT_MAX`, and monitor server audit logs. |
-
-Action endpoints reject cross-origin POST origins, apply a small in-memory per-IP rate limit, and write sanitized audit lines without logging the token.
-
 When data remains stale past `STALE_DATA_MAX_AGE_MINUTES`, the server sends an operator alert through configured Telegram/Discord channels after failed or degraded sweep attempts. `STALE_ALERT_COOLDOWN_MINUTES` prevents repeated stale alerts from spamming every refresh interval. Set `DASHBOARD_URL` to the Pangolin/public URL you want included in those alerts.
 
-#### Memory And Prediction Loop
+The dashboard Terminal Actions panel can trigger `status`, `sweep`, and `brief` through `/api/action`. Leave `TERMINAL_ACTIONS_ENABLED=true` for a private home-server deployment. For an internet-exposed deployment, set `SWEEP_TOKEN` and pass it through trusted automation, or set `TERMINAL_ACTIONS_ENABLED=false` to disable browser-triggered actions. If you protect actions with `SWEEP_TOKEN`, the browser can send it from `localStorage.crucix_sweep_token`.
 
-Crucix stores longitudinal memory in `runs/intelligence.db` when the current Node.js build exposes `node:sqlite`. If SQLite is unavailable, the file is created as a harmless placeholder and `/api/health` reports the memory store as unavailable instead of failing the sweep.
+#### Reverse Proxy SSE
 
-The memory layer persists:
+The dashboard receives live sweep updates from `GET /events` using Server-Sent Events. The server sends `retry: 10000` reconnect guidance and lightweight heartbeat comments every `SSE_HEARTBEAT_INTERVAL_MS` milliseconds so reverse proxies do not close an otherwise idle stream between 15-minute sweeps.
 
-| Table | Purpose |
+Recommended proxy settings:
+
+| Proxy | Setting |
 | --- | --- |
-| `runs` | Sweep timestamps, source health counts, and delta direction summaries. |
-| `entities` | Stable entity IDs for recurring countries, regions, and locations. |
-| `events` | Stable event IDs for conflict, OSINT, urgent news, and new delta signals across sweeps. |
-| `predictions` | Trade/intelligence hypotheses with evidence, confidence, horizon, outcome state, and latest grading. |
+| Pangolin / Traefik-style frontends | Keep response streaming enabled and set idle timeouts above `SSE_HEARTBEAT_INTERVAL_MS`. |
+| Nginx | Disable proxy buffering for `/events`, keep `proxy_read_timeout` above the heartbeat interval, and preserve `Connection: keep-alive`. |
+| Cloudflare-style proxies | Keep the heartbeat below common idle cutoffs; the default 25s is intentionally conservative. |
 
-Query endpoints:
-
-```text
-GET /api/memory/search?q=iran&limit=25
-GET /api/memory/predictions?state=open&limit=25
-```
-
-Memory endpoints use the same operator authorization gate as Terminal Actions. The dashboard Terminal Actions panel includes a `Memory` action for a quick operator-facing view of recent events and prediction states.
-
-Retention, backup, and privacy expectations:
-
-- Treat `runs/intelligence.db` as operator data. It can contain source excerpts, headlines, generated hypotheses, and URLs from your configured feeds.
-- Back up `runs/` with the rest of your Dockge volume if you want longitudinal learning to survive container replacement.
-- Delete `runs/intelligence.db` to reset SQLite memory; the next sweep recreates the schema.
-- Do not commit `runs/` or `.env`. API credentials stay in `.env`; memory stores derived observations, not secrets.
-- If you expose the dashboard through a reverse proxy, protect Terminal Actions and memory queries behind your normal authentication boundary.
+If you raise the heartbeat interval, keep it shorter than the lowest idle timeout in the proxy chain.
 
 #### Scenario Watchlist
 

apis/briefing.mjs

@@ -59,8 +59,7 @@ export async function runSource(name, fn, ...args) {
     });
     const data = await Promise.race([dataPromise, timeoutPromise]);
     const hasError = Boolean(data?.error);
-    const degradedStatuses = ['no_credentials', 'no_key', 'disabled', 'degraded', 'failed', 'error'];
-    const isDegraded = hasError || degradedStatuses.includes(data?.status);
+    const isDegraded = hasError || ['no_credentials', 'degraded', 'failed'].includes(data?.status);
     return {
       name,
       status: isDegraded ? 'degraded' : 'ok',

apis/sources/adsb.mjs

@@ -1,8 +1,7 @@
 // ADS-B Exchange — Unfiltered Flight Tracking (including Military)
 // Unlike FlightRadar24/FlightAware, ADS-B Exchange does NOT filter military aircraft.
 // Public feed access varies; RapidAPI tier available for programmatic use.
-// This module reports explicit disabled/degraded state instead of making
-// unavailable aircraft data look live.
+// This module attempts the public endpoints and falls back to a documented stub.
 
 import { safeFetch } from '../utils/fetch.mjs';
 
@@ -141,7 +140,6 @@ async function fetchViaRapidApi(apiKey) {
   // Get all military aircraft
   const data = await safeFetch(`${ENDPOINTS.rapidApi}/mil`, {
     timeout: 20000,
-    source: 'adsb-rapidapi',
     headers: {
       'X-RapidAPI-Key': apiKey,
       'X-RapidAPI-Host': 'adsbexchange-com1.p.rapidapi.com',
@@ -153,26 +151,21 @@ async function fetchViaRapidApi(apiKey) {
 
 // Attempt to fetch from public feed
 async function fetchPublicFeed() {
-  const data = await safeFetch(ENDPOINTS.publicFeed, { timeout: 15000, source: 'adsb-public' });
+  const data = await safeFetch(ENDPOINTS.publicFeed, { timeout: 15000 });
   return data;
 }
 
-async function getMilitaryAircraftResult(apiKey) {
-  const failures = [];
-
+// Get military aircraft from available sources
+export async function getMilitaryAircraft(apiKey) {
   // Try RapidAPI first if key available
   if (apiKey) {
     const data = await fetchViaRapidApi(apiKey);
     if (data && !data.error) {
       const aircraft = data.ac || data.aircraft || [];
       if (Array.isArray(aircraft)) {
-        return {
-          provider: 'rapidapi',
-          aircraft: aircraft.map(classifyAircraft).filter(a => a.isMilitary),
-        };
+        return aircraft.map(classifyAircraft).filter(a => a.isMilitary);
       }
     }
-    failures.push({ provider: 'rapidapi', error: data?.error || 'RapidAPI returned an unsupported payload' });
   }
 
   // Try public feed
@@ -180,21 +173,11 @@ async function getMilitaryAircraftResult(apiKey) {
   if (pubData && !pubData.error) {
     const aircraft = pubData.ac || pubData.aircraft || pubData.states || [];
     if (Array.isArray(aircraft)) {
-      return {
-        provider: 'public-feed',
-        aircraft: aircraft.map(classifyAircraft).filter(a => a.isMilitary),
-      };
+      return aircraft.map(classifyAircraft).filter(a => a.isMilitary);
     }
   }
-  failures.push({ provider: 'public-feed', error: pubData?.error || 'Public feed returned an unsupported payload' });
 
-  return { provider: null, aircraft: null, failures };
-}
-
-// Get military aircraft from available sources
-export async function getMilitaryAircraft(apiKey) {
-  const result = await getMilitaryAircraftResult(apiKey);
-  return result.aircraft;
+  return null; // all sources failed
 }
 
 // Get all aircraft in a geographic bounding box via RapidAPI
@@ -225,8 +208,7 @@ export async function getAircraftInArea(lat, lon, radiusNm = 250, apiKey) {
 // Briefing — attempt to get military flight data, document what's available
 export async function briefing() {
   const apiKey = process.env.ADSB_API_KEY || process.env.RAPIDAPI_KEY || null;
-  const result = await getMilitaryAircraftResult(apiKey);
-  const militaryAircraft = result.aircraft;
+  const militaryAircraft = await getMilitaryAircraft(apiKey);
 
   // If we got data, analyze it
   if (militaryAircraft && militaryAircraft.length > 0) {
@@ -273,7 +255,6 @@ export async function briefing() {
       source: 'ADS-B Exchange',
       timestamp: new Date().toISOString(),
       status: 'live',
-      provider: result.provider,
       totalMilitary: militaryAircraft.length,
       byCountry,
       categories: {
@@ -288,18 +269,10 @@ export async function briefing() {
   }
 
   // No data available — return stub with integration documentation
-  const status = apiKey ? 'degraded' : 'disabled';
-  const error = apiKey
-    ? 'ADS-B providers returned no usable aircraft data'
-    : 'ADSB_API_KEY or RAPIDAPI_KEY is not configured';
-
   return {
     source: 'ADS-B Exchange',
     timestamp: new Date().toISOString(),
-    status,
-    provider: result.provider,
-    error,
-    failures: result.failures,
+    status: apiKey ? 'error' : 'no_key',
     militaryAircraft: [],
     message: apiKey
       ? 'ADS-B Exchange API returned no data. The endpoint may be temporarily unavailable.'

crucix.config.mjs

@@ -26,9 +26,8 @@ export default {
   staleAlertCooldownMinutes: intEnv('STALE_ALERT_COOLDOWN_MINUTES', 60),
   dashboardUrl: process.env.DASHBOARD_URL || null,
   sweepToken: process.env.SWEEP_TOKEN || null,
-  terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', !!process.env.SWEEP_TOKEN || process.env.NODE_ENV !== 'production'),
-  terminalActionRateLimitWindowMs: intEnv('TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS', 60_000),
-  terminalActionRateLimitMax: intEnv('TERMINAL_ACTION_RATE_LIMIT_MAX', 10),
+  terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', true),
+  sseHeartbeatIntervalMs: intEnv('SSE_HEARTBEAT_INTERVAL_MS', 25000),
 
   llm: {
     provider: process.env.LLM_PROVIDER || null, // anthropic | openai | gemini | codex | openrouter | minimax | mistral | ollama | grok

docs/sources/README.md

@@ -16,5 +16,4 @@ Source docs:
 - [Telegram](telegram.md)
 - [FIRMS](firms.md)
 - [Maritime](maritime.md)
-- [ADS-B](adsb.md)
 - [Reddit](reddit.md)

docs/sources/adsb.md

@@ -1,24 +0,0 @@
-# ADS-B Source
-
-ADS-B Exchange support is optional and intended for unfiltered aircraft and military-flight awareness.
-
-- Source module: `apis/sources/adsb.mjs`
-- Preferred provider: ADS-B Exchange via RapidAPI
-- Credentials: `ADSB_API_KEY` or `RAPIDAPI_KEY`
-- Runtime status without credentials: `disabled`
-- Runtime status when providers fail: `degraded`
-- Runtime status with usable aircraft payloads: `live`
-
-The source does not treat a missing key or unavailable public feed as normal live data. `/api/health` and `/api/metrics` surface the degraded source state through the sweep source summary.
-
-Known failure modes:
-
-- Missing `ADSB_API_KEY` / `RAPIDAPI_KEY`: source is disabled with operator guidance.
-- RapidAPI rejects or rate-limits the request: source is degraded and records provider failure detail.
-- Public feed is blocked, rate-limited, or changes shape: source remains degraded instead of returning stale-looking data.
-
-Register for the provider documented in the README, then set:
-
-```env
-ADSB_API_KEY=<rapidapi-key>
-```

lib/intelligence-store.mjs

@@ -2,9 +2,6 @@
 
 import { existsSync, mkdirSync, writeFileSync } from 'fs';
 import { dirname, join } from 'path';
-import { createHash } from 'crypto';
-
-const PREDICTION_STATES = new Set(['open', 'monitoring', 'observed', 'expired_unverified', 'invalidated']);
 
 export class IntelligenceStore {
   constructor(dbPath) {
@@ -33,24 +30,15 @@ export class IntelligenceStore {
         );
         CREATE TABLE IF NOT EXISTS predictions (
           id INTEGER PRIMARY KEY AUTOINCREMENT,
-          stable_id TEXT UNIQUE,
           created_at TEXT NOT NULL,
-          updated_at TEXT,
           title TEXT NOT NULL,
           type TEXT,
-          hypothesis TEXT,
-          evidence_json TEXT,
           confidence TEXT,
-          horizon TEXT,
-          outcome_state TEXT DEFAULT 'open',
-          outcome_json TEXT,
-          last_evaluated_at TEXT,
           source TEXT,
           payload_json TEXT NOT NULL
         );
         CREATE TABLE IF NOT EXISTS entities (
           id INTEGER PRIMARY KEY AUTOINCREMENT,
-          stable_id TEXT UNIQUE,
           first_seen TEXT NOT NULL,
           last_seen TEXT NOT NULL,
           name TEXT NOT NULL,
@@ -58,21 +46,7 @@ export class IntelligenceStore {
           count INTEGER DEFAULT 1,
           UNIQUE(name, kind)
         );
-        CREATE TABLE IF NOT EXISTS events (
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          stable_id TEXT NOT NULL UNIQUE,
-          first_seen TEXT NOT NULL,
-          last_seen TEXT NOT NULL,
-          kind TEXT NOT NULL,
-          name TEXT NOT NULL,
-          region TEXT,
-          severity TEXT,
-          source TEXT,
-          evidence_json TEXT NOT NULL,
-          count INTEGER DEFAULT 1
-        );
       `);
-      this._migrate();
       this.available = true;
     } catch (err) {
       this.available = false;
@@ -97,139 +71,22 @@ export class IntelligenceStore {
       delta?.summary?.direction || null,
       JSON.stringify({ meta, delta: delta?.summary || null }),
     );
-    this._recordEntities(data, timestamp);
-    this._recordEvents(data, delta, timestamp);
-    this.evaluatePredictions(data, timestamp);
-    this._recordPredictions(data, timestamp);
-  }
-
-  status() {
-    return { available: this.available, path: this.dbPath, reason: this.reason };
-  }
-
-  queryMemory({ q = '', limit = 25 } = {}) {
-    if (!this.available || !this.db) return { available: false, reason: this.reason, results: [] };
-    const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
-    const term = String(q || '').trim();
-    const like = `%${term}%`;
-    const where = term
-      ? 'WHERE name LIKE ? OR region LIKE ? OR source LIKE ? OR kind LIKE ?'
-      : '';
-    const params = term ? [like, like, like, like, safeLimit] : [safeLimit];
-    const events = this.db.prepare(`
-      SELECT stable_id, first_seen, last_seen, kind, name, region, severity, source, count, evidence_json
-      FROM events
-      ${where}
-      ORDER BY last_seen DESC
-      LIMIT ?
-    `).all(...params).map(row => ({ ...row, evidence: parseJson(row.evidence_json, {}) }));
-    return { available: true, q: term, results: events };
-  }
-
-  listPredictions({ state = null, limit = 25 } = {}) {
-    if (!this.available || !this.db) return { available: false, reason: this.reason, predictions: [] };
-    const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
-    const normalizedState = state && PREDICTION_STATES.has(String(state)) ? String(state) : null;
-    const rows = normalizedState
-      ? this.db.prepare(`SELECT * FROM predictions WHERE outcome_state = ? ORDER BY created_at DESC LIMIT ?`).all(normalizedState, safeLimit)
-      : this.db.prepare(`SELECT * FROM predictions ORDER BY created_at DESC LIMIT ?`).all(safeLimit);
-    return {
-      available: true,
-      predictions: rows.map(row => ({
-        stable_id: row.stable_id,
-        created_at: row.created_at,
-        updated_at: row.updated_at,
-        title: row.title,
-        type: row.type,
-        hypothesis: row.hypothesis,
-        confidence: row.confidence,
-        horizon: row.horizon,
-        outcome_state: row.outcome_state,
-        last_evaluated_at: row.last_evaluated_at,
-        source: row.source,
-        evidence: parseJson(row.evidence_json, []),
-        outcome: parseJson(row.outcome_json, null),
-      })),
-    };
-  }
-
-  evaluatePredictions(data, timestamp = new Date().toISOString()) {
-    if (!this.available || !this.db) return;
-    const rows = this.db.prepare(`
-      SELECT id, created_at, title, type, horizon, outcome_state, payload_json
-      FROM predictions
-      WHERE outcome_state IN ('open', 'monitoring')
-      ORDER BY created_at ASC
-      LIMIT 200
-    `).all();
-    for (const row of rows) {
-      const payload = parseJson(row.payload_json, {});
-      const evaluation = evaluatePredictionAgainstSweep(row, payload, data, timestamp);
-      this.db.prepare(`UPDATE predictions
-        SET outcome_state = ?, outcome_json = ?, last_evaluated_at = ?, updated_at = ?
-        WHERE id = ?`).run(
-        evaluation.state,
-        JSON.stringify(evaluation),
-        timestamp,
-        timestamp,
-        row.id,
-      );
-    }
-  }
-
-  _migrate() {
-    const columns = {
-      predictions: [
-        ['stable_id', 'TEXT'],
-        ['updated_at', 'TEXT'],
-        ['hypothesis', 'TEXT'],
-        ['evidence_json', 'TEXT'],
-        ['horizon', 'TEXT'],
-        ['outcome_state', "TEXT DEFAULT 'open'"],
-        ['outcome_json', 'TEXT'],
-        ['last_evaluated_at', 'TEXT'],
-      ],
-      entities: [
-        ['stable_id', 'TEXT'],
-      ],
-    };
-    for (const [table, defs] of Object.entries(columns)) {
-      for (const [name, type] of defs) {
-        try { this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${name} ${type}`); } catch { }
-      }
-    }
-    try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_predictions_stable_id ON predictions(stable_id)`); } catch { }
-    try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_entities_stable_id ON entities(stable_id)`); } catch { }
-  }
-
-  _recordPredictions(data, timestamp) {
     for (const idea of data.ideas || []) {
-      const title = idea.title || 'Untitled idea';
-      const stableId = stableId('prediction', title, idea.type || '', idea.ticker || '', idea.horizon || '');
-      const evidence = Array.isArray(idea.signals) ? idea.signals : [];
-      this.db.prepare(`INSERT INTO predictions (
-          stable_id, created_at, updated_at, title, type, hypothesis, evidence_json, confidence,
-          horizon, outcome_state, source, payload_json
-        )
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, ?)
-        ON CONFLICT(stable_id) DO UPDATE SET
-          updated_at=excluded.updated_at,
-          confidence=excluded.confidence,
-          evidence_json=excluded.evidence_json,
-          payload_json=excluded.payload_json`).run(
-        stableId,
+      this.db.prepare(`INSERT INTO predictions (created_at, title, type, confidence, source, payload_json)
+        VALUES (?, ?, ?, ?, ?, ?)`).run(
         timestamp,
-        timestamp,
-        title,
+        idea.title || 'Untitled idea',
         idea.type || null,
-        idea.rationale || idea.text || title,
-        JSON.stringify(evidence),
         idea.confidence || null,
-        idea.horizon || null,
         idea.source || data.ideasSource || null,
         JSON.stringify(idea),
       );
     }
+    this._recordEntities(data, timestamp);
+  }
+
+  status() {
+    return { available: this.available, path: this.dbPath, reason: this.reason };
   }
 
   _recordEntities(data, timestamp) {
@@ -242,154 +99,14 @@ export class IntelligenceStore {
       if (item.region) names.push([item.region, 'region']);
     }
     for (const [name, kind] of names.slice(0, 200)) {
-      const cleanName = String(name).slice(0, 160);
-      this.db.prepare(`INSERT INTO entities (stable_id, first_seen, last_seen, name, kind, count)
-        VALUES (?, ?, ?, ?, ?, 1)
+      this.db.prepare(`INSERT INTO entities (first_seen, last_seen, name, kind, count)
+        VALUES (?, ?, ?, ?, 1)
         ON CONFLICT(name, kind) DO UPDATE SET last_seen=excluded.last_seen, count=count+1`).run(
-        stableId('entity', kind, cleanName),
         timestamp,
         timestamp,
-        cleanName,
+        String(name).slice(0, 160),
         kind,
       );
     }
   }
-
-  _recordEvents(data, delta, timestamp) {
-    const events = extractEvents(data, delta);
-    for (const event of events.slice(0, 300)) {
-      this.db.prepare(`INSERT INTO events (
-          stable_id, first_seen, last_seen, kind, name, region, severity, source, evidence_json, count
-        )
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
-        ON CONFLICT(stable_id) DO UPDATE SET
-          last_seen=excluded.last_seen,
-          severity=COALESCE(excluded.severity, severity),
-          evidence_json=excluded.evidence_json,
-          count=count+1`).run(
-        event.stable_id,
-        timestamp,
-        timestamp,
-        event.kind,
-        event.name,
-        event.region || null,
-        event.severity || null,
-        event.source || null,
-        JSON.stringify(event.evidence || {}),
-      );
-    }
-  }
-}
-
-function stableId(...parts) {
-  const input = parts.map(part => String(part || '').trim().toLowerCase()).join('|');
-  return createHash('sha256').update(input).digest('hex').slice(0, 24);
-}
-
-function parseJson(value, fallback) {
-  try { return value ? JSON.parse(value) : fallback; } catch { return fallback; }
-}
-
-function extractEvents(data, delta) {
-  const events = [];
-  const push = ({ kind, name, region, severity, source, evidence }) => {
-    if (!kind || !name) return;
-    events.push({
-      stable_id: stableId('event', kind, name, region || source || ''),
-      kind,
-      name: String(name).slice(0, 240),
-      region: region ? String(region).slice(0, 120) : null,
-      severity: severity || null,
-      source: source || null,
-      evidence: evidence || {},
-    });
-  };
-
-  for (const item of data.acled?.deadliestEvents || []) {
-    push({
-      kind: 'conflict',
-      name: item.event_type || item.sub_event_type || item.location || item.country,
-      region: item.country || item.location,
-      severity: Number(item.fatalities || 0) > 0 ? 'high' : 'medium',
-      source: 'ACLED',
-      evidence: item,
-    });
-  }
-  for (const item of data.tg?.urgent || []) {
-    push({
-      kind: 'osint',
-      name: (item.text || '').slice(0, 120),
-      region: item.region || 'OSINT',
-      severity: 'high',
-      source: item.channel || item.chat || 'telegram',
-      evidence: item,
-    });
-  }
-  for (const item of data.newsFeed || data.news || []) {
-    if (!item.urgent) continue;
-    push({
-      kind: 'news',
-      name: item.headline || item.title,
-      region: item.region,
-      severity: 'medium',
-      source: item.source,
-      evidence: item,
-    });
-  }
-  for (const signal of delta?.signals?.new || []) {
-    push({
-      kind: 'delta',
-      name: signal.label || signal.reason || signal.key,
-      region: signal.region,
-      severity: signal.severity || 'medium',
-      source: 'delta',
-      evidence: signal,
-    });
-  }
-  return events;
-}
-
-function evaluatePredictionAgainstSweep(row, payload, data, timestamp) {
-  const terms = [
-    row.title,
-    payload.ticker,
-    ...(Array.isArray(payload.signals) ? payload.signals : []),
-  ].filter(Boolean).map(v => String(v).toLowerCase());
-  const evidenceText = [
-    ...(data.tSignals || []),
-    ...(data.newsFeed || []).slice(0, 40).map(n => `${n.source || ''} ${n.headline || n.title || ''}`),
-    ...(data.tg?.urgent || []).slice(0, 20).map(p => p.text || ''),
-  ].join('\n').toLowerCase();
-  const matched = terms.filter(term => term.length >= 4 && evidenceText.includes(term.slice(0, 60)));
-  const expired = predictionExpired(row.created_at, row.horizon, timestamp);
-  const state = matched.length
-    ? 'observed'
-    : expired
-      ? 'expired_unverified'
-      : 'monitoring';
-  return {
-    state,
-    evaluated_at: timestamp,
-    matched_terms: matched.slice(0, 10),
-    expired,
-    reason: matched.length
-      ? 'Current sweep contains matching evidence terms.'
-      : expired
-        ? 'Prediction horizon elapsed without matching evidence.'
-        : 'Prediction remains open for future sweeps.',
-  };
-}
-
-function predictionExpired(createdAt, horizon, nowIso) {
-  const created = new Date(createdAt).getTime();
-  const now = new Date(nowIso).getTime();
-  if (!Number.isFinite(created) || !Number.isFinite(now)) return false;
-  const text = String(horizon || '').toLowerCase();
-  const days = text.includes('intraday') ? 1
-    : text.includes('day') ? 7
-      : text.includes('week') ? 45
-        : text.includes('month') ? 180
-          : text.includes('strategic') ? 365
-            : 30;
-  return now - created > days * 24 * 60 * 60 * 1000;
 }

package.json

@@ -12,7 +12,7 @@
     "brief:save": "node apis/save-briefing.mjs",
     "diag": "node diag.mjs",
     "test": "npm run test:unit",
-    "test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs test/acled-source.test.mjs test/mojibake-text.test.mjs test/adsb.test.mjs test/dashboard-geotagging.test.mjs",
+    "test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs test/acled-source.test.mjs test/mojibake-text.test.mjs test/dashboard-geotagging.test.mjs",
     "compose:config": "docker compose config",
     "clean": "node scripts/clean.mjs",
     "fresh-start": "npm run clean && npm start"

server.mjs

@@ -41,7 +41,6 @@ let sweepStartedAt = null; // Timestamp when current/last sweep started
 let sweepInProgress = false;
 const startTime = Date.now();
 const sseClients = new Set();
-const terminalActionBuckets = new Map();
 const staleAlertState = {};
 
 // === Delta/Memory ===
@@ -292,67 +291,29 @@ app.get('/api/metrics', (req, res) => {
   });
 });
 
-app.get('/api/memory/search', (req, res) => {
-  const guard = authorizeTerminalAction(req, res, 'memory:search');
-  if (!guard.ok) return;
-  auditTerminalAction(req, 'memory:search', 'ok');
-  res.json(intelligenceStore.queryMemory({
-    q: req.query.q || '',
-    limit: req.query.limit || 25,
-  }));
-});
-
-app.get('/api/memory/predictions', (req, res) => {
-  const guard = authorizeTerminalAction(req, res, 'memory:predictions');
-  if (!guard.ok) return;
-  auditTerminalAction(req, 'memory:predictions', 'ok');
-  res.json(intelligenceStore.listPredictions({
-    state: req.query.state || null,
-    limit: req.query.limit || 25,
-  }));
-});
-
 app.post('/api/sweep', express.json(), (req, res) => {
-  const guard = authorizeTerminalAction(req, res, 'sweep');
-  if (!guard.ok) return;
-  triggerSweepAction(req, res, 'sweep');
+  if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
+  triggerSweep(res);
 });
 
-app.post('/api/action', express.json(), (req, res) => {
-  const action = String(req.body?.action || req.body?.command || '').trim().toLowerCase();
-  const guard = authorizeTerminalAction(req, res, action || 'unknown');
-  if (!guard.ok) return;
+app.post('/api/action', express.json(), async (req, res) => {
+  if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
+  const action = String(req.body?.action || req.query.action || '').toLowerCase();
 
   if (action === 'status') {
-    auditTerminalAction(req, 'status', 'ok');
-    return res.json({ ok: true, action, status: 'ok', health: buildHealth() });
+    return res.json({ ok: true, action, health: buildHealth() });
   }
 
   if (action === 'brief') {
-    if (!currentData) {
-      auditTerminalAction(req, 'brief', 'rejected', 'no_data');
-      return res.status(503).json({ ok: false, action, error: 'No data yet - first sweep in progress' });
-    }
-    auditTerminalAction(req, 'brief', 'ok');
-    const brief = buildBrief(currentData);
-    return res.json({ ok: true, action, status: 'ok', brief, text: brief });
+    if (!currentData) return res.status(503).json({ ok: false, action, error: 'No data yet — first sweep in progress' });
+    return res.json({ ok: true, action, text: buildBrief(currentData) });
   }
 
-  if (action === 'memory') {
-    auditTerminalAction(req, 'memory', 'ok');
-    return res.json({
-      ok: true,
-      action,
-      memory: intelligenceStore.status(),
-      recentEvents: intelligenceStore.queryMemory({ q: req.body?.q || '', limit: 8 }).results,
-      predictions: intelligenceStore.listPredictions({ limit: 8 }).predictions,
-    });
+  if (action === 'sweep') {
+    return triggerSweep(res);
   }
 
-  if (action === 'sweep') return triggerSweepAction(req, res, 'action:sweep');
-
-  auditTerminalAction(req, action || 'unknown', 'rejected', 'unknown_action');
-  return res.status(400).json({ ok: false, error: 'Unknown action', allowed: ['status', 'brief', 'memory', 'sweep'], actions: ['status', 'brief', 'memory', 'sweep'] });
+  res.status(400).json({ ok: false, error: 'Unknown action', actions: ['status', 'brief', 'sweep'] });
 });
 
 // API: available locales
@@ -370,10 +331,24 @@ app.get('/events', (req, res) => {
     'Cache-Control': 'no-cache',
     'Connection': 'keep-alive',
     'Access-Control-Allow-Origin': '*',
+    'X-Accel-Buffering': 'no',
   });
+  res.write('retry: 10000\n');
   res.write('data: {"type":"connected"}\n\n');
+  const heartbeatMs = Math.max(5000, config.sseHeartbeatIntervalMs || 25000);
+  const heartbeat = setInterval(() => {
+    try {
+      res.write(`: heartbeat ${new Date().toISOString()}\n\n`);
+    } catch {
+      clearInterval(heartbeat);
+      sseClients.delete(res);
+    }
+  }, heartbeatMs);
   sseClients.add(res);
-  req.on('close', () => sseClients.delete(res));
+  req.on('close', () => {
+    clearInterval(heartbeat);
+    sseClients.delete(res);
+  });
 });
 
 function broadcast(data) {
@@ -383,114 +358,26 @@ function broadcast(data) {
   }
 }
 
-function requestIp(req) {
-  return req.ip || req.socket?.remoteAddress || 'unknown';
-}
-
-function isLocalRequest(req) {
-  const remote = requestIp(req);
-  return remote === '::1'
-    || remote === '127.0.0.1'
-    || remote === '::ffff:127.0.0.1'
-    || remote.startsWith('127.')
-    || remote === 'localhost';
-}
-
-function sameOriginPost(req) {
-  const origin = req.get('origin');
-  if (!origin) return true;
-  try {
-    const originUrl = new URL(origin);
-    const host = req.get('host');
-    return host && originUrl.host === host;
-  } catch {
-    return false;
-  }
-}
-
-function actionToken(req) {
-  return req.get('x-crucix-token') || req.body?.token || null;
-}
-
-function auditTerminalAction(req, action, outcome, detail = null) {
-  const suffix = detail ? ` detail=${detail}` : '';
-  console.log(`[Crucix][audit] terminal_action action=${action || 'unknown'} outcome=${outcome} ip=${requestIp(req)}${suffix}`);
-}
-
-function rateLimitTerminalAction(req, action) {
-  const now = Date.now();
-  const windowMs = Math.max(1000, config.terminalActionRateLimitWindowMs || 60_000);
-  const max = Math.max(1, config.terminalActionRateLimitMax || 10);
-  const key = `${requestIp(req)}:${action}`;
-  const bucket = terminalActionBuckets.get(key);
-  if (!bucket || now > bucket.resetAt) {
-    terminalActionBuckets.set(key, { count: 1, resetAt: now + windowMs });
-    return { ok: true };
-  }
-  bucket.count += 1;
-  if (bucket.count > max) {
-    return { ok: false, retryAfterSeconds: Math.ceil((bucket.resetAt - now) / 1000) };
-  }
-  return { ok: true };
-}
-
-function authorizeTerminalAction(req, res, action) {
-  const rate = rateLimitTerminalAction(req, action);
-  if (!rate.ok) {
-    auditTerminalAction(req, action, 'rejected', 'rate_limited');
-    res.set('Retry-After', String(rate.retryAfterSeconds));
-    res.status(429).json({ error: 'Too many terminal actions', retryAfterSeconds: rate.retryAfterSeconds });
-    return { ok: false };
-  }
-
-  if (!sameOriginPost(req)) {
-    auditTerminalAction(req, action, 'rejected', 'csrf_origin');
-    res.status(403).json({ error: 'Origin mismatch' });
-    return { ok: false };
-  }
-
-  const local = isLocalRequest(req);
-  const token = actionToken(req);
-  if (!config.terminalActionsEnabled) {
-    auditTerminalAction(req, action, 'rejected', 'disabled');
-    res.status(403).json({ error: 'Terminal actions are disabled' });
-    return { ok: false };
-  }
-
-  if (config.sweepToken) {
-    if (token !== config.sweepToken) {
-      auditTerminalAction(req, action, 'rejected', 'invalid_token');
-      res.status(401).json({ error: 'Invalid terminal action token' });
-      return { ok: false };
-    }
-    return { ok: true };
-  }
-
-  if (!local) {
-    auditTerminalAction(req, action, 'rejected', 'missing_token');
-    res.status(403).json({ error: 'Terminal actions are local-only unless SWEEP_TOKEN is set' });
-    return { ok: false };
-  }
-
-  return { ok: true };
-}
-
-function triggerSweepAction(req, res, auditAction) {
-  if (sweepInProgress) {
-    auditTerminalAction(req, auditAction, 'rejected', 'already_running');
-    return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
-  }
-  auditTerminalAction(req, auditAction, 'accepted');
-  runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
-  return res.status(202).json({ ok: true, status: 'accepted' });
-}
-
 function dataAgeMs() {
   const ts = currentData?.meta?.timestamp || lastSuccessfulSweepTime || lastSweepTime;
   const ms = ts ? Date.now() - new Date(ts).getTime() : null;
   return Number.isFinite(ms) ? ms : null;
 }
 
+function canRunTerminalAction(req) {
+  const remote = req.ip || '';
+  const local = remote.includes('127.0.0.1') || remote === '::1' || remote === '::ffff:127.0.0.1';
+  const token = req.get('x-crucix-token') || req.query.token || req.body?.token;
+  if (config.sweepToken) return token === config.sweepToken;
+  return Boolean(config.terminalActionsEnabled || local);
+}
+
+function triggerSweep(res) {
+  if (sweepInProgress) return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
+  runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
+  return res.status(202).json({ ok: true, status: 'accepted' });
+}
+
 function getLLMStatus() {
   if (!config.llm.provider) return { state: 'disabled' };
   if (!llmProvider) return { state: 'misconfigured', provider: config.llm.provider };
@@ -534,8 +421,7 @@ function buildHealth() {
     llm: getLLMStatus(),
     telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId),
     discordEnabled: !!(config.discord?.botToken || config.discord?.webhookUrl),
-    terminalActionsEnabled: config.terminalActionsEnabled,
-    terminalActionsTokenRequired: !!config.sweepToken,
+    terminalActionsEnabled: Boolean(config.terminalActionsEnabled || config.sweepToken),
     refreshIntervalMinutes: config.refreshIntervalMinutes,
     language: currentLanguage,
     memory: intelligenceStore.status(),

test/adsb.test.mjs

@@ -1,82 +0,0 @@
-import test from 'node:test';
-import assert from 'node:assert/strict';
-
-async function withFetch(mockFetch, fn) {
-  const originalFetch = globalThis.fetch;
-  const originalAdsbKey = process.env.ADSB_API_KEY;
-  const originalRapidKey = process.env.RAPIDAPI_KEY;
-  globalThis.fetch = mockFetch;
-  delete process.env.ADSB_API_KEY;
-  delete process.env.RAPIDAPI_KEY;
-  try {
-    return await fn();
-  } finally {
-    globalThis.fetch = originalFetch;
-    if (originalAdsbKey === undefined) delete process.env.ADSB_API_KEY;
-    else process.env.ADSB_API_KEY = originalAdsbKey;
-    if (originalRapidKey === undefined) delete process.env.RAPIDAPI_KEY;
-    else process.env.RAPIDAPI_KEY = originalRapidKey;
-  }
-}
-
-function jsonResponse(payload, ok = true, status = 200) {
-  return {
-    ok,
-    status,
-    headers: { get: () => 'application/json' },
-    text: async () => JSON.stringify(payload),
-  };
-}
-
-test('ADS-B reports disabled when no key is configured and public feed fails', async () => {
-  await withFetch(async () => jsonResponse({ error: 'blocked' }, false, 403), async () => {
-    const { briefing } = await import('../apis/sources/adsb.mjs');
-    const data = await briefing();
-
-    assert.equal(data.status, 'disabled');
-    assert.match(data.error, /ADSB_API_KEY|RAPIDAPI_KEY/);
-    assert.equal(data.militaryAircraft.length, 0);
-  });
-});
-
-test('ADS-B reports degraded when RapidAPI and public feed fail', async () => {
-  await withFetch(async () => jsonResponse({ error: 'unavailable' }, false, 503), async () => {
-    process.env.ADSB_API_KEY = 'test-key';
-    const { briefing } = await import('../apis/sources/adsb.mjs');
-    const data = await briefing();
-
-    assert.equal(data.status, 'degraded');
-    assert.match(data.error, /providers returned no usable/);
-    assert.equal(data.failures.length, 2);
-  });
-});
-
-test('ADS-B returns live RapidAPI military aircraft payloads', async () => {
-  await withFetch(async () => jsonResponse({
-    ac: [{
-      hex: 'AE1234',
-      flight: 'RCH123',
-      t: 'KC135',
-      lat: 50,
-      lon: 8,
-      mil: true,
-    }],
-  }), async () => {
-    process.env.ADSB_API_KEY = 'test-key';
-    const { briefing } = await import('../apis/sources/adsb.mjs');
-    const data = await briefing();
-
-    assert.equal(data.status, 'live');
-    assert.equal(data.provider, 'rapidapi');
-    assert.equal(data.totalMilitary, 1);
-    assert.equal(data.militaryAircraft[0].callsign, 'RCH123');
-  });
-});
-
-test('runSource treats disabled source status as degraded health', async () => {
-  const { runSource } = await import('../apis/briefing.mjs');
-  const result = await runSource('ADS-B', async () => ({ status: 'disabled', message: 'missing key' }));
-
-  assert.equal(result.status, 'degraded');
-  assert.equal(result.error, null);
-});

test/fetch-utils.test.mjs

@@ -101,62 +101,15 @@ test('safeFetchText returns text and byte count', async () => {
   }
 });
 
-test('intelligence store defines durable memory and prediction lifecycle tables', () => {
-  const store = readFileSync(new URL('../lib/intelligence-store.mjs', import.meta.url), 'utf8');
-  assert.match(store, /CREATE TABLE IF NOT EXISTS events/);
-  assert.match(store, /stable_id TEXT NOT NULL UNIQUE/);
-  assert.match(store, /hypothesis TEXT/);
-  assert.match(store, /evidence_json TEXT/);
-  assert.match(store, /outcome_state TEXT DEFAULT 'open'/);
-  assert.match(store, /evaluatePredictions/);
-  assert.match(store, /queryMemory/);
-  assert.match(store, /listPredictions/);
-});
-
-test('server exposes memory-backed query APIs and dashboard memory action', () => {
+test('SSE endpoint sends reconnect guidance and clears heartbeat timer', () => {
   const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
-  const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
-  assert.match(server, /\/api\/memory\/search/);
-  assert.match(server, /\/api\/memory\/predictions/);
-  assert.match(server, /action === 'memory'/);
-  assert.match(html, /runTerminalAction\('memory'\)/);
-});
-
-test('terminal action endpoints avoid URL tokens and include hardening gates', () => {
-  const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
-  assert.match(server, /app\.post\('\/api\/action'/);
-  assert.match(server, /app\.post\('\/api\/sweep'/);
-  assert.match(server, /x-crucix-token/);
-  assert.match(server, /sameOriginPost/);
-  assert.match(server, /rateLimitTerminalAction/);
-  assert.match(server, /auditTerminalAction/);
-  assert.doesNotMatch(server, /req\.query\.token/);
-});
-
-test('dashboard exposes token configuration flow without devtools edits', () => {
-  const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
-  assert.match(html, /configureTerminalActionToken/);
-  assert.match(html, /crucix_sweep_token/);
-  assert.match(html, /x-crucix-token/);
-  assert.match(html, /SET TOKEN/);
-});
-
-test('server dashboard shell does not embed an operational snapshot', () => {
-  const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
-  assert.match(html, /let D = createDashboardShellData\(\);/);
-  assert.doesNotMatch(html, /2026-04-03T16:18:10\.188Z/);
-  assert.doesNotMatch(html, /Trump announced new strikes on Iran/);
-});
-
-test('server dashboard fetches api data before initialization', () => {
-  const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
-  const serverMode = html.indexOf('if (canProbeApi)');
-  const apiFetch = html.indexOf("fetch('/api/data')");
-  const firstInitAfterServerMode = html.indexOf('init();', serverMode);
-
-  assert.ok(serverMode > -1);
-  assert.ok(apiFetch > serverMode);
-  assert.ok(firstInitAfterServerMode > apiFetch);
+  const config = readFileSync(new URL('../crucix.config.mjs', import.meta.url), 'utf8');
+  assert.match(config, /sseHeartbeatIntervalMs/);
+  assert.match(server, /retry: 10000\\n/);
+  assert.match(server, /setInterval\(\(\) =>/);
+  assert.match(server, /: heartbeat/);
+  assert.match(server, /clearInterval\(heartbeat\)/);
+  assert.match(server, /X-Accel-Buffering/);
 });
 
 test('stale alert is skipped for fresh health and resets active key', () => {