Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-22-source-fetch-instrumentation

# Conflicts:
#	test/fetch-utils.test.mjs

.env.example

@@ -6,8 +6,13 @@ PORT=3117
 REFRESH_INTERVAL_MINUTES=15
 AUTO_OPEN_BROWSER=false
 STALE_DATA_MAX_AGE_MINUTES=60
+STALE_ALERT_COOLDOWN_MINUTES=60
+DASHBOARD_URL=
 TERMINAL_ACTIONS_ENABLED=true
 SWEEP_TOKEN=
+SSE_HEARTBEAT_INTERVAL_MS=25000
+TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
+TERMINAL_ACTION_RATE_LIMIT_MAX=10
 BRIEF_VERBOSITY=standard
 
 # LLM layer

README.md

@@ -130,8 +130,13 @@ PORT=3117
 REFRESH_INTERVAL_MINUTES=15
 AUTO_OPEN_BROWSER=false
 STALE_DATA_MAX_AGE_MINUTES=60
+STALE_ALERT_COOLDOWN_MINUTES=60
+DASHBOARD_URL=https://intelligence.example.internal
 TERMINAL_ACTIONS_ENABLED=true
 SWEEP_TOKEN=
+SSE_HEARTBEAT_INTERVAL_MS=25000
+TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
+TERMINAL_ACTION_RATE_LIMIT_MAX=10
 BRIEF_VERBOSITY=standard
 
 LLM_PROVIDER=openrouter
@@ -183,7 +188,68 @@ LLM_MODEL=your-model
 
 For Pangolin or another reverse proxy, forward HTTP traffic to `intelligence-terminal:3117` (or the `PORT` you set). Missing API keys do not crash sweeps; affected sources are reported as degraded in `/api/health`.
 
-The dashboard Terminal Actions panel can trigger `status`, `sweep`, and `brief` through `/api/action`. Leave `TERMINAL_ACTIONS_ENABLED=true` for a private home-server deployment. For an internet-exposed deployment, set `SWEEP_TOKEN` and pass it through trusted automation, or set `TERMINAL_ACTIONS_ENABLED=false` to disable browser-triggered actions. If you protect actions with `SWEEP_TOKEN`, the browser can send it from `localStorage.crucix_sweep_token`.
+#### Terminal Action Exposure
+
+`POST /api/action` and `POST /api/sweep` can trigger operational actions such as manual sweeps. The dashboard has a **SET TOKEN** control that stores your `SWEEP_TOKEN` in browser local storage and sends it as the `x-crucix-token` header; do not put action tokens in URLs.
+
+Recommended settings:
+
+| Deployment | Settings |
+| --- | --- |
+| Private local machine | `NODE_ENV=development`, optional `SWEEP_TOKEN`, optional `TERMINAL_ACTIONS_ENABLED=true`. Localhost can run actions without a token for development. |
+| Private LAN / Dockge | Set a strong `SWEEP_TOKEN`, keep `TERMINAL_ACTIONS_ENABLED=true`, expose only to trusted clients. |
+| Pangolin-authenticated reverse proxy | Set a strong `SWEEP_TOKEN`, keep Pangolin auth in front, use the dashboard **SET TOKEN** flow once per browser. |
+| Public internet | Do not expose Terminal Actions directly. If exposure is unavoidable, require `SWEEP_TOKEN`, keep proxy authentication enabled, lower `TERMINAL_ACTION_RATE_LIMIT_MAX`, and monitor server audit logs. |
+
+Action endpoints reject cross-origin POST origins, apply a small in-memory per-IP rate limit, and write sanitized audit lines without logging the token.
+
+When data remains stale past `STALE_DATA_MAX_AGE_MINUTES`, the server sends an operator alert through configured Telegram/Discord channels after failed or degraded sweep attempts. `STALE_ALERT_COOLDOWN_MINUTES` prevents repeated stale alerts from spamming every refresh interval. Set `DASHBOARD_URL` to the Pangolin/public URL you want included in those alerts.
+
+#### Memory And Prediction Loop
+
+Crucix stores longitudinal memory in `runs/intelligence.db` when the current Node.js build exposes `node:sqlite`. If SQLite is unavailable, the file is created as a harmless placeholder and `/api/health` reports the memory store as unavailable instead of failing the sweep.
+
+The memory layer persists:
+
+| Table | Purpose |
+| --- | --- |
+| `runs` | Sweep timestamps, source health counts, and delta direction summaries. |
+| `entities` | Stable entity IDs for recurring countries, regions, and locations. |
+| `events` | Stable event IDs for conflict, OSINT, urgent news, and new delta signals across sweeps. |
+| `predictions` | Trade/intelligence hypotheses with evidence, confidence, horizon, outcome state, and latest grading. |
+
+Query endpoints:
+
+```text
+GET /api/memory/search?q=iran&limit=25
+GET /api/memory/predictions?state=open&limit=25
+```
+
+Memory endpoints use the same operator authorization gate as Terminal Actions. The dashboard Terminal Actions panel includes a `Memory` action for a quick operator-facing view of recent events and prediction states.
+
+Retention, backup, and privacy expectations:
+
+- Treat `runs/intelligence.db` as operator data. It can contain source excerpts, headlines, generated hypotheses, and URLs from your configured feeds.
+- Back up `runs/` with the rest of your Dockge volume if you want longitudinal learning to survive container replacement.
+- Delete `runs/intelligence.db` to reset SQLite memory; the next sweep recreates the schema.
+- Do not commit `runs/` or `.env`. API credentials stay in `.env`; memory stores derived observations, not secrets.
+- If you expose the dashboard through a reverse proxy, protect Terminal Actions and memory queries behind your normal authentication boundary.
+
+#### Reverse Proxy SSE
+
+The dashboard receives live sweep updates from `GET /events` using Server-Sent Events. The server sends `retry: 10000` reconnect guidance and lightweight heartbeat comments every `SSE_HEARTBEAT_INTERVAL_MS` milliseconds so reverse proxies do not close an otherwise idle stream between 15-minute sweeps.
+
+Recommended proxy settings:
+
+| Proxy | Setting |
+| --- | --- |
+| Pangolin / Traefik-style frontends | Keep response streaming enabled and set idle timeouts above `SSE_HEARTBEAT_INTERVAL_MS`. |
+| Nginx | Disable proxy buffering for `/events`, keep `proxy_read_timeout` above the heartbeat interval, and preserve `Connection: keep-alive`. |
+| Cloudflare-style proxies | Keep the heartbeat below common idle cutoffs; the default 25s is intentionally conservative. |
+
+If you raise the heartbeat interval, keep it shorter than the lowest idle timeout in the proxy chain.
+
+`/api/metrics` includes network health grouped by host and source/provider. Source modules should use `safeFetch(url, { source: 'SourceName' })`; when omitted, the shared helper infers a stable provider bucket from the URL host instead of grouping normal source traffic under `unknown`. Raw fetch exceptions are documented in [Source Fetch Instrumentation](docs/source-fetch-instrumentation.md).
 
 #### Scenario Watchlist
 
@@ -556,6 +622,9 @@ All settings are in `.env` with sensible defaults:
 |----------|---------|-------------|
 | `PORT` | `3117` | Dashboard server port |
 | `REFRESH_INTERVAL_MINUTES` | `15` | Auto-refresh interval |
+| `STALE_DATA_MAX_AGE_MINUTES` | `60` | Data age threshold for stale health state |
+| `STALE_ALERT_COOLDOWN_MINUTES` | `60` | Minimum time between repeated operator stale-data alerts |
+| `DASHBOARD_URL` | local URL | Dashboard URL included in operator alerts |
 | `LLM_PROVIDER` | disabled | `anthropic`, `openai`, `gemini`, `codex`, `openrouter`, `minimax`, `mistral`, or `grok` |
 | `LLM_API_KEY` | — | API key (not needed for codex) |
 | `LLM_MODEL` | per-provider default | Override model selection |

apis/briefing.mjs

@@ -59,7 +59,8 @@ export async function runSource(name, fn, ...args) {
     });
     const data = await Promise.race([dataPromise, timeoutPromise]);
     const hasError = Boolean(data?.error);
-    const isDegraded = hasError || ['no_credentials', 'degraded', 'failed'].includes(data?.status);
+    const degradedStatuses = ['no_credentials', 'no_key', 'disabled', 'degraded', 'failed', 'error'];
+    const isDegraded = hasError || degradedStatuses.includes(data?.status);
     return {
       name,
       status: isDegraded ? 'degraded' : 'ok',

apis/sources/adsb.mjs

@@ -1,7 +1,8 @@
 // ADS-B Exchange — Unfiltered Flight Tracking (including Military)
 // Unlike FlightRadar24/FlightAware, ADS-B Exchange does NOT filter military aircraft.
 // Public feed access varies; RapidAPI tier available for programmatic use.
-// This module attempts the public endpoints and falls back to a documented stub.
+// This module reports explicit disabled/degraded state instead of making
+// unavailable aircraft data look live.
 
 import { safeFetch } from '../utils/fetch.mjs';
 
@@ -140,6 +141,7 @@ async function fetchViaRapidApi(apiKey) {
   // Get all military aircraft
   const data = await safeFetch(`${ENDPOINTS.rapidApi}/mil`, {
     timeout: 20000,
+    source: 'adsb-rapidapi',
     headers: {
       'X-RapidAPI-Key': apiKey,
       'X-RapidAPI-Host': 'adsbexchange-com1.p.rapidapi.com',
@@ -151,21 +153,26 @@ async function fetchViaRapidApi(apiKey) {
 
 // Attempt to fetch from public feed
 async function fetchPublicFeed() {
-  const data = await safeFetch(ENDPOINTS.publicFeed, { timeout: 15000 });
+  const data = await safeFetch(ENDPOINTS.publicFeed, { timeout: 15000, source: 'adsb-public' });
   return data;
 }
 
-// Get military aircraft from available sources
-export async function getMilitaryAircraft(apiKey) {
+async function getMilitaryAircraftResult(apiKey) {
+  const failures = [];
+
   // Try RapidAPI first if key available
   if (apiKey) {
     const data = await fetchViaRapidApi(apiKey);
     if (data && !data.error) {
       const aircraft = data.ac || data.aircraft || [];
       if (Array.isArray(aircraft)) {
-        return aircraft.map(classifyAircraft).filter(a => a.isMilitary);
+        return {
+          provider: 'rapidapi',
+          aircraft: aircraft.map(classifyAircraft).filter(a => a.isMilitary),
+        };
       }
     }
+    failures.push({ provider: 'rapidapi', error: data?.error || 'RapidAPI returned an unsupported payload' });
   }
 
   // Try public feed
@@ -173,11 +180,21 @@ export async function getMilitaryAircraft(apiKey) {
   if (pubData && !pubData.error) {
     const aircraft = pubData.ac || pubData.aircraft || pubData.states || [];
     if (Array.isArray(aircraft)) {
-      return aircraft.map(classifyAircraft).filter(a => a.isMilitary);
+      return {
+        provider: 'public-feed',
+        aircraft: aircraft.map(classifyAircraft).filter(a => a.isMilitary),
+      };
     }
   }
+  failures.push({ provider: 'public-feed', error: pubData?.error || 'Public feed returned an unsupported payload' });
 
-  return null; // all sources failed
+  return { provider: null, aircraft: null, failures };
+}
+
+// Get military aircraft from available sources
+export async function getMilitaryAircraft(apiKey) {
+  const result = await getMilitaryAircraftResult(apiKey);
+  return result.aircraft;
 }
 
 // Get all aircraft in a geographic bounding box via RapidAPI
@@ -208,7 +225,8 @@ export async function getAircraftInArea(lat, lon, radiusNm = 250, apiKey) {
 // Briefing — attempt to get military flight data, document what's available
 export async function briefing() {
   const apiKey = process.env.ADSB_API_KEY || process.env.RAPIDAPI_KEY || null;
-  const militaryAircraft = await getMilitaryAircraft(apiKey);
+  const result = await getMilitaryAircraftResult(apiKey);
+  const militaryAircraft = result.aircraft;
 
   // If we got data, analyze it
   if (militaryAircraft && militaryAircraft.length > 0) {
@@ -255,6 +273,7 @@ export async function briefing() {
       source: 'ADS-B Exchange',
       timestamp: new Date().toISOString(),
       status: 'live',
+      provider: result.provider,
       totalMilitary: militaryAircraft.length,
       byCountry,
       categories: {
@@ -269,10 +288,18 @@ export async function briefing() {
   }
 
   // No data available — return stub with integration documentation
+  const status = apiKey ? 'degraded' : 'disabled';
+  const error = apiKey
+    ? 'ADS-B providers returned no usable aircraft data'
+    : 'ADSB_API_KEY or RAPIDAPI_KEY is not configured';
+
   return {
     source: 'ADS-B Exchange',
     timestamp: new Date().toISOString(),
-    status: apiKey ? 'error' : 'no_key',
+    status,
+    provider: result.provider,
+    error,
+    failures: result.failures,
     militaryAircraft: [],
     message: apiKey
       ? 'ADS-B Exchange API returned no data. The endpoint may be temporarily unavailable.'

apis/utils/fetch.mjs

@@ -10,6 +10,44 @@ const fetchMetrics = {
   recent: [],
 };
 
+const SOURCE_BY_HOST = [
+  [/api\.bls\.gov$/i, 'BLS'],
+  [/api\.fred\.stlouisfed\.org$/i, 'FRED'],
+  [/api\.eia\.gov$/i, 'EIA'],
+  [/api\.gdeltproject\.org$/i, 'GDELT'],
+  [/api\.weather\.gov$/i, 'NOAA'],
+  [/api\.open-notify\.org$/i, 'OpenNotify'],
+  [/opensky-network\.org$/i, 'OpenSky'],
+  [/firms\.modaps\.eosdis\.nasa\.gov$/i, 'FIRMS'],
+  [/api\.acleddata\.com$/i, 'ACLED'],
+  [/api\.reliefweb\.int$/i, 'ReliefWeb'],
+  [/receiverbook\.de$/i, 'KiwiSDR'],
+  [/safecast\.org$/i, 'Safecast'],
+  [/api\.patentsview\.org$/i, 'PatentsView'],
+  [/api\.trade\.gov$/i, 'Comtrade'],
+  [/api\.usaspending\.gov$/i, 'USASpending'],
+  [/api\.telegram\.org$/i, 'Telegram'],
+  [/oauth\.reddit\.com$/i, 'Reddit'],
+  [/reddit\.com$/i, 'Reddit'],
+  [/api\.bsky\.app$/i, 'Bluesky'],
+  [/api\.yahoo\.com$/i, 'YahooFinance'],
+  [/query\d?\.finance\.yahoo\.com$/i, 'YahooFinance'],
+  [/api\.cloudflare\.com$/i, 'CloudflareRadar'],
+  [/api\.opensanctions\.org$/i, 'OpenSanctions'],
+  [/home\.treasury\.gov$/i, 'Treasury'],
+  [/fiscaldata\.treasury\.gov$/i, 'Treasury'],
+  [/who\.int$/i, 'WHO'],
+];
+
+export function inferFetchSource(url) {
+  let host = 'unknown';
+  try { host = new URL(url).host.toLowerCase(); } catch { return 'unknown'; }
+  for (const [pattern, source] of SOURCE_BY_HOST) {
+    if (pattern.test(host)) return source;
+  }
+  return host;
+}
+
 function metricBucket(map, key) {
   if (!map[key]) map[key] = { requests: 0, ok: 0, failed: 0, bytes: 0, lastStatus: null, lastError: null, lastMs: 0 };
   return map[key];
@@ -38,7 +76,7 @@ export function getFetchMetrics() {
 }
 
 export async function safeFetch(url, opts = {}) {
-  const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts;
+  const { timeout = 15000, retries = 1, headers = {}, source = inferFetchSource(url) } = opts;
   let lastError;
   for (let i = 0; i <= retries; i++) {
     const started = Date.now();
@@ -79,11 +117,11 @@ export async function safeFetch(url, opts = {}) {
       if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
     }
   }
-  return { error: lastError?.message || 'Unknown error', source: url };
+  return { error: lastError?.message || 'Unknown error', source };
 }
 
 export async function safeFetchText(url, opts = {}) {
-  const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts;
+  const { timeout = 15000, retries = 1, headers = {}, source = inferFetchSource(url) } = opts;
   let lastError;
   for (let i = 0; i <= retries; i++) {
     const started = Date.now();

crucix.config.mjs

@@ -23,8 +23,13 @@ export default {
   refreshIntervalMinutes: intEnv('REFRESH_INTERVAL_MINUTES', 15),
   autoOpenBrowser: boolEnv('AUTO_OPEN_BROWSER', false),
   staleDataMaxAgeMinutes: intEnv('STALE_DATA_MAX_AGE_MINUTES', 60),
+  staleAlertCooldownMinutes: intEnv('STALE_ALERT_COOLDOWN_MINUTES', 60),
+  dashboardUrl: process.env.DASHBOARD_URL || null,
   sweepToken: process.env.SWEEP_TOKEN || null,
-  terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', true),
+  terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', !!process.env.SWEEP_TOKEN || process.env.NODE_ENV !== 'production'),
+  terminalActionRateLimitWindowMs: intEnv('TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS', 60_000),
+  terminalActionRateLimitMax: intEnv('TERMINAL_ACTION_RATE_LIMIT_MAX', 10),
+  sseHeartbeatIntervalMs: intEnv('SSE_HEARTBEAT_INTERVAL_MS', 25000),
 
   llm: {
     provider: process.env.LLM_PROVIDER || null, // anthropic | openai | gemini | codex | openrouter | minimax | mistral | ollama | grok

dashboard/inject.mjs

@@ -83,16 +83,48 @@ const geoKeywords = {
   'IMF':[38.9,-77],'World Bank':[38.9,-77],'UN':[40.7,-74],
 };
 
-function geoTagText(text) {
+function escapeRegex(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function geoKeywordRegex(keyword) {
+  const flags = keyword.length <= 3 && keyword === keyword.toUpperCase() ? 'u' : 'iu';
+  return new RegExp(`(^|[^\\p{L}\\p{N}])${escapeRegex(keyword)}(?=$|[^\\p{L}\\p{N}])`, flags);
+}
+
+const geoKeywordEntries = Object.entries(geoKeywords)
+  .sort((a, b) => b[0].length - a[0].length)
+  .map(([keyword, coords]) => ({ keyword, coords, pattern: geoKeywordRegex(keyword) }));
+
+export function geoTagText(text) {
   if (!text) return null;
-  for (const [keyword, [lat, lon]] of Object.entries(geoKeywords)) {
-    if (text.includes(keyword)) {
+  for (const { keyword, coords, pattern } of geoKeywordEntries) {
+    if (pattern.test(text)) {
+      const [lat, lon] = coords;
       return { lat, lon, region: keyword };
     }
   }
   return null;
 }
 
+function stableHash(value) {
+  let hash = 2166136261;
+  for (let i = 0; i < value.length; i++) {
+    hash ^= value.charCodeAt(i);
+    hash = Math.imul(hash, 16777619);
+  }
+  return hash >>> 0;
+}
+
+export function stableGeoJitter(key, axis) {
+  const bucket = stableHash(`${axis}:${key}`) / 0xffffffff;
+  return (bucket - 0.5) * 2;
+}
+
+function newsGeoKey(item) {
+  return `${item.source || ''}|${item.title || ''}|${item.date || ''}|${item.url || ''}`;
+}
+
 function sanitizeExternalUrl(raw) {
   if (!raw) return undefined;
   try {
@@ -235,8 +267,8 @@ export async function fetchAllNews() {
         source: item.source,
         date: item.date,
         url: item.url,
-        lat: geo.lat + (Math.random() - 0.5) * 2,
-        lon: geo.lon + (Math.random() - 0.5) * 2,
+        lat: geo.lat + stableGeoJitter(newsGeoKey(item), 'lat'),
+        lon: geo.lon + stableGeoJitter(newsGeoKey(item), 'lon'),
         region: geo.region
       });
     }

docs/source-fetch-instrumentation.md

@@ -0,0 +1,21 @@
+# Source Fetch Instrumentation
+
+`safeFetch()` and `safeFetchText()` attribute requests to `/api/metrics.fetch.bySource`.
+
+Rules:
+
+- Prefer passing an explicit `source` option from source modules when the call has a clear Crucix source name.
+- If `source` is omitted, the shared helper infers a stable provider name from the request host.
+- Unknown hosts fall back to the lowercase host instead of the old `unknown` bucket.
+- Raw `fetch()` calls should be limited to cases where the shared helper cannot represent the protocol cleanly.
+
+Current raw-fetch exceptions:
+
+| Area | Reason |
+| --- | --- |
+| OAuth/session handshakes | Token exchange calls often need custom form bodies, credential headers, or status-specific diagnostics. |
+| Bot and alert delivery | Telegram/Discord alert calls are outbound operator notifications, not intelligence source health. |
+| LLM providers | Provider clients already track model/provider status separately from source fetch health. |
+| Dashboard browser calls | Browser-side `/api/*` and asset fetches are UI behavior, not source provider health. |
+
+When adding a new intelligence source, use `safeFetch(url, { source: 'SourceName' })` unless there is a documented exception.

docs/sources/README.md

@@ -16,4 +16,5 @@ Source docs:
 - [Telegram](telegram.md)
 - [FIRMS](firms.md)
 - [Maritime](maritime.md)
+- [ADS-B](adsb.md)
 - [Reddit](reddit.md)

docs/sources/adsb.md

@@ -0,0 +1,24 @@
+# ADS-B Source
+
+ADS-B Exchange support is optional and intended for unfiltered aircraft and military-flight awareness.
+
+- Source module: `apis/sources/adsb.mjs`
+- Preferred provider: ADS-B Exchange via RapidAPI
+- Credentials: `ADSB_API_KEY` or `RAPIDAPI_KEY`
+- Runtime status without credentials: `disabled`
+- Runtime status when providers fail: `degraded`
+- Runtime status with usable aircraft payloads: `live`
+
+The source does not treat a missing key or unavailable public feed as normal live data. `/api/health` and `/api/metrics` surface the degraded source state through the sweep source summary.
+
+Known failure modes:
+
+- Missing `ADSB_API_KEY` / `RAPIDAPI_KEY`: source is disabled with operator guidance.
+- RapidAPI rejects or rate-limits the request: source is degraded and records provider failure detail.
+- Public feed is blocked, rate-limited, or changes shape: source remains degraded instead of returning stale-looking data.
+
+Register for the provider documented in the README, then set:
+
+```env
+ADSB_API_KEY=<rapidapi-key>
+```

lib/intelligence-store.mjs

@@ -2,6 +2,9 @@
 
 import { existsSync, mkdirSync, writeFileSync } from 'fs';
 import { dirname, join } from 'path';
+import { createHash } from 'crypto';
+
+const PREDICTION_STATES = new Set(['open', 'monitoring', 'observed', 'expired_unverified', 'invalidated']);
 
 export class IntelligenceStore {
   constructor(dbPath) {
@@ -30,15 +33,24 @@ export class IntelligenceStore {
         );
         CREATE TABLE IF NOT EXISTS predictions (
           id INTEGER PRIMARY KEY AUTOINCREMENT,
+          stable_id TEXT UNIQUE,
           created_at TEXT NOT NULL,
+          updated_at TEXT,
           title TEXT NOT NULL,
           type TEXT,
+          hypothesis TEXT,
+          evidence_json TEXT,
           confidence TEXT,
+          horizon TEXT,
+          outcome_state TEXT DEFAULT 'open',
+          outcome_json TEXT,
+          last_evaluated_at TEXT,
           source TEXT,
           payload_json TEXT NOT NULL
         );
         CREATE TABLE IF NOT EXISTS entities (
           id INTEGER PRIMARY KEY AUTOINCREMENT,
+          stable_id TEXT UNIQUE,
           first_seen TEXT NOT NULL,
           last_seen TEXT NOT NULL,
           name TEXT NOT NULL,
@@ -46,7 +58,21 @@ export class IntelligenceStore {
           count INTEGER DEFAULT 1,
           UNIQUE(name, kind)
         );
+        CREATE TABLE IF NOT EXISTS events (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          stable_id TEXT NOT NULL UNIQUE,
+          first_seen TEXT NOT NULL,
+          last_seen TEXT NOT NULL,
+          kind TEXT NOT NULL,
+          name TEXT NOT NULL,
+          region TEXT,
+          severity TEXT,
+          source TEXT,
+          evidence_json TEXT NOT NULL,
+          count INTEGER DEFAULT 1
+        );
       `);
+      this._migrate();
       this.available = true;
     } catch (err) {
       this.available = false;
@@ -71,24 +97,141 @@ export class IntelligenceStore {
       delta?.summary?.direction || null,
       JSON.stringify({ meta, delta: delta?.summary || null }),
     );
-    for (const idea of data.ideas || []) {
-      this.db.prepare(`INSERT INTO predictions (created_at, title, type, confidence, source, payload_json)
-        VALUES (?, ?, ?, ?, ?, ?)`).run(
-        timestamp,
-        idea.title || 'Untitled idea',
-        idea.type || null,
-        idea.confidence || null,
-        idea.source || data.ideasSource || null,
-        JSON.stringify(idea),
-      );
-    }
     this._recordEntities(data, timestamp);
+    this._recordEvents(data, delta, timestamp);
+    this.evaluatePredictions(data, timestamp);
+    this._recordPredictions(data, timestamp);
   }
 
   status() {
     return { available: this.available, path: this.dbPath, reason: this.reason };
   }
 
+  queryMemory({ q = '', limit = 25 } = {}) {
+    if (!this.available || !this.db) return { available: false, reason: this.reason, results: [] };
+    const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
+    const term = String(q || '').trim();
+    const like = `%${term}%`;
+    const where = term
+      ? 'WHERE name LIKE ? OR region LIKE ? OR source LIKE ? OR kind LIKE ?'
+      : '';
+    const params = term ? [like, like, like, like, safeLimit] : [safeLimit];
+    const events = this.db.prepare(`
+      SELECT stable_id, first_seen, last_seen, kind, name, region, severity, source, count, evidence_json
+      FROM events
+      ${where}
+      ORDER BY last_seen DESC
+      LIMIT ?
+    `).all(...params).map(row => ({ ...row, evidence: parseJson(row.evidence_json, {}) }));
+    return { available: true, q: term, results: events };
+  }
+
+  listPredictions({ state = null, limit = 25 } = {}) {
+    if (!this.available || !this.db) return { available: false, reason: this.reason, predictions: [] };
+    const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
+    const normalizedState = state && PREDICTION_STATES.has(String(state)) ? String(state) : null;
+    const rows = normalizedState
+      ? this.db.prepare(`SELECT * FROM predictions WHERE outcome_state = ? ORDER BY created_at DESC LIMIT ?`).all(normalizedState, safeLimit)
+      : this.db.prepare(`SELECT * FROM predictions ORDER BY created_at DESC LIMIT ?`).all(safeLimit);
+    return {
+      available: true,
+      predictions: rows.map(row => ({
+        stable_id: row.stable_id,
+        created_at: row.created_at,
+        updated_at: row.updated_at,
+        title: row.title,
+        type: row.type,
+        hypothesis: row.hypothesis,
+        confidence: row.confidence,
+        horizon: row.horizon,
+        outcome_state: row.outcome_state,
+        last_evaluated_at: row.last_evaluated_at,
+        source: row.source,
+        evidence: parseJson(row.evidence_json, []),
+        outcome: parseJson(row.outcome_json, null),
+      })),
+    };
+  }
+
+  evaluatePredictions(data, timestamp = new Date().toISOString()) {
+    if (!this.available || !this.db) return;
+    const rows = this.db.prepare(`
+      SELECT id, created_at, title, type, horizon, outcome_state, payload_json
+      FROM predictions
+      WHERE outcome_state IN ('open', 'monitoring')
+      ORDER BY created_at ASC
+      LIMIT 200
+    `).all();
+    for (const row of rows) {
+      const payload = parseJson(row.payload_json, {});
+      const evaluation = evaluatePredictionAgainstSweep(row, payload, data, timestamp);
+      this.db.prepare(`UPDATE predictions
+        SET outcome_state = ?, outcome_json = ?, last_evaluated_at = ?, updated_at = ?
+        WHERE id = ?`).run(
+        evaluation.state,
+        JSON.stringify(evaluation),
+        timestamp,
+        timestamp,
+        row.id,
+      );
+    }
+  }
+
+  _migrate() {
+    const columns = {
+      predictions: [
+        ['stable_id', 'TEXT'],
+        ['updated_at', 'TEXT'],
+        ['hypothesis', 'TEXT'],
+        ['evidence_json', 'TEXT'],
+        ['horizon', 'TEXT'],
+        ['outcome_state', "TEXT DEFAULT 'open'"],
+        ['outcome_json', 'TEXT'],
+        ['last_evaluated_at', 'TEXT'],
+      ],
+      entities: [
+        ['stable_id', 'TEXT'],
+      ],
+    };
+    for (const [table, defs] of Object.entries(columns)) {
+      for (const [name, type] of defs) {
+        try { this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${name} ${type}`); } catch { }
+      }
+    }
+    try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_predictions_stable_id ON predictions(stable_id)`); } catch { }
+    try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_entities_stable_id ON entities(stable_id)`); } catch { }
+  }
+
+  _recordPredictions(data, timestamp) {
+    for (const idea of data.ideas || []) {
+      const title = idea.title || 'Untitled idea';
+      const stableId = stableId('prediction', title, idea.type || '', idea.ticker || '', idea.horizon || '');
+      const evidence = Array.isArray(idea.signals) ? idea.signals : [];
+      this.db.prepare(`INSERT INTO predictions (
+          stable_id, created_at, updated_at, title, type, hypothesis, evidence_json, confidence,
+          horizon, outcome_state, source, payload_json
+        )
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, ?)
+        ON CONFLICT(stable_id) DO UPDATE SET
+          updated_at=excluded.updated_at,
+          confidence=excluded.confidence,
+          evidence_json=excluded.evidence_json,
+          payload_json=excluded.payload_json`).run(
+        stableId,
+        timestamp,
+        timestamp,
+        title,
+        idea.type || null,
+        idea.rationale || idea.text || title,
+        JSON.stringify(evidence),
+        idea.confidence || null,
+        idea.horizon || null,
+        idea.source || data.ideasSource || null,
+        JSON.stringify(idea),
+      );
+    }
+  }
+
   _recordEntities(data, timestamp) {
     const names = [];
     for (const item of data.acled?.deadliestEvents || []) {
@@ -99,14 +242,154 @@ export class IntelligenceStore {
       if (item.region) names.push([item.region, 'region']);
     }
     for (const [name, kind] of names.slice(0, 200)) {
-      this.db.prepare(`INSERT INTO entities (first_seen, last_seen, name, kind, count)
-        VALUES (?, ?, ?, ?, 1)
+      const cleanName = String(name).slice(0, 160);
+      this.db.prepare(`INSERT INTO entities (stable_id, first_seen, last_seen, name, kind, count)
+        VALUES (?, ?, ?, ?, ?, 1)
         ON CONFLICT(name, kind) DO UPDATE SET last_seen=excluded.last_seen, count=count+1`).run(
+        stableId('entity', kind, cleanName),
         timestamp,
         timestamp,
-        String(name).slice(0, 160),
+        cleanName,
         kind,
       );
     }
   }
+
+  _recordEvents(data, delta, timestamp) {
+    const events = extractEvents(data, delta);
+    for (const event of events.slice(0, 300)) {
+      this.db.prepare(`INSERT INTO events (
+          stable_id, first_seen, last_seen, kind, name, region, severity, source, evidence_json, count
+        )
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
+        ON CONFLICT(stable_id) DO UPDATE SET
+          last_seen=excluded.last_seen,
+          severity=COALESCE(excluded.severity, severity),
+          evidence_json=excluded.evidence_json,
+          count=count+1`).run(
+        event.stable_id,
+        timestamp,
+        timestamp,
+        event.kind,
+        event.name,
+        event.region || null,
+        event.severity || null,
+        event.source || null,
+        JSON.stringify(event.evidence || {}),
+      );
+    }
+  }
+}
+
+function stableId(...parts) {
+  const input = parts.map(part => String(part || '').trim().toLowerCase()).join('|');
+  return createHash('sha256').update(input).digest('hex').slice(0, 24);
+}
+
+function parseJson(value, fallback) {
+  try { return value ? JSON.parse(value) : fallback; } catch { return fallback; }
+}
+
+function extractEvents(data, delta) {
+  const events = [];
+  const push = ({ kind, name, region, severity, source, evidence }) => {
+    if (!kind || !name) return;
+    events.push({
+      stable_id: stableId('event', kind, name, region || source || ''),
+      kind,
+      name: String(name).slice(0, 240),
+      region: region ? String(region).slice(0, 120) : null,
+      severity: severity || null,
+      source: source || null,
+      evidence: evidence || {},
+    });
+  };
+
+  for (const item of data.acled?.deadliestEvents || []) {
+    push({
+      kind: 'conflict',
+      name: item.event_type || item.sub_event_type || item.location || item.country,
+      region: item.country || item.location,
+      severity: Number(item.fatalities || 0) > 0 ? 'high' : 'medium',
+      source: 'ACLED',
+      evidence: item,
+    });
+  }
+  for (const item of data.tg?.urgent || []) {
+    push({
+      kind: 'osint',
+      name: (item.text || '').slice(0, 120),
+      region: item.region || 'OSINT',
+      severity: 'high',
+      source: item.channel || item.chat || 'telegram',
+      evidence: item,
+    });
+  }
+  for (const item of data.newsFeed || data.news || []) {
+    if (!item.urgent) continue;
+    push({
+      kind: 'news',
+      name: item.headline || item.title,
+      region: item.region,
+      severity: 'medium',
+      source: item.source,
+      evidence: item,
+    });
+  }
+  for (const signal of delta?.signals?.new || []) {
+    push({
+      kind: 'delta',
+      name: signal.label || signal.reason || signal.key,
+      region: signal.region,
+      severity: signal.severity || 'medium',
+      source: 'delta',
+      evidence: signal,
+    });
+  }
+  return events;
+}
+
+function evaluatePredictionAgainstSweep(row, payload, data, timestamp) {
+  const terms = [
+    row.title,
+    payload.ticker,
+    ...(Array.isArray(payload.signals) ? payload.signals : []),
+  ].filter(Boolean).map(v => String(v).toLowerCase());
+  const evidenceText = [
+    ...(data.tSignals || []),
+    ...(data.newsFeed || []).slice(0, 40).map(n => `${n.source || ''} ${n.headline || n.title || ''}`),
+    ...(data.tg?.urgent || []).slice(0, 20).map(p => p.text || ''),
+  ].join('\n').toLowerCase();
+  const matched = terms.filter(term => term.length >= 4 && evidenceText.includes(term.slice(0, 60)));
+  const expired = predictionExpired(row.created_at, row.horizon, timestamp);
+  const state = matched.length
+    ? 'observed'
+    : expired
+      ? 'expired_unverified'
+      : 'monitoring';
+  return {
+    state,
+    evaluated_at: timestamp,
+    matched_terms: matched.slice(0, 10),
+    expired,
+    reason: matched.length
+      ? 'Current sweep contains matching evidence terms.'
+      : expired
+        ? 'Prediction horizon elapsed without matching evidence.'
+        : 'Prediction remains open for future sweeps.',
+  };
+}
+
+function predictionExpired(createdAt, horizon, nowIso) {
+  const created = new Date(createdAt).getTime();
+  const now = new Date(nowIso).getTime();
+  if (!Number.isFinite(created) || !Number.isFinite(now)) return false;
+  const text = String(horizon || '').toLowerCase();
+  const days = text.includes('intraday') ? 1
+    : text.includes('day') ? 7
+      : text.includes('week') ? 45
+        : text.includes('month') ? 180
+          : text.includes('strategic') ? 365
+            : 30;
+  return now - created > days * 24 * 60 * 60 * 1000;
 }

lib/stale-alerts.mjs

@@ -0,0 +1,52 @@
+const DEFAULT_COOLDOWN_MS = 60 * 60 * 1000;
+
+export function shouldSendStaleAlert(health, state = {}, opts = {}) {
+  const now = opts.now ?? Date.now();
+  const cooldownMs = opts.cooldownMs ?? DEFAULT_COOLDOWN_MS;
+  if (!health?.stale) {
+    state.lastStaleAlertKey = null;
+    return { send: false, reason: 'not_stale' };
+  }
+
+  const key = [
+    health.lastSuccessfulSweep || 'never',
+    health.lastSweepError || 'no-error',
+    health.sourcesFailed || 0,
+    health.sourcesDegraded || 0,
+  ].join('|');
+
+  if (state.lastStaleAlertKey === key && now - (state.lastStaleAlertAt || 0) < cooldownMs) {
+    return { send: false, reason: 'cooldown', key };
+  }
+
+  state.lastStaleAlertKey = key;
+  state.lastStaleAlertAt = now;
+  return { send: true, reason: 'stale', key };
+}
+
+export function formatStaleAlert(health, opts = {}) {
+  const dashboardUrl = opts.dashboardUrl || 'http://localhost:3117';
+  const context = opts.context || 'scheduled sweep';
+  const ageMinutes = health.dataAgeSeconds == null ? 'unknown' : Math.floor(health.dataAgeSeconds / 60);
+  const affected = (health.sourceHealth || [])
+    .filter(s => (s.status && s.status !== 'ok') || s.error)
+    .slice(0, 6)
+    .map(s => `- ${s.name || s.n || 'source'}: ${s.status || 'degraded'}${s.error ? ` (${String(s.error).slice(0, 100)})` : ''}`);
+
+  return [
+    '*CRUCIX STALE DATA ALERT*',
+    '',
+    `Context: ${context}`,
+    `Status: ${health.status || 'unknown'}`,
+    `Data age: ${ageMinutes} minutes`,
+    `Last successful sweep: ${health.lastSuccessfulSweep || 'never'}`,
+    `Last attempted sweep: ${health.lastSweep || 'never'}`,
+    `Last error: ${health.lastSweepError || 'none'}`,
+    `Sources: ${health.sourcesOk || 0} OK / ${health.sourcesDegraded || 0} degraded / ${health.sourcesFailed || 0} failed`,
+    '',
+    '*Affected sources*',
+    affected.length ? affected.join('\n') : '- No per-source errors available',
+    '',
+    `Dashboard: ${dashboardUrl}`,
+  ].join('\n');
+}

package.json

@@ -12,7 +12,7 @@
     "brief:save": "node apis/save-briefing.mjs",
     "diag": "node diag.mjs",
     "test": "npm run test:unit",
-    "test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs test/acled-source.test.mjs test/mojibake-text.test.mjs",
+    "test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs test/acled-source.test.mjs test/mojibake-text.test.mjs test/adsb.test.mjs test/dashboard-geotagging.test.mjs",
     "compose:config": "docker compose config",
     "clean": "node scripts/clean.mjs",
     "fresh-start": "npm run clean && npm start"

server.mjs

@@ -18,6 +18,7 @@ import { TelegramAlerter } from './lib/alerts/telegram.mjs';
 import { DiscordAlerter } from './lib/alerts/discord.mjs';
 import { getFetchMetrics } from './apis/utils/fetch.mjs';
 import { IntelligenceStore } from './lib/intelligence-store.mjs';
+import { formatStaleAlert, shouldSendStaleAlert } from './lib/stale-alerts.mjs';
 import { evaluateScenarios } from './lib/scenarios.mjs';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -40,6 +41,8 @@ let sweepStartedAt = null; // Timestamp when current/last sweep started
 let sweepInProgress = false;
 const startTime = Date.now();
 const sseClients = new Set();
+const terminalActionBuckets = new Map();
+const staleAlertState = {};
 
 // === Delta/Memory ===
 const memory = new MemoryManager(RUNS_DIR);
@@ -289,29 +292,67 @@ app.get('/api/metrics', (req, res) => {
   });
 });
 
-app.post('/api/sweep', express.json(), (req, res) => {
-  if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
-  triggerSweep(res);
+app.get('/api/memory/search', (req, res) => {
+  const guard = authorizeTerminalAction(req, res, 'memory:search');
+  if (!guard.ok) return;
+  auditTerminalAction(req, 'memory:search', 'ok');
+  res.json(intelligenceStore.queryMemory({
+    q: req.query.q || '',
+    limit: req.query.limit || 25,
+  }));
 });
 
-app.post('/api/action', express.json(), async (req, res) => {
-  if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
-  const action = String(req.body?.action || req.query.action || '').toLowerCase();
+app.get('/api/memory/predictions', (req, res) => {
+  const guard = authorizeTerminalAction(req, res, 'memory:predictions');
+  if (!guard.ok) return;
+  auditTerminalAction(req, 'memory:predictions', 'ok');
+  res.json(intelligenceStore.listPredictions({
+    state: req.query.state || null,
+    limit: req.query.limit || 25,
+  }));
+});
+
+app.post('/api/sweep', express.json(), (req, res) => {
+  const guard = authorizeTerminalAction(req, res, 'sweep');
+  if (!guard.ok) return;
+  triggerSweepAction(req, res, 'sweep');
+});
+
+app.post('/api/action', express.json(), (req, res) => {
+  const action = String(req.body?.action || req.body?.command || '').trim().toLowerCase();
+  const guard = authorizeTerminalAction(req, res, action || 'unknown');
+  if (!guard.ok) return;
 
   if (action === 'status') {
-    return res.json({ ok: true, action, health: buildHealth() });
+    auditTerminalAction(req, 'status', 'ok');
+    return res.json({ ok: true, action, status: 'ok', health: buildHealth() });
   }
 
   if (action === 'brief') {
-    if (!currentData) return res.status(503).json({ ok: false, action, error: 'No data yet — first sweep in progress' });
-    return res.json({ ok: true, action, text: buildBrief(currentData) });
+    if (!currentData) {
+      auditTerminalAction(req, 'brief', 'rejected', 'no_data');
+      return res.status(503).json({ ok: false, action, error: 'No data yet - first sweep in progress' });
+    }
+    auditTerminalAction(req, 'brief', 'ok');
+    const brief = buildBrief(currentData);
+    return res.json({ ok: true, action, status: 'ok', brief, text: brief });
   }
 
-  if (action === 'sweep') {
-    return triggerSweep(res);
+  if (action === 'memory') {
+    auditTerminalAction(req, 'memory', 'ok');
+    return res.json({
+      ok: true,
+      action,
+      memory: intelligenceStore.status(),
+      recentEvents: intelligenceStore.queryMemory({ q: req.body?.q || '', limit: 8 }).results,
+      predictions: intelligenceStore.listPredictions({ limit: 8 }).predictions,
+    });
   }
 
-  res.status(400).json({ ok: false, error: 'Unknown action', actions: ['status', 'brief', 'sweep'] });
+  if (action === 'sweep') return triggerSweepAction(req, res, 'action:sweep');
+
+  auditTerminalAction(req, action || 'unknown', 'rejected', 'unknown_action');
+  return res.status(400).json({ ok: false, error: 'Unknown action', allowed: ['status', 'brief', 'memory', 'sweep'], actions: ['status', 'brief', 'memory', 'sweep'] });
 });
 
 // API: available locales
@@ -329,10 +370,24 @@ app.get('/events', (req, res) => {
     'Cache-Control': 'no-cache',
     'Connection': 'keep-alive',
     'Access-Control-Allow-Origin': '*',
+    'X-Accel-Buffering': 'no',
   });
+  res.write('retry: 10000\n');
   res.write('data: {"type":"connected"}\n\n');
+  const heartbeatMs = Math.max(5000, config.sseHeartbeatIntervalMs || 25000);
+  const heartbeat = setInterval(() => {
+    try {
+      res.write(`: heartbeat ${new Date().toISOString()}\n\n`);
+    } catch {
+      clearInterval(heartbeat);
+      sseClients.delete(res);
+    }
+  }, heartbeatMs);
   sseClients.add(res);
-  req.on('close', () => sseClients.delete(res));
+  req.on('close', () => {
+    clearInterval(heartbeat);
+    sseClients.delete(res);
+  });
 });
 
 function broadcast(data) {
@@ -342,26 +397,114 @@ function broadcast(data) {
   }
 }
 
+function requestIp(req) {
+  return req.ip || req.socket?.remoteAddress || 'unknown';
+}
+
+function isLocalRequest(req) {
+  const remote = requestIp(req);
+  return remote === '::1'
+    || remote === '127.0.0.1'
+    || remote === '::ffff:127.0.0.1'
+    || remote.startsWith('127.')
+    || remote === 'localhost';
+}
+
+function sameOriginPost(req) {
+  const origin = req.get('origin');
+  if (!origin) return true;
+  try {
+    const originUrl = new URL(origin);
+    const host = req.get('host');
+    return host && originUrl.host === host;
+  } catch {
+    return false;
+  }
+}
+
+function actionToken(req) {
+  return req.get('x-crucix-token') || req.body?.token || null;
+}
+
+function auditTerminalAction(req, action, outcome, detail = null) {
+  const suffix = detail ? ` detail=${detail}` : '';
+  console.log(`[Crucix][audit] terminal_action action=${action || 'unknown'} outcome=${outcome} ip=${requestIp(req)}${suffix}`);
+}
+
+function rateLimitTerminalAction(req, action) {
+  const now = Date.now();
+  const windowMs = Math.max(1000, config.terminalActionRateLimitWindowMs || 60_000);
+  const max = Math.max(1, config.terminalActionRateLimitMax || 10);
+  const key = `${requestIp(req)}:${action}`;
+  const bucket = terminalActionBuckets.get(key);
+  if (!bucket || now > bucket.resetAt) {
+    terminalActionBuckets.set(key, { count: 1, resetAt: now + windowMs });
+    return { ok: true };
+  }
+  bucket.count += 1;
+  if (bucket.count > max) {
+    return { ok: false, retryAfterSeconds: Math.ceil((bucket.resetAt - now) / 1000) };
+  }
+  return { ok: true };
+}
+
+function authorizeTerminalAction(req, res, action) {
+  const rate = rateLimitTerminalAction(req, action);
+  if (!rate.ok) {
+    auditTerminalAction(req, action, 'rejected', 'rate_limited');
+    res.set('Retry-After', String(rate.retryAfterSeconds));
+    res.status(429).json({ error: 'Too many terminal actions', retryAfterSeconds: rate.retryAfterSeconds });
+    return { ok: false };
+  }
+
+  if (!sameOriginPost(req)) {
+    auditTerminalAction(req, action, 'rejected', 'csrf_origin');
+    res.status(403).json({ error: 'Origin mismatch' });
+    return { ok: false };
+  }
+
+  const local = isLocalRequest(req);
+  const token = actionToken(req);
+  if (!config.terminalActionsEnabled) {
+    auditTerminalAction(req, action, 'rejected', 'disabled');
+    res.status(403).json({ error: 'Terminal actions are disabled' });
+    return { ok: false };
+  }
+
+  if (config.sweepToken) {
+    if (token !== config.sweepToken) {
+      auditTerminalAction(req, action, 'rejected', 'invalid_token');
+      res.status(401).json({ error: 'Invalid terminal action token' });
+      return { ok: false };
+    }
+    return { ok: true };
+  }
+
+  if (!local) {
+    auditTerminalAction(req, action, 'rejected', 'missing_token');
+    res.status(403).json({ error: 'Terminal actions are local-only unless SWEEP_TOKEN is set' });
+    return { ok: false };
+  }
+
+  return { ok: true };
+}
+
+function triggerSweepAction(req, res, auditAction) {
+  if (sweepInProgress) {
+    auditTerminalAction(req, auditAction, 'rejected', 'already_running');
+    return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
+  }
+  auditTerminalAction(req, auditAction, 'accepted');
+  runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
+  return res.status(202).json({ ok: true, status: 'accepted' });
+}
+
 function dataAgeMs() {
   const ts = currentData?.meta?.timestamp || lastSuccessfulSweepTime || lastSweepTime;
   const ms = ts ? Date.now() - new Date(ts).getTime() : null;
   return Number.isFinite(ms) ? ms : null;
 }
 
-function canRunTerminalAction(req) {
-  const remote = req.ip || '';
-  const local = remote.includes('127.0.0.1') || remote === '::1' || remote === '::ffff:127.0.0.1';
-  const token = req.get('x-crucix-token') || req.query.token || req.body?.token;
-  if (config.sweepToken) return token === config.sweepToken;
-  return Boolean(config.terminalActionsEnabled || local);
-}
-
-function triggerSweep(res) {
-  if (sweepInProgress) return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
-  runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
-  return res.status(202).json({ ok: true, status: 'accepted' });
-}
-
 function getLLMStatus() {
   if (!config.llm.provider) return { state: 'disabled' };
   if (!llmProvider) return { state: 'misconfigured', provider: config.llm.provider };
@@ -405,13 +548,39 @@ function buildHealth() {
     llm: getLLMStatus(),
     telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId),
     discordEnabled: !!(config.discord?.botToken || config.discord?.webhookUrl),
-    terminalActionsEnabled: Boolean(config.terminalActionsEnabled || config.sweepToken),
+    terminalActionsEnabled: config.terminalActionsEnabled,
+    terminalActionsTokenRequired: !!config.sweepToken,
     refreshIntervalMinutes: config.refreshIntervalMinutes,
     language: currentLanguage,
     memory: intelligenceStore.status(),
   };
 }
 
+async function notifyIfDataStale(context = 'scheduled sweep') {
+  const health = buildHealth();
+  const decision = shouldSendStaleAlert(health, staleAlertState, {
+    cooldownMs: config.staleAlertCooldownMinutes * 60 * 1000,
+  });
+  if (!decision.send) return false;
+
+  const dashboardUrl = config.dashboardUrl || `http://localhost:${config.port}`;
+  const message = formatStaleAlert(health, { dashboardUrl, context });
+  const sends = [];
+  if (telegramAlerter.isConfigured) sends.push(telegramAlerter.sendMessage(message));
+  if (discordAlerter.isConfigured) sends.push(discordAlerter.sendAlert(message));
+
+  if (sends.length === 0) {
+    console.warn('[Crucix] Data is stale but no operator alert channel is configured');
+    return false;
+  }
+
+  const results = await Promise.allSettled(sends);
+  const sent = results.some(r => r.status === 'fulfilled' && (r.value === true || r.value?.ok === true));
+  if (sent) console.warn('[Crucix] Operator stale-data alert sent');
+  else console.warn('[Crucix] Operator stale-data alert attempted but no channel accepted it');
+  return sent;
+}
+
 function buildBrief(data) {
   const verbosity = config.telegram.briefVerbosity || 'standard';
   const delta = memory.getLastDelta();
@@ -562,6 +731,9 @@ async function runSweepCycle() {
     broadcast({ type: 'sweep_error', error: err.message });
   } finally {
     sweepInProgress = false;
+    await notifyIfDataStale(lastSweepError ? 'failed sweep' : 'completed sweep').catch(err => {
+      console.error('[Crucix] Stale-data operator alert failed:', err.message);
+    });
   }
 }
 

test/adsb.test.mjs

@@ -0,0 +1,82 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+async function withFetch(mockFetch, fn) {
+  const originalFetch = globalThis.fetch;
+  const originalAdsbKey = process.env.ADSB_API_KEY;
+  const originalRapidKey = process.env.RAPIDAPI_KEY;
+  globalThis.fetch = mockFetch;
+  delete process.env.ADSB_API_KEY;
+  delete process.env.RAPIDAPI_KEY;
+  try {
+    return await fn();
+  } finally {
+    globalThis.fetch = originalFetch;
+    if (originalAdsbKey === undefined) delete process.env.ADSB_API_KEY;
+    else process.env.ADSB_API_KEY = originalAdsbKey;
+    if (originalRapidKey === undefined) delete process.env.RAPIDAPI_KEY;
+    else process.env.RAPIDAPI_KEY = originalRapidKey;
+  }
+}
+
+function jsonResponse(payload, ok = true, status = 200) {
+  return {
+    ok,
+    status,
+    headers: { get: () => 'application/json' },
+    text: async () => JSON.stringify(payload),
+  };
+}
+
+test('ADS-B reports disabled when no key is configured and public feed fails', async () => {
+  await withFetch(async () => jsonResponse({ error: 'blocked' }, false, 403), async () => {
+    const { briefing } = await import('../apis/sources/adsb.mjs');
+    const data = await briefing();
+
+    assert.equal(data.status, 'disabled');
+    assert.match(data.error, /ADSB_API_KEY|RAPIDAPI_KEY/);
+    assert.equal(data.militaryAircraft.length, 0);
+  });
+});
+
+test('ADS-B reports degraded when RapidAPI and public feed fail', async () => {
+  await withFetch(async () => jsonResponse({ error: 'unavailable' }, false, 503), async () => {
+    process.env.ADSB_API_KEY = 'test-key';
+    const { briefing } = await import('../apis/sources/adsb.mjs');
+    const data = await briefing();
+
+    assert.equal(data.status, 'degraded');
+    assert.match(data.error, /providers returned no usable/);
+    assert.equal(data.failures.length, 2);
+  });
+});
+
+test('ADS-B returns live RapidAPI military aircraft payloads', async () => {
+  await withFetch(async () => jsonResponse({
+    ac: [{
+      hex: 'AE1234',
+      flight: 'RCH123',
+      t: 'KC135',
+      lat: 50,
+      lon: 8,
+      mil: true,
+    }],
+  }), async () => {
+    process.env.ADSB_API_KEY = 'test-key';
+    const { briefing } = await import('../apis/sources/adsb.mjs');
+    const data = await briefing();
+
+    assert.equal(data.status, 'live');
+    assert.equal(data.provider, 'rapidapi');
+    assert.equal(data.totalMilitary, 1);
+    assert.equal(data.militaryAircraft[0].callsign, 'RCH123');
+  });
+});
+
+test('runSource treats disabled source status as degraded health', async () => {
+  const { runSource } = await import('../apis/briefing.mjs');
+  const result = await runSource('ADS-B', async () => ({ status: 'disabled', message: 'missing key' }));
+
+  assert.equal(result.status, 'degraded');
+  assert.equal(result.error, null);
+});

test/dashboard-geotagging.test.mjs

@@ -0,0 +1,47 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { geoTagText, stableGeoJitter } from '../dashboard/inject.mjs';
+
+test('geoTagText matches headlines case-insensitively', () => {
+  assert.deepEqual(geoTagText('ukraine reports new air defense activity'), {
+    lat: 49,
+    lon: 32,
+    region: 'Ukraine',
+  });
+
+  assert.deepEqual(geoTagText('flooding disrupts são paulo transport'), {
+    lat: -23.5,
+    lon: -46.6,
+    region: 'São Paulo',
+  });
+});
+
+test('geoTagText prefers longer place names before broad countries', () => {
+  assert.deepEqual(geoTagText('New York markets react before wider US session'), {
+    lat: 40.7,
+    lon: -74,
+    region: 'New York',
+  });
+});
+
+test('geoTagText uses word boundaries to reduce false positives', () => {
+  assert.equal(geoTagText('A music festival announces its lineup'), null);
+  assert.equal(geoTagText('Officials discuss a new focus for aid'), null);
+  assert.deepEqual(geoTagText('US officials discuss a new aid package'), {
+    lat: 39,
+    lon: -98,
+    region: 'US',
+  });
+});
+
+test('stableGeoJitter is deterministic and bounded', () => {
+  const key = 'BBC|lower-case ukraine headline|Sun, 17 May 2026 12:00:00 GMT|https://example.test/a';
+  const latA = stableGeoJitter(key, 'lat');
+  const latB = stableGeoJitter(key, 'lat');
+  const lon = stableGeoJitter(key, 'lon');
+
+  assert.equal(latA, latB);
+  assert.notEqual(latA, lon);
+  assert.ok(latA >= -1 && latA <= 1);
+  assert.ok(lon >= -1 && lon <= 1);
+});

test/fetch-utils.test.mjs

@@ -1,7 +1,8 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import { readFileSync } from 'node:fs';
-import { safeFetch, safeFetchText, getFetchMetrics } from '../apis/utils/fetch.mjs';
+import { safeFetch, safeFetchText, getFetchMetrics, inferFetchSource } from '../apis/utils/fetch.mjs';
+import { formatStaleAlert, shouldSendStaleAlert } from '../lib/stale-alerts.mjs';
 
 test('safeFetch reports HTML as degraded JSON response', async () => {
   const originalFetch = globalThis.fetch;
@@ -100,6 +101,157 @@ test('safeFetchText returns text and byte count', async () => {
   }
 });
 
+test('safeFetch attributes unlabelled requests to a stable provider source', async () => {
+  const originalFetch = globalThis.fetch;
+  globalThis.fetch = async () => ({
+    ok: true,
+    status: 200,
+    headers: { get: () => 'application/json' },
+    text: async () => '{"observations":[]}',
+  });
+  try {
+    const data = await safeFetch('https://api.fred.stlouisfed.org/fred/series/observations?series_id=VIXCLS', { retries: 0 });
+    assert.deepEqual(data, { observations: [] });
+    const bucket = getFetchMetrics().bySource.FRED;
+    assert.ok(bucket.requests >= 1);
+    assert.equal(bucket.lastStatus, 200);
+  } finally {
+    globalThis.fetch = originalFetch;
+  }
+});
+
+test('inferFetchSource returns provider names and host fallback', () => {
+  assert.equal(inferFetchSource('https://api.bls.gov/publicAPI/v2/timeseries/data/CPI'), 'BLS');
+  assert.equal(inferFetchSource('https://query1.finance.yahoo.com/v8/finance/chart/%5EGSPC'), 'YahooFinance');
+  assert.equal(inferFetchSource('https://unknown.example.test/path'), 'unknown.example.test');
+});
+
+test('SSE endpoint sends reconnect guidance and clears heartbeat timer', () => {
+  const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
+  const config = readFileSync(new URL('../crucix.config.mjs', import.meta.url), 'utf8');
+  assert.match(config, /sseHeartbeatIntervalMs/);
+  assert.match(server, /retry: 10000\\n/);
+  assert.match(server, /setInterval\(\(\) =>/);
+  assert.match(server, /: heartbeat/);
+  assert.match(server, /clearInterval\(heartbeat\)/);
+  assert.match(server, /X-Accel-Buffering/);
+});
+
+test('intelligence store defines durable memory and prediction lifecycle tables', () => {
+  const store = readFileSync(new URL('../lib/intelligence-store.mjs', import.meta.url), 'utf8');
+  assert.match(store, /CREATE TABLE IF NOT EXISTS events/);
+  assert.match(store, /stable_id TEXT NOT NULL UNIQUE/);
+  assert.match(store, /hypothesis TEXT/);
+  assert.match(store, /evidence_json TEXT/);
+  assert.match(store, /outcome_state TEXT DEFAULT 'open'/);
+  assert.match(store, /evaluatePredictions/);
+  assert.match(store, /queryMemory/);
+  assert.match(store, /listPredictions/);
+});
+
+test('server exposes memory-backed query APIs and dashboard memory action', () => {
+  const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
+  const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
+  assert.match(server, /\/api\/memory\/search/);
+  assert.match(server, /\/api\/memory\/predictions/);
+  assert.match(server, /action === 'memory'/);
+  assert.match(html, /runTerminalAction\('memory'\)/);
+});
+
+test('terminal action endpoints avoid URL tokens and include hardening gates', () => {
+  const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
+  assert.match(server, /app\.post\('\/api\/action'/);
+  assert.match(server, /app\.post\('\/api\/sweep'/);
+  assert.match(server, /x-crucix-token/);
+  assert.match(server, /sameOriginPost/);
+  assert.match(server, /rateLimitTerminalAction/);
+  assert.match(server, /auditTerminalAction/);
+  assert.doesNotMatch(server, /req\.query\.token/);
+});
+
+test('dashboard exposes token configuration flow without devtools edits', () => {
+  const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
+  assert.match(html, /configureTerminalActionToken/);
+  assert.match(html, /crucix_sweep_token/);
+  assert.match(html, /x-crucix-token/);
+  assert.match(html, /SET TOKEN/);
+});
+
+test('server dashboard shell does not embed an operational snapshot', () => {
+  const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
+  assert.match(html, /let D = createDashboardShellData\(\);/);
+  assert.doesNotMatch(html, /2026-04-03T16:18:10\.188Z/);
+  assert.doesNotMatch(html, /Trump announced new strikes on Iran/);
+});
+
+test('server dashboard fetches api data before initialization', () => {
+  const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
+  const serverMode = html.indexOf('if (canProbeApi)');
+  const apiFetch = html.indexOf("fetch('/api/data')");
+  const firstInitAfterServerMode = html.indexOf('init();', serverMode);
+
+  assert.ok(serverMode > -1);
+  assert.ok(apiFetch > serverMode);
+  assert.ok(firstInitAfterServerMode > apiFetch);
+});
+
+test('stale alert is skipped for fresh health and resets active key', () => {
+  const state = { lastStaleAlertKey: 'old', lastStaleAlertAt: 100 };
+  const decision = shouldSendStaleAlert({ stale: false }, state, { now: 200 });
+  assert.equal(decision.send, false);
+  assert.equal(decision.reason, 'not_stale');
+  assert.equal(state.lastStaleAlertKey, null);
+});
+
+test('stale alert sends once and deduplicates during cooldown', () => {
+  const state = {};
+  const health = {
+    stale: true,
+    lastSuccessfulSweep: '2026-05-17T08:00:00.000Z',
+    lastSweepError: 'network timeout',
+    sourcesFailed: 2,
+    sourcesDegraded: 1,
+  };
+
+  const first = shouldSendStaleAlert(health, state, { now: 1_000, cooldownMs: 60_000 });
+  const second = shouldSendStaleAlert(health, state, { now: 2_000, cooldownMs: 60_000 });
+
+  assert.equal(first.send, true);
+  assert.equal(second.send, false);
+  assert.equal(second.reason, 'cooldown');
+});
+
+test('stale alert repeats after cooldown', () => {
+  const state = {};
+  const health = { stale: true, lastSuccessfulSweep: 'a', lastSweepError: 'timeout', sourcesFailed: 1 };
+
+  assert.equal(shouldSendStaleAlert(health, state, { now: 1_000, cooldownMs: 60_000 }).send, true);
+  assert.equal(shouldSendStaleAlert(health, state, { now: 62_000, cooldownMs: 60_000 }).send, true);
+});
+
+test('stale alert message includes operator context and affected sources', () => {
+  const message = formatStaleAlert({
+    status: 'stale',
+    stale: true,
+    dataAgeSeconds: 7200,
+    lastSuccessfulSweep: '2026-05-17T08:00:00.000Z',
+    lastSweep: '2026-05-17T10:00:00.000Z',
+    lastSweepError: 'GDELT timeout',
+    sourcesOk: 20,
+    sourcesDegraded: 3,
+    sourcesFailed: 2,
+    sourceHealth: [
+      { name: 'GDELT', status: 'degraded', error: 'timeout' },
+      { name: 'Reddit', status: 'no_credentials' },
+    ],
+  }, { dashboardUrl: 'https://terminal.example.test', context: 'failed sweep' });
+
+  assert.match(message, /CRUCIX STALE DATA ALERT/);
+  assert.match(message, /Data age: 120 minutes/);
+  assert.match(message, /GDELT: degraded \(timeout\)/);
+  assert.match(message, /Dashboard: https:\/\/terminal\.example\.test/);
+});
+
 test('scenario watchlist feature is wired into sweep, briefing, and dashboard', () => {
   const scenarios = readFileSync(new URL('../lib/scenarios.mjs', import.meta.url), 'utf8');
   const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');