Merge pull request 'feat: extend memory prediction loop' (#32) from codex/issue-4-memory-prediction-loop into codex/production-intelligence-terminal

README.md

@@ -204,6 +204,36 @@ Action endpoints reject cross-origin POST origins, apply a small in-memory per-I
 
 When data remains stale past `STALE_DATA_MAX_AGE_MINUTES`, the server sends an operator alert through configured Telegram/Discord channels after failed or degraded sweep attempts. `STALE_ALERT_COOLDOWN_MINUTES` prevents repeated stale alerts from spamming every refresh interval. Set `DASHBOARD_URL` to the Pangolin/public URL you want included in those alerts.
 
+#### Memory And Prediction Loop
+
+Crucix stores longitudinal memory in `runs/intelligence.db` when the current Node.js build exposes `node:sqlite`. If SQLite is unavailable, the file is created as a harmless placeholder and `/api/health` reports the memory store as unavailable instead of failing the sweep.
+
+The memory layer persists:
+
+| Table | Purpose |
+| --- | --- |
+| `runs` | Sweep timestamps, source health counts, and delta direction summaries. |
+| `entities` | Stable entity IDs for recurring countries, regions, and locations. |
+| `events` | Stable event IDs for conflict, OSINT, urgent news, and new delta signals across sweeps. |
+| `predictions` | Trade/intelligence hypotheses with evidence, confidence, horizon, outcome state, and latest grading. |
+
+Query endpoints:
+
+```text
+GET /api/memory/search?q=iran&limit=25
+GET /api/memory/predictions?state=open&limit=25
+```
+
+Memory endpoints use the same operator authorization gate as Terminal Actions. The dashboard Terminal Actions panel includes a `Memory` action for a quick operator-facing view of recent events and prediction states.
+
+Retention, backup, and privacy expectations:
+
+- Treat `runs/intelligence.db` as operator data. It can contain source excerpts, headlines, generated hypotheses, and URLs from your configured feeds.
+- Back up `runs/` with the rest of your Dockge volume if you want longitudinal learning to survive container replacement.
+- Delete `runs/intelligence.db` to reset SQLite memory; the next sweep recreates the schema.
+- Do not commit `runs/` or `.env`. API credentials stay in `.env`; memory stores derived observations, not secrets.
+- If you expose the dashboard through a reverse proxy, protect Terminal Actions and memory queries behind your normal authentication boundary.
+
 #### Scenario Watchlist
 
 Intelligence Terminal can track operator hypotheses across sweeps with a runtime scenario file at `runs/scenarios.json`. On first run, the server creates three disabled starter examples:

dashboard/public/jarvis.html

@@ -83,7 +83,7 @@ html,body{height:100%;background:var(--bg);color:var(--text);font-family:var(--s
 .sensor-actions{display:flex;gap:6px;align-items:center}
 .mini-btn{border:1px solid rgba(100,240,200,0.18);background:rgba(100,240,200,0.04);color:var(--dim);font-family:var(--mono);font-size:9px;padding:3px 6px;cursor:pointer}
 .mini-btn:hover{color:var(--accent);border-color:rgba(100,240,200,0.4)}
-.action-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:6px;margin-bottom:10px}
+.action-grid{display:grid;grid-template-columns:repeat(4,1fr);gap:6px;margin-bottom:10px}
 .action-btn{border:1px solid rgba(68,204,255,0.24);background:rgba(68,204,255,0.06);color:var(--text);font-family:var(--mono);font-size:9px;padding:7px 6px;cursor:pointer;text-transform:uppercase;letter-spacing:.08em}
 .action-btn:hover{border-color:rgba(68,204,255,0.55);color:var(--accent2);background:rgba(68,204,255,0.12)}
 .action-btn[disabled]{opacity:.45;cursor:wait}
@@ -1641,6 +1641,17 @@ async function runTerminalAction(action){
       ].join('\n');
     }else if(action === 'brief'){
       terminalOutput = `> brief\n${payload.text || 'No briefing text returned.'}`;
+    }else if(action === 'memory'){
+      const events = payload.recentEvents || [];
+      const predictions = payload.predictions || [];
+      terminalOutput = [
+        '> memory',
+        `Store: ${payload.memory?.available ? 'available' : 'unavailable'}`,
+        `Recent events: ${events.length}`,
+        ...events.slice(0,4).map(e => `- ${e.kind}: ${e.name}${e.region ? ' [' + e.region + ']' : ''}`),
+        `Predictions: ${predictions.length}`,
+        ...predictions.slice(0,4).map(p => `- ${p.outcome_state || 'open'}: ${p.title}`)
+      ].join('\n');
     }else if(action === 'sweep'){
       terminalOutput = `> sweep\n${payload.status === 'already_running' ? 'Sweep already running.' : 'Sweep accepted. The dashboard will update when the sweep finishes.'}`;
     }
@@ -1707,6 +1718,7 @@ function renderRight(){
         <button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('status')">Status</button>
         <button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('sweep')">Sweep</button>
         <button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('brief')">Brief</button>
+        <button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('memory')">Memory</button>
       </div>
       <button class="mini-btn" style="margin-bottom:8px" onclick="configureTerminalActionToken()">Configure token</button>
       <div class="terminal-output">${terminalOutput.replace(/[&<>]/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;'}[c])).replace(/\n/g,'<br>')}</div>

lib/intelligence-store.mjs

@@ -2,6 +2,9 @@
 
 import { existsSync, mkdirSync, writeFileSync } from 'fs';
 import { dirname, join } from 'path';
+import { createHash } from 'crypto';
+
+const PREDICTION_STATES = new Set(['open', 'monitoring', 'observed', 'expired_unverified', 'invalidated']);
 
 export class IntelligenceStore {
   constructor(dbPath) {
@@ -30,15 +33,24 @@ export class IntelligenceStore {
         );
         CREATE TABLE IF NOT EXISTS predictions (
           id INTEGER PRIMARY KEY AUTOINCREMENT,
+          stable_id TEXT UNIQUE,
           created_at TEXT NOT NULL,
+          updated_at TEXT,
           title TEXT NOT NULL,
           type TEXT,
+          hypothesis TEXT,
+          evidence_json TEXT,
           confidence TEXT,
+          horizon TEXT,
+          outcome_state TEXT DEFAULT 'open',
+          outcome_json TEXT,
+          last_evaluated_at TEXT,
           source TEXT,
           payload_json TEXT NOT NULL
         );
         CREATE TABLE IF NOT EXISTS entities (
           id INTEGER PRIMARY KEY AUTOINCREMENT,
+          stable_id TEXT UNIQUE,
           first_seen TEXT NOT NULL,
           last_seen TEXT NOT NULL,
           name TEXT NOT NULL,
@@ -46,7 +58,21 @@ export class IntelligenceStore {
           count INTEGER DEFAULT 1,
           UNIQUE(name, kind)
         );
+        CREATE TABLE IF NOT EXISTS events (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          stable_id TEXT NOT NULL UNIQUE,
+          first_seen TEXT NOT NULL,
+          last_seen TEXT NOT NULL,
+          kind TEXT NOT NULL,
+          name TEXT NOT NULL,
+          region TEXT,
+          severity TEXT,
+          source TEXT,
+          evidence_json TEXT NOT NULL,
+          count INTEGER DEFAULT 1
+        );
       `);
+      this._migrate();
       this.available = true;
     } catch (err) {
       this.available = false;
@@ -71,24 +97,141 @@ export class IntelligenceStore {
       delta?.summary?.direction || null,
       JSON.stringify({ meta, delta: delta?.summary || null }),
     );
-    for (const idea of data.ideas || []) {
-      this.db.prepare(`INSERT INTO predictions (created_at, title, type, confidence, source, payload_json)
-        VALUES (?, ?, ?, ?, ?, ?)`).run(
-        timestamp,
-        idea.title || 'Untitled idea',
-        idea.type || null,
-        idea.confidence || null,
-        idea.source || data.ideasSource || null,
-        JSON.stringify(idea),
-      );
-    }
     this._recordEntities(data, timestamp);
+    this._recordEvents(data, delta, timestamp);
+    this.evaluatePredictions(data, timestamp);
+    this._recordPredictions(data, timestamp);
   }
 
   status() {
     return { available: this.available, path: this.dbPath, reason: this.reason };
   }
 
+  queryMemory({ q = '', limit = 25 } = {}) {
+    if (!this.available || !this.db) return { available: false, reason: this.reason, results: [] };
+    const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
+    const term = String(q || '').trim();
+    const like = `%${term}%`;
+    const where = term
+      ? 'WHERE name LIKE ? OR region LIKE ? OR source LIKE ? OR kind LIKE ?'
+      : '';
+    const params = term ? [like, like, like, like, safeLimit] : [safeLimit];
+    const events = this.db.prepare(`
+      SELECT stable_id, first_seen, last_seen, kind, name, region, severity, source, count, evidence_json
+      FROM events
+      ${where}
+      ORDER BY last_seen DESC
+      LIMIT ?
+    `).all(...params).map(row => ({ ...row, evidence: parseJson(row.evidence_json, {}) }));
+    return { available: true, q: term, results: events };
+  }
+
+  listPredictions({ state = null, limit = 25 } = {}) {
+    if (!this.available || !this.db) return { available: false, reason: this.reason, predictions: [] };
+    const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
+    const normalizedState = state && PREDICTION_STATES.has(String(state)) ? String(state) : null;
+    const rows = normalizedState
+      ? this.db.prepare(`SELECT * FROM predictions WHERE outcome_state = ? ORDER BY created_at DESC LIMIT ?`).all(normalizedState, safeLimit)
+      : this.db.prepare(`SELECT * FROM predictions ORDER BY created_at DESC LIMIT ?`).all(safeLimit);
+    return {
+      available: true,
+      predictions: rows.map(row => ({
+        stable_id: row.stable_id,
+        created_at: row.created_at,
+        updated_at: row.updated_at,
+        title: row.title,
+        type: row.type,
+        hypothesis: row.hypothesis,
+        confidence: row.confidence,
+        horizon: row.horizon,
+        outcome_state: row.outcome_state,
+        last_evaluated_at: row.last_evaluated_at,
+        source: row.source,
+        evidence: parseJson(row.evidence_json, []),
+        outcome: parseJson(row.outcome_json, null),
+      })),
+    };
+  }
+
+  evaluatePredictions(data, timestamp = new Date().toISOString()) {
+    if (!this.available || !this.db) return;
+    const rows = this.db.prepare(`
+      SELECT id, created_at, title, type, horizon, outcome_state, payload_json
+      FROM predictions
+      WHERE outcome_state IN ('open', 'monitoring')
+      ORDER BY created_at ASC
+      LIMIT 200
+    `).all();
+    for (const row of rows) {
+      const payload = parseJson(row.payload_json, {});
+      const evaluation = evaluatePredictionAgainstSweep(row, payload, data, timestamp);
+      this.db.prepare(`UPDATE predictions
+        SET outcome_state = ?, outcome_json = ?, last_evaluated_at = ?, updated_at = ?
+        WHERE id = ?`).run(
+        evaluation.state,
+        JSON.stringify(evaluation),
+        timestamp,
+        timestamp,
+        row.id,
+      );
+    }
+  }
+
+  _migrate() {
+    const columns = {
+      predictions: [
+        ['stable_id', 'TEXT'],
+        ['updated_at', 'TEXT'],
+        ['hypothesis', 'TEXT'],
+        ['evidence_json', 'TEXT'],
+        ['horizon', 'TEXT'],
+        ['outcome_state', "TEXT DEFAULT 'open'"],
+        ['outcome_json', 'TEXT'],
+        ['last_evaluated_at', 'TEXT'],
+      ],
+      entities: [
+        ['stable_id', 'TEXT'],
+      ],
+    };
+    for (const [table, defs] of Object.entries(columns)) {
+      for (const [name, type] of defs) {
+        try { this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${name} ${type}`); } catch { }
+      }
+    }
+    try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_predictions_stable_id ON predictions(stable_id)`); } catch { }
+    try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_entities_stable_id ON entities(stable_id)`); } catch { }
+  }
+
+  _recordPredictions(data, timestamp) {
+    for (const idea of data.ideas || []) {
+      const title = idea.title || 'Untitled idea';
+      const stableId = stableId('prediction', title, idea.type || '', idea.ticker || '', idea.horizon || '');
+      const evidence = Array.isArray(idea.signals) ? idea.signals : [];
+      this.db.prepare(`INSERT INTO predictions (
+          stable_id, created_at, updated_at, title, type, hypothesis, evidence_json, confidence,
+          horizon, outcome_state, source, payload_json
+        )
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, ?)
+        ON CONFLICT(stable_id) DO UPDATE SET
+          updated_at=excluded.updated_at,
+          confidence=excluded.confidence,
+          evidence_json=excluded.evidence_json,
+          payload_json=excluded.payload_json`).run(
+        stableId,
+        timestamp,
+        timestamp,
+        title,
+        idea.type || null,
+        idea.rationale || idea.text || title,
+        JSON.stringify(evidence),
+        idea.confidence || null,
+        idea.horizon || null,
+        idea.source || data.ideasSource || null,
+        JSON.stringify(idea),
+      );
+    }
+  }
+
   _recordEntities(data, timestamp) {
     const names = [];
     for (const item of data.acled?.deadliestEvents || []) {
@@ -99,14 +242,154 @@ export class IntelligenceStore {
       if (item.region) names.push([item.region, 'region']);
     }
     for (const [name, kind] of names.slice(0, 200)) {
-      this.db.prepare(`INSERT INTO entities (first_seen, last_seen, name, kind, count)
-        VALUES (?, ?, ?, ?, 1)
+      const cleanName = String(name).slice(0, 160);
+      this.db.prepare(`INSERT INTO entities (stable_id, first_seen, last_seen, name, kind, count)
+        VALUES (?, ?, ?, ?, ?, 1)
         ON CONFLICT(name, kind) DO UPDATE SET last_seen=excluded.last_seen, count=count+1`).run(
+        stableId('entity', kind, cleanName),
         timestamp,
         timestamp,
-        String(name).slice(0, 160),
+        cleanName,
         kind,
       );
     }
   }
+
+  _recordEvents(data, delta, timestamp) {
+    const events = extractEvents(data, delta);
+    for (const event of events.slice(0, 300)) {
+      this.db.prepare(`INSERT INTO events (
+          stable_id, first_seen, last_seen, kind, name, region, severity, source, evidence_json, count
+        )
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
+        ON CONFLICT(stable_id) DO UPDATE SET
+          last_seen=excluded.last_seen,
+          severity=COALESCE(excluded.severity, severity),
+          evidence_json=excluded.evidence_json,
+          count=count+1`).run(
+        event.stable_id,
+        timestamp,
+        timestamp,
+        event.kind,
+        event.name,
+        event.region || null,
+        event.severity || null,
+        event.source || null,
+        JSON.stringify(event.evidence || {}),
+      );
+    }
+  }
+}
+
+function stableId(...parts) {
+  const input = parts.map(part => String(part || '').trim().toLowerCase()).join('|');
+  return createHash('sha256').update(input).digest('hex').slice(0, 24);
+}
+
+function parseJson(value, fallback) {
+  try { return value ? JSON.parse(value) : fallback; } catch { return fallback; }
+}
+
+function extractEvents(data, delta) {
+  const events = [];
+  const push = ({ kind, name, region, severity, source, evidence }) => {
+    if (!kind || !name) return;
+    events.push({
+      stable_id: stableId('event', kind, name, region || source || ''),
+      kind,
+      name: String(name).slice(0, 240),
+      region: region ? String(region).slice(0, 120) : null,
+      severity: severity || null,
+      source: source || null,
+      evidence: evidence || {},
+    });
+  };
+
+  for (const item of data.acled?.deadliestEvents || []) {
+    push({
+      kind: 'conflict',
+      name: item.event_type || item.sub_event_type || item.location || item.country,
+      region: item.country || item.location,
+      severity: Number(item.fatalities || 0) > 0 ? 'high' : 'medium',
+      source: 'ACLED',
+      evidence: item,
+    });
+  }
+  for (const item of data.tg?.urgent || []) {
+    push({
+      kind: 'osint',
+      name: (item.text || '').slice(0, 120),
+      region: item.region || 'OSINT',
+      severity: 'high',
+      source: item.channel || item.chat || 'telegram',
+      evidence: item,
+    });
+  }
+  for (const item of data.newsFeed || data.news || []) {
+    if (!item.urgent) continue;
+    push({
+      kind: 'news',
+      name: item.headline || item.title,
+      region: item.region,
+      severity: 'medium',
+      source: item.source,
+      evidence: item,
+    });
+  }
+  for (const signal of delta?.signals?.new || []) {
+    push({
+      kind: 'delta',
+      name: signal.label || signal.reason || signal.key,
+      region: signal.region,
+      severity: signal.severity || 'medium',
+      source: 'delta',
+      evidence: signal,
+    });
+  }
+  return events;
+}
+
+function evaluatePredictionAgainstSweep(row, payload, data, timestamp) {
+  const terms = [
+    row.title,
+    payload.ticker,
+    ...(Array.isArray(payload.signals) ? payload.signals : []),
+  ].filter(Boolean).map(v => String(v).toLowerCase());
+  const evidenceText = [
+    ...(data.tSignals || []),
+    ...(data.newsFeed || []).slice(0, 40).map(n => `${n.source || ''} ${n.headline || n.title || ''}`),
+    ...(data.tg?.urgent || []).slice(0, 20).map(p => p.text || ''),
+  ].join('\n').toLowerCase();
+  const matched = terms.filter(term => term.length >= 4 && evidenceText.includes(term.slice(0, 60)));
+  const expired = predictionExpired(row.created_at, row.horizon, timestamp);
+  const state = matched.length
+    ? 'observed'
+    : expired
+      ? 'expired_unverified'
+      : 'monitoring';
+  return {
+    state,
+    evaluated_at: timestamp,
+    matched_terms: matched.slice(0, 10),
+    expired,
+    reason: matched.length
+      ? 'Current sweep contains matching evidence terms.'
+      : expired
+        ? 'Prediction horizon elapsed without matching evidence.'
+        : 'Prediction remains open for future sweeps.',
+  };
+}
+
+function predictionExpired(createdAt, horizon, nowIso) {
+  const created = new Date(createdAt).getTime();
+  const now = new Date(nowIso).getTime();
+  if (!Number.isFinite(created) || !Number.isFinite(now)) return false;
+  const text = String(horizon || '').toLowerCase();
+  const days = text.includes('intraday') ? 1
+    : text.includes('day') ? 7
+      : text.includes('week') ? 45
+        : text.includes('month') ? 180
+          : text.includes('strategic') ? 365
+            : 30;
+  return now - created > days * 24 * 60 * 60 * 1000;
 }

server.mjs

@@ -292,6 +292,26 @@ app.get('/api/metrics', (req, res) => {
   });
 });
 
+app.get('/api/memory/search', (req, res) => {
+  const guard = authorizeTerminalAction(req, res, 'memory:search');
+  if (!guard.ok) return;
+  auditTerminalAction(req, 'memory:search', 'ok');
+  res.json(intelligenceStore.queryMemory({
+    q: req.query.q || '',
+    limit: req.query.limit || 25,
+  }));
+});
+
+app.get('/api/memory/predictions', (req, res) => {
+  const guard = authorizeTerminalAction(req, res, 'memory:predictions');
+  if (!guard.ok) return;
+  auditTerminalAction(req, 'memory:predictions', 'ok');
+  res.json(intelligenceStore.listPredictions({
+    state: req.query.state || null,
+    limit: req.query.limit || 25,
+  }));
+});
+
 app.post('/api/sweep', express.json(), (req, res) => {
   const guard = authorizeTerminalAction(req, res, 'sweep');
   if (!guard.ok) return;
@@ -318,10 +338,21 @@ app.post('/api/action', express.json(), (req, res) => {
     return res.json({ ok: true, action, status: 'ok', brief, text: brief });
   }
 
+  if (action === 'memory') {
+    auditTerminalAction(req, 'memory', 'ok');
+    return res.json({
+      ok: true,
+      action,
+      memory: intelligenceStore.status(),
+      recentEvents: intelligenceStore.queryMemory({ q: req.body?.q || '', limit: 8 }).results,
+      predictions: intelligenceStore.listPredictions({ limit: 8 }).predictions,
+    });
+  }
+
   if (action === 'sweep') return triggerSweepAction(req, res, 'action:sweep');
 
   auditTerminalAction(req, action || 'unknown', 'rejected', 'unknown_action');
-  return res.status(400).json({ ok: false, error: 'Unknown action', allowed: ['status', 'brief', 'sweep'], actions: ['status', 'brief', 'sweep'] });
+  return res.status(400).json({ ok: false, error: 'Unknown action', allowed: ['status', 'brief', 'memory', 'sweep'], actions: ['status', 'brief', 'memory', 'sweep'] });
 });
 
 // API: available locales

test/fetch-utils.test.mjs

@@ -101,6 +101,27 @@ test('safeFetchText returns text and byte count', async () => {
   }
 });
 
+test('intelligence store defines durable memory and prediction lifecycle tables', () => {
+  const store = readFileSync(new URL('../lib/intelligence-store.mjs', import.meta.url), 'utf8');
+  assert.match(store, /CREATE TABLE IF NOT EXISTS events/);
+  assert.match(store, /stable_id TEXT NOT NULL UNIQUE/);
+  assert.match(store, /hypothesis TEXT/);
+  assert.match(store, /evidence_json TEXT/);
+  assert.match(store, /outcome_state TEXT DEFAULT 'open'/);
+  assert.match(store, /evaluatePredictions/);
+  assert.match(store, /queryMemory/);
+  assert.match(store, /listPredictions/);
+});
+
+test('server exposes memory-backed query APIs and dashboard memory action', () => {
+  const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
+  const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
+  assert.match(server, /\/api\/memory\/search/);
+  assert.match(server, /\/api\/memory\/predictions/);
+  assert.match(server, /action === 'memory'/);
+  assert.match(html, /runTerminalAction\('memory'\)/);
+});
+
 test('terminal action endpoints avoid URL tokens and include hardening gates', () => {
   const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
   assert.match(server, /app\.post\('\/api\/action'/);