diff --git a/README.md b/README.md index 55e9e91..fffd910 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,36 @@ Action endpoints reject cross-origin POST origins, apply a small in-memory per-I When data remains stale past `STALE_DATA_MAX_AGE_MINUTES`, the server sends an operator alert through configured Telegram/Discord channels after failed or degraded sweep attempts. `STALE_ALERT_COOLDOWN_MINUTES` prevents repeated stale alerts from spamming every refresh interval. Set `DASHBOARD_URL` to the Pangolin/public URL you want included in those alerts. +#### Memory And Prediction Loop + +Crucix stores longitudinal memory in `runs/intelligence.db` when the current Node.js build exposes `node:sqlite`. If SQLite is unavailable, the file is created as a harmless placeholder and `/api/health` reports the memory store as unavailable instead of failing the sweep. + +The memory layer persists: + +| Table | Purpose | +| --- | --- | +| `runs` | Sweep timestamps, source health counts, and delta direction summaries. | +| `entities` | Stable entity IDs for recurring countries, regions, and locations. | +| `events` | Stable event IDs for conflict, OSINT, urgent news, and new delta signals across sweeps. | +| `predictions` | Trade/intelligence hypotheses with evidence, confidence, horizon, outcome state, and latest grading. | + +Query endpoints: + +```text +GET /api/memory/search?q=iran&limit=25 +GET /api/memory/predictions?state=open&limit=25 +``` + +Memory endpoints use the same operator authorization gate as Terminal Actions. The dashboard Terminal Actions panel includes a `Memory` action for a quick operator-facing view of recent events and prediction states. + +Retention, backup, and privacy expectations: + +- Treat `runs/intelligence.db` as operator data. It can contain source excerpts, headlines, generated hypotheses, and URLs from your configured feeds. +- Back up `runs/` with the rest of your Dockge volume if you want longitudinal learning to survive container replacement. +- Delete `runs/intelligence.db` to reset SQLite memory; the next sweep recreates the schema. +- Do not commit `runs/` or `.env`. API credentials stay in `.env`; memory stores derived observations, not secrets. +- If you expose the dashboard through a reverse proxy, protect Terminal Actions and memory queries behind your normal authentication boundary. + #### Scenario Watchlist Intelligence Terminal can track operator hypotheses across sweeps with a runtime scenario file at `runs/scenarios.json`. On first run, the server creates three disabled starter examples: diff --git a/dashboard/public/jarvis.html b/dashboard/public/jarvis.html index 4f2f7bd..cb0b228 100644 --- a/dashboard/public/jarvis.html +++ b/dashboard/public/jarvis.html @@ -83,7 +83,7 @@ html,body{height:100%;background:var(--bg);color:var(--text);font-family:var(--s .sensor-actions{display:flex;gap:6px;align-items:center} .mini-btn{border:1px solid rgba(100,240,200,0.18);background:rgba(100,240,200,0.04);color:var(--dim);font-family:var(--mono);font-size:9px;padding:3px 6px;cursor:pointer} .mini-btn:hover{color:var(--accent);border-color:rgba(100,240,200,0.4)} -.action-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:6px;margin-bottom:10px} +.action-grid{display:grid;grid-template-columns:repeat(4,1fr);gap:6px;margin-bottom:10px} .action-btn{border:1px solid rgba(68,204,255,0.24);background:rgba(68,204,255,0.06);color:var(--text);font-family:var(--mono);font-size:9px;padding:7px 6px;cursor:pointer;text-transform:uppercase;letter-spacing:.08em} .action-btn:hover{border-color:rgba(68,204,255,0.55);color:var(--accent2);background:rgba(68,204,255,0.12)} .action-btn[disabled]{opacity:.45;cursor:wait} @@ -1641,6 +1641,17 @@ async function runTerminalAction(action){ ].join('\n'); }else if(action === 'brief'){ terminalOutput = `> brief\n${payload.text || 'No briefing text returned.'}`; + }else if(action === 'memory'){ + const events = payload.recentEvents || []; + const predictions = payload.predictions || []; + terminalOutput = [ + '> memory', + `Store: ${payload.memory?.available ? 'available' : 'unavailable'}`, + `Recent events: ${events.length}`, + ...events.slice(0,4).map(e => `- ${e.kind}: ${e.name}${e.region ? ' [' + e.region + ']' : ''}`), + `Predictions: ${predictions.length}`, + ...predictions.slice(0,4).map(p => `- ${p.outcome_state || 'open'}: ${p.title}`) + ].join('\n'); }else if(action === 'sweep'){ terminalOutput = `> sweep\n${payload.status === 'already_running' ? 'Sweep already running.' : 'Sweep accepted. The dashboard will update when the sweep finishes.'}`; } @@ -1707,6 +1718,7 @@ function renderRight(){ +
${terminalOutput.replace(/[&<>]/g,c=>({'&':'&','<':'<','>':'>'}[c])).replace(/\n/g,'
')}
diff --git a/lib/intelligence-store.mjs b/lib/intelligence-store.mjs index 10daf40..fff20d3 100644 --- a/lib/intelligence-store.mjs +++ b/lib/intelligence-store.mjs @@ -2,6 +2,9 @@ import { existsSync, mkdirSync, writeFileSync } from 'fs'; import { dirname, join } from 'path'; +import { createHash } from 'crypto'; + +const PREDICTION_STATES = new Set(['open', 'monitoring', 'observed', 'expired_unverified', 'invalidated']); export class IntelligenceStore { constructor(dbPath) { @@ -30,15 +33,24 @@ export class IntelligenceStore { ); CREATE TABLE IF NOT EXISTS predictions ( id INTEGER PRIMARY KEY AUTOINCREMENT, + stable_id TEXT UNIQUE, created_at TEXT NOT NULL, + updated_at TEXT, title TEXT NOT NULL, type TEXT, + hypothesis TEXT, + evidence_json TEXT, confidence TEXT, + horizon TEXT, + outcome_state TEXT DEFAULT 'open', + outcome_json TEXT, + last_evaluated_at TEXT, source TEXT, payload_json TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS entities ( id INTEGER PRIMARY KEY AUTOINCREMENT, + stable_id TEXT UNIQUE, first_seen TEXT NOT NULL, last_seen TEXT NOT NULL, name TEXT NOT NULL, @@ -46,7 +58,21 @@ export class IntelligenceStore { count INTEGER DEFAULT 1, UNIQUE(name, kind) ); + CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + stable_id TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL, + kind TEXT NOT NULL, + name TEXT NOT NULL, + region TEXT, + severity TEXT, + source TEXT, + evidence_json TEXT NOT NULL, + count INTEGER DEFAULT 1 + ); `); + this._migrate(); this.available = true; } catch (err) { this.available = false; @@ -71,24 +97,141 @@ export class IntelligenceStore { delta?.summary?.direction || null, JSON.stringify({ meta, delta: delta?.summary || null }), ); - for (const idea of data.ideas || []) { - this.db.prepare(`INSERT INTO predictions (created_at, title, type, confidence, source, payload_json) - VALUES (?, ?, ?, ?, ?, ?)`).run( - timestamp, - idea.title || 'Untitled idea', - idea.type || null, - idea.confidence || null, - idea.source || data.ideasSource || null, - JSON.stringify(idea), - ); - } this._recordEntities(data, timestamp); + this._recordEvents(data, delta, timestamp); + this.evaluatePredictions(data, timestamp); + this._recordPredictions(data, timestamp); } status() { return { available: this.available, path: this.dbPath, reason: this.reason }; } + queryMemory({ q = '', limit = 25 } = {}) { + if (!this.available || !this.db) return { available: false, reason: this.reason, results: [] }; + const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25)); + const term = String(q || '').trim(); + const like = `%${term}%`; + const where = term + ? 'WHERE name LIKE ? OR region LIKE ? OR source LIKE ? OR kind LIKE ?' + : ''; + const params = term ? [like, like, like, like, safeLimit] : [safeLimit]; + const events = this.db.prepare(` + SELECT stable_id, first_seen, last_seen, kind, name, region, severity, source, count, evidence_json + FROM events + ${where} + ORDER BY last_seen DESC + LIMIT ? + `).all(...params).map(row => ({ ...row, evidence: parseJson(row.evidence_json, {}) })); + return { available: true, q: term, results: events }; + } + + listPredictions({ state = null, limit = 25 } = {}) { + if (!this.available || !this.db) return { available: false, reason: this.reason, predictions: [] }; + const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25)); + const normalizedState = state && PREDICTION_STATES.has(String(state)) ? String(state) : null; + const rows = normalizedState + ? this.db.prepare(`SELECT * FROM predictions WHERE outcome_state = ? ORDER BY created_at DESC LIMIT ?`).all(normalizedState, safeLimit) + : this.db.prepare(`SELECT * FROM predictions ORDER BY created_at DESC LIMIT ?`).all(safeLimit); + return { + available: true, + predictions: rows.map(row => ({ + stable_id: row.stable_id, + created_at: row.created_at, + updated_at: row.updated_at, + title: row.title, + type: row.type, + hypothesis: row.hypothesis, + confidence: row.confidence, + horizon: row.horizon, + outcome_state: row.outcome_state, + last_evaluated_at: row.last_evaluated_at, + source: row.source, + evidence: parseJson(row.evidence_json, []), + outcome: parseJson(row.outcome_json, null), + })), + }; + } + + evaluatePredictions(data, timestamp = new Date().toISOString()) { + if (!this.available || !this.db) return; + const rows = this.db.prepare(` + SELECT id, created_at, title, type, horizon, outcome_state, payload_json + FROM predictions + WHERE outcome_state IN ('open', 'monitoring') + ORDER BY created_at ASC + LIMIT 200 + `).all(); + for (const row of rows) { + const payload = parseJson(row.payload_json, {}); + const evaluation = evaluatePredictionAgainstSweep(row, payload, data, timestamp); + this.db.prepare(`UPDATE predictions + SET outcome_state = ?, outcome_json = ?, last_evaluated_at = ?, updated_at = ? + WHERE id = ?`).run( + evaluation.state, + JSON.stringify(evaluation), + timestamp, + timestamp, + row.id, + ); + } + } + + _migrate() { + const columns = { + predictions: [ + ['stable_id', 'TEXT'], + ['updated_at', 'TEXT'], + ['hypothesis', 'TEXT'], + ['evidence_json', 'TEXT'], + ['horizon', 'TEXT'], + ['outcome_state', "TEXT DEFAULT 'open'"], + ['outcome_json', 'TEXT'], + ['last_evaluated_at', 'TEXT'], + ], + entities: [ + ['stable_id', 'TEXT'], + ], + }; + for (const [table, defs] of Object.entries(columns)) { + for (const [name, type] of defs) { + try { this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${name} ${type}`); } catch { } + } + } + try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_predictions_stable_id ON predictions(stable_id)`); } catch { } + try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_entities_stable_id ON entities(stable_id)`); } catch { } + } + + _recordPredictions(data, timestamp) { + for (const idea of data.ideas || []) { + const title = idea.title || 'Untitled idea'; + const stableId = stableId('prediction', title, idea.type || '', idea.ticker || '', idea.horizon || ''); + const evidence = Array.isArray(idea.signals) ? idea.signals : []; + this.db.prepare(`INSERT INTO predictions ( + stable_id, created_at, updated_at, title, type, hypothesis, evidence_json, confidence, + horizon, outcome_state, source, payload_json + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, ?) + ON CONFLICT(stable_id) DO UPDATE SET + updated_at=excluded.updated_at, + confidence=excluded.confidence, + evidence_json=excluded.evidence_json, + payload_json=excluded.payload_json`).run( + stableId, + timestamp, + timestamp, + title, + idea.type || null, + idea.rationale || idea.text || title, + JSON.stringify(evidence), + idea.confidence || null, + idea.horizon || null, + idea.source || data.ideasSource || null, + JSON.stringify(idea), + ); + } + } + _recordEntities(data, timestamp) { const names = []; for (const item of data.acled?.deadliestEvents || []) { @@ -99,14 +242,154 @@ export class IntelligenceStore { if (item.region) names.push([item.region, 'region']); } for (const [name, kind] of names.slice(0, 200)) { - this.db.prepare(`INSERT INTO entities (first_seen, last_seen, name, kind, count) - VALUES (?, ?, ?, ?, 1) + const cleanName = String(name).slice(0, 160); + this.db.prepare(`INSERT INTO entities (stable_id, first_seen, last_seen, name, kind, count) + VALUES (?, ?, ?, ?, ?, 1) ON CONFLICT(name, kind) DO UPDATE SET last_seen=excluded.last_seen, count=count+1`).run( + stableId('entity', kind, cleanName), timestamp, timestamp, - String(name).slice(0, 160), + cleanName, kind, ); } } + + _recordEvents(data, delta, timestamp) { + const events = extractEvents(data, delta); + for (const event of events.slice(0, 300)) { + this.db.prepare(`INSERT INTO events ( + stable_id, first_seen, last_seen, kind, name, region, severity, source, evidence_json, count + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1) + ON CONFLICT(stable_id) DO UPDATE SET + last_seen=excluded.last_seen, + severity=COALESCE(excluded.severity, severity), + evidence_json=excluded.evidence_json, + count=count+1`).run( + event.stable_id, + timestamp, + timestamp, + event.kind, + event.name, + event.region || null, + event.severity || null, + event.source || null, + JSON.stringify(event.evidence || {}), + ); + } + } +} + +function stableId(...parts) { + const input = parts.map(part => String(part || '').trim().toLowerCase()).join('|'); + return createHash('sha256').update(input).digest('hex').slice(0, 24); +} + +function parseJson(value, fallback) { + try { return value ? JSON.parse(value) : fallback; } catch { return fallback; } +} + +function extractEvents(data, delta) { + const events = []; + const push = ({ kind, name, region, severity, source, evidence }) => { + if (!kind || !name) return; + events.push({ + stable_id: stableId('event', kind, name, region || source || ''), + kind, + name: String(name).slice(0, 240), + region: region ? String(region).slice(0, 120) : null, + severity: severity || null, + source: source || null, + evidence: evidence || {}, + }); + }; + + for (const item of data.acled?.deadliestEvents || []) { + push({ + kind: 'conflict', + name: item.event_type || item.sub_event_type || item.location || item.country, + region: item.country || item.location, + severity: Number(item.fatalities || 0) > 0 ? 'high' : 'medium', + source: 'ACLED', + evidence: item, + }); + } + for (const item of data.tg?.urgent || []) { + push({ + kind: 'osint', + name: (item.text || '').slice(0, 120), + region: item.region || 'OSINT', + severity: 'high', + source: item.channel || item.chat || 'telegram', + evidence: item, + }); + } + for (const item of data.newsFeed || data.news || []) { + if (!item.urgent) continue; + push({ + kind: 'news', + name: item.headline || item.title, + region: item.region, + severity: 'medium', + source: item.source, + evidence: item, + }); + } + for (const signal of delta?.signals?.new || []) { + push({ + kind: 'delta', + name: signal.label || signal.reason || signal.key, + region: signal.region, + severity: signal.severity || 'medium', + source: 'delta', + evidence: signal, + }); + } + return events; +} + +function evaluatePredictionAgainstSweep(row, payload, data, timestamp) { + const terms = [ + row.title, + payload.ticker, + ...(Array.isArray(payload.signals) ? payload.signals : []), + ].filter(Boolean).map(v => String(v).toLowerCase()); + const evidenceText = [ + ...(data.tSignals || []), + ...(data.newsFeed || []).slice(0, 40).map(n => `${n.source || ''} ${n.headline || n.title || ''}`), + ...(data.tg?.urgent || []).slice(0, 20).map(p => p.text || ''), + ].join('\n').toLowerCase(); + const matched = terms.filter(term => term.length >= 4 && evidenceText.includes(term.slice(0, 60))); + const expired = predictionExpired(row.created_at, row.horizon, timestamp); + const state = matched.length + ? 'observed' + : expired + ? 'expired_unverified' + : 'monitoring'; + return { + state, + evaluated_at: timestamp, + matched_terms: matched.slice(0, 10), + expired, + reason: matched.length + ? 'Current sweep contains matching evidence terms.' + : expired + ? 'Prediction horizon elapsed without matching evidence.' + : 'Prediction remains open for future sweeps.', + }; +} + +function predictionExpired(createdAt, horizon, nowIso) { + const created = new Date(createdAt).getTime(); + const now = new Date(nowIso).getTime(); + if (!Number.isFinite(created) || !Number.isFinite(now)) return false; + const text = String(horizon || '').toLowerCase(); + const days = text.includes('intraday') ? 1 + : text.includes('day') ? 7 + : text.includes('week') ? 45 + : text.includes('month') ? 180 + : text.includes('strategic') ? 365 + : 30; + return now - created > days * 24 * 60 * 60 * 1000; } diff --git a/server.mjs b/server.mjs index 180a550..d9e0a3f 100644 --- a/server.mjs +++ b/server.mjs @@ -292,6 +292,26 @@ app.get('/api/metrics', (req, res) => { }); }); +app.get('/api/memory/search', (req, res) => { + const guard = authorizeTerminalAction(req, res, 'memory:search'); + if (!guard.ok) return; + auditTerminalAction(req, 'memory:search', 'ok'); + res.json(intelligenceStore.queryMemory({ + q: req.query.q || '', + limit: req.query.limit || 25, + })); +}); + +app.get('/api/memory/predictions', (req, res) => { + const guard = authorizeTerminalAction(req, res, 'memory:predictions'); + if (!guard.ok) return; + auditTerminalAction(req, 'memory:predictions', 'ok'); + res.json(intelligenceStore.listPredictions({ + state: req.query.state || null, + limit: req.query.limit || 25, + })); +}); + app.post('/api/sweep', express.json(), (req, res) => { const guard = authorizeTerminalAction(req, res, 'sweep'); if (!guard.ok) return; @@ -318,10 +338,21 @@ app.post('/api/action', express.json(), (req, res) => { return res.json({ ok: true, action, status: 'ok', brief, text: brief }); } + if (action === 'memory') { + auditTerminalAction(req, 'memory', 'ok'); + return res.json({ + ok: true, + action, + memory: intelligenceStore.status(), + recentEvents: intelligenceStore.queryMemory({ q: req.body?.q || '', limit: 8 }).results, + predictions: intelligenceStore.listPredictions({ limit: 8 }).predictions, + }); + } + if (action === 'sweep') return triggerSweepAction(req, res, 'action:sweep'); auditTerminalAction(req, action || 'unknown', 'rejected', 'unknown_action'); - return res.status(400).json({ ok: false, error: 'Unknown action', allowed: ['status', 'brief', 'sweep'], actions: ['status', 'brief', 'sweep'] }); + return res.status(400).json({ ok: false, error: 'Unknown action', allowed: ['status', 'brief', 'memory', 'sweep'], actions: ['status', 'brief', 'memory', 'sweep'] }); }); // API: available locales diff --git a/test/fetch-utils.test.mjs b/test/fetch-utils.test.mjs index 27d34ff..95aaab2 100644 --- a/test/fetch-utils.test.mjs +++ b/test/fetch-utils.test.mjs @@ -101,6 +101,27 @@ test('safeFetchText returns text and byte count', async () => { } }); +test('intelligence store defines durable memory and prediction lifecycle tables', () => { + const store = readFileSync(new URL('../lib/intelligence-store.mjs', import.meta.url), 'utf8'); + assert.match(store, /CREATE TABLE IF NOT EXISTS events/); + assert.match(store, /stable_id TEXT NOT NULL UNIQUE/); + assert.match(store, /hypothesis TEXT/); + assert.match(store, /evidence_json TEXT/); + assert.match(store, /outcome_state TEXT DEFAULT 'open'/); + assert.match(store, /evaluatePredictions/); + assert.match(store, /queryMemory/); + assert.match(store, /listPredictions/); +}); + +test('server exposes memory-backed query APIs and dashboard memory action', () => { + const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8'); + const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8'); + assert.match(server, /\/api\/memory\/search/); + assert.match(server, /\/api\/memory\/predictions/); + assert.match(server, /action === 'memory'/); + assert.match(html, /runTerminalAction\('memory'\)/); +}); + test('terminal action endpoints avoid URL tokens and include hardening gates', () => { const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8'); assert.match(server, /app\.post\('\/api\/action'/);