396 lines
13 KiB
JavaScript
396 lines
13 KiB
JavaScript
// Phase-1 intelligence memory. Uses node:sqlite when available and degrades to no-op.
|
|
|
|
import { existsSync, mkdirSync, writeFileSync } from 'fs';
|
|
import { dirname, join } from 'path';
|
|
import { createHash } from 'crypto';
|
|
|
|
const PREDICTION_STATES = new Set(['open', 'monitoring', 'observed', 'expired_unverified', 'invalidated']);
|
|
|
|
export class IntelligenceStore {
|
|
constructor(dbPath) {
|
|
this.dbPath = dbPath;
|
|
this.db = null;
|
|
this.available = false;
|
|
this.reason = null;
|
|
}
|
|
|
|
async init() {
|
|
const dir = dirname(this.dbPath);
|
|
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
try {
|
|
const sqlite = await import('node:sqlite');
|
|
const DatabaseSync = sqlite.DatabaseSync;
|
|
this.db = new DatabaseSync(this.dbPath);
|
|
this.db.exec(`
|
|
CREATE TABLE IF NOT EXISTS runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
timestamp TEXT NOT NULL,
|
|
sources_ok INTEGER DEFAULT 0,
|
|
sources_degraded INTEGER DEFAULT 0,
|
|
sources_failed INTEGER DEFAULT 0,
|
|
direction TEXT,
|
|
summary_json TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS predictions (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
stable_id TEXT UNIQUE,
|
|
created_at TEXT NOT NULL,
|
|
updated_at TEXT,
|
|
title TEXT NOT NULL,
|
|
type TEXT,
|
|
hypothesis TEXT,
|
|
evidence_json TEXT,
|
|
confidence TEXT,
|
|
horizon TEXT,
|
|
outcome_state TEXT DEFAULT 'open',
|
|
outcome_json TEXT,
|
|
last_evaluated_at TEXT,
|
|
source TEXT,
|
|
payload_json TEXT NOT NULL
|
|
);
|
|
CREATE TABLE IF NOT EXISTS entities (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
stable_id TEXT UNIQUE,
|
|
first_seen TEXT NOT NULL,
|
|
last_seen TEXT NOT NULL,
|
|
name TEXT NOT NULL,
|
|
kind TEXT NOT NULL,
|
|
count INTEGER DEFAULT 1,
|
|
UNIQUE(name, kind)
|
|
);
|
|
CREATE TABLE IF NOT EXISTS events (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
stable_id TEXT NOT NULL UNIQUE,
|
|
first_seen TEXT NOT NULL,
|
|
last_seen TEXT NOT NULL,
|
|
kind TEXT NOT NULL,
|
|
name TEXT NOT NULL,
|
|
region TEXT,
|
|
severity TEXT,
|
|
source TEXT,
|
|
evidence_json TEXT NOT NULL,
|
|
count INTEGER DEFAULT 1
|
|
);
|
|
`);
|
|
this._migrate();
|
|
this.available = true;
|
|
} catch (err) {
|
|
this.available = false;
|
|
this.reason = err.message;
|
|
if (!existsSync(this.dbPath)) {
|
|
writeFileSync(this.dbPath, '');
|
|
}
|
|
}
|
|
return this;
|
|
}
|
|
|
|
recordRun(data, delta) {
|
|
if (!this.available || !this.db) return;
|
|
const meta = data.meta || {};
|
|
const timestamp = meta.timestamp || new Date().toISOString();
|
|
this.db.prepare(`INSERT INTO runs (timestamp, sources_ok, sources_degraded, sources_failed, direction, summary_json)
|
|
VALUES (?, ?, ?, ?, ?, ?)`).run(
|
|
timestamp,
|
|
meta.sourcesOk || 0,
|
|
meta.sourcesDegraded || 0,
|
|
meta.sourcesFailed || 0,
|
|
delta?.summary?.direction || null,
|
|
JSON.stringify({ meta, delta: delta?.summary || null }),
|
|
);
|
|
this._recordEntities(data, timestamp);
|
|
this._recordEvents(data, delta, timestamp);
|
|
this.evaluatePredictions(data, timestamp);
|
|
this._recordPredictions(data, timestamp);
|
|
}
|
|
|
|
status() {
|
|
return { available: this.available, path: this.dbPath, reason: this.reason };
|
|
}
|
|
|
|
queryMemory({ q = '', limit = 25 } = {}) {
|
|
if (!this.available || !this.db) return { available: false, reason: this.reason, results: [] };
|
|
const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
|
|
const term = String(q || '').trim();
|
|
const like = `%${term}%`;
|
|
const where = term
|
|
? 'WHERE name LIKE ? OR region LIKE ? OR source LIKE ? OR kind LIKE ?'
|
|
: '';
|
|
const params = term ? [like, like, like, like, safeLimit] : [safeLimit];
|
|
const events = this.db.prepare(`
|
|
SELECT stable_id, first_seen, last_seen, kind, name, region, severity, source, count, evidence_json
|
|
FROM events
|
|
${where}
|
|
ORDER BY last_seen DESC
|
|
LIMIT ?
|
|
`).all(...params).map(row => ({ ...row, evidence: parseJson(row.evidence_json, {}) }));
|
|
return { available: true, q: term, results: events };
|
|
}
|
|
|
|
listPredictions({ state = null, limit = 25 } = {}) {
|
|
if (!this.available || !this.db) return { available: false, reason: this.reason, predictions: [] };
|
|
const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
|
|
const normalizedState = state && PREDICTION_STATES.has(String(state)) ? String(state) : null;
|
|
const rows = normalizedState
|
|
? this.db.prepare(`SELECT * FROM predictions WHERE outcome_state = ? ORDER BY created_at DESC LIMIT ?`).all(normalizedState, safeLimit)
|
|
: this.db.prepare(`SELECT * FROM predictions ORDER BY created_at DESC LIMIT ?`).all(safeLimit);
|
|
return {
|
|
available: true,
|
|
predictions: rows.map(row => ({
|
|
stable_id: row.stable_id,
|
|
created_at: row.created_at,
|
|
updated_at: row.updated_at,
|
|
title: row.title,
|
|
type: row.type,
|
|
hypothesis: row.hypothesis,
|
|
confidence: row.confidence,
|
|
horizon: row.horizon,
|
|
outcome_state: row.outcome_state,
|
|
last_evaluated_at: row.last_evaluated_at,
|
|
source: row.source,
|
|
evidence: parseJson(row.evidence_json, []),
|
|
outcome: parseJson(row.outcome_json, null),
|
|
})),
|
|
};
|
|
}
|
|
|
|
evaluatePredictions(data, timestamp = new Date().toISOString()) {
|
|
if (!this.available || !this.db) return;
|
|
const rows = this.db.prepare(`
|
|
SELECT id, created_at, title, type, horizon, outcome_state, payload_json
|
|
FROM predictions
|
|
WHERE outcome_state IN ('open', 'monitoring')
|
|
ORDER BY created_at ASC
|
|
LIMIT 200
|
|
`).all();
|
|
for (const row of rows) {
|
|
const payload = parseJson(row.payload_json, {});
|
|
const evaluation = evaluatePredictionAgainstSweep(row, payload, data, timestamp);
|
|
this.db.prepare(`UPDATE predictions
|
|
SET outcome_state = ?, outcome_json = ?, last_evaluated_at = ?, updated_at = ?
|
|
WHERE id = ?`).run(
|
|
evaluation.state,
|
|
JSON.stringify(evaluation),
|
|
timestamp,
|
|
timestamp,
|
|
row.id,
|
|
);
|
|
}
|
|
}
|
|
|
|
_migrate() {
|
|
const columns = {
|
|
predictions: [
|
|
['stable_id', 'TEXT'],
|
|
['updated_at', 'TEXT'],
|
|
['hypothesis', 'TEXT'],
|
|
['evidence_json', 'TEXT'],
|
|
['horizon', 'TEXT'],
|
|
['outcome_state', "TEXT DEFAULT 'open'"],
|
|
['outcome_json', 'TEXT'],
|
|
['last_evaluated_at', 'TEXT'],
|
|
],
|
|
entities: [
|
|
['stable_id', 'TEXT'],
|
|
],
|
|
};
|
|
for (const [table, defs] of Object.entries(columns)) {
|
|
for (const [name, type] of defs) {
|
|
try { this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${name} ${type}`); } catch { }
|
|
}
|
|
}
|
|
try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_predictions_stable_id ON predictions(stable_id)`); } catch { }
|
|
try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_entities_stable_id ON entities(stable_id)`); } catch { }
|
|
}
|
|
|
|
_recordPredictions(data, timestamp) {
|
|
for (const idea of data.ideas || []) {
|
|
const title = idea.title || 'Untitled idea';
|
|
const stableId = stableId('prediction', title, idea.type || '', idea.ticker || '', idea.horizon || '');
|
|
const evidence = Array.isArray(idea.signals) ? idea.signals : [];
|
|
this.db.prepare(`INSERT INTO predictions (
|
|
stable_id, created_at, updated_at, title, type, hypothesis, evidence_json, confidence,
|
|
horizon, outcome_state, source, payload_json
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, ?)
|
|
ON CONFLICT(stable_id) DO UPDATE SET
|
|
updated_at=excluded.updated_at,
|
|
confidence=excluded.confidence,
|
|
evidence_json=excluded.evidence_json,
|
|
payload_json=excluded.payload_json`).run(
|
|
stableId,
|
|
timestamp,
|
|
timestamp,
|
|
title,
|
|
idea.type || null,
|
|
idea.rationale || idea.text || title,
|
|
JSON.stringify(evidence),
|
|
idea.confidence || null,
|
|
idea.horizon || null,
|
|
idea.source || data.ideasSource || null,
|
|
JSON.stringify(idea),
|
|
);
|
|
}
|
|
}
|
|
|
|
_recordEntities(data, timestamp) {
|
|
const names = [];
|
|
for (const item of data.acled?.deadliestEvents || []) {
|
|
if (item.country) names.push([item.country, 'country']);
|
|
if (item.location) names.push([item.location, 'location']);
|
|
}
|
|
for (const item of data.news || []) {
|
|
if (item.region) names.push([item.region, 'region']);
|
|
}
|
|
for (const [name, kind] of names.slice(0, 200)) {
|
|
const cleanName = String(name).slice(0, 160);
|
|
this.db.prepare(`INSERT INTO entities (stable_id, first_seen, last_seen, name, kind, count)
|
|
VALUES (?, ?, ?, ?, ?, 1)
|
|
ON CONFLICT(name, kind) DO UPDATE SET last_seen=excluded.last_seen, count=count+1`).run(
|
|
stableId('entity', kind, cleanName),
|
|
timestamp,
|
|
timestamp,
|
|
cleanName,
|
|
kind,
|
|
);
|
|
}
|
|
}
|
|
|
|
_recordEvents(data, delta, timestamp) {
|
|
const events = extractEvents(data, delta);
|
|
for (const event of events.slice(0, 300)) {
|
|
this.db.prepare(`INSERT INTO events (
|
|
stable_id, first_seen, last_seen, kind, name, region, severity, source, evidence_json, count
|
|
)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
|
|
ON CONFLICT(stable_id) DO UPDATE SET
|
|
last_seen=excluded.last_seen,
|
|
severity=COALESCE(excluded.severity, severity),
|
|
evidence_json=excluded.evidence_json,
|
|
count=count+1`).run(
|
|
event.stable_id,
|
|
timestamp,
|
|
timestamp,
|
|
event.kind,
|
|
event.name,
|
|
event.region || null,
|
|
event.severity || null,
|
|
event.source || null,
|
|
JSON.stringify(event.evidence || {}),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
function stableId(...parts) {
|
|
const input = parts.map(part => String(part || '').trim().toLowerCase()).join('|');
|
|
return createHash('sha256').update(input).digest('hex').slice(0, 24);
|
|
}
|
|
|
|
function parseJson(value, fallback) {
|
|
try { return value ? JSON.parse(value) : fallback; } catch { return fallback; }
|
|
}
|
|
|
|
function extractEvents(data, delta) {
|
|
const events = [];
|
|
const push = ({ kind, name, region, severity, source, evidence }) => {
|
|
if (!kind || !name) return;
|
|
events.push({
|
|
stable_id: stableId('event', kind, name, region || source || ''),
|
|
kind,
|
|
name: String(name).slice(0, 240),
|
|
region: region ? String(region).slice(0, 120) : null,
|
|
severity: severity || null,
|
|
source: source || null,
|
|
evidence: evidence || {},
|
|
});
|
|
};
|
|
|
|
for (const item of data.acled?.deadliestEvents || []) {
|
|
push({
|
|
kind: 'conflict',
|
|
name: item.event_type || item.sub_event_type || item.location || item.country,
|
|
region: item.country || item.location,
|
|
severity: Number(item.fatalities || 0) > 0 ? 'high' : 'medium',
|
|
source: 'ACLED',
|
|
evidence: item,
|
|
});
|
|
}
|
|
for (const item of data.tg?.urgent || []) {
|
|
push({
|
|
kind: 'osint',
|
|
name: (item.text || '').slice(0, 120),
|
|
region: item.region || 'OSINT',
|
|
severity: 'high',
|
|
source: item.channel || item.chat || 'telegram',
|
|
evidence: item,
|
|
});
|
|
}
|
|
for (const item of data.newsFeed || data.news || []) {
|
|
if (!item.urgent) continue;
|
|
push({
|
|
kind: 'news',
|
|
name: item.headline || item.title,
|
|
region: item.region,
|
|
severity: 'medium',
|
|
source: item.source,
|
|
evidence: item,
|
|
});
|
|
}
|
|
for (const signal of delta?.signals?.new || []) {
|
|
push({
|
|
kind: 'delta',
|
|
name: signal.label || signal.reason || signal.key,
|
|
region: signal.region,
|
|
severity: signal.severity || 'medium',
|
|
source: 'delta',
|
|
evidence: signal,
|
|
});
|
|
}
|
|
return events;
|
|
}
|
|
|
|
function evaluatePredictionAgainstSweep(row, payload, data, timestamp) {
|
|
const terms = [
|
|
row.title,
|
|
payload.ticker,
|
|
...(Array.isArray(payload.signals) ? payload.signals : []),
|
|
].filter(Boolean).map(v => String(v).toLowerCase());
|
|
const evidenceText = [
|
|
...(data.tSignals || []),
|
|
...(data.newsFeed || []).slice(0, 40).map(n => `${n.source || ''} ${n.headline || n.title || ''}`),
|
|
...(data.tg?.urgent || []).slice(0, 20).map(p => p.text || ''),
|
|
].join('\n').toLowerCase();
|
|
const matched = terms.filter(term => term.length >= 4 && evidenceText.includes(term.slice(0, 60)));
|
|
const expired = predictionExpired(row.created_at, row.horizon, timestamp);
|
|
const state = matched.length
|
|
? 'observed'
|
|
: expired
|
|
? 'expired_unverified'
|
|
: 'monitoring';
|
|
return {
|
|
state,
|
|
evaluated_at: timestamp,
|
|
matched_terms: matched.slice(0, 10),
|
|
expired,
|
|
reason: matched.length
|
|
? 'Current sweep contains matching evidence terms.'
|
|
: expired
|
|
? 'Prediction horizon elapsed without matching evidence.'
|
|
: 'Prediction remains open for future sweeps.',
|
|
};
|
|
}
|
|
|
|
function predictionExpired(createdAt, horizon, nowIso) {
|
|
const created = new Date(createdAt).getTime();
|
|
const now = new Date(nowIso).getTime();
|
|
if (!Number.isFinite(created) || !Number.isFinite(now)) return false;
|
|
const text = String(horizon || '').toLowerCase();
|
|
const days = text.includes('intraday') ? 1
|
|
: text.includes('day') ? 7
|
|
: text.includes('week') ? 45
|
|
: text.includes('month') ? 180
|
|
: text.includes('strategic') ? 365
|
|
: 30;
|
|
return now - created > days * 24 * 60 * 60 * 1000;
|
|
}
|