Initial release — Crucix Intelligence Engine v2.0.0
26-source OSINT intelligence engine with live Jarvis dashboard, auto-refresh via SSE, optional LLM layer (4 providers), delta/memory system, and Telegram breaking news alerts. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
calesthio
117
lib/delta/engine.mjs
Normal file
117
lib/delta/engine.mjs
Normal file
@@ -0,0 +1,117 @@
|
||||
// Delta Engine — compares two synthesized sweep results and produces structured changes
|
||||
|
||||
// Metrics we track for delta computation
|
||||
const NUMERIC_METRICS = [
|
||||
{ key: 'vix', extract: d => d.fred?.find(f => f.id === 'VIXCLS')?.value, label: 'VIX', threshold: 5 },
|
||||
{ key: 'hy_spread', extract: d => d.fred?.find(f => f.id === 'BAMLH0A0HYM2')?.value, label: 'HY Spread', threshold: 5 },
|
||||
{ key: '10y2y', extract: d => d.fred?.find(f => f.id === 'T10Y2Y')?.value, label: '10Y-2Y Spread', threshold: 10 },
|
||||
{ key: 'wti', extract: d => d.energy?.wti, label: 'WTI Crude', threshold: 3 },
|
||||
{ key: 'brent', extract: d => d.energy?.brent, label: 'Brent Crude', threshold: 3 },
|
||||
{ key: 'natgas', extract: d => d.energy?.natgas, label: 'Natural Gas', threshold: 5 },
|
||||
{ key: 'unemployment', extract: d => d.bls?.find(b => b.id === 'LNS14000000' || b.id === 'UNRATE')?.value, label: 'Unemployment', threshold: 2 },
|
||||
{ key: 'fed_funds', extract: d => d.fred?.find(f => f.id === 'DFF')?.value, label: 'Fed Funds Rate', threshold: 1 },
|
||||
{ key: '10y_yield', extract: d => d.fred?.find(f => f.id === 'DGS10')?.value, label: '10Y Yield', threshold: 3 },
|
||||
{ key: 'usd_index', extract: d => d.fred?.find(f => f.id === 'DTWEXBGS')?.value, label: 'USD Index', threshold: 1 },
|
||||
{ key: 'mortgage', extract: d => d.fred?.find(f => f.id === 'MORTGAGE30US')?.value, label: '30Y Mortgage', threshold: 2 },
|
||||
];
|
||||
|
||||
const COUNT_METRICS = [
|
||||
{ key: 'urgent_posts', extract: d => d.tg?.urgent?.length || 0, label: 'Urgent OSINT Posts' },
|
||||
{ key: 'thermal_total', extract: d => d.thermal?.reduce((s, t) => s + t.det, 0) || 0, label: 'Thermal Detections' },
|
||||
{ key: 'air_total', extract: d => d.air?.reduce((s, a) => s + a.total, 0) || 0, label: 'Air Activity' },
|
||||
{ key: 'who_alerts', extract: d => d.who?.length || 0, label: 'WHO Alerts' },
|
||||
{ key: 'conflict_events', extract: d => d.acled?.totalEvents || 0, label: 'Conflict Events' },
|
||||
{ key: 'conflict_fatalities', extract: d => d.acled?.totalFatalities || 0, label: 'Conflict Fatalities' },
|
||||
{ key: 'sdr_online', extract: d => d.sdr?.online || 0, label: 'SDR Receivers' },
|
||||
{ key: 'news_count', extract: d => d.news?.length || 0, label: 'News Items' },
|
||||
{ key: 'sources_ok', extract: d => d.meta?.sourcesOk || 0, label: 'Sources OK' },
|
||||
];
|
||||
|
||||
export function computeDelta(current, previous) {
|
||||
if (!previous) return null;
|
||||
|
||||
const signals = { new: [], escalated: [], deescalated: [], unchanged: [] };
|
||||
let criticalChanges = 0;
|
||||
|
||||
// Numeric metrics: track % change
|
||||
for (const m of NUMERIC_METRICS) {
|
||||
const curr = m.extract(current);
|
||||
const prev = m.extract(previous);
|
||||
if (curr == null || prev == null) continue;
|
||||
|
||||
const pctChange = prev !== 0 ? ((curr - prev) / Math.abs(prev)) * 100 : 0;
|
||||
|
||||
if (Math.abs(pctChange) > m.threshold) {
|
||||
const entry = {
|
||||
key: m.key, label: m.label, from: prev, to: curr,
|
||||
pctChange: parseFloat(pctChange.toFixed(2)),
|
||||
direction: pctChange > 0 ? 'up' : 'down',
|
||||
};
|
||||
if (pctChange > 0) signals.escalated.push(entry);
|
||||
else signals.deescalated.push(entry);
|
||||
if (Math.abs(pctChange) > 10) criticalChanges++;
|
||||
} else {
|
||||
signals.unchanged.push(m.key);
|
||||
}
|
||||
}
|
||||
|
||||
// Count metrics: track absolute change
|
||||
for (const m of COUNT_METRICS) {
|
||||
const curr = m.extract(current);
|
||||
const prev = m.extract(previous);
|
||||
const diff = curr - prev;
|
||||
|
||||
if (Math.abs(diff) > 0) {
|
||||
const entry = {
|
||||
key: m.key, label: m.label, from: prev, to: curr,
|
||||
change: diff, direction: diff > 0 ? 'up' : 'down',
|
||||
};
|
||||
if (diff > 0) signals.escalated.push(entry);
|
||||
else signals.deescalated.push(entry);
|
||||
} else {
|
||||
signals.unchanged.push(m.key);
|
||||
}
|
||||
}
|
||||
|
||||
// New urgent posts (check by text content)
|
||||
const prevUrgentTexts = new Set((previous.tg?.urgent || []).map(p => p.text?.substring(0, 60)));
|
||||
for (const post of (current.tg?.urgent || [])) {
|
||||
const key = post.text?.substring(0, 60);
|
||||
if (key && !prevUrgentTexts.has(key)) {
|
||||
signals.new.push({ key: 'tg_urgent', item: post, reason: 'New urgent OSINT post' });
|
||||
criticalChanges++;
|
||||
}
|
||||
}
|
||||
|
||||
// Nuclear anomaly change
|
||||
const currAnom = current.nuke?.some(n => n.anom) || false;
|
||||
const prevAnom = previous.nuke?.some(n => n.anom) || false;
|
||||
if (currAnom && !prevAnom) {
|
||||
signals.new.push({ key: 'nuke_anomaly', reason: 'Nuclear anomaly detected' });
|
||||
criticalChanges += 5; // Critical
|
||||
} else if (!currAnom && prevAnom) {
|
||||
signals.deescalated.push({ key: 'nuke_anomaly', label: 'Nuclear Anomaly', direction: 'resolved' });
|
||||
}
|
||||
|
||||
// Determine overall direction
|
||||
let direction = 'mixed';
|
||||
const riskUp = signals.escalated.filter(s =>
|
||||
['vix', 'hy_spread', 'urgent_posts', 'conflict_events', 'thermal_total'].includes(s.key)
|
||||
).length;
|
||||
const riskDown = signals.deescalated.filter(s =>
|
||||
['vix', 'hy_spread', 'urgent_posts', 'conflict_events', 'thermal_total'].includes(s.key)
|
||||
).length;
|
||||
if (riskUp > riskDown + 1) direction = 'risk-off';
|
||||
else if (riskDown > riskUp + 1) direction = 'risk-on';
|
||||
|
||||
return {
|
||||
timestamp: current.meta?.timestamp || new Date().toISOString(),
|
||||
previous: previous.meta?.timestamp || null,
|
||||
signals,
|
||||
summary: {
|
||||
totalChanges: signals.new.length + signals.escalated.length + signals.deescalated.length,
|
||||
criticalChanges,
|
||||
direction,
|
||||
},
|
||||
};
|
||||
}
|
||||
2
lib/delta/index.mjs
Normal file
2
lib/delta/index.mjs
Normal file
@@ -0,0 +1,2 @@
|
||||
export { computeDelta } from './engine.mjs';
|
||||
export { MemoryManager } from './memory.mjs';
|
||||
139
lib/delta/memory.mjs
Normal file
139
lib/delta/memory.mjs
Normal file
@@ -0,0 +1,139 @@
|
||||
// Memory Manager — hot/cold storage for sweep history and alert tracking
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { computeDelta } from './engine.mjs';
|
||||
|
||||
const MAX_HOT_RUNS = 3;
|
||||
|
||||
export class MemoryManager {
|
||||
constructor(runsDir) {
|
||||
this.runsDir = runsDir;
|
||||
this.memoryDir = join(runsDir, 'memory');
|
||||
this.hotPath = join(this.memoryDir, 'hot.json');
|
||||
this.coldDir = join(this.memoryDir, 'cold');
|
||||
|
||||
// Ensure dirs exist
|
||||
for (const dir of [this.memoryDir, this.coldDir]) {
|
||||
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
// Load hot memory from disk
|
||||
this.hot = this._loadHot();
|
||||
}
|
||||
|
||||
_loadHot() {
|
||||
try {
|
||||
return JSON.parse(readFileSync(this.hotPath, 'utf8'));
|
||||
} catch {
|
||||
return { runs: [], alertedSignals: {} };
|
||||
}
|
||||
}
|
||||
|
||||
_saveHot() {
|
||||
try {
|
||||
writeFileSync(this.hotPath, JSON.stringify(this.hot, null, 2));
|
||||
} catch (err) {
|
||||
console.error('[Memory] Failed to save hot memory:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Add a new run to hot memory
|
||||
addRun(synthesizedData) {
|
||||
const previous = this.getLastRun();
|
||||
const delta = computeDelta(synthesizedData, previous);
|
||||
|
||||
// Compact the data for storage (strip large arrays)
|
||||
const compact = this._compactForStorage(synthesizedData);
|
||||
|
||||
this.hot.runs.unshift({
|
||||
timestamp: synthesizedData.meta?.timestamp || new Date().toISOString(),
|
||||
data: compact,
|
||||
delta,
|
||||
});
|
||||
|
||||
// Keep only MAX_HOT_RUNS
|
||||
if (this.hot.runs.length > MAX_HOT_RUNS) {
|
||||
const archived = this.hot.runs.splice(MAX_HOT_RUNS);
|
||||
this._archiveToCold(archived);
|
||||
}
|
||||
|
||||
this._saveHot();
|
||||
return delta;
|
||||
}
|
||||
|
||||
// Get last run's synthesized data
|
||||
getLastRun() {
|
||||
if (this.hot.runs.length === 0) return null;
|
||||
return this.hot.runs[0].data;
|
||||
}
|
||||
|
||||
// Get last N runs
|
||||
getRunHistory(n = 3) {
|
||||
return this.hot.runs.slice(0, n);
|
||||
}
|
||||
|
||||
// Get the delta from the most recent run
|
||||
getLastDelta() {
|
||||
if (this.hot.runs.length === 0) return null;
|
||||
return this.hot.runs[0].delta;
|
||||
}
|
||||
|
||||
// Track what signals have been alerted on
|
||||
getAlertedSignals() {
|
||||
return this.hot.alertedSignals || {};
|
||||
}
|
||||
|
||||
markAsAlerted(signalKey, timestamp) {
|
||||
this.hot.alertedSignals[signalKey] = timestamp || new Date().toISOString();
|
||||
this._saveHot();
|
||||
}
|
||||
|
||||
// Clean up old alerted signals (older than 24h)
|
||||
pruneAlertedSignals() {
|
||||
const cutoff = Date.now() - 24 * 60 * 60 * 1000;
|
||||
for (const [key, ts] of Object.entries(this.hot.alertedSignals)) {
|
||||
if (new Date(ts).getTime() < cutoff) {
|
||||
delete this.hot.alertedSignals[key];
|
||||
}
|
||||
}
|
||||
this._saveHot();
|
||||
}
|
||||
|
||||
// Compact data for storage — strip heavy arrays
|
||||
_compactForStorage(data) {
|
||||
return {
|
||||
meta: data.meta,
|
||||
fred: data.fred,
|
||||
energy: data.energy,
|
||||
bls: data.bls,
|
||||
treasury: data.treasury,
|
||||
gscpi: data.gscpi,
|
||||
tg: { posts: data.tg?.posts, urgent: (data.tg?.urgent || []).map(p => ({ text: p.text?.substring(0, 80), date: p.date })) },
|
||||
thermal: (data.thermal || []).map(t => ({ region: t.region, det: t.det, night: t.night, hc: t.hc })),
|
||||
air: (data.air || []).map(a => ({ region: a.region, total: a.total })),
|
||||
nuke: (data.nuke || []).map(n => ({ site: n.site, anom: n.anom, cpm: n.cpm })),
|
||||
who: (data.who || []).map(w => ({ title: w.title })),
|
||||
acled: { totalEvents: data.acled?.totalEvents, totalFatalities: data.acled?.totalFatalities },
|
||||
sdr: { total: data.sdr?.total, online: data.sdr?.online },
|
||||
ideas: (data.ideas || []).map(i => ({ title: i.title, type: i.type, confidence: i.confidence })),
|
||||
};
|
||||
}
|
||||
|
||||
// Archive old runs to cold storage
|
||||
_archiveToCold(runs) {
|
||||
if (runs.length === 0) return;
|
||||
const dateKey = new Date().toISOString().split('T')[0]; // YYYY-MM-DD
|
||||
const coldPath = join(this.coldDir, `${dateKey}.json`);
|
||||
|
||||
let existing = [];
|
||||
try { existing = JSON.parse(readFileSync(coldPath, 'utf8')); } catch { }
|
||||
|
||||
existing.push(...runs);
|
||||
try {
|
||||
writeFileSync(coldPath, JSON.stringify(existing, null, 2));
|
||||
} catch (err) {
|
||||
console.error('[Memory] Failed to archive to cold storage:', err.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user