MrSphay/intelligence-terminal
1
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases Wiki Activity

Crucix — agent with dashboard, delta engine, Telegram/Discord bots

Browse Source
This commit is contained in:
calesthio
2026-03-14 00:35:31 -07:00
parent ef2c6470fb
commit 3674fcb4f7
23 changed files with 2143 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
lib/delta/engine.mjs
View File

@@ -1,18 +1,51 @@
// Delta Engine — compares two synthesized sweep results and produces structured changes
// Delta Engine v2 — compares two synthesized sweep results and produces structured changes
// Improvements: count metric thresholds, semantic TG dedup, configurable thresholds, null-safety
import { createHash } from 'crypto';
// ─── Default Thresholds ──────────────────────────────────────────────────────
// Override via config.delta.thresholds in crucix.config.mjs
const DEFAULT_NUMERIC_THRESHOLDS = {
vix: 5, // % change to flag
hy_spread: 5,
'10y2y': 10,
wti: 3,
brent: 3,
natgas: 5,
unemployment: 2,
fed_funds: 1,
'10y_yield': 3,
usd_index: 1,
mortgage: 2,
};
const DEFAULT_COUNT_THRESHOLDS = {
urgent_posts: 2, // need ±2 to matter (was 0 — any change)
thermal_total: 500, // ±500 detections (was 0 — +1 was noise)
air_total: 50, // ±50 aircraft
who_alerts: 1, // any new WHO alert matters
conflict_events: 5, // ±5 ACLED events
conflict_fatalities: 10, // ±10 fatalities
sdr_online: 3, // ±3 receivers
news_count: 5, // ±5 news items
sources_ok: 1, // any source going down matters
};
// ─── Metric Definitions ──────────────────────────────────────────────────────
// Metrics we track for delta computation
const NUMERIC_METRICS = [
{ key: 'vix', extract: d => d.fred?.find(f => f.id === 'VIXCLS')?.value, label: 'VIX', threshold: 5 },
{ key: 'hy_spread', extract: d => d.fred?.find(f => f.id === 'BAMLH0A0HYM2')?.value, label: 'HY Spread', threshold: 5 },
{ key: '10y2y', extract: d => d.fred?.find(f => f.id === 'T10Y2Y')?.value, label: '10Y-2Y Spread', threshold: 10 },
{ key: 'wti', extract: d => d.energy?.wti, label: 'WTI Crude', threshold: 3 },
{ key: 'brent', extract: d => d.energy?.brent, label: 'Brent Crude', threshold: 3 },
{ key: 'natgas', extract: d => d.energy?.natgas, label: 'Natural Gas', threshold: 5 },
{ key: 'unemployment', extract: d => d.bls?.find(b => b.id === 'LNS14000000' || b.id === 'UNRATE')?.value, label: 'Unemployment', threshold: 2 },
{ key: 'fed_funds', extract: d => d.fred?.find(f => f.id === 'DFF')?.value, label: 'Fed Funds Rate', threshold: 1 },
{ key: '10y_yield', extract: d => d.fred?.find(f => f.id === 'DGS10')?.value, label: '10Y Yield', threshold: 3 },
{ key: 'usd_index', extract: d => d.fred?.find(f => f.id === 'DTWEXBGS')?.value, label: 'USD Index', threshold: 1 },
{ key: 'mortgage', extract: d => d.fred?.find(f => f.id === 'MORTGAGE30US')?.value, label: '30Y Mortgage', threshold: 2 },
{ key: 'vix', extract: d => d.fred?.find(f => f.id === 'VIXCLS')?.value, label: 'VIX' },
{ key: 'hy_spread', extract: d => d.fred?.find(f => f.id === 'BAMLH0A0HYM2')?.value, label: 'HY Spread' },
{ key: '10y2y', extract: d => d.fred?.find(f => f.id === 'T10Y2Y')?.value, label: '10Y-2Y Spread' },
{ key: 'wti', extract: d => d.energy?.wti, label: 'WTI Crude' },
{ key: 'brent', extract: d => d.energy?.brent, label: 'Brent Crude' },
{ key: 'natgas', extract: d => d.energy?.natgas, label: 'Natural Gas' },
{ key: 'unemployment', extract: d => d.bls?.find(b => b.id === 'LNS14000000' || b.id === 'UNRATE')?.value, label: 'Unemployment' },
{ key: 'fed_funds', extract: d => d.fred?.find(f => f.id === 'DFF')?.value, label: 'Fed Funds Rate' },
{ key: '10y_yield', extract: d => d.fred?.find(f => f.id === 'DGS10')?.value, label: '10Y Yield' },
{ key: 'usd_index', extract: d => d.fred?.find(f => f.id === 'DTWEXBGS')?.value, label: 'USD Index' },
{ key: 'mortgage', extract: d => d.fred?.find(f => f.id === 'MORTGAGE30US')?.value, label: '30Y Mortgage' },
];
const COUNT_METRICS = [
@@ -23,29 +56,66 @@ const COUNT_METRICS = [
{ key: 'conflict_events', extract: d => d.acled?.totalEvents || 0, label: 'Conflict Events' },
{ key: 'conflict_fatalities', extract: d => d.acled?.totalFatalities || 0, label: 'Conflict Fatalities' },
{ key: 'sdr_online', extract: d => d.sdr?.online || 0, label: 'SDR Receivers' },
{ key: 'news_count', extract: d => d.news?.length || 0, label: 'News Items' },
{ key: 'news_count', extract: d => (d.news?.length ?? d.news?.count) || 0, label: 'News Items' },
{ key: 'sources_ok', extract: d => d.meta?.sourcesOk || 0, label: 'Sources OK' },
];
export function computeDelta(current, previous) {
// Risk-sensitive keys: used for determining overall direction
const RISK_KEYS = ['vix', 'hy_spread', 'urgent_posts', 'conflict_events', 'thermal_total'];
// ─── Semantic Hashing for Telegram Posts ─────────────────────────────────────
/**
* Produce a normalized hash of a post's content.
* Strips timestamps, normalizes numbers, lowercases — so "BREAKING: 5 missiles at 14:32"
* and "Breaking: 7 missiles at 15:01" produce the same hash (both are "missile strike" signals).
*/
function contentHash(text) {
if (!text) return '';
const normalized = text
.toLowerCase()
.replace(/\d{1,2}:\d{2}(:\d{2})?/g, '') // strip times
.replace(/\d+/g, 'N') // normalize all numbers
.replace(/[^\w\s]/g, '') // strip punctuation
.replace(/\s+/g, ' ')
.trim()
.substring(0, 100);
return createHash('sha256').update(normalized).digest('hex').substring(0, 12);
}
// ─── Core Delta Computation ──────────────────────────────────────────────────
/**
* @param {object} current - current sweep's synthesized data
* @param {object|null} previous - previous sweep's synthesized data (null on first run)
* @param {object} [thresholdOverrides] - optional: { numeric: {...}, count: {...} }
*/
export function computeDelta(current, previous, thresholdOverrides = {}) {
if (!previous) return null;
if (!current) return null;
const numThresholds = { ...DEFAULT_NUMERIC_THRESHOLDS, ...(thresholdOverrides.numeric || {}) };
const cntThresholds = { ...DEFAULT_COUNT_THRESHOLDS, ...(thresholdOverrides.count || {}) };
const signals = { new: [], escalated: [], deescalated: [], unchanged: [] };
let criticalChanges = 0;
// Numeric metrics: track % change
// ─── Numeric metrics: track % change ─────────────────────────────────
for (const m of NUMERIC_METRICS) {
const curr = m.extract(current);
const prev = m.extract(previous);
if (curr == null || prev == null) continue;
const threshold = numThresholds[m.key] ?? 5;
const pctChange = prev !== 0 ? ((curr - prev) / Math.abs(prev)) * 100 : 0;
if (Math.abs(pctChange) > m.threshold) {
if (Math.abs(pctChange) > threshold) {
const entry = {
key: m.key, label: m.label, from: prev, to: curr,
pctChange: parseFloat(pctChange.toFixed(2)),
direction: pctChange > 0 ? 'up' : 'down',
severity: Math.abs(pctChange) > threshold * 3 ? 'critical' : Math.abs(pctChange) > threshold * 2 ? 'high' : 'moderate',
};
if (pctChange > 0) signals.escalated.push(entry);
else signals.deescalated.push(entry);
@@ -55,52 +125,78 @@ export function computeDelta(current, previous) {
}
}
// Count metrics: track absolute change
// ─── Count metrics: track absolute change (with minimum thresholds) ──
for (const m of COUNT_METRICS) {
const curr = m.extract(current);
const prev = m.extract(previous);
const diff = curr - prev;
const threshold = cntThresholds[m.key] ?? 1;
if (Math.abs(diff) > 0) {
if (Math.abs(diff) >= threshold) {
const pctChange = prev > 0 ? ((diff / prev) * 100) : (diff > 0 ? 100 : 0);
const entry = {
key: m.key, label: m.label, from: prev, to: curr,
change: diff, direction: diff > 0 ? 'up' : 'down',
pctChange: parseFloat(pctChange.toFixed(1)),
severity: Math.abs(diff) >= threshold * 5 ? 'critical' : Math.abs(diff) >= threshold * 2 ? 'high' : 'moderate',
};
if (diff > 0) signals.escalated.push(entry);
else signals.deescalated.push(entry);
// Count metrics only critical if the change is extreme
if (entry.severity === 'critical') criticalChanges++;
} else {
signals.unchanged.push(m.key);
}
}
// New urgent posts (check by text content)
const prevUrgentTexts = new Set((previous.tg?.urgent || []).map(p => p.text?.substring(0, 60)));
// ─── New urgent Telegram posts (semantic dedup) ──────────────────────
const prevHashes = new Set(
(previous.tg?.urgent || []).map(p => contentHash(p.text))
);
for (const post of (current.tg?.urgent || [])) {
const key = post.text?.substring(0, 60);
if (key && !prevUrgentTexts.has(key)) {
signals.new.push({ key: 'tg_urgent', item: post, reason: 'New urgent OSINT post' });
const hash = contentHash(post.text);
if (hash && !prevHashes.has(hash)) {
signals.new.push({
key: `tg_urgent:${hash}`,
text: post.text?.substring(0, 120),
item: post,
reason: 'New urgent OSINT post',
});
criticalChanges++;
}
}
// Nuclear anomaly change
// ─── Nuclear anomaly state change ────────────────────────────────────
const currAnom = current.nuke?.some(n => n.anom) || false;
const prevAnom = previous.nuke?.some(n => n.anom) || false;
if (currAnom && !prevAnom) {
signals.new.push({ key: 'nuke_anomaly', reason: 'Nuclear anomaly detected' });
criticalChanges += 5; // Critical
signals.new.push({ key: 'nuke_anomaly', reason: 'Nuclear anomaly detected', severity: 'critical' });
criticalChanges += 5;
} else if (!currAnom && prevAnom) {
signals.deescalated.push({ key: 'nuke_anomaly', label: 'Nuclear Anomaly', direction: 'resolved' });
signals.deescalated.push({ key: 'nuke_anomaly', label: 'Nuclear Anomaly', direction: 'resolved', severity: 'high' });
}
// Determine overall direction
// ─── Source health degradation ───────────────────────────────────────
const currSourcesDown = current.health?.filter(s => s.err).length || 0;
const prevSourcesDown = previous.health?.filter(s => s.err).length || 0;
if (currSourcesDown > prevSourcesDown + 2) {
signals.new.push({
key: 'source_degradation',
reason: `${currSourcesDown - prevSourcesDown} additional sources failing (${currSourcesDown} total down)`,
severity: currSourcesDown > 5 ? 'critical' : 'moderate',
});
}
// ─── Overall direction ───────────────────────────────────────────────
let direction = 'mixed';
const riskUp = signals.escalated.filter(s =>
['vix', 'hy_spread', 'urgent_posts', 'conflict_events', 'thermal_total'].includes(s.key)
).length;
const riskDown = signals.deescalated.filter(s =>
['vix', 'hy_spread', 'urgent_posts', 'conflict_events', 'thermal_total'].includes(s.key)
).length;
const riskUp = signals.escalated.filter(s => RISK_KEYS.includes(s.key)).length;
const riskDown = signals.deescalated.filter(s => RISK_KEYS.includes(s.key)).length;
if (riskUp > riskDown + 1) direction = 'risk-off';
else if (riskDown > riskUp + 1) direction = 'risk-on';
@@ -112,6 +208,15 @@ export function computeDelta(current, previous) {
totalChanges: signals.new.length + signals.escalated.length + signals.deescalated.length,
criticalChanges,
direction,
signalBreakdown: {
new: signals.new.length,
escalated: signals.escalated.length,
deescalated: signals.deescalated.length,
unchanged: signals.unchanged.length,
},
},
};
}
// Export thresholds for external config
export { DEFAULT_NUMERIC_THRESHOLDS, DEFAULT_COUNT_THRESHOLDS };

121
lib/delta/memory.mjs
View File

@@ -1,11 +1,16 @@
// Memory Manager — hot/cold storage for sweep history and alert tracking
// v2: Atomic writes, decay-based alert cooldowns, configurable retention
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
import { readFileSync, writeFileSync, mkdirSync, existsSync, renameSync, unlinkSync } from 'fs';
import { join } from 'path';
import { computeDelta } from './engine.mjs';
const MAX_HOT_RUNS = 3;
// Alert cooldown tiers — repeated signals get progressively longer suppression
// First alert: 0h wait. Second occurrence within 24h: 6h cooldown. Third: 12h. Fourth+: 24h.
const ALERT_DECAY_TIERS = [0, 6, 12, 24]; // hours
export class MemoryManager {
constructor(runsDir) {
this.runsDir = runsDir;
@@ -23,18 +28,46 @@ export class MemoryManager {
}
_loadHot() {
try {
return JSON.parse(readFileSync(this.hotPath, 'utf8'));
} catch {
return { runs: [], alertedSignals: {} };
// Try primary file first, then backup
for (const path of [this.hotPath, this.hotPath + '.bak']) {
try {
const raw = readFileSync(path, 'utf8');
const data = JSON.parse(raw);
// Validate structure
if (data && Array.isArray(data.runs) && typeof data.alertedSignals === 'object') {
return data;
}
} catch { /* try next */ }
}
console.warn('[Memory] No valid hot memory found — starting fresh');
return { runs: [], alertedSignals: {} };
}
/**
* Atomic write: write to .tmp, then rename over target.
* Keeps a .bak of the previous version for crash recovery.
*/
_saveHot() {
const tmpPath = this.hotPath + '.tmp';
const bakPath = this.hotPath + '.bak';
try {
writeFileSync(this.hotPath, JSON.stringify(this.hot, null, 2));
// 1. Write to temp file (if this crashes, original is untouched)
writeFileSync(tmpPath, JSON.stringify(this.hot, null, 2));
// 2. Back up current file (if it exists)
try {
if (existsSync(this.hotPath)) {
// Copy current → .bak (overwrite previous backup)
renameSync(this.hotPath, bakPath);
}
} catch { /* backup failure is non-fatal */ }
// 3. Atomic rename: .tmp → hot.json
renameSync(tmpPath, this.hotPath);
} catch (err) {
console.error('[Memory] Failed to save hot memory:', err.message);
// Clean up tmp if it exists
try { unlinkSync(tmpPath); } catch { }
}
}
@@ -79,21 +112,78 @@ export class MemoryManager {
return this.hot.runs[0].delta;
}
// Track what signals have been alerted on
// ─── Alert Signal Tracking (Decay-Based) ───────────────────────────────
getAlertedSignals() {
return this.hot.alertedSignals || {};
}
/**
* Check if a signal should be suppressed based on decay-based cooldown.
* Returns true if the signal is still in cooldown.
*/
isSignalSuppressed(signalKey) {
const entry = this.hot.alertedSignals[signalKey];
if (!entry) return false;
const now = Date.now();
const occurrences = typeof entry === 'object' ? (entry.count || 1) : 1;
const lastAlerted = typeof entry === 'object' ? new Date(entry.lastAlerted).getTime() : new Date(entry).getTime();
// Pick cooldown tier based on how many times this signal has fired
const tierIndex = Math.min(occurrences, ALERT_DECAY_TIERS.length - 1);
const cooldownHours = ALERT_DECAY_TIERS[tierIndex];
const cooldownMs = cooldownHours * 60 * 60 * 1000;
return (now - lastAlerted) < cooldownMs;
}
/**
* Mark a signal as alerted, incrementing its occurrence counter.
* Supports both legacy (string timestamp) and new (object with count) formats.
*/
markAsAlerted(signalKey, timestamp) {
this.hot.alertedSignals[signalKey] = timestamp || new Date().toISOString();
const now = timestamp || new Date().toISOString();
const existing = this.hot.alertedSignals[signalKey];
if (existing && typeof existing === 'object') {
// Increment existing
existing.count = (existing.count || 1) + 1;
existing.lastAlerted = now;
existing.firstSeen = existing.firstSeen || now;
} else {
// New entry (or migrate from legacy string format)
this.hot.alertedSignals[signalKey] = {
firstSeen: typeof existing === 'string' ? existing : now,
lastAlerted: now,
count: typeof existing === 'string' ? 2 : 1,
};
}
this._saveHot();
}
// Clean up old alerted signals (older than 24h)
/**
* Prune stale alerted signals.
* Signals with 1 occurrence: pruned after 24h.
* Signals with 2+ occurrences: pruned after 48h from last alert.
* This prevents infinite accumulation while keeping recurring signal awareness.
*/
pruneAlertedSignals() {
const cutoff = Date.now() - 24 * 60 * 60 * 1000;
for (const [key, ts] of Object.entries(this.hot.alertedSignals)) {
if (new Date(ts).getTime() < cutoff) {
const now = Date.now();
for (const [key, entry] of Object.entries(this.hot.alertedSignals)) {
let lastTime, count;
if (typeof entry === 'object') {
lastTime = new Date(entry.lastAlerted).getTime();
count = entry.count || 1;
} else {
// Legacy string format
lastTime = new Date(entry).getTime();
count = 1;
}
const maxAge = count >= 2 ? 48 * 60 * 60 * 1000 : 24 * 60 * 60 * 1000;
if ((now - lastTime) > maxAge) {
delete this.hot.alertedSignals[key];
}
}
@@ -116,6 +206,7 @@ export class MemoryManager {
who: (data.who || []).map(w => ({ title: w.title })),
acled: { totalEvents: data.acled?.totalEvents, totalFatalities: data.acled?.totalFatalities },
sdr: { total: data.sdr?.total, online: data.sdr?.online },
news: { count: data.news?.length || 0 },
ideas: (data.ideas || []).map(i => ({ title: i.title, type: i.type, confidence: i.confidence })),
};
}
@@ -130,10 +221,14 @@ export class MemoryManager {
try { existing = JSON.parse(readFileSync(coldPath, 'utf8')); } catch { }
existing.push(...runs);
// Use atomic write for cold storage too
const tmpPath = coldPath + '.tmp';
try {
writeFileSync(coldPath, JSON.stringify(existing, null, 2));
writeFileSync(tmpPath, JSON.stringify(existing, null, 2));
renameSync(tmpPath, coldPath);
} catch (err) {
console.error('[Memory] Failed to archive to cold storage:', err.message);
try { unlinkSync(tmpPath); } catch { }
}
}
}