feat: extend memory prediction loop #32
@@ -10,6 +10,8 @@ STALE_ALERT_COOLDOWN_MINUTES=60
|
||||
DASHBOARD_URL=
|
||||
TERMINAL_ACTIONS_ENABLED=true
|
||||
SWEEP_TOKEN=
|
||||
TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
|
||||
TERMINAL_ACTION_RATE_LIMIT_MAX=10
|
||||
BRIEF_VERBOSITY=standard
|
||||
|
||||
# LLM layer
|
||||
|
||||
19
README.md
19
README.md
@@ -134,6 +134,8 @@ STALE_ALERT_COOLDOWN_MINUTES=60
|
||||
DASHBOARD_URL=https://intelligence.example.internal
|
||||
TERMINAL_ACTIONS_ENABLED=true
|
||||
SWEEP_TOKEN=
|
||||
TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
|
||||
TERMINAL_ACTION_RATE_LIMIT_MAX=10
|
||||
BRIEF_VERBOSITY=standard
|
||||
|
||||
LLM_PROVIDER=openrouter
|
||||
@@ -185,9 +187,22 @@ LLM_MODEL=your-model
|
||||
|
||||
For Pangolin or another reverse proxy, forward HTTP traffic to `intelligence-terminal:3117` (or the `PORT` you set). Missing API keys do not crash sweeps; affected sources are reported as degraded in `/api/health`.
|
||||
|
||||
When data remains stale past `STALE_DATA_MAX_AGE_MINUTES`, the server sends an operator alert through configured Telegram/Discord channels after failed or degraded sweep attempts. `STALE_ALERT_COOLDOWN_MINUTES` prevents repeated stale alerts from spamming every refresh interval. Set `DASHBOARD_URL` to the Pangolin/public URL you want included in those alerts.
|
||||
#### Terminal Action Exposure
|
||||
|
||||
The dashboard Terminal Actions panel can trigger `status`, `sweep`, and `brief` through `/api/action`. Leave `TERMINAL_ACTIONS_ENABLED=true` for a private home-server deployment. For an internet-exposed deployment, set `SWEEP_TOKEN` and pass it through trusted automation, or set `TERMINAL_ACTIONS_ENABLED=false` to disable browser-triggered actions. If you protect actions with `SWEEP_TOKEN`, the browser can send it from `localStorage.crucix_sweep_token`.
|
||||
`POST /api/action` and `POST /api/sweep` can trigger operational actions such as manual sweeps. The dashboard has a **SET TOKEN** control that stores your `SWEEP_TOKEN` in browser local storage and sends it as the `x-crucix-token` header; do not put action tokens in URLs.
|
||||
|
||||
Recommended settings:
|
||||
|
||||
| Deployment | Settings |
|
||||
| --- | --- |
|
||||
| Private local machine | `NODE_ENV=development`, optional `SWEEP_TOKEN`, optional `TERMINAL_ACTIONS_ENABLED=true`. Localhost can run actions without a token for development. |
|
||||
| Private LAN / Dockge | Set a strong `SWEEP_TOKEN`, keep `TERMINAL_ACTIONS_ENABLED=true`, expose only to trusted clients. |
|
||||
| Pangolin-authenticated reverse proxy | Set a strong `SWEEP_TOKEN`, keep Pangolin auth in front, use the dashboard **SET TOKEN** flow once per browser. |
|
||||
| Public internet | Do not expose Terminal Actions directly. If exposure is unavoidable, require `SWEEP_TOKEN`, keep proxy authentication enabled, lower `TERMINAL_ACTION_RATE_LIMIT_MAX`, and monitor server audit logs. |
|
||||
|
||||
Action endpoints reject cross-origin POST origins, apply a small in-memory per-IP rate limit, and write sanitized audit lines without logging the token.
|
||||
|
||||
When data remains stale past `STALE_DATA_MAX_AGE_MINUTES`, the server sends an operator alert through configured Telegram/Discord channels after failed or degraded sweep attempts. `STALE_ALERT_COOLDOWN_MINUTES` prevents repeated stale alerts from spamming every refresh interval. Set `DASHBOARD_URL` to the Pangolin/public URL you want included in those alerts.
|
||||
|
||||
#### Memory And Prediction Loop
|
||||
|
||||
|
||||
@@ -26,7 +26,9 @@ export default {
|
||||
staleAlertCooldownMinutes: intEnv('STALE_ALERT_COOLDOWN_MINUTES', 60),
|
||||
dashboardUrl: process.env.DASHBOARD_URL || null,
|
||||
sweepToken: process.env.SWEEP_TOKEN || null,
|
||||
terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', true),
|
||||
terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', !!process.env.SWEEP_TOKEN || process.env.NODE_ENV !== 'production'),
|
||||
terminalActionRateLimitWindowMs: intEnv('TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS', 60_000),
|
||||
terminalActionRateLimitMax: intEnv('TERMINAL_ACTION_RATE_LIMIT_MAX', 10),
|
||||
|
||||
llm: {
|
||||
provider: process.env.LLM_PROVIDER || null, // anthropic | openai | gemini | codex | openrouter | minimax | mistral | ollama | grok
|
||||
|
||||
@@ -432,6 +432,7 @@ let terminalOutput = 'Ready. Live data is loaded from /api/data in server mode.'
|
||||
let terminalBusy = false;
|
||||
let currentRegion = 'world';
|
||||
let flatSvg, flatProjection, flatPath, flatG, flatZoom, flatW, flatH;
|
||||
const terminalActionTokenKey = 'crucix_sweep_token';
|
||||
|
||||
const layerTypeMap = {
|
||||
air: ['air'],
|
||||
@@ -632,6 +633,7 @@ function renderTopbar(){
|
||||
const ts = new Date(D.meta.timestamp);
|
||||
const d = ts.toLocaleDateString('en-US',{month:'short',day:'numeric',year:'numeric'}).toUpperCase();
|
||||
const timeStr = ts.toLocaleTimeString('en-US',{hour:'2-digit',minute:'2-digit',hour12:true});
|
||||
const hasActionToken = !!getTerminalActionToken();
|
||||
document.getElementById('topbar').innerHTML=`
|
||||
<div class="top-left">
|
||||
<span class="brand">CRUCIX MONITOR</span>
|
||||
@@ -644,12 +646,26 @@ function renderTopbar(){
|
||||
<span class="meta-pill">${d} <span class="v">${timeStr}</span></span>
|
||||
<span class="meta-pill">${t('dashboard.sources','SOURCES')} <span class="v">${D.meta.sourcesOk}/${D.meta.sourcesQueried}</span></span>
|
||||
${D.delta?.summary ? `<span class="meta-pill">${t('dashboard.delta','DELTA')} <span class="v">${D.delta.summary.direction==='risk-off'?'▲ '+t('dashboard.riskOff','RISK-OFF'):D.delta.summary.direction==='risk-on'?'▼ '+t('dashboard.riskOn','RISK-ON'):'◆ '+t('dashboard.mixed','MIXED')}</span></span>` : ''}
|
||||
<button class="guide-btn" onclick="configureTerminalActionToken()" title="Configure SWEEP_TOKEN for protected terminal actions">${hasActionToken?'TOKEN SET':'SET TOKEN'}</button>
|
||||
<button class="guide-btn" onclick="openGlossary()">${t('dashboard.guideBtn','What Signals Mean')}</button>
|
||||
<span class="alert-badge">${t('dashboard.highAlert','HIGH ALERT')}</span>
|
||||
</div>`;
|
||||
renderRegionControls();
|
||||
}
|
||||
|
||||
function getTerminalActionToken(){
|
||||
return localStorage.getItem(terminalActionTokenKey) || localStorage.getItem('crucix_terminal_action_token') || '';
|
||||
}
|
||||
|
||||
function configureTerminalActionToken(){
|
||||
const next = window.prompt('Terminal action token (SWEEP_TOKEN). Leave empty to clear.', getTerminalActionToken());
|
||||
if(next === null) return;
|
||||
const clean = next.trim();
|
||||
if(clean) localStorage.setItem(terminalActionTokenKey, clean);
|
||||
else localStorage.removeItem(terminalActionTokenKey);
|
||||
renderTopbar();
|
||||
}
|
||||
|
||||
// === LEFT RAIL ===
|
||||
function layerMode(key){ return layerModes[key] || 'normal'; }
|
||||
function layerModeLabel(key){ return layerMode(key) === 'focus' ? 'focused' : layerMode(key) === 'hidden' ? 'hidden' : 'normal'; }
|
||||
@@ -1592,6 +1608,12 @@ function renderLower(){
|
||||
|
||||
async function runTerminalAction(action){
|
||||
if(terminalBusy) return;
|
||||
let token = getTerminalActionToken();
|
||||
if(!token && location.hostname !== 'localhost' && location.hostname !== '127.0.0.1'){
|
||||
configureTerminalActionToken();
|
||||
token = getTerminalActionToken();
|
||||
if(!token) return;
|
||||
}
|
||||
terminalBusy = true;
|
||||
terminalOutput = `> ${action}\nRunning...`;
|
||||
renderRight();
|
||||
@@ -1600,7 +1622,7 @@ async function runTerminalAction(action){
|
||||
method:'POST',
|
||||
headers:{
|
||||
'Content-Type':'application/json',
|
||||
...(localStorage.getItem('crucix_sweep_token') ? {'x-crucix-token': localStorage.getItem('crucix_sweep_token')} : {})
|
||||
...(token ? {'x-crucix-token': token} : {})
|
||||
},
|
||||
body:JSON.stringify({action})
|
||||
});
|
||||
@@ -1698,6 +1720,7 @@ function renderRight(){
|
||||
<button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('brief')">Brief</button>
|
||||
<button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('memory')">Memory</button>
|
||||
</div>
|
||||
<button class="mini-btn" style="margin-bottom:8px" onclick="configureTerminalActionToken()">Configure token</button>
|
||||
<div class="terminal-output">${terminalOutput.replace(/[&<>]/g,c=>({'&':'&','<':'<','>':'>'}[c])).replace(/\n/g,'<br>')}</div>
|
||||
</div>
|
||||
<div class="g-panel right-signals">
|
||||
|
||||
160
server.mjs
160
server.mjs
@@ -41,6 +41,7 @@ let sweepStartedAt = null; // Timestamp when current/last sweep started
|
||||
let sweepInProgress = false;
|
||||
const startTime = Date.now();
|
||||
const sseClients = new Set();
|
||||
const terminalActionBuckets = new Map();
|
||||
const staleAlertState = {};
|
||||
|
||||
// === Delta/Memory ===
|
||||
@@ -292,7 +293,9 @@ app.get('/api/metrics', (req, res) => {
|
||||
});
|
||||
|
||||
app.get('/api/memory/search', (req, res) => {
|
||||
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Memory queries disabled or unauthorized' });
|
||||
const guard = authorizeTerminalAction(req, res, 'memory:search');
|
||||
if (!guard.ok) return;
|
||||
auditTerminalAction(req, 'memory:search', 'ok');
|
||||
res.json(intelligenceStore.queryMemory({
|
||||
q: req.query.q || '',
|
||||
limit: req.query.limit || 25,
|
||||
@@ -300,7 +303,9 @@ app.get('/api/memory/search', (req, res) => {
|
||||
});
|
||||
|
||||
app.get('/api/memory/predictions', (req, res) => {
|
||||
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Memory queries disabled or unauthorized' });
|
||||
const guard = authorizeTerminalAction(req, res, 'memory:predictions');
|
||||
if (!guard.ok) return;
|
||||
auditTerminalAction(req, 'memory:predictions', 'ok');
|
||||
res.json(intelligenceStore.listPredictions({
|
||||
state: req.query.state || null,
|
||||
limit: req.query.limit || 25,
|
||||
@@ -308,24 +313,33 @@ app.get('/api/memory/predictions', (req, res) => {
|
||||
});
|
||||
|
||||
app.post('/api/sweep', express.json(), (req, res) => {
|
||||
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
|
||||
triggerSweep(res);
|
||||
const guard = authorizeTerminalAction(req, res, 'sweep');
|
||||
if (!guard.ok) return;
|
||||
triggerSweepAction(req, res, 'sweep');
|
||||
});
|
||||
|
||||
app.post('/api/action', express.json(), async (req, res) => {
|
||||
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
|
||||
const action = String(req.body?.action || req.query.action || '').toLowerCase();
|
||||
app.post('/api/action', express.json(), (req, res) => {
|
||||
const action = String(req.body?.action || req.body?.command || '').trim().toLowerCase();
|
||||
const guard = authorizeTerminalAction(req, res, action || 'unknown');
|
||||
if (!guard.ok) return;
|
||||
|
||||
if (action === 'status') {
|
||||
return res.json({ ok: true, action, health: buildHealth() });
|
||||
auditTerminalAction(req, 'status', 'ok');
|
||||
return res.json({ ok: true, action, status: 'ok', health: buildHealth() });
|
||||
}
|
||||
|
||||
if (action === 'brief') {
|
||||
if (!currentData) return res.status(503).json({ ok: false, action, error: 'No data yet — first sweep in progress' });
|
||||
return res.json({ ok: true, action, text: buildBrief(currentData) });
|
||||
if (!currentData) {
|
||||
auditTerminalAction(req, 'brief', 'rejected', 'no_data');
|
||||
return res.status(503).json({ ok: false, action, error: 'No data yet - first sweep in progress' });
|
||||
}
|
||||
auditTerminalAction(req, 'brief', 'ok');
|
||||
const brief = buildBrief(currentData);
|
||||
return res.json({ ok: true, action, status: 'ok', brief, text: brief });
|
||||
}
|
||||
|
||||
if (action === 'memory') {
|
||||
auditTerminalAction(req, 'memory', 'ok');
|
||||
return res.json({
|
||||
ok: true,
|
||||
action,
|
||||
@@ -335,11 +349,10 @@ app.post('/api/action', express.json(), async (req, res) => {
|
||||
});
|
||||
}
|
||||
|
||||
if (action === 'sweep') {
|
||||
return triggerSweep(res);
|
||||
}
|
||||
if (action === 'sweep') return triggerSweepAction(req, res, 'action:sweep');
|
||||
|
||||
res.status(400).json({ ok: false, error: 'Unknown action', actions: ['status', 'brief', 'memory', 'sweep'] });
|
||||
auditTerminalAction(req, action || 'unknown', 'rejected', 'unknown_action');
|
||||
return res.status(400).json({ ok: false, error: 'Unknown action', allowed: ['status', 'brief', 'memory', 'sweep'], actions: ['status', 'brief', 'memory', 'sweep'] });
|
||||
});
|
||||
|
||||
// API: available locales
|
||||
@@ -370,26 +383,114 @@ function broadcast(data) {
|
||||
}
|
||||
}
|
||||
|
||||
function requestIp(req) {
|
||||
return req.ip || req.socket?.remoteAddress || 'unknown';
|
||||
}
|
||||
|
||||
function isLocalRequest(req) {
|
||||
const remote = requestIp(req);
|
||||
return remote === '::1'
|
||||
|| remote === '127.0.0.1'
|
||||
|| remote === '::ffff:127.0.0.1'
|
||||
|| remote.startsWith('127.')
|
||||
|| remote === 'localhost';
|
||||
}
|
||||
|
||||
function sameOriginPost(req) {
|
||||
const origin = req.get('origin');
|
||||
if (!origin) return true;
|
||||
try {
|
||||
const originUrl = new URL(origin);
|
||||
const host = req.get('host');
|
||||
return host && originUrl.host === host;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function actionToken(req) {
|
||||
return req.get('x-crucix-token') || req.body?.token || null;
|
||||
}
|
||||
|
||||
function auditTerminalAction(req, action, outcome, detail = null) {
|
||||
const suffix = detail ? ` detail=${detail}` : '';
|
||||
console.log(`[Crucix][audit] terminal_action action=${action || 'unknown'} outcome=${outcome} ip=${requestIp(req)}${suffix}`);
|
||||
}
|
||||
|
||||
function rateLimitTerminalAction(req, action) {
|
||||
const now = Date.now();
|
||||
const windowMs = Math.max(1000, config.terminalActionRateLimitWindowMs || 60_000);
|
||||
const max = Math.max(1, config.terminalActionRateLimitMax || 10);
|
||||
const key = `${requestIp(req)}:${action}`;
|
||||
const bucket = terminalActionBuckets.get(key);
|
||||
if (!bucket || now > bucket.resetAt) {
|
||||
terminalActionBuckets.set(key, { count: 1, resetAt: now + windowMs });
|
||||
return { ok: true };
|
||||
}
|
||||
bucket.count += 1;
|
||||
if (bucket.count > max) {
|
||||
return { ok: false, retryAfterSeconds: Math.ceil((bucket.resetAt - now) / 1000) };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
function authorizeTerminalAction(req, res, action) {
|
||||
const rate = rateLimitTerminalAction(req, action);
|
||||
if (!rate.ok) {
|
||||
auditTerminalAction(req, action, 'rejected', 'rate_limited');
|
||||
res.set('Retry-After', String(rate.retryAfterSeconds));
|
||||
res.status(429).json({ error: 'Too many terminal actions', retryAfterSeconds: rate.retryAfterSeconds });
|
||||
return { ok: false };
|
||||
}
|
||||
|
||||
if (!sameOriginPost(req)) {
|
||||
auditTerminalAction(req, action, 'rejected', 'csrf_origin');
|
||||
res.status(403).json({ error: 'Origin mismatch' });
|
||||
return { ok: false };
|
||||
}
|
||||
|
||||
const local = isLocalRequest(req);
|
||||
const token = actionToken(req);
|
||||
if (!config.terminalActionsEnabled) {
|
||||
auditTerminalAction(req, action, 'rejected', 'disabled');
|
||||
res.status(403).json({ error: 'Terminal actions are disabled' });
|
||||
return { ok: false };
|
||||
}
|
||||
|
||||
if (config.sweepToken) {
|
||||
if (token !== config.sweepToken) {
|
||||
auditTerminalAction(req, action, 'rejected', 'invalid_token');
|
||||
res.status(401).json({ error: 'Invalid terminal action token' });
|
||||
return { ok: false };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
if (!local) {
|
||||
auditTerminalAction(req, action, 'rejected', 'missing_token');
|
||||
res.status(403).json({ error: 'Terminal actions are local-only unless SWEEP_TOKEN is set' });
|
||||
return { ok: false };
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
function triggerSweepAction(req, res, auditAction) {
|
||||
if (sweepInProgress) {
|
||||
auditTerminalAction(req, auditAction, 'rejected', 'already_running');
|
||||
return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
|
||||
}
|
||||
auditTerminalAction(req, auditAction, 'accepted');
|
||||
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
|
||||
return res.status(202).json({ ok: true, status: 'accepted' });
|
||||
}
|
||||
|
||||
function dataAgeMs() {
|
||||
const ts = currentData?.meta?.timestamp || lastSuccessfulSweepTime || lastSweepTime;
|
||||
const ms = ts ? Date.now() - new Date(ts).getTime() : null;
|
||||
return Number.isFinite(ms) ? ms : null;
|
||||
}
|
||||
|
||||
function canRunTerminalAction(req) {
|
||||
const remote = req.ip || '';
|
||||
const local = remote.includes('127.0.0.1') || remote === '::1' || remote === '::ffff:127.0.0.1';
|
||||
const token = req.get('x-crucix-token') || req.query.token || req.body?.token;
|
||||
if (config.sweepToken) return token === config.sweepToken;
|
||||
return Boolean(config.terminalActionsEnabled || local);
|
||||
}
|
||||
|
||||
function triggerSweep(res) {
|
||||
if (sweepInProgress) return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
|
||||
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
|
||||
return res.status(202).json({ ok: true, status: 'accepted' });
|
||||
}
|
||||
|
||||
function getLLMStatus() {
|
||||
if (!config.llm.provider) return { state: 'disabled' };
|
||||
if (!llmProvider) return { state: 'misconfigured', provider: config.llm.provider };
|
||||
@@ -433,7 +534,8 @@ function buildHealth() {
|
||||
llm: getLLMStatus(),
|
||||
telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId),
|
||||
discordEnabled: !!(config.discord?.botToken || config.discord?.webhookUrl),
|
||||
terminalActionsEnabled: Boolean(config.terminalActionsEnabled || config.sweepToken),
|
||||
terminalActionsEnabled: config.terminalActionsEnabled,
|
||||
terminalActionsTokenRequired: !!config.sweepToken,
|
||||
refreshIntervalMinutes: config.refreshIntervalMinutes,
|
||||
language: currentLanguage,
|
||||
memory: intelligenceStore.status(),
|
||||
|
||||
@@ -122,6 +122,25 @@ test('server exposes memory-backed query APIs and dashboard memory action', () =
|
||||
assert.match(html, /runTerminalAction\('memory'\)/);
|
||||
});
|
||||
|
||||
test('terminal action endpoints avoid URL tokens and include hardening gates', () => {
|
||||
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
|
||||
assert.match(server, /app\.post\('\/api\/action'/);
|
||||
assert.match(server, /app\.post\('\/api\/sweep'/);
|
||||
assert.match(server, /x-crucix-token/);
|
||||
assert.match(server, /sameOriginPost/);
|
||||
assert.match(server, /rateLimitTerminalAction/);
|
||||
assert.match(server, /auditTerminalAction/);
|
||||
assert.doesNotMatch(server, /req\.query\.token/);
|
||||
});
|
||||
|
||||
test('dashboard exposes token configuration flow without devtools edits', () => {
|
||||
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
|
||||
assert.match(html, /configureTerminalActionToken/);
|
||||
assert.match(html, /crucix_sweep_token/);
|
||||
assert.match(html, /x-crucix-token/);
|
||||
assert.match(html, /SET TOKEN/);
|
||||
});
|
||||
|
||||
test('server dashboard shell does not embed an operational snapshot', () => {
|
||||
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
|
||||
assert.match(html, /let D = createDashboardShellData\(\);/);
|
||||
|
||||
Reference in New Issue
Block a user