Merge branch 'codex/production-intelligence-terminal' into codex/issue-8-doc-cleanup
This commit is contained in:
@@ -6,6 +6,7 @@ PORT=3117
|
||||
REFRESH_INTERVAL_MINUTES=15
|
||||
AUTO_OPEN_BROWSER=false
|
||||
STALE_DATA_MAX_AGE_MINUTES=60
|
||||
TERMINAL_ACTIONS_ENABLED=true
|
||||
SWEEP_TOKEN=
|
||||
BRIEF_VERBOSITY=standard
|
||||
|
||||
@@ -35,6 +36,8 @@ ACLED_EMAIL=
|
||||
ACLED_PASSWORD=
|
||||
CLOUDFLARE_API_TOKEN=
|
||||
BLS_API_KEY=
|
||||
REDDIT_CLIENT_ID=
|
||||
REDDIT_CLIENT_SECRET=
|
||||
|
||||
# Telegram bot and alerts
|
||||
TELEGRAM_BOT_TOKEN=
|
||||
|
||||
@@ -38,7 +38,12 @@ jobs:
|
||||
run: docker compose config
|
||||
|
||||
- name: Build Docker image
|
||||
run: docker build -t "${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${IMAGE_NAME}:${GITHUB_SHA}" .
|
||||
shell: bash
|
||||
run: |
|
||||
image="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${IMAGE_NAME}"
|
||||
build_tag="build-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_NUMBER:-0}"
|
||||
echo "BUILD_IMAGE=${image}:${build_tag}" >> "$GITHUB_ENV"
|
||||
docker build -t "${image}:${build_tag}" .
|
||||
|
||||
- name: Publish Docker image
|
||||
if: ${{ env.REGISTRY_TOKEN != '' }}
|
||||
@@ -47,8 +52,9 @@ jobs:
|
||||
image="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${IMAGE_NAME}"
|
||||
date_tag="$(date -u +%Y%m%d)"
|
||||
echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY_HOST}" -u "${REGISTRY_USERNAME}" --password-stdin
|
||||
docker tag "${image}:${GITHUB_SHA}" "${image}:latest"
|
||||
docker tag "${image}:${GITHUB_SHA}" "${image}:${date_tag}"
|
||||
docker tag "${BUILD_IMAGE}" "${image}:${GITHUB_SHA}"
|
||||
docker tag "${BUILD_IMAGE}" "${image}:latest"
|
||||
docker tag "${BUILD_IMAGE}" "${image}:${date_tag}"
|
||||
docker push "${image}:${GITHUB_SHA}"
|
||||
docker push "${image}:latest"
|
||||
docker push "${image}:${date_tag}"
|
||||
|
||||
39
README.md
39
README.md
@@ -123,6 +123,7 @@ PORT=3117
|
||||
REFRESH_INTERVAL_MINUTES=15
|
||||
AUTO_OPEN_BROWSER=false
|
||||
STALE_DATA_MAX_AGE_MINUTES=60
|
||||
TERMINAL_ACTIONS_ENABLED=true
|
||||
SWEEP_TOKEN=
|
||||
BRIEF_VERBOSITY=standard
|
||||
|
||||
@@ -175,6 +176,41 @@ LLM_MODEL=your-model
|
||||
|
||||
For Pangolin or another reverse proxy, forward HTTP traffic to `intelligence-terminal:3117` (or the `PORT` you set). Missing API keys do not crash sweeps; affected sources are reported as degraded in `/api/health`.
|
||||
|
||||
The dashboard Terminal Actions panel can trigger `status`, `sweep`, and `brief` through `/api/action`. Leave `TERMINAL_ACTIONS_ENABLED=true` for a private home-server deployment. For an internet-exposed deployment, set `SWEEP_TOKEN` and pass it through trusted automation, or set `TERMINAL_ACTIONS_ENABLED=false` to disable browser-triggered actions. If you protect actions with `SWEEP_TOKEN`, the browser can send it from `localStorage.crucix_sweep_token`.
|
||||
|
||||
#### Scenario Watchlist
|
||||
|
||||
Crucix can track operator hypotheses across sweeps with a runtime scenario file at `runs/scenarios.json`. On first run, the server creates three disabled starter examples:
|
||||
|
||||
- Middle East energy shock
|
||||
- Macro stress spillover
|
||||
- Regional escalation risk
|
||||
|
||||
Enable or add scenarios by editing `runs/scenarios.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "middle-east-energy-shock",
|
||||
"enabled": true,
|
||||
"name": "Middle East energy shock",
|
||||
"description": "Energy supply risk building from regional conflict.",
|
||||
"regions": ["Middle East", "Iran", "Strait of Hormuz"],
|
||||
"categories": ["osint", "energy", "maritime"],
|
||||
"keywords": ["missile", "strike", "hormuz", "oil"],
|
||||
"thresholds": { "watching": 2, "building": 4, "confirmed": 7 },
|
||||
"invalidation": "WTI normalizes and urgent regional signals fade."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Malformed scenario config degrades safely: sweeps continue and the dashboard shows the watchlist as a config issue. Scenario state is persisted in `runs/scenario-state.json`; delete that file to reset state transitions without deleting definitions.
|
||||
|
||||
Scenario states are `dormant`, `watching`, `building`, and `confirmed`. The dashboard shows active scenario state, confidence, score, and recent trigger time. Briefings include a `Scenario Watchlist` section when one or more scenarios change state.
|
||||
|
||||
#### Build And Publish Your Gitea Image
|
||||
|
||||
```bash
|
||||
@@ -315,6 +351,9 @@ These three unlock the most valuable economic and satellite data. Each takes abo
|
||||
| `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2 |
|
||||
| `AISSTREAM_API_KEY` | Maritime AIS vessel tracking | [aisstream.io](https://aisstream.io/) — free |
|
||||
| `ADSB_API_KEY` | Unfiltered flight tracking | [RapidAPI](https://rapidapi.com/adsbexchange/api/adsbexchange-com1) — ~$10/mo |
|
||||
| `REDDIT_CLIENT_ID` + `REDDIT_CLIENT_SECRET` | Reddit social sentiment | [reddit.com/prefs/apps](https://www.reddit.com/prefs/apps/) — create a script app |
|
||||
|
||||
Reddit is OAuth-only in this fork. If the Reddit credentials are missing or rejected, the Reddit source is reported as degraded and no unauthenticated `reddit.com/.../hot.json` fallback is used.
|
||||
|
||||
### LLM Provider (optional, for AI-enhanced ideas)
|
||||
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
// Reddit — social sentiment intelligence
|
||||
// Reddit now requires OAuth for API access (public JSON API returns 403).
|
||||
// Gracefully degrades when not authenticated.
|
||||
// To enable: register an app at https://www.reddit.com/prefs/apps/ and set
|
||||
// REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET in .env
|
||||
// Reddit social sentiment intelligence.
|
||||
// Reddit API access requires OAuth. Runtime sweeps intentionally do not use
|
||||
// unauthenticated reddit.com .json scraping because it is unreliable and not
|
||||
// acceptable for production operation.
|
||||
|
||||
import { safeFetch } from '../utils/fetch.mjs';
|
||||
import '../utils/env.mjs';
|
||||
|
||||
function delay(ms) { return new Promise(r => setTimeout(r, ms)); }
|
||||
|
||||
const USER_AGENT = 'Crucix/2.0 intelligence-engine';
|
||||
|
||||
const SUBREDDITS = [
|
||||
'worldnews',
|
||||
'geopolitics',
|
||||
@@ -17,48 +18,95 @@ const SUBREDDITS = [
|
||||
'commodities',
|
||||
];
|
||||
|
||||
// Get OAuth token using client credentials flow (application-only)
|
||||
async function getToken() {
|
||||
const clientId = process.env.REDDIT_CLIENT_ID;
|
||||
const clientSecret = process.env.REDDIT_CLIENT_SECRET;
|
||||
if (!clientId || !clientSecret) return null;
|
||||
export function getRedditConfig(env = process.env) {
|
||||
const clientId = env.REDDIT_CLIENT_ID || '';
|
||||
const clientSecret = env.REDDIT_CLIENT_SECRET || '';
|
||||
const missing = [];
|
||||
if (!clientId) missing.push('REDDIT_CLIENT_ID');
|
||||
if (!clientSecret) missing.push('REDDIT_CLIENT_SECRET');
|
||||
return {
|
||||
clientId,
|
||||
clientSecret,
|
||||
configured: missing.length === 0,
|
||||
missing,
|
||||
};
|
||||
}
|
||||
|
||||
function credentialsMessage(missing) {
|
||||
return `Reddit requires OAuth. Register a script app at https://www.reddit.com/prefs/apps/ and set ${missing.join(' and ')} in .env`;
|
||||
}
|
||||
|
||||
export async function getToken({ env = process.env, fetchImpl = globalThis.fetch } = {}) {
|
||||
const config = getRedditConfig(env);
|
||||
if (!config.configured) {
|
||||
return {
|
||||
ok: false,
|
||||
status: 'no_credentials',
|
||||
missing: config.missing,
|
||||
error: 'missing_reddit_oauth_credentials',
|
||||
message: credentialsMessage(config.missing),
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const auth = Buffer.from(`${clientId}:${clientSecret}`).toString('base64');
|
||||
const res = await fetch('https://www.reddit.com/api/v1/access_token', {
|
||||
const auth = Buffer.from(`${config.clientId}:${config.clientSecret}`).toString('base64');
|
||||
const res = await fetchImpl('https://www.reddit.com/api/v1/access_token', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Basic ${auth}`,
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'User-Agent': 'Crucix/1.0 intelligence-engine',
|
||||
'User-Agent': USER_AGENT,
|
||||
},
|
||||
body: 'grant_type=client_credentials',
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
if (!res.ok) {
|
||||
const body = await res.text().catch(() => '');
|
||||
return {
|
||||
ok: false,
|
||||
status: 'auth_failed',
|
||||
error: `reddit_oauth_http_${res.status}`,
|
||||
message: `Reddit OAuth token request failed with HTTP ${res.status}`,
|
||||
detail: body.slice(0, 200),
|
||||
};
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
return data.access_token || null;
|
||||
} catch {
|
||||
return null;
|
||||
if (!data.access_token) {
|
||||
return {
|
||||
ok: false,
|
||||
status: 'auth_failed',
|
||||
error: 'reddit_oauth_missing_access_token',
|
||||
message: 'Reddit OAuth token response did not include an access token',
|
||||
};
|
||||
}
|
||||
return { ok: true, status: 'ok', token: data.access_token };
|
||||
} catch (e) {
|
||||
return {
|
||||
ok: false,
|
||||
status: 'auth_failed',
|
||||
error: 'reddit_oauth_request_failed',
|
||||
message: e.message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch hot posts — tries OAuth first, then falls back to public endpoint
|
||||
export async function getHot(subreddit, opts = {}) {
|
||||
const { limit = 10, token = null } = opts;
|
||||
|
||||
if (token) {
|
||||
// Use OAuth endpoint
|
||||
return safeFetch(`https://oauth.reddit.com/r/${subreddit}/hot?limit=${limit}&raw_json=1`, {
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'User-Agent': 'Crucix/1.0 intelligence-engine',
|
||||
},
|
||||
});
|
||||
if (!token) {
|
||||
return {
|
||||
status: 'no_credentials',
|
||||
error: 'reddit_oauth_required',
|
||||
message: 'Reddit source requires OAuth; unauthenticated reddit.com .json scraping is disabled',
|
||||
};
|
||||
}
|
||||
|
||||
// Try public endpoint (may 403)
|
||||
return safeFetch(`https://www.reddit.com/r/${subreddit}/hot.json?limit=${limit}&raw_json=1`, {
|
||||
headers: { 'User-Agent': 'Crucix/1.0 intelligence-engine' },
|
||||
return safeFetch(`https://oauth.reddit.com/r/${subreddit}/hot?limit=${limit}&raw_json=1`, {
|
||||
source: 'Reddit',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'User-Agent': USER_AGENT,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -74,29 +122,46 @@ function compactPost(child) {
|
||||
};
|
||||
}
|
||||
|
||||
export async function briefing() {
|
||||
const token = await getToken();
|
||||
export async function briefing(opts = {}) {
|
||||
const {
|
||||
env = process.env,
|
||||
subreddits = SUBREDDITS,
|
||||
delayMs = 1000,
|
||||
fetchImpl = globalThis.fetch,
|
||||
} = opts;
|
||||
const tokenResult = await getToken({ env, fetchImpl });
|
||||
|
||||
if (!token && !process.env.REDDIT_CLIENT_ID) {
|
||||
if (!tokenResult.ok) {
|
||||
return {
|
||||
source: 'Reddit',
|
||||
timestamp: new Date().toISOString(),
|
||||
status: 'no_key',
|
||||
message: 'Reddit requires OAuth. Register at https://www.reddit.com/prefs/apps/ (script type), set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET in .env',
|
||||
status: tokenResult.status,
|
||||
error: tokenResult.error,
|
||||
message: tokenResult.message,
|
||||
missing: tokenResult.missing || [],
|
||||
};
|
||||
}
|
||||
|
||||
const subredditResults = {};
|
||||
for (const sub of SUBREDDITS) {
|
||||
const result = await getHot(sub, { limit: 10, token });
|
||||
const errors = [];
|
||||
for (const sub of subreddits) {
|
||||
const result = await getHot(sub, { limit: 10, token: tokenResult.token });
|
||||
if (result?.error) {
|
||||
errors.push({ subreddit: sub, error: result.error });
|
||||
subredditResults[sub] = [];
|
||||
if (delayMs > 0) await delay(delayMs);
|
||||
continue;
|
||||
}
|
||||
const children = result?.data?.children || [];
|
||||
subredditResults[sub] = children.map(compactPost).filter(Boolean);
|
||||
await delay(token ? 1000 : 2000);
|
||||
if (delayMs > 0) await delay(delayMs);
|
||||
}
|
||||
|
||||
return {
|
||||
source: 'Reddit',
|
||||
timestamp: new Date().toISOString(),
|
||||
status: errors.length > 0 ? 'degraded' : 'ok',
|
||||
...(errors.length > 0 ? { error: 'reddit_subreddit_fetch_failed', errors } : {}),
|
||||
subreddits: subredditResults,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ export default {
|
||||
autoOpenBrowser: boolEnv('AUTO_OPEN_BROWSER', false),
|
||||
staleDataMaxAgeMinutes: intEnv('STALE_DATA_MAX_AGE_MINUTES', 60),
|
||||
sweepToken: process.env.SWEEP_TOKEN || null,
|
||||
terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', true),
|
||||
|
||||
llm: {
|
||||
provider: process.env.LLM_PROVIDER || null, // anthropic | openai | gemini | codex | openrouter | minimax | mistral | ollama | grok
|
||||
|
||||
@@ -83,6 +83,13 @@ html,body{height:100%;background:var(--bg);color:var(--text);font-family:var(--s
|
||||
.sensor-actions{display:flex;gap:6px;align-items:center}
|
||||
.mini-btn{border:1px solid rgba(100,240,200,0.18);background:rgba(100,240,200,0.04);color:var(--dim);font-family:var(--mono);font-size:9px;padding:3px 6px;cursor:pointer}
|
||||
.mini-btn:hover{color:var(--accent);border-color:rgba(100,240,200,0.4)}
|
||||
.action-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:6px;margin-bottom:10px}
|
||||
.action-btn{border:1px solid rgba(68,204,255,0.24);background:rgba(68,204,255,0.06);color:var(--text);font-family:var(--mono);font-size:9px;padding:7px 6px;cursor:pointer;text-transform:uppercase;letter-spacing:.08em}
|
||||
.action-btn:hover{border-color:rgba(68,204,255,0.55);color:var(--accent2);background:rgba(68,204,255,0.12)}
|
||||
.action-btn[disabled]{opacity:.45;cursor:wait}
|
||||
.terminal-output{min-height:58px;max-height:180px;overflow:auto;border:1px solid rgba(255,255,255,0.05);background:rgba(0,0,0,0.22);padding:8px;font-family:var(--mono);font-size:10px;line-height:1.45;color:var(--dim);white-space:pre-wrap}
|
||||
.terminal-output strong{color:var(--accent)}
|
||||
.terminal-output .err{color:var(--danger)}
|
||||
.layer-left{display:flex;align-items:center;gap:8px}
|
||||
.ldot{width:10px;height:10px;border-radius:50%;flex-shrink:0}
|
||||
.ldot.air{background:var(--accent);box-shadow:0 0 6px rgba(100,240,200,0.4)}
|
||||
@@ -404,6 +411,8 @@ let lowPerfMode = localStorage.getItem('crucix_low_perf') === 'true';
|
||||
let isFlat = shouldStartFlat();
|
||||
let layerModes = JSON.parse(localStorage.getItem('crucix_layer_modes') || '{}');
|
||||
let spaceDisplayMode = localStorage.getItem('crucix_space_display') || 'icons';
|
||||
let terminalOutput = 'Ready. Live data is loaded from /api/data in server mode.';
|
||||
let terminalBusy = false;
|
||||
let currentRegion = 'world';
|
||||
let flatSvg, flatProjection, flatPath, flatG, flatZoom, flatW, flatH;
|
||||
|
||||
@@ -1564,6 +1573,46 @@ function renderLower(){
|
||||
document.getElementById('lowerGrid').innerHTML=`${tickerPanel}${osintPanel}${macroPanel}${ideasPanel}`;
|
||||
}
|
||||
|
||||
async function runTerminalAction(action){
|
||||
if(terminalBusy) return;
|
||||
terminalBusy = true;
|
||||
terminalOutput = `> ${action}\nRunning...`;
|
||||
renderRight();
|
||||
try{
|
||||
const res = await fetch('/api/action', {
|
||||
method:'POST',
|
||||
headers:{
|
||||
'Content-Type':'application/json',
|
||||
...(localStorage.getItem('crucix_sweep_token') ? {'x-crucix-token': localStorage.getItem('crucix_sweep_token')} : {})
|
||||
},
|
||||
body:JSON.stringify({action})
|
||||
});
|
||||
const payload = await res.json().catch(()=>({error:'Invalid server response'}));
|
||||
if(!res.ok) throw new Error(payload.error || `HTTP ${res.status}`);
|
||||
if(action === 'status'){
|
||||
const h = payload.health || {};
|
||||
terminalOutput = [
|
||||
'> status',
|
||||
`State: ${h.status || '--'}`,
|
||||
`Last sweep: ${h.lastSuccessfulSweep || h.lastSweep || '--'}`,
|
||||
`Data age: ${h.dataAgeSeconds != null ? h.dataAgeSeconds + 's' : '--'}`,
|
||||
`Sources: ${h.sourcesOk || 0} ok / ${h.sourcesDegraded || 0} degraded / ${h.sourcesFailed || 0} failed`,
|
||||
`LLM: ${h.llm?.state || '--'}`,
|
||||
`Sweep active: ${h.sweepInProgress ? 'yes' : 'no'}`
|
||||
].join('\n');
|
||||
}else if(action === 'brief'){
|
||||
terminalOutput = `> brief\n${payload.text || 'No briefing text returned.'}`;
|
||||
}else if(action === 'sweep'){
|
||||
terminalOutput = `> sweep\n${payload.status === 'already_running' ? 'Sweep already running.' : 'Sweep accepted. The dashboard will update when the sweep finishes.'}`;
|
||||
}
|
||||
}catch(err){
|
||||
terminalOutput = `> ${action}\nERROR: ${err.message}`;
|
||||
}finally{
|
||||
terminalBusy = false;
|
||||
renderRight();
|
||||
}
|
||||
}
|
||||
|
||||
// === RIGHT RAIL ===
|
||||
function renderRight(){
|
||||
const mobile = isMobileLayout();
|
||||
@@ -1603,12 +1652,33 @@ function renderRight(){
|
||||
deltaRows.push(`<div class="delta-row"><span class="delta-badge down">▼</span><span class="delta-label">${s.label||s.key}</span><span class="delta-val">${s.from}→${s.to} (${val})</span></div>`);
|
||||
}
|
||||
const deltaHtml = hasDelta ? deltaRows.join('') : `<div style="padding:12px;text-align:center;color:var(--dim);font-family:var(--mono);font-size:10px">${t('delta.noChanges','No changes since last sweep')}</div>`;
|
||||
const scenarioItems = (D.scenarios?.items || []).filter(s => s.enabled || s.state !== 'dormant').slice(0,4);
|
||||
const scenarioHtml = scenarioItems.length ? scenarioItems.map(s => `
|
||||
<div class="signal-row">
|
||||
<strong>${s.name} <span class="delta-badge ${s.changed?'new':''}">${(s.state||'dormant').toUpperCase()}</span></strong>
|
||||
<p>${s.description || ''}</p>
|
||||
<div class="layer-sub">${s.confidence || 0}% confidence · score ${s.score || 0}${s.lastTriggerTime ? ' · ' + getAge(s.lastTriggerTime) : ''}</div>
|
||||
</div>
|
||||
`).join('') : `<div style="padding:12px;text-align:center;color:var(--dim);font-family:var(--mono);font-size:10px">No active scenario watchlist items</div>`;
|
||||
|
||||
document.getElementById('rightRail').innerHTML=`
|
||||
<div class="g-panel right-actions">
|
||||
<div class="sec-head"><h3>Terminal Actions</h3><span class="badge">${terminalBusy?'RUNNING':'READY'}</span></div>
|
||||
<div class="action-grid">
|
||||
<button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('status')">Status</button>
|
||||
<button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('sweep')">Sweep</button>
|
||||
<button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('brief')">Brief</button>
|
||||
</div>
|
||||
<div class="terminal-output">${terminalOutput.replace(/[&<>]/g,c=>({'&':'&','<':'<','>':'>'}[c])).replace(/\n/g,'<br>')}</div>
|
||||
</div>
|
||||
<div class="g-panel right-signals">
|
||||
<div class="sec-head"><h3>${t('panels.crossSourceSignals','Cross-Source Signals')}</h3><span class="badge">${t('badges.worldview','WORLDVIEW')}</span></div>
|
||||
${signals}
|
||||
</div>
|
||||
<div class="g-panel right-scenarios">
|
||||
<div class="sec-head"><h3>Scenario Watchlist</h3><span class="badge">${D.scenarios?.available===false?'CONFIG':'LIVE'}</span></div>
|
||||
${scenarioHtml}
|
||||
</div>
|
||||
${mobile ? '' : buildOsintPanel('right-osint', 260)}
|
||||
<div class="g-panel right-core">
|
||||
<div class="sec-head"><h3>${t('panels.signalCore','Signal Core')}</h3><span class="badge">${t('badges.hotMetrics','HOT METRICS')}</span></div>
|
||||
@@ -1839,10 +1909,10 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
const hasInlineData = !!(D && D.meta);
|
||||
const canProbeApi = location.protocol !== 'file:';
|
||||
|
||||
if (canProbeApi && !hasInlineData) {
|
||||
if (canProbeApi) {
|
||||
// Server mode: always fetch live data from API (ignore any stale inline D)
|
||||
fetch('/api/data')
|
||||
.then(r => r.json())
|
||||
.then(r => { if(!r.ok) throw new Error(`HTTP ${r.status}`); return r.json(); })
|
||||
.then(data => { D = data; init(); connectSSE(); })
|
||||
.catch(() => {
|
||||
// Should not reach here — server routes to loading.html when no data
|
||||
|
||||
@@ -1,18 +1,489 @@
|
||||
# Agent Handoff
|
||||
|
||||
## Current Release Goal
|
||||
Last updated: 2026-05-17
|
||||
|
||||
Source branch: `codex/production-intelligence-terminal`
|
||||
## Repository State
|
||||
|
||||
Registry image:
|
||||
Project: Crucix fork / Intelligence Terminal
|
||||
|
||||
Local workspace:
|
||||
|
||||
```text
|
||||
C:\Users\MrSphay\Documents\Codex\Crucix\intelligence-terminal
|
||||
```
|
||||
|
||||
Remotes:
|
||||
|
||||
```text
|
||||
origin https://git.wilkensxl.de/MrSphay/intelligence-terminal.git
|
||||
upstream https://github.com/calesthio/Crucix.git
|
||||
```
|
||||
|
||||
Current branch tip:
|
||||
|
||||
```text
|
||||
Run `git rev-parse HEAD` after clone/pull. This handoff was updated by the `docs: sync issue tracker and handoff` commit after the implementation commit below.
|
||||
```
|
||||
|
||||
Latest implementation commit before issue-sync documentation:
|
||||
|
||||
```text
|
||||
53470cc701ec322080a89d220aef449b25850590
|
||||
```
|
||||
|
||||
Both pushed branches currently point to this commit:
|
||||
|
||||
```text
|
||||
origin/codex/production-intelligence-terminal
|
||||
origin/main
|
||||
```
|
||||
|
||||
Gitea repository:
|
||||
|
||||
```text
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal
|
||||
```
|
||||
|
||||
Default branch observed through the Gitea API:
|
||||
|
||||
```text
|
||||
codex/production-intelligence-terminal
|
||||
```
|
||||
|
||||
## Agent Kit Requirements Applied
|
||||
|
||||
The mandatory kit was cloned and reviewed first:
|
||||
|
||||
```text
|
||||
C:\Users\MrSphay\Documents\Codex\Crucix\agent-kit
|
||||
```
|
||||
|
||||
Rules applied from the kit:
|
||||
|
||||
- Keep agent context in source control: `AGENTS.md`, `.codex/project.md`, and this handoff file.
|
||||
- Use Gitea Ubuntu runners for heavy verification and package publishing.
|
||||
- Keep Docker/Dockge operation first-class.
|
||||
- Do not commit secrets, `.env`, private logs, tokens, or generated `runs/` data.
|
||||
- Add report-only maintenance workflows for security, dependency checks, repo cleanup, release dry runs, and template compliance.
|
||||
- Poll pushed Gitea Actions until terminal state when a token is available.
|
||||
|
||||
## What Was Implemented
|
||||
|
||||
### Docker And Runtime
|
||||
|
||||
- Docker image is Docker-first and Dockge/Pangolin suitable.
|
||||
- Browser auto-open is disabled by default through `AUTO_OPEN_BROWSER=false`.
|
||||
- Runtime health checks now work in the container without `wget` or host browser tools.
|
||||
- `runs` is persisted through a volume.
|
||||
- A later fix added `docker-entrypoint.sh` to prepare `/app/runs` before dropping privileges, so mounted volumes work with the non-root Node runtime.
|
||||
- `docker-compose.yml` uses the Gitea Registry image by default:
|
||||
|
||||
```text
|
||||
git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
## Notes
|
||||
### API And Health
|
||||
|
||||
- The repository is Docker-first and should stay suitable for Dockge/Pangolin.
|
||||
- Use `.env.example` as the operator-facing source of truth for configuration.
|
||||
- Source health and network metrics are available through `/api/health` and `/api/metrics`.
|
||||
- If Gitea Registry authentication is unavailable locally, build and push with the commands documented in `README.md`.
|
||||
Added or hardened:
|
||||
|
||||
- `GET /api/health`
|
||||
- `GET /api/data`
|
||||
- `GET /api/metrics`
|
||||
- `POST /api/sweep`
|
||||
- `POST /api/action`
|
||||
|
||||
Health now reports:
|
||||
|
||||
- `starting`
|
||||
- `healthy`
|
||||
- `degraded`
|
||||
- `stale`
|
||||
- `error`
|
||||
|
||||
It also reports:
|
||||
|
||||
- last sweep timestamps
|
||||
- stale/bootstrap state
|
||||
- data age
|
||||
- source health
|
||||
- source errors
|
||||
- LLM configuration state
|
||||
- Telegram/Discord enabled state
|
||||
- memory store state
|
||||
|
||||
### Live Data And Source Degradation
|
||||
|
||||
- Existing `runs/latest.json` is only treated as bootstrap/stale data until a real sweep completes.
|
||||
- Sweeps update `sourceHealth`, SSE/API data, and memory state.
|
||||
- RSS/news feed failures no longer silently look like fresh valid data.
|
||||
- `safeFetch` now tracks request counts, failures, bytes, source labels, hosts, and recent fetch events.
|
||||
- `safeFetch` has better timeout/retry/backoff/error behavior and reports HTML-as-API-error cases.
|
||||
- Yahoo Finance fetches are more explicit about source errors and HTML/API failures.
|
||||
- ACLED missing credentials now degrade transparently.
|
||||
- Telegram polling has quieter network-error backoff logs.
|
||||
|
||||
### LLM Integration
|
||||
|
||||
Added unified OpenAI-compatible provider layer:
|
||||
|
||||
```text
|
||||
lib/llm/openai-compatible.mjs
|
||||
```
|
||||
|
||||
Supported provider paths include:
|
||||
|
||||
- `openrouter`
|
||||
- `openai`
|
||||
- `openai-compatible`
|
||||
- `local-openai`
|
||||
- `lmstudio`
|
||||
- `lm-studio`
|
||||
- `ollama`
|
||||
|
||||
Relevant environment keys:
|
||||
|
||||
```text
|
||||
LLM_PROVIDER
|
||||
LLM_BASE_URL
|
||||
LLM_API_KEY
|
||||
LLM_MODEL
|
||||
LLM_TEMPERATURE
|
||||
LLM_MAX_TOKENS
|
||||
LLM_TIMEOUT_MS
|
||||
OPENROUTER_SITE_URL
|
||||
OPENROUTER_APP_NAME
|
||||
```
|
||||
|
||||
OpenRouter Free and local OpenAI-compatible endpoints are documented in `README.md` and `.env.example`.
|
||||
|
||||
### Memory
|
||||
|
||||
Added Phase-1 SQLite memory:
|
||||
|
||||
```text
|
||||
lib/intelligence-store.mjs
|
||||
runs/intelligence.db
|
||||
```
|
||||
|
||||
It uses `node:sqlite` when available and gracefully falls back when unavailable.
|
||||
|
||||
### Dashboard
|
||||
|
||||
Implemented:
|
||||
|
||||
- interactive Sensor Grid layer modes
|
||||
- focus/hide/normal states persisted in `localStorage`
|
||||
- Space Watch icon/orbit toggle
|
||||
- map/globe filtering consistency
|
||||
- flat map label redraw handling
|
||||
- live server-mode data loading from `/api/data` even when `jarvis.html` still contains an offline inline snapshot
|
||||
- Terminal Actions panel with `Status`, `Sweep`, and `Brief` buttons
|
||||
|
||||
Important UI markers in the final code:
|
||||
|
||||
```text
|
||||
layerModes
|
||||
spaceDisplayMode
|
||||
toggleSpaceDisplay()
|
||||
shouldShowType()
|
||||
runTerminalAction()
|
||||
```
|
||||
|
||||
### Briefings
|
||||
|
||||
Brief output now includes:
|
||||
|
||||
- Source Integrity
|
||||
- evidence links
|
||||
- event IDs
|
||||
- configurable verbosity through `BRIEF_VERBOSITY`
|
||||
|
||||
### Documentation
|
||||
|
||||
Updated:
|
||||
|
||||
- `README.md`
|
||||
- `.env.example`
|
||||
- `docs/sources/README.md`
|
||||
- `docs/sources/opensky.md`
|
||||
- `docs/sources/acled.md`
|
||||
- `docs/sources/telegram.md`
|
||||
- `docs/sources/firms.md`
|
||||
- `docs/sources/maritime.md`
|
||||
- `docs/security-review.md`
|
||||
- `docs/release-checklist.md`
|
||||
|
||||
README includes:
|
||||
|
||||
- Gitea Registry pull example
|
||||
- Dockge-compatible compose example
|
||||
- full `.env` examples
|
||||
- OpenRouter Free setup
|
||||
- LM Studio setup
|
||||
- Ollama setup
|
||||
- local OpenAI-compatible setup
|
||||
- Pangolin/reverse proxy notes
|
||||
|
||||
## Registry And Images
|
||||
|
||||
Registry image:
|
||||
|
||||
```text
|
||||
git.wilkensxl.de/mrsphay/intelligence-terminal
|
||||
```
|
||||
|
||||
Verified package tags through Gitea API:
|
||||
|
||||
```text
|
||||
latest
|
||||
20260517
|
||||
e933586b220656a2858d2215b934b22d1f08a908
|
||||
53470cc701ec322080a89d220aef449b25850590
|
||||
```
|
||||
|
||||
Successful pull test:
|
||||
|
||||
```bash
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
Observed digest:
|
||||
|
||||
```text
|
||||
sha256:780a41413921bd9a676461eca1cd1372591f523be4b7c9513d9bc085cbe7922d
|
||||
```
|
||||
|
||||
## Gitea Actions
|
||||
|
||||
Workflows present:
|
||||
|
||||
```text
|
||||
.gitea/workflows/build.yml
|
||||
.gitea/workflows/security-scan.yml
|
||||
.gitea/workflows/repo-cleanup.yml
|
||||
.gitea/workflows/dependency-check.yml
|
||||
.gitea/workflows/release-dry-run.yml
|
||||
.gitea/workflows/template-compliance.yml
|
||||
```
|
||||
|
||||
Final runs for commit `53470cc701ec322080a89d220aef449b25850590` were polled through the Gitea API and succeeded:
|
||||
|
||||
```text
|
||||
build.yml on main: success
|
||||
build.yml on codex/production-intelligence-terminal: success
|
||||
release-dry-run.yml on main: success
|
||||
release-dry-run.yml on codex/production-intelligence-terminal: success
|
||||
template-compliance.yml on main: success
|
||||
template-compliance.yml on codex/production-intelligence-terminal: success
|
||||
```
|
||||
|
||||
Relevant run URLs:
|
||||
|
||||
```text
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/23
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/24
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/25
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/26
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/27
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/28
|
||||
```
|
||||
|
||||
Repository secret expected by the registry publish workflow:
|
||||
|
||||
```text
|
||||
REGISTRY_TOKEN
|
||||
```
|
||||
|
||||
Local token note:
|
||||
|
||||
- `GITEA_TOKEN` was visible in the final Codex process.
|
||||
- It was used only for Gitea API checks and not printed.
|
||||
|
||||
## Issue Sync
|
||||
|
||||
Open upstream GitHub issues were reviewed on 2026-05-17 from:
|
||||
|
||||
```text
|
||||
https://github.com/calesthio/Crucix/issues
|
||||
```
|
||||
|
||||
The upstream list contained 24 open issues. Issues already handled by this fork were not copied as open work, including the Docker stale-dashboard incident (#105), map label redraw (#70), Sensor Grid controls (#72), space display toggle (#51), source docs (#52), Dockge/CasaOS docs (#78), LLM timeout (#87), inject/static helper confusion (#100), network metrics (#101), Telegram polling backoff (#104), and briefing/evidence context (#75).
|
||||
|
||||
Issues not relevant to this fork were also not copied, including the Wallpaper Engine redesign (#41), the fork-inflation discussion (#107), empty/unclear placeholders (#79/#80), and the general use-case discussion (#93).
|
||||
|
||||
The following Gitea issues were created for real remaining work:
|
||||
|
||||
```text
|
||||
#1 Reddit source must stop unauthenticated .json scraping
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/1
|
||||
|
||||
#2 Send operator alerts when dashboard data remains stale
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/2
|
||||
|
||||
#3 ACLED credentialed integration needs regression test and diagnostics
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/3
|
||||
|
||||
#4 Complete memory and prediction loop beyond Phase-1 SQLite
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/4
|
||||
|
||||
#5 Remove old inline dashboard snapshot from production builds
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/5
|
||||
|
||||
#6 Harden Terminal Actions for public reverse-proxy deployments
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/6
|
||||
|
||||
#7 Replace ADS-B stub with real disabled/degraded source handling
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/7
|
||||
|
||||
#8 Clean inherited public-demo and upstream marketing references
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/8
|
||||
```
|
||||
|
||||
## Verification Already Performed
|
||||
|
||||
Local lightweight checks:
|
||||
|
||||
```bash
|
||||
npm run test:unit
|
||||
npm audit --omit=dev --audit-level=high
|
||||
docker compose --env-file .env.example config
|
||||
node --check server.mjs
|
||||
node --check dashboard/inject.mjs
|
||||
node --check lib/llm/openai-compatible.mjs
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Unit test result:
|
||||
|
||||
```text
|
||||
21 tests passing
|
||||
0 failing
|
||||
```
|
||||
|
||||
Audit result:
|
||||
|
||||
```text
|
||||
0 high vulnerabilities
|
||||
```
|
||||
|
||||
Docker build and smoke test were performed locally earlier:
|
||||
|
||||
```bash
|
||||
docker build -t git.wilkensxl.de/mrsphay/intelligence-terminal:latest .
|
||||
docker run --rm -d --name intelligence-terminal-smoke -p 127.0.0.1::3117 -e AUTO_OPEN_BROWSER=false git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
Smoke test observations:
|
||||
|
||||
- Server booted.
|
||||
- No `xdg-open` error.
|
||||
- Initial sweep completed.
|
||||
- `/api/health` moved from `starting` to `degraded` with transparent source errors.
|
||||
- Degraded state was expected without all optional API keys.
|
||||
|
||||
Additional checks after fixing the dashboard live-data bug and Terminal Actions:
|
||||
|
||||
```bash
|
||||
node --check server.mjs
|
||||
npm run test:unit
|
||||
docker compose --env-file .env.example config
|
||||
git diff --check
|
||||
```
|
||||
|
||||
The dashboard script was also syntax-checked after extracting script blocks from `dashboard/public/jarvis.html`.
|
||||
|
||||
## Important Commits
|
||||
|
||||
```text
|
||||
7e85a54 chore: apply agent kit project structure
|
||||
85f97bb feat: harden intelligence runtime and llm providers
|
||||
42b7fc2 docs: add registry dockge and dashboard operations
|
||||
d072390 ci: align gitea workflows with agent kit
|
||||
0559481 ci: fix gitea registry publish login
|
||||
f3c9331 ci: fix agent kit compliance checks
|
||||
c2d572e fix: prepare runs volume before dropping privileges
|
||||
8e096b2 ci: harden gitea workflow reruns
|
||||
e933586 merge: reconcile main with production branch
|
||||
4262c7e docs: expand agent handoff
|
||||
53470cc fix: load live dashboard data and add terminal actions
|
||||
```
|
||||
|
||||
The large implementation commit `85f97bb` and the dashboard/action fix `53470cc` are contained in both:
|
||||
|
||||
```text
|
||||
origin/codex/production-intelligence-terminal
|
||||
origin/main
|
||||
```
|
||||
|
||||
## How To Continue In A Fresh Codex Environment
|
||||
|
||||
1. Clone the Gitea repository:
|
||||
|
||||
```bash
|
||||
git clone https://git.wilkensxl.de/MrSphay/intelligence-terminal.git
|
||||
cd intelligence-terminal
|
||||
git checkout codex/production-intelligence-terminal
|
||||
```
|
||||
|
||||
2. Confirm the expected commit:
|
||||
|
||||
```bash
|
||||
git rev-parse HEAD
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
```text
|
||||
The branch tip should include commit 53470cc701ec322080a89d220aef449b25850590 and the later `docs: sync issue tracker and handoff` commit.
|
||||
```
|
||||
|
||||
3. Read these files first:
|
||||
|
||||
```text
|
||||
AGENTS.md
|
||||
.codex/project.md
|
||||
docs/agent-handoff.md
|
||||
README.md
|
||||
.env.example
|
||||
```
|
||||
|
||||
4. If checking Actions, use `GITEA_TOKEN` from the environment. Do not print it.
|
||||
|
||||
PowerShell check:
|
||||
|
||||
```powershell
|
||||
if ($env:GITEA_TOKEN) { "GITEA_TOKEN=set" } else { "GITEA_TOKEN=missing" }
|
||||
```
|
||||
|
||||
5. Useful commands:
|
||||
|
||||
```bash
|
||||
npm run test:unit
|
||||
docker compose --env-file .env.example config
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
6. Start with Dockge/Pangolin using the README compose example and a `.env` based on `.env.example`.
|
||||
|
||||
## Remaining Risks And Follow-Ups
|
||||
|
||||
- Some sources will report `degraded` until optional keys are set, especially ACLED, FRED, EIA, and Cloudflare Radar.
|
||||
- OpenSky can rate-limit with HTTP 429; this is now visible in health instead of hidden.
|
||||
- GDELT/OFAC can time out under runner/network conditions; health reports this explicitly.
|
||||
- Browser-level visual verification of the full dashboard should be repeated after any future UI change.
|
||||
- The project still inherits the original Crucix broad source surface. Future work should prefer focused source-by-source tests over broad refactors.
|
||||
- If a new Codex environment sees non-fast-forward branch pushes, fetch first and preserve remote commits. Do not force-push without explicit approval.
|
||||
|
||||
## Operator Pull Command
|
||||
|
||||
For deployment:
|
||||
|
||||
```bash
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
For a pinned deployment:
|
||||
|
||||
```bash
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:20260517
|
||||
```
|
||||
|
||||
@@ -5,12 +5,21 @@
|
||||
- Shell execution: browser auto-open is gated by `AUTO_OPEN_BROWSER` and defaults to false.
|
||||
- Secrets: `.env` remains ignored; `.env.example` contains no real keys.
|
||||
- External network calls: source fetches use timeout/retry diagnostics and expose degraded state.
|
||||
- Manual actions: `/api/sweep` is local-only unless `SWEEP_TOKEN` is configured.
|
||||
- Manual actions: `/api/sweep` and `/api/action` are gated by `TERMINAL_ACTIONS_ENABLED` and local-only or `SWEEP_TOKEN` authorization.
|
||||
- File writes: runtime writes are limited to `runs/`.
|
||||
- HTML injection: dashboard data is JSON-injected only by the CLI path; server mode serves data through API/SSE.
|
||||
|
||||
## Terminal Actions
|
||||
|
||||
- `TERMINAL_ACTIONS_ENABLED=true` enables dashboard-triggered `status`, `sweep`, and `brief` actions through `POST /api/action`.
|
||||
- If `SWEEP_TOKEN` is set, callers must send the token through `x-sweep-token`, `Authorization: Bearer ...`, or the `token` request body field.
|
||||
- If `SWEEP_TOKEN` is empty, actions are accepted only from local loopback addresses.
|
||||
- For private Dockge/LAN deployments, this is intended to make the terminal operable from the browser.
|
||||
- For Pangolin or other internet-exposed deployments, set `SWEEP_TOKEN` or `TERMINAL_ACTIONS_ENABLED=false` until the public reverse-proxy hardening issue is completed.
|
||||
|
||||
## Residual Risk
|
||||
|
||||
- External feeds can return malformed, stale, or adversarial content. UI rendering should continue to sanitize titles and URLs.
|
||||
- LLM outputs are advisory only and must not be treated as financial advice.
|
||||
- `node:sqlite` availability depends on the Node 22 build; when unavailable the memory database degrades to a no-op placeholder.
|
||||
- Browser-stored sweep tokens are acceptable for a trusted home-server UI, but should not be treated as a strong auth boundary on a public endpoint.
|
||||
|
||||
@@ -16,3 +16,4 @@ Source docs:
|
||||
- [Telegram](telegram.md)
|
||||
- [FIRMS](firms.md)
|
||||
- [Maritime](maritime.md)
|
||||
- [Reddit](reddit.md)
|
||||
|
||||
33
docs/sources/reddit.md
Normal file
33
docs/sources/reddit.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Reddit Source
|
||||
|
||||
Reddit is used as a social sentiment input for selected geopolitical and market subreddits.
|
||||
|
||||
## Configuration
|
||||
|
||||
Create a Reddit script app at:
|
||||
|
||||
```text
|
||||
https://www.reddit.com/prefs/apps/
|
||||
```
|
||||
|
||||
Then set:
|
||||
|
||||
```env
|
||||
REDDIT_CLIENT_ID=
|
||||
REDDIT_CLIENT_SECRET=
|
||||
```
|
||||
|
||||
## Runtime Behavior
|
||||
|
||||
- The source uses the OAuth client credentials flow and then reads `https://oauth.reddit.com`.
|
||||
- Unauthenticated `reddit.com/.../hot.json` scraping is intentionally disabled.
|
||||
- Missing credentials return `status: no_credentials` and are surfaced as source degradation.
|
||||
- OAuth failures return `status: auth_failed` without logging or returning the client secret.
|
||||
- Subreddit fetch failures return `status: degraded` with per-subreddit errors.
|
||||
|
||||
## Test
|
||||
|
||||
```bash
|
||||
node apis/sources/reddit.mjs
|
||||
npm run test:unit
|
||||
```
|
||||
212
lib/scenarios.mjs
Normal file
212
lib/scenarios.mjs
Normal file
@@ -0,0 +1,212 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
const DEFAULT_SCENARIOS = [
|
||||
{
|
||||
id: 'middle-east-energy-shock',
|
||||
enabled: false,
|
||||
name: 'Middle East energy shock',
|
||||
description: 'Energy supply risk building from Middle East conflict or chokepoint pressure.',
|
||||
regions: ['Middle East', 'Iran', 'Israel', 'Strait of Hormuz'],
|
||||
categories: ['osint', 'energy', 'maritime'],
|
||||
keywords: ['missile', 'strike', 'hormuz', 'oil', 'energy', 'blockade'],
|
||||
thresholds: { watching: 2, building: 4, confirmed: 7 },
|
||||
invalidation: 'WTI normalizes and regional urgent signals fade for several sweeps.',
|
||||
},
|
||||
{
|
||||
id: 'macro-stress-spillover',
|
||||
enabled: false,
|
||||
name: 'Macro stress spillover',
|
||||
description: 'Market stress spreads from volatility into credit, rates, or commodities.',
|
||||
regions: ['US', 'Global'],
|
||||
categories: ['macro', 'markets'],
|
||||
keywords: ['vix', 'spread', 'credit', 'yield', 'inflation', 'gold'],
|
||||
thresholds: { watching: 2, building: 4, confirmed: 6 },
|
||||
invalidation: 'VIX and credit stress both normalize while source health remains stable.',
|
||||
},
|
||||
{
|
||||
id: 'regional-escalation-risk',
|
||||
enabled: false,
|
||||
name: 'Regional escalation risk',
|
||||
description: 'Local conflict signals broaden across adjacent regions or source categories.',
|
||||
regions: ['Ukraine', 'Taiwan', 'Africa', 'Middle East'],
|
||||
categories: ['conflict', 'thermal', 'osint', 'air'],
|
||||
keywords: ['mobilization', 'intercept', 'drone', 'ballistic', 'fatalities', 'border'],
|
||||
thresholds: { watching: 2, building: 5, confirmed: 8 },
|
||||
invalidation: 'No fresh cross-source escalation signals appear inside the configured horizon.',
|
||||
},
|
||||
];
|
||||
|
||||
export function evaluateScenarios(data, delta, runsDir) {
|
||||
const loaded = loadScenarioDefinitions(runsDir);
|
||||
if (!loaded.ok) {
|
||||
return { available: false, error: loaded.error, items: [], changed: [] };
|
||||
}
|
||||
|
||||
const statePath = join(runsDir, 'scenario-state.json');
|
||||
const previous = readJson(statePath, {});
|
||||
const evaluatedAt = data.meta?.timestamp || new Date().toISOString();
|
||||
const corpus = buildCorpus(data, delta);
|
||||
const items = loaded.scenarios.map(def => evaluateScenario(def, corpus, previous[def.id], evaluatedAt));
|
||||
const changed = items.filter(item => item.changed);
|
||||
|
||||
writeJson(statePath, Object.fromEntries(items.map(item => [item.id, {
|
||||
state: item.state,
|
||||
score: item.score,
|
||||
confidence: item.confidence,
|
||||
lastTriggerTime: item.lastTriggerTime,
|
||||
updatedAt: evaluatedAt,
|
||||
}])));
|
||||
|
||||
return {
|
||||
available: true,
|
||||
path: loaded.path,
|
||||
items,
|
||||
changed,
|
||||
};
|
||||
}
|
||||
|
||||
export function loadScenarioDefinitions(runsDir) {
|
||||
const path = join(runsDir, 'scenarios.json');
|
||||
try {
|
||||
if (!existsSync(runsDir)) mkdirSync(runsDir, { recursive: true });
|
||||
if (!existsSync(path)) {
|
||||
writeJson(path, {
|
||||
version: 1,
|
||||
scenarios: DEFAULT_SCENARIOS,
|
||||
});
|
||||
}
|
||||
const raw = JSON.parse(readFileSync(path, 'utf8'));
|
||||
if (!raw || !Array.isArray(raw.scenarios)) throw new Error('scenarios must be an array');
|
||||
const scenarios = raw.scenarios
|
||||
.map(normalizeScenario)
|
||||
.filter(Boolean);
|
||||
return { ok: true, path, scenarios };
|
||||
} catch (err) {
|
||||
return { ok: false, path, error: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeScenario(input) {
|
||||
if (!input || typeof input !== 'object') return null;
|
||||
const id = String(input.id || input.name || '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
|
||||
const name = String(input.name || input.id || '').trim();
|
||||
if (!id || !name) return null;
|
||||
const thresholds = input.thresholds || {};
|
||||
return {
|
||||
id,
|
||||
enabled: input.enabled === true,
|
||||
name,
|
||||
description: String(input.description || ''),
|
||||
regions: arrayOfStrings(input.regions),
|
||||
categories: arrayOfStrings(input.categories),
|
||||
keywords: arrayOfStrings(input.keywords).map(s => s.toLowerCase()),
|
||||
thresholds: {
|
||||
watching: Number(thresholds.watching || 2),
|
||||
building: Number(thresholds.building || 4),
|
||||
confirmed: Number(thresholds.confirmed || 7),
|
||||
},
|
||||
invalidation: String(input.invalidation || ''),
|
||||
};
|
||||
}
|
||||
|
||||
function evaluateScenario(def, corpus, previous, evaluatedAt) {
|
||||
if (!def.enabled) {
|
||||
return {
|
||||
...publicScenario(def),
|
||||
state: 'dormant',
|
||||
score: 0,
|
||||
confidence: 0,
|
||||
evidence: [],
|
||||
changed: previous?.state && previous.state !== 'dormant',
|
||||
lastTriggerTime: previous?.lastTriggerTime || null,
|
||||
};
|
||||
}
|
||||
|
||||
const evidence = [];
|
||||
let score = 0;
|
||||
for (const keyword of def.keywords) {
|
||||
const hit = corpus.entries.find(entry => entry.text.includes(keyword));
|
||||
if (hit) {
|
||||
score += 1;
|
||||
evidence.push({ type: 'keyword', label: keyword, source: hit.source, text: hit.original.slice(0, 180) });
|
||||
}
|
||||
}
|
||||
for (const region of def.regions) {
|
||||
const needle = region.toLowerCase();
|
||||
const hit = corpus.entries.find(entry => entry.text.includes(needle));
|
||||
if (hit) {
|
||||
score += 1;
|
||||
evidence.push({ type: 'region', label: region, source: hit.source, text: hit.original.slice(0, 180) });
|
||||
}
|
||||
}
|
||||
for (const category of def.categories) {
|
||||
if (corpus.categories.has(category.toLowerCase())) {
|
||||
score += 1;
|
||||
evidence.push({ type: 'category', label: category, source: 'sweep', text: `${category} category active` });
|
||||
}
|
||||
}
|
||||
|
||||
const state = score >= def.thresholds.confirmed ? 'confirmed'
|
||||
: score >= def.thresholds.building ? 'building'
|
||||
: score >= def.thresholds.watching ? 'watching'
|
||||
: 'dormant';
|
||||
const confidence = Math.min(100, Math.round((score / Math.max(1, def.thresholds.confirmed)) * 100));
|
||||
const changed = previous?.state ? previous.state !== state : state !== 'dormant';
|
||||
return {
|
||||
...publicScenario(def),
|
||||
state,
|
||||
score,
|
||||
confidence,
|
||||
evidence: evidence.slice(0, 6),
|
||||
changed,
|
||||
lastTriggerTime: state === 'dormant' ? (previous?.lastTriggerTime || null) : evaluatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function publicScenario(def) {
|
||||
return {
|
||||
id: def.id,
|
||||
name: def.name,
|
||||
description: def.description,
|
||||
enabled: def.enabled,
|
||||
invalidation: def.invalidation,
|
||||
};
|
||||
}
|
||||
|
||||
function buildCorpus(data, delta) {
|
||||
const entries = [];
|
||||
const categories = new Set();
|
||||
const push = (source, text, category) => {
|
||||
if (!text) return;
|
||||
entries.push({ source, original: String(text), text: String(text).toLowerCase() });
|
||||
if (category) categories.add(category);
|
||||
};
|
||||
|
||||
for (const signal of data.tSignals || []) push('thermal', signal, 'thermal');
|
||||
for (const post of data.tg?.urgent || []) push(post.channel || 'telegram', post.text, 'osint');
|
||||
for (const item of data.newsFeed || []) push(item.source || 'news', item.headline || item.title, 'news');
|
||||
for (const item of data.news || []) push(item.source || 'news', item.headline || item.title, 'news');
|
||||
for (const item of data.acled?.deadliestEvents || []) push('ACLED', `${item.country || ''} ${item.location || ''} ${item.event_type || ''} ${item.fatalities || ''}`, 'conflict');
|
||||
for (const item of data.air || []) push('OpenSky', `${item.region} ${item.total} aircraft`, 'air');
|
||||
for (const item of data.chokepoints || []) push('Maritime', `${item.label} ${item.note}`, 'maritime');
|
||||
if (data.energy?.wti || data.energy?.brent) push('energy', `WTI ${data.energy.wti} Brent ${data.energy.brent}`, 'energy');
|
||||
if (data.markets?.vix || data.fred?.some(f => f.id === 'VIXCLS')) push('markets', 'VIX volatility market stress', 'markets');
|
||||
if (delta?.summary) push('delta', `${delta.summary.direction} ${delta.summary.totalChanges} changes ${delta.summary.criticalChanges} critical`, 'delta');
|
||||
for (const signal of delta?.signals?.new || []) push('delta', signal.label || signal.reason || signal.key, 'delta');
|
||||
for (const signal of delta?.signals?.escalated || []) push('delta', signal.label || signal.reason || signal.key, 'delta');
|
||||
|
||||
return { entries, categories };
|
||||
}
|
||||
|
||||
function arrayOfStrings(value) {
|
||||
return Array.isArray(value) ? value.map(v => String(v).trim()).filter(Boolean) : [];
|
||||
}
|
||||
|
||||
function readJson(path, fallback) {
|
||||
try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return fallback; }
|
||||
}
|
||||
|
||||
function writeJson(path, value) {
|
||||
writeFileSync(path, JSON.stringify(value, null, 2));
|
||||
}
|
||||
@@ -12,7 +12,7 @@
|
||||
"brief:save": "node apis/save-briefing.mjs",
|
||||
"diag": "node diag.mjs",
|
||||
"test": "npm run test:unit",
|
||||
"test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs",
|
||||
"test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs",
|
||||
"compose:config": "docker compose config",
|
||||
"clean": "node scripts/clean.mjs",
|
||||
"fresh-start": "npm run clean && npm start"
|
||||
|
||||
54
server.mjs
54
server.mjs
@@ -18,6 +18,7 @@ import { TelegramAlerter } from './lib/alerts/telegram.mjs';
|
||||
import { DiscordAlerter } from './lib/alerts/discord.mjs';
|
||||
import { getFetchMetrics } from './apis/utils/fetch.mjs';
|
||||
import { IntelligenceStore } from './lib/intelligence-store.mjs';
|
||||
import { evaluateScenarios } from './lib/scenarios.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = __dirname;
|
||||
@@ -289,14 +290,28 @@ app.get('/api/metrics', (req, res) => {
|
||||
});
|
||||
|
||||
app.post('/api/sweep', express.json(), (req, res) => {
|
||||
const remote = req.ip || '';
|
||||
const local = remote.includes('127.0.0.1') || remote === '::1' || remote === '::ffff:127.0.0.1';
|
||||
const token = req.get('x-crucix-token') || req.query.token || req.body?.token;
|
||||
if (config.sweepToken && token !== config.sweepToken) return res.status(401).json({ error: 'Invalid sweep token' });
|
||||
if (!config.sweepToken && !local) return res.status(403).json({ error: 'Manual sweep is local-only unless SWEEP_TOKEN is set' });
|
||||
if (sweepInProgress) return res.status(409).json({ status: 'already_running', sweepStartedAt });
|
||||
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
|
||||
res.status(202).json({ status: 'accepted' });
|
||||
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
|
||||
triggerSweep(res);
|
||||
});
|
||||
|
||||
app.post('/api/action', express.json(), async (req, res) => {
|
||||
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
|
||||
const action = String(req.body?.action || req.query.action || '').toLowerCase();
|
||||
|
||||
if (action === 'status') {
|
||||
return res.json({ ok: true, action, health: buildHealth() });
|
||||
}
|
||||
|
||||
if (action === 'brief') {
|
||||
if (!currentData) return res.status(503).json({ ok: false, action, error: 'No data yet — first sweep in progress' });
|
||||
return res.json({ ok: true, action, text: buildBrief(currentData) });
|
||||
}
|
||||
|
||||
if (action === 'sweep') {
|
||||
return triggerSweep(res);
|
||||
}
|
||||
|
||||
res.status(400).json({ ok: false, error: 'Unknown action', actions: ['status', 'brief', 'sweep'] });
|
||||
});
|
||||
|
||||
// API: available locales
|
||||
@@ -333,6 +348,20 @@ function dataAgeMs() {
|
||||
return Number.isFinite(ms) ? ms : null;
|
||||
}
|
||||
|
||||
function canRunTerminalAction(req) {
|
||||
const remote = req.ip || '';
|
||||
const local = remote.includes('127.0.0.1') || remote === '::1' || remote === '::ffff:127.0.0.1';
|
||||
const token = req.get('x-crucix-token') || req.query.token || req.body?.token;
|
||||
if (config.sweepToken) return token === config.sweepToken;
|
||||
return Boolean(config.terminalActionsEnabled || local);
|
||||
}
|
||||
|
||||
function triggerSweep(res) {
|
||||
if (sweepInProgress) return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
|
||||
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
|
||||
return res.status(202).json({ ok: true, status: 'accepted' });
|
||||
}
|
||||
|
||||
function getLLMStatus() {
|
||||
if (!config.llm.provider) return { state: 'disabled' };
|
||||
if (!llmProvider) return { state: 'misconfigured', provider: config.llm.provider };
|
||||
@@ -376,6 +405,7 @@ function buildHealth() {
|
||||
llm: getLLMStatus(),
|
||||
telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId),
|
||||
discordEnabled: !!(config.discord?.botToken || config.discord?.webhookUrl),
|
||||
terminalActionsEnabled: Boolean(config.terminalActionsEnabled || config.sweepToken),
|
||||
refreshIntervalMinutes: config.refreshIntervalMinutes,
|
||||
language: currentLanguage,
|
||||
memory: intelligenceStore.status(),
|
||||
@@ -418,6 +448,13 @@ function buildBrief(data) {
|
||||
lines.push('', '*Why This Matters*');
|
||||
for (const idea of ideas) lines.push(`- ${idea.title}: ${(idea.rationale || idea.text || '').slice(0, 140)}`);
|
||||
}
|
||||
const scenarioChanges = data.scenarios?.changed || [];
|
||||
if (scenarioChanges.length) {
|
||||
lines.push('', '*Scenario Watchlist*');
|
||||
for (const scenario of scenarioChanges.slice(0, 4)) {
|
||||
lines.push(`- ${scenario.name}: ${scenario.state.toUpperCase()} (${scenario.confidence}% confidence)`);
|
||||
}
|
||||
}
|
||||
lines.push('', '*What To Do Next*', '- Open the dashboard, verify the evidence links, and compare source health before acting.');
|
||||
return lines.join('\n');
|
||||
}
|
||||
@@ -464,6 +501,7 @@ async function runSweepCycle() {
|
||||
// 4. Delta computation + memory
|
||||
const delta = memory.addRun(synthesized);
|
||||
synthesized.delta = delta;
|
||||
synthesized.scenarios = evaluateScenarios(synthesized, delta, RUNS_DIR);
|
||||
|
||||
// 5. LLM-powered trade ideas (LLM-only feature) — isolated so failures don't kill sweep
|
||||
if (llmProvider?.isConfigured) {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { safeFetch, safeFetchText, getFetchMetrics } from '../apis/utils/fetch.mjs';
|
||||
|
||||
test('safeFetch reports HTML as degraded JSON response', async () => {
|
||||
@@ -34,3 +35,18 @@ test('safeFetchText returns text and byte count', async () => {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('scenario watchlist feature is wired into sweep, briefing, and dashboard', () => {
|
||||
const scenarios = readFileSync(new URL('../lib/scenarios.mjs', import.meta.url), 'utf8');
|
||||
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
|
||||
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
|
||||
const readme = readFileSync(new URL('../README.md', import.meta.url), 'utf8');
|
||||
assert.match(scenarios, /DEFAULT_SCENARIOS/);
|
||||
assert.match(scenarios, /runsDir, 'scenarios\.json'/);
|
||||
assert.match(scenarios, /scenario-state\.json/);
|
||||
assert.match(scenarios, /watching.*building.*confirmed/s);
|
||||
assert.match(server, /evaluateScenarios\(synthesized, delta, RUNS_DIR\)/);
|
||||
assert.match(server, /\*Scenario Watchlist\*/);
|
||||
assert.match(html, /Scenario Watchlist/);
|
||||
assert.match(readme, /runs\/scenarios\.json/);
|
||||
});
|
||||
|
||||
109
test/reddit-source.test.mjs
Normal file
109
test/reddit-source.test.mjs
Normal file
@@ -0,0 +1,109 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { briefing, getHot, getRedditConfig, getToken } from '../apis/sources/reddit.mjs';
|
||||
|
||||
test('Reddit reports missing OAuth credentials without network access', async () => {
|
||||
let calls = 0;
|
||||
const data = await briefing({
|
||||
env: {},
|
||||
delayMs: 0,
|
||||
fetchImpl: async () => {
|
||||
calls++;
|
||||
throw new Error('unexpected network access');
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(calls, 0);
|
||||
assert.equal(data.status, 'no_credentials');
|
||||
assert.equal(data.error, 'missing_reddit_oauth_credentials');
|
||||
assert.deepEqual(data.missing, ['REDDIT_CLIENT_ID', 'REDDIT_CLIENT_SECRET']);
|
||||
});
|
||||
|
||||
test('Reddit hot posts require OAuth token and never use public JSON fallback', async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
let calledUrl = null;
|
||||
globalThis.fetch = async url => {
|
||||
calledUrl = url;
|
||||
throw new Error('unexpected public fallback');
|
||||
};
|
||||
|
||||
try {
|
||||
const data = await getHot('worldnews');
|
||||
assert.equal(calledUrl, null);
|
||||
assert.equal(data.status, 'no_credentials');
|
||||
assert.equal(data.error, 'reddit_oauth_required');
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('Reddit classifies OAuth HTTP failure without exposing secrets', async () => {
|
||||
const result = await getToken({
|
||||
env: { REDDIT_CLIENT_ID: 'client-id', REDDIT_CLIENT_SECRET: 'client-secret' },
|
||||
fetchImpl: async () => ({
|
||||
ok: false,
|
||||
status: 401,
|
||||
text: async () => 'invalid client',
|
||||
}),
|
||||
});
|
||||
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(result.status, 'auth_failed');
|
||||
assert.equal(result.error, 'reddit_oauth_http_401');
|
||||
assert.doesNotMatch(JSON.stringify(result), /client-secret/);
|
||||
});
|
||||
|
||||
test('Reddit fetches hot posts through oauth.reddit.com when configured', async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const urls = [];
|
||||
globalThis.fetch = async url => {
|
||||
urls.push(String(url));
|
||||
if (String(url).includes('/api/v1/access_token')) {
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
json: async () => ({ access_token: 'test-token' }),
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
status: 200,
|
||||
headers: { get: () => 'application/json' },
|
||||
text: async () => JSON.stringify({
|
||||
data: {
|
||||
children: [
|
||||
{
|
||||
data: {
|
||||
title: 'Market stress headline',
|
||||
score: 42,
|
||||
num_comments: 7,
|
||||
url: 'https://example.test/post',
|
||||
created_utc: 1700000000,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
try {
|
||||
const data = await briefing({
|
||||
env: { REDDIT_CLIENT_ID: 'client-id', REDDIT_CLIENT_SECRET: 'client-secret' },
|
||||
subreddits: ['worldnews'],
|
||||
delayMs: 0,
|
||||
});
|
||||
|
||||
assert.equal(data.status, 'ok');
|
||||
assert.equal(data.subreddits.worldnews[0].title, 'Market stress headline');
|
||||
assert.ok(urls.some(url => url === 'https://www.reddit.com/api/v1/access_token'));
|
||||
assert.ok(urls.some(url => url.startsWith('https://oauth.reddit.com/r/worldnews/hot')));
|
||||
assert.equal(urls.some(url => url.includes('hot.json')), false);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('Reddit config reports partial credential state', () => {
|
||||
assert.deepEqual(getRedditConfig({ REDDIT_CLIENT_ID: 'id' }).missing, ['REDDIT_CLIENT_SECRET']);
|
||||
});
|
||||
Reference in New Issue
Block a user