37 Commits

Author SHA1 Message Date
MrSphay
331175ae3c Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-4-memory-prediction-loop
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 57s
# Conflicts:
#	server.mjs
#	test/fetch-utils.test.mjs
2026-05-17 20:46:02 +02:00
e288881c41 Merge pull request 'fix: harden terminal action endpoints' (#25) from codex/issue-6-terminal-actions-hardening into codex/production-intelligence-terminal
All checks were successful
Release Dry Run / release-dry-run (push) Successful in 14s
Build / test-and-image (push) Successful in 27s
Codex Template Compliance / template-compliance (push) Successful in 4s
Reviewed-on: #25
2026-05-17 18:43:21 +00:00
MrSphay
3069114ffd Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-4-memory-prediction-loop
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 6s
Build / test-and-image (pull_request) Successful in 1m3s
# Conflicts:
#	test/fetch-utils.test.mjs
2026-05-17 20:39:38 +02:00
MrSphay
09df127e06 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-6-terminal-actions-hardening
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m4s
# Conflicts:
#	test/fetch-utils.test.mjs
2026-05-17 20:39:04 +02:00
49176b42fd Merge pull request 'fix: remove embedded dashboard snapshot' (#20) from codex/issue-5-dashboard-shell into codex/production-intelligence-terminal
All checks were successful
Build / test-and-image (push) Successful in 25s
Release Dry Run / release-dry-run (push) Successful in 14s
Codex Template Compliance / template-compliance (push) Successful in 6s
Reviewed-on: #20
2026-05-17 18:35:58 +00:00
MrSphay
090e90ea70 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-4-memory-prediction-loop
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 55s
# Conflicts:
#	README.md
#	test/fetch-utils.test.mjs
2026-05-17 20:35:44 +02:00
e70801ae98 Merge branch 'codex/production-intelligence-terminal' into codex/issue-5-dashboard-shell
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 6s
Build / test-and-image (pull_request) Successful in 54s
2026-05-17 18:34:34 +00:00
703670e7a0 Merge pull request 'fix: report adsb unavailable state as degraded' (#11) from codex/issue-7-adsb-degraded into codex/production-intelligence-terminal
All checks were successful
Build / test-and-image (push) Successful in 28s
Release Dry Run / release-dry-run (push) Successful in 14s
Codex Template Compliance / template-compliance (push) Successful in 5s
Reviewed-on: #11
2026-05-17 18:33:51 +00:00
MrSphay
1423dca199 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-6-terminal-actions-hardening
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 56s
# Conflicts:
#	README.md
#	server.mjs
#	test/fetch-utils.test.mjs
2026-05-17 20:33:45 +02:00
MrSphay
5b013947b4 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-5-dashboard-shell
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 53s
# Conflicts:
#	test/fetch-utils.test.mjs
2026-05-17 20:32:04 +02:00
MrSphay
5113e341b2 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-7-adsb-degraded
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m14s
# Conflicts:
#	package.json
2026-05-17 20:30:53 +02:00
d625bffd4a Merge pull request 'Alert operators when dashboard data is stale' (#12) from codex/issue-2-stale-alerts into codex/production-intelligence-terminal
All checks were successful
Build / test-and-image (push) Successful in 22s
Codex Template Compliance / template-compliance (push) Successful in 5s
Release Dry Run / release-dry-run (push) Successful in 10s
Reviewed-on: #12
2026-05-17 17:41:32 +00:00
776d200853 Merge branch 'codex/production-intelligence-terminal' into codex/issue-2-stale-alerts
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 55s
2026-05-17 17:38:56 +00:00
b8f34d3d19 Merge pull request 'fix: make news geotagging deterministic' (#41) from codex/issue-19-deterministic-news-geotags into codex/production-intelligence-terminal
All checks were successful
Release Dry Run / release-dry-run (push) Successful in 10s
Codex Template Compliance / template-compliance (push) Successful in 6s
Build / test-and-image (push) Successful in 23s
Reviewed-on: #41
2026-05-17 17:37:56 +00:00
d7f10bf545 merge: update adsb degraded branch
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 6s
Build / test-and-image (pull_request) Successful in 1m13s
2026-05-17 19:03:49 +02:00
533abb914c Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-19-deterministic-news-geotags
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m11s
# Conflicts:
#	package.json
2026-05-17 18:58:49 +02:00
a6e1026aef Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-2-stale-alerts
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 6s
Build / test-and-image (pull_request) Successful in 52s
2026-05-17 18:56:32 +02:00
2f2a10609f Merge pull request 'test: guard dashboard text against mojibake' (#40) from codex/issue-23-mojibake-locale-check into codex/production-intelligence-terminal
All checks were successful
Release Dry Run / release-dry-run (push) Successful in 11s
Codex Template Compliance / template-compliance (push) Successful in 6s
Build / test-and-image (push) Successful in 21s
Reviewed-on: #40
2026-05-17 16:55:34 +00:00
900f43ba13 fix: make news geotagging deterministic
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m4s
2026-05-17 18:54:35 +02:00
47ecb34a4d test: scope mojibake check to locales
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 52s
2026-05-17 18:52:51 +02:00
6a45bf9ce6 test: guard dashboard text against mojibake
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m3s
2026-05-17 18:49:59 +02:00
a809a55881 Merge pull request 'test: include ACLED source regression in unit suite' (#39) from codex/issue-3-acled-diagnostics-auth-tests into codex/production-intelligence-terminal
All checks were successful
Release Dry Run / release-dry-run (push) Successful in 11s
Codex Template Compliance / template-compliance (push) Successful in 5s
Build / test-and-image (push) Successful in 22s
Reviewed-on: #39
2026-05-17 16:43:13 +00:00
fb41d52101 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-3-acled-diagnostics-auth-tests
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m5s
# Conflicts:
#	package.json
2026-05-17 18:37:31 +02:00
3e3e3d57c7 test: include acled source regression
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m6s
2026-05-17 18:32:51 +02:00
e0345e2de3 Merge pull request 'fix: avoid double-counting fetch failures' (#33) from codex/issue-15-fetch-metrics into codex/production-intelligence-terminal
All checks were successful
Build / test-and-image (push) Successful in 22s
Codex Template Compliance / template-compliance (push) Successful in 5s
Release Dry Run / release-dry-run (push) Successful in 9s
Reviewed-on: #33
2026-05-17 15:59:27 +00:00
18b2e61678 Merge branch 'codex/production-intelligence-terminal' into codex/issue-15-fetch-metrics
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 55s
2026-05-17 15:56:09 +00:00
7a562bc8cf Merge pull request 'Improve ACLED auth diagnostics and tests' (#14) from codex/issue-3-acled-diagnostics-auth-tests into codex/production-intelligence-terminal
All checks were successful
Build / test-and-image (push) Successful in 18s
Release Dry Run / release-dry-run (push) Successful in 8s
Codex Template Compliance / template-compliance (push) Successful in 5s
Reviewed-on: #14
2026-05-17 15:48:03 +00:00
833b7dedd7 Merge branch 'codex/production-intelligence-terminal' into codex/issue-3-acled-diagnostics-auth-tests
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 48s
2026-05-17 15:46:10 +00:00
da180831bf Merge pull request 'docs(marketing): finalize Intelligence Terminal product communication' (#38) from codex/marketing-product-communication into codex/production-intelligence-terminal
All checks were successful
Build / test-and-image (push) Successful in 20s
Codex Template Compliance / template-compliance (push) Successful in 5s
Release Dry Run / release-dry-run (push) Successful in 8s
Reviewed-on: #38
2026-05-17 15:36:11 +00:00
43a5b642e4 Merge branch 'codex/production-intelligence-terminal' into codex/issue-3-acled-diagnostics-auth-tests
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 50s
2026-05-17 14:45:19 +00:00
MrSphay
08d24594c1 fix: avoid double-counting fetch failures
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 6s
Build / test-and-image (pull_request) Successful in 49s
2026-05-17 14:39:46 +02:00
MrSphay
6096a0ad03 fix: remove embedded dashboard snapshot
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 49s
2026-05-17 14:33:52 +02:00
MrSphay
267af03b22 feat: extend memory prediction loop
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 50s
2026-05-17 14:30:39 +02:00
MrSphay
d7df2e4aee fix: harden terminal action endpoints
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 49s
2026-05-17 14:19:28 +02:00
bb139799d7 fix: improve acled auth diagnostics
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 49s
2026-05-17 14:05:01 +02:00
e574ad1c3d feat: alert operators on stale data
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 53s
2026-05-17 13:58:32 +02:00
MrSphay
b2f604b120 fix: report adsb unavailable state as degraded
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m13s
2026-05-17 13:55:42 +02:00
21 changed files with 1382 additions and 198 deletions

View File

@@ -6,8 +6,12 @@ PORT=3117
REFRESH_INTERVAL_MINUTES=15
AUTO_OPEN_BROWSER=false
STALE_DATA_MAX_AGE_MINUTES=60
STALE_ALERT_COOLDOWN_MINUTES=60
DASHBOARD_URL=
TERMINAL_ACTIONS_ENABLED=true
SWEEP_TOKEN=
TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
TERMINAL_ACTION_RATE_LIMIT_MAX=10
BRIEF_VERBOSITY=standard
# LLM layer

View File

@@ -130,8 +130,12 @@ PORT=3117
REFRESH_INTERVAL_MINUTES=15
AUTO_OPEN_BROWSER=false
STALE_DATA_MAX_AGE_MINUTES=60
STALE_ALERT_COOLDOWN_MINUTES=60
DASHBOARD_URL=https://intelligence.example.internal
TERMINAL_ACTIONS_ENABLED=true
SWEEP_TOKEN=
TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
TERMINAL_ACTION_RATE_LIMIT_MAX=10
BRIEF_VERBOSITY=standard
LLM_PROVIDER=openrouter
@@ -183,7 +187,52 @@ LLM_MODEL=your-model
For Pangolin or another reverse proxy, forward HTTP traffic to `intelligence-terminal:3117` (or the `PORT` you set). Missing API keys do not crash sweeps; affected sources are reported as degraded in `/api/health`.
The dashboard Terminal Actions panel can trigger `status`, `sweep`, and `brief` through `/api/action`. Leave `TERMINAL_ACTIONS_ENABLED=true` for a private home-server deployment. For an internet-exposed deployment, set `SWEEP_TOKEN` and pass it through trusted automation, or set `TERMINAL_ACTIONS_ENABLED=false` to disable browser-triggered actions. If you protect actions with `SWEEP_TOKEN`, the browser can send it from `localStorage.crucix_sweep_token`.
#### Terminal Action Exposure
`POST /api/action` and `POST /api/sweep` can trigger operational actions such as manual sweeps. The dashboard has a **SET TOKEN** control that stores your `SWEEP_TOKEN` in browser local storage and sends it as the `x-crucix-token` header; do not put action tokens in URLs.
Recommended settings:
| Deployment | Settings |
| --- | --- |
| Private local machine | `NODE_ENV=development`, optional `SWEEP_TOKEN`, optional `TERMINAL_ACTIONS_ENABLED=true`. Localhost can run actions without a token for development. |
| Private LAN / Dockge | Set a strong `SWEEP_TOKEN`, keep `TERMINAL_ACTIONS_ENABLED=true`, expose only to trusted clients. |
| Pangolin-authenticated reverse proxy | Set a strong `SWEEP_TOKEN`, keep Pangolin auth in front, use the dashboard **SET TOKEN** flow once per browser. |
| Public internet | Do not expose Terminal Actions directly. If exposure is unavoidable, require `SWEEP_TOKEN`, keep proxy authentication enabled, lower `TERMINAL_ACTION_RATE_LIMIT_MAX`, and monitor server audit logs. |
Action endpoints reject cross-origin POST origins, apply a small in-memory per-IP rate limit, and write sanitized audit lines without logging the token.
When data remains stale past `STALE_DATA_MAX_AGE_MINUTES`, the server sends an operator alert through configured Telegram/Discord channels after failed or degraded sweep attempts. `STALE_ALERT_COOLDOWN_MINUTES` prevents repeated stale alerts from spamming every refresh interval. Set `DASHBOARD_URL` to the Pangolin/public URL you want included in those alerts.
#### Memory And Prediction Loop
Crucix stores longitudinal memory in `runs/intelligence.db` when the current Node.js build exposes `node:sqlite`. If SQLite is unavailable, the file is created as a harmless placeholder and `/api/health` reports the memory store as unavailable instead of failing the sweep.
The memory layer persists:
| Table | Purpose |
| --- | --- |
| `runs` | Sweep timestamps, source health counts, and delta direction summaries. |
| `entities` | Stable entity IDs for recurring countries, regions, and locations. |
| `events` | Stable event IDs for conflict, OSINT, urgent news, and new delta signals across sweeps. |
| `predictions` | Trade/intelligence hypotheses with evidence, confidence, horizon, outcome state, and latest grading. |
Query endpoints:
```text
GET /api/memory/search?q=iran&limit=25
GET /api/memory/predictions?state=open&limit=25
```
Memory endpoints use the same operator authorization gate as Terminal Actions. The dashboard Terminal Actions panel includes a `Memory` action for a quick operator-facing view of recent events and prediction states.
Retention, backup, and privacy expectations:
- Treat `runs/intelligence.db` as operator data. It can contain source excerpts, headlines, generated hypotheses, and URLs from your configured feeds.
- Back up `runs/` with the rest of your Dockge volume if you want longitudinal learning to survive container replacement.
- Delete `runs/intelligence.db` to reset SQLite memory; the next sweep recreates the schema.
- Do not commit `runs/` or `.env`. API credentials stay in `.env`; memory stores derived observations, not secrets.
- If you expose the dashboard through a reverse proxy, protect Terminal Actions and memory queries behind your normal authentication boundary.
#### Scenario Watchlist
@@ -355,7 +404,7 @@ These three unlock the most valuable economic and satellite data. Each takes abo
| Key | Source | How to Get |
|-----|--------|------------|
| `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2 |
| `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2. `ACLED_USER` / `ACLED_USERNAME` are accepted as email aliases |
| `AISSTREAM_API_KEY` | Maritime AIS vessel tracking | [aisstream.io](https://aisstream.io/) — free |
| `ADSB_API_KEY` | Unfiltered flight tracking | [RapidAPI](https://rapidapi.com/adsbexchange/api/adsbexchange-com1) — ~$10/mo |
| `REDDIT_CLIENT_ID` + `REDDIT_CLIENT_SECRET` | Reddit social sentiment | [reddit.com/prefs/apps](https://www.reddit.com/prefs/apps/) — create a script app |
@@ -556,6 +605,9 @@ All settings are in `.env` with sensible defaults:
|----------|---------|-------------|
| `PORT` | `3117` | Dashboard server port |
| `REFRESH_INTERVAL_MINUTES` | `15` | Auto-refresh interval |
| `STALE_DATA_MAX_AGE_MINUTES` | `60` | Data age threshold for stale health state |
| `STALE_ALERT_COOLDOWN_MINUTES` | `60` | Minimum time between repeated operator stale-data alerts |
| `DASHBOARD_URL` | local URL | Dashboard URL included in operator alerts |
| `LLM_PROVIDER` | disabled | `anthropic`, `openai`, `gemini`, `codex`, `openrouter`, `minimax`, `mistral`, or `grok` |
| `LLM_API_KEY` | — | API key (not needed for codex) |
| `LLM_MODEL` | per-provider default | Override model selection |

View File

@@ -59,7 +59,8 @@ export async function runSource(name, fn, ...args) {
});
const data = await Promise.race([dataPromise, timeoutPromise]);
const hasError = Boolean(data?.error);
const isDegraded = hasError || ['no_credentials', 'degraded', 'failed'].includes(data?.status);
const degradedStatuses = ['no_credentials', 'no_key', 'disabled', 'degraded', 'failed', 'error'];
const isDegraded = hasError || degradedStatuses.includes(data?.status);
return {
name,
status: isDegraded ? 'degraded' : 'ok',

View File

@@ -1,9 +1,9 @@
// ACLED Armed Conflict Location & Event Data
// ACLED - Armed Conflict Location & Event Data.
// Auth strategy (tries in order):
// 1. Cookie-based session: POST /user/login?_format=json → session cookie
// 2. OAuth Bearer token: POST /oauth/token → Authorization header
// Set ACLED_EMAIL and ACLED_PASSWORD in .env (your myACLED login credentials).
// Data endpoint: GET https://acleddata.com/api/acled/read
// 1. OAuth Bearer token: POST /oauth/token -> Authorization header
// 2. Cookie-based session: POST /user/login?_format=json -> session cookie
// Set ACLED_EMAIL and ACLED_PASSWORD in .env. ACLED_USER or ACLED_USERNAME are
// accepted as aliases for ACLED_EMAIL.
import { daysAgo } from '../utils/fetch.mjs';
import '../utils/env.mjs';
@@ -12,124 +12,135 @@ const LOGIN_URL = 'https://acleddata.com/user/login?_format=json';
const TOKEN_URL = 'https://acleddata.com/oauth/token';
const API_BASE = 'https://acleddata.com/api/acled/read';
// Session cache
let sessionCache = { cookies: null, token: null, method: null, expires: 0 };
// Strategy 1: Cookie-based session login (mirrors browser login)
async function loginCookie(email, password) {
export function resetAcledSessionCache() {
sessionCache = { cookies: null, token: null, method: null, expires: 0 };
}
export function getAcledConfig(env = process.env) {
const email = env.ACLED_EMAIL || env.ACLED_USER || env.ACLED_USERNAME || '';
const password = env.ACLED_PASSWORD || '';
const missing = [];
if (!email) missing.push('ACLED_EMAIL');
if (!password) missing.push('ACLED_PASSWORD');
return { email, password, configured: missing.length === 0, missing };
}
function acledError(status, error, message, extra = {}) {
return { status, error, message, ...extra };
}
function safeText(value, max = 200) {
return String(value || '').replace(/Bearer\s+[A-Za-z0-9._-]+/gi, 'Bearer [redacted]').slice(0, max);
}
async function fetchWithTimeout(fetchImpl, url, init, timeoutMs) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 15000);
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const res = await fetch(LOGIN_URL, {
return await fetchImpl(url, { ...init, signal: controller.signal });
} finally {
clearTimeout(timer);
}
}
export async function loginCookie(email, password, opts = {}) {
const fetchImpl = opts.fetchImpl || globalThis.fetch;
try {
const res = await fetchWithTimeout(fetchImpl, LOGIN_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: email, pass: password }),
redirect: 'manual',
signal: controller.signal,
});
clearTimeout(timer);
}, 15000);
// Collect Set-Cookie headers
const setCookies = res.headers.getSetCookie?.() || [];
const cookieStr = setCookies.map(c => c.split(';')[0]).join('; ');
if (res.ok && cookieStr) {
return { cookies: cookieStr };
}
// Some Drupal sites return 303 redirect on successful login — cookies still set
if (res.status >= 300 && res.status < 400 && cookieStr) {
return { cookies: cookieStr };
if ((res.ok || (res.status >= 300 && res.status < 400)) && cookieStr) {
return { ok: true, cookies: cookieStr };
}
const errText = await res.text().catch(() => '');
return { error: `Cookie login failed (HTTP ${res.status}): ${errText.slice(0, 200)}` };
return acledError('auth_failed', `acled_cookie_http_${res.status}`, `ACLED cookie login failed with HTTP ${res.status}`, {
detail: safeText(errText),
});
} catch (e) {
clearTimeout(timer);
const cause = e.cause ? `${e.cause.message || e.cause.code || e.cause}` : '';
return { error: `Cookie login error: ${e.message}${cause}` };
return acledError('auth_failed', 'acled_cookie_request_failed', `ACLED cookie login error: ${e.message}`);
}
}
// Strategy 2: OAuth2 password grant
async function loginOAuth(email, password) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 15000);
export async function loginOAuth(email, password, opts = {}) {
const fetchImpl = opts.fetchImpl || globalThis.fetch;
try {
const body = new URLSearchParams({
username: email,
password: password,
password,
grant_type: 'password',
client_id: 'acled',
});
const res = await fetch(TOKEN_URL, {
const res = await fetchWithTimeout(fetchImpl, TOKEN_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: body.toString(),
signal: controller.signal,
});
clearTimeout(timer);
}, 15000);
if (!res.ok) {
const errText = await res.text().catch(() => '');
return { error: `OAuth failed (HTTP ${res.status}): ${errText.slice(0, 200)}` };
return acledError('auth_failed', `acled_oauth_http_${res.status}`, `ACLED OAuth failed with HTTP ${res.status}`, {
detail: safeText(errText),
});
}
const data = await res.json();
if (!data.access_token) {
return { error: `OAuth response missing access_token: ${JSON.stringify(data).slice(0, 200)}` };
return acledError('auth_failed', 'acled_oauth_missing_access_token', 'ACLED OAuth response did not include access_token');
}
return { token: data.access_token };
return { ok: true, token: data.access_token };
} catch (e) {
clearTimeout(timer);
const cause = e.cause ? `${e.cause.message || e.cause.code || e.cause}` : '';
return { error: `OAuth error: ${e.message}${cause}` };
return acledError('auth_failed', 'acled_oauth_request_failed', `ACLED OAuth error: ${e.message}`);
}
}
// Try both auth strategies
async function authenticate() {
const email = process.env.ACLED_EMAIL;
const password = process.env.ACLED_PASSWORD;
if (!email || !password) {
return { error: 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.' };
export async function authenticate(opts = {}) {
const env = opts.env || process.env;
const fetchImpl = opts.fetchImpl || globalThis.fetch;
const config = getAcledConfig(env);
if (!config.configured) {
return acledError('no_credentials', 'missing_acled_credentials', 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.', {
missing: config.missing,
});
}
// Return cached session if still valid
if (sessionCache.method && Date.now() < sessionCache.expires) {
return sessionCache;
}
const errors = [];
const debug = process.argv.includes('--debug');
// Try OAuth first (official programmatic method per ACLED docs)
const oauthResult = await loginOAuth(email, password);
if (oauthResult.token) {
if (debug) console.error(`[ACLED DEBUG] OAuth OK — token: ${oauthResult.token.slice(0, 20)}...`);
const diagnostics = [];
const oauthResult = await loginOAuth(config.email, config.password, { fetchImpl });
if (oauthResult.ok) {
sessionCache = { cookies: null, token: oauthResult.token, method: 'oauth', expires: Date.now() + 23 * 60 * 60 * 1000 };
return sessionCache;
}
errors.push(`OAuth: ${oauthResult.error}`);
if (debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
diagnostics.push({ method: 'oauth', status: oauthResult.status, error: oauthResult.error, message: oauthResult.message });
if (opts.debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
// Fall back to cookie-based session
const cookieResult = await loginCookie(email, password);
if (cookieResult.cookies) {
if (debug) console.error(`[ACLED DEBUG] Cookie OK — cookies: ${cookieResult.cookies.slice(0, 80)}...`);
const cookieResult = await loginCookie(config.email, config.password, { fetchImpl });
if (cookieResult.ok) {
sessionCache = { cookies: cookieResult.cookies, token: null, method: 'cookie', expires: Date.now() + 12 * 60 * 60 * 1000 };
return sessionCache;
}
errors.push(`Cookie: ${cookieResult.error}`);
diagnostics.push({ method: 'cookie', status: cookieResult.status, error: cookieResult.error, message: cookieResult.message });
if (opts.debug) console.error(`[ACLED DEBUG] Cookie login failed: ${cookieResult.error}`);
return { error: `All ACLED auth methods failed.\n${errors.join('\n')}` };
return acledError('auth_failed', 'acled_auth_failed', 'All ACLED auth methods failed.', { diagnostics });
}
// Build headers based on auth method
function authHeaders(session) {
const headers = { 'User-Agent': 'Crucix/1.0', 'Content-Type': 'application/json' };
const headers = { 'User-Agent': 'Crucix/2.0', 'Content-Type': 'application/json' };
if (session.method === 'cookie' && session.cookies) {
headers['Cookie'] = session.cookies;
} else if (session.method === 'oauth' && session.token) {
@@ -138,7 +149,6 @@ function authHeaders(session) {
return headers;
}
// Event type constants
export const EVENT_TYPES = [
'Battles',
'Explosions/Remote violence',
@@ -148,7 +158,6 @@ export const EVENT_TYPES = [
'Strategic developments',
];
// Query conflict events with flexible filters
export async function getEvents(opts = {}) {
const {
limit = 500,
@@ -157,10 +166,13 @@ export async function getEvents(opts = {}) {
eventType,
country,
region,
env = process.env,
fetchImpl = globalThis.fetch,
debug = process.argv.includes('--debug'),
} = opts;
const session = await authenticate();
if (session.error) return { error: session.error };
const session = await authenticate({ env, fetchImpl, debug });
if (session.error) return session;
const params = new URLSearchParams({ _format: 'json', limit: String(limit) });
if (eventDateStart && eventDateEnd) {
@@ -171,59 +183,43 @@ export async function getEvents(opts = {}) {
if (country) params.set('country', country);
if (region) params.set('region', String(region));
const debug = process.argv.includes('--debug');
try {
const url = `${API_BASE}?${params}`;
const hdrs = authHeaders(session);
if (debug) {
console.error(`[ACLED DEBUG] Data request: GET ${url}`);
console.error(`[ACLED DEBUG] Headers: ${JSON.stringify(hdrs)}`);
}
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 25000);
const res = await fetch(url, {
headers: hdrs,
signal: controller.signal,
});
clearTimeout(timer);
if (debug) console.error(`[ACLED DEBUG] Data request: GET ${url}`);
const res = await fetchWithTimeout(fetchImpl, url, { headers: authHeaders(session) }, 25000);
if (debug) console.error(`[ACLED DEBUG] Data response: HTTP ${res.status}`);
if (!res.ok) {
const errText = await res.text().catch(() => '');
if (debug) console.error(`[ACLED DEBUG] Error body: ${errText.slice(0, 500)}`);
if (res.status === 401 || res.status === 403) {
// Clear cache and report
sessionCache = { cookies: null, token: null, method: null, expires: 0 };
const hint = res.status === 403
? '\n→ Fix: Log in at https://acleddata.com/user/login, then:\n'
+ ' 1. Accept Terms of Use (profile → Terms of Use button → check the box)\n'
+ ' 2. Complete all required profile fields\n'
+ ' 3. Ensure your account has the "API" access group\n'
+ ' Contact access@acleddata.com if issues persist.'
: '';
return { error: `ACLED data access denied (HTTP ${res.status}, auth method: ${session.method}). Response: ${errText.slice(0, 300)}${hint}` };
return acledError('access_denied', `acled_data_http_${res.status}`, `ACLED data access denied with HTTP ${res.status}`, {
authMethod: session.method,
detail: safeText(errText, 300),
hint: 'Accept ACLED terms, complete profile fields, and confirm API access for the account.',
});
}
return { error: `HTTP ${res.status}: ${errText.slice(0, 200)}` };
return acledError('api_failed', `acled_data_http_${res.status}`, `ACLED data request failed with HTTP ${res.status}`, {
detail: safeText(errText),
});
}
const data = await res.json();
// ACLED may return a 200 with an error status in the body
if (data?.status && data.status !== 200) {
return { error: `ACLED API error: status ${data.status}${data.message || 'Unknown error'}` };
return acledError('api_failed', `acled_api_status_${data.status}`, `ACLED API returned status ${data.status}`, {
detail: safeText(data.message),
});
}
return data;
} catch (e) {
if (e.name === 'AbortError') {
return { error: 'ACLED data request timed out (25s)' };
return acledError('api_failed', 'acled_data_timeout', 'ACLED data request timed out after 25s');
}
const rootCause = e.cause ? `${e.cause.message || e.cause.code || e.cause}` : '';
return { error: `ACLED data error: ${e.message}${rootCause ? ' → ' + rootCause : ''}` };
return acledError('api_failed', 'acled_data_request_failed', `ACLED data error: ${e.message}`);
}
}
// Summarize events by a given field
function groupBy(events, field) {
const map = {};
for (const e of events) {
@@ -235,33 +231,47 @@ function groupBy(events, field) {
return map;
}
// Briefing — last 7 days of global conflict events
export async function briefing() {
if (!process.env.ACLED_EMAIL || !process.env.ACLED_PASSWORD) {
export async function briefing(opts = {}) {
const env = opts.env || process.env;
const fetchImpl = opts.fetchImpl || globalThis.fetch;
const config = getAcledConfig(env);
if (!config.configured) {
return {
source: 'ACLED',
timestamp: new Date().toISOString(),
status: 'no_credentials',
error: 'missing_acled_credentials',
missing: config.missing,
message: 'Set ACLED_EMAIL and ACLED_PASSWORD in .env. Register at https://acleddata.com/user/register',
};
}
const start = daysAgo(7);
const end = daysAgo(0);
const end = daysAgo(0);
const data = await getEvents({
eventDateStart: start,
eventDateEnd: end,
limit: 2000,
env,
fetchImpl,
debug: opts.debug,
});
if (data?.error) {
return { source: 'ACLED', timestamp: new Date().toISOString(), error: data.error };
return {
source: 'ACLED',
timestamp: new Date().toISOString(),
status: data.status || 'api_failed',
error: data.error,
message: data.message,
detail: data.detail,
hint: data.hint,
diagnostics: data.diagnostics,
};
}
let events = data?.data || [];
// Enrich all events with numeric lat/lon
events = events.map(e => ({
...e,
lat: parseFloat(e.latitude) || null,
@@ -272,10 +282,9 @@ export async function briefing() {
(sum, e) => sum + (parseInt(e.fatalities, 10) || 0), 0
);
const byRegion = groupBy(events, 'region');
const byType = groupBy(events, 'event_type');
const byRegion = groupBy(events, 'region');
const byType = groupBy(events, 'event_type');
const byCountry = groupBy(events, 'country');
const topCountries = Object.entries(byCountry)
.sort((a, b) => b[1].count - a[1].count)
.slice(0, 10)
@@ -286,20 +295,21 @@ export async function briefing() {
.sort((a, b) => (parseInt(b.fatalities, 10) || 0) - (parseInt(a.fatalities, 10) || 0))
.slice(0, 15)
.map(e => ({
date: e.event_date,
type: e.event_type,
subType: e.sub_event_type,
country: e.country,
location: e.location,
date: e.event_date,
type: e.event_type,
subType: e.sub_event_type,
country: e.country,
location: e.location,
fatalities: parseInt(e.fatalities, 10) || 0,
lat: parseFloat(e.latitude) || null,
lon: parseFloat(e.longitude) || null,
notes: e.notes?.slice(0, 200),
lat: parseFloat(e.latitude) || null,
lon: parseFloat(e.longitude) || null,
notes: e.notes?.slice(0, 200),
}));
return {
source: 'ACLED',
timestamp: new Date().toISOString(),
status: 'ok',
period: { start, end },
totalEvents: events.length,
totalFatalities,

View File

@@ -1,7 +1,8 @@
// ADS-B Exchange — Unfiltered Flight Tracking (including Military)
// Unlike FlightRadar24/FlightAware, ADS-B Exchange does NOT filter military aircraft.
// Public feed access varies; RapidAPI tier available for programmatic use.
// This module attempts the public endpoints and falls back to a documented stub.
// This module reports explicit disabled/degraded state instead of making
// unavailable aircraft data look live.
import { safeFetch } from '../utils/fetch.mjs';
@@ -140,6 +141,7 @@ async function fetchViaRapidApi(apiKey) {
// Get all military aircraft
const data = await safeFetch(`${ENDPOINTS.rapidApi}/mil`, {
timeout: 20000,
source: 'adsb-rapidapi',
headers: {
'X-RapidAPI-Key': apiKey,
'X-RapidAPI-Host': 'adsbexchange-com1.p.rapidapi.com',
@@ -151,21 +153,26 @@ async function fetchViaRapidApi(apiKey) {
// Attempt to fetch from public feed
async function fetchPublicFeed() {
const data = await safeFetch(ENDPOINTS.publicFeed, { timeout: 15000 });
const data = await safeFetch(ENDPOINTS.publicFeed, { timeout: 15000, source: 'adsb-public' });
return data;
}
// Get military aircraft from available sources
export async function getMilitaryAircraft(apiKey) {
async function getMilitaryAircraftResult(apiKey) {
const failures = [];
// Try RapidAPI first if key available
if (apiKey) {
const data = await fetchViaRapidApi(apiKey);
if (data && !data.error) {
const aircraft = data.ac || data.aircraft || [];
if (Array.isArray(aircraft)) {
return aircraft.map(classifyAircraft).filter(a => a.isMilitary);
return {
provider: 'rapidapi',
aircraft: aircraft.map(classifyAircraft).filter(a => a.isMilitary),
};
}
}
failures.push({ provider: 'rapidapi', error: data?.error || 'RapidAPI returned an unsupported payload' });
}
// Try public feed
@@ -173,11 +180,21 @@ export async function getMilitaryAircraft(apiKey) {
if (pubData && !pubData.error) {
const aircraft = pubData.ac || pubData.aircraft || pubData.states || [];
if (Array.isArray(aircraft)) {
return aircraft.map(classifyAircraft).filter(a => a.isMilitary);
return {
provider: 'public-feed',
aircraft: aircraft.map(classifyAircraft).filter(a => a.isMilitary),
};
}
}
failures.push({ provider: 'public-feed', error: pubData?.error || 'Public feed returned an unsupported payload' });
return null; // all sources failed
return { provider: null, aircraft: null, failures };
}
// Get military aircraft from available sources
export async function getMilitaryAircraft(apiKey) {
const result = await getMilitaryAircraftResult(apiKey);
return result.aircraft;
}
// Get all aircraft in a geographic bounding box via RapidAPI
@@ -208,7 +225,8 @@ export async function getAircraftInArea(lat, lon, radiusNm = 250, apiKey) {
// Briefing — attempt to get military flight data, document what's available
export async function briefing() {
const apiKey = process.env.ADSB_API_KEY || process.env.RAPIDAPI_KEY || null;
const militaryAircraft = await getMilitaryAircraft(apiKey);
const result = await getMilitaryAircraftResult(apiKey);
const militaryAircraft = result.aircraft;
// If we got data, analyze it
if (militaryAircraft && militaryAircraft.length > 0) {
@@ -255,6 +273,7 @@ export async function briefing() {
source: 'ADS-B Exchange',
timestamp: new Date().toISOString(),
status: 'live',
provider: result.provider,
totalMilitary: militaryAircraft.length,
byCountry,
categories: {
@@ -269,10 +288,18 @@ export async function briefing() {
}
// No data available — return stub with integration documentation
const status = apiKey ? 'degraded' : 'disabled';
const error = apiKey
? 'ADS-B providers returned no usable aircraft data'
: 'ADSB_API_KEY or RAPIDAPI_KEY is not configured';
return {
source: 'ADS-B Exchange',
timestamp: new Date().toISOString(),
status: apiKey ? 'error' : 'no_key',
status,
provider: result.provider,
error,
failures: result.failures,
militaryAircraft: [],
message: apiKey
? 'ADS-B Exchange API returned no data. The endpoint may be temporarily unavailable.'

View File

@@ -42,6 +42,7 @@ export async function safeFetch(url, opts = {}) {
let lastError;
for (let i = 0; i <= retries; i++) {
const started = Date.now();
let metricRecorded = false;
try {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeout);
@@ -51,22 +52,29 @@ export async function safeFetch(url, opts = {}) {
});
clearTimeout(timer);
const status = res.status;
if (!res.ok) {
const body = await res.text().catch(() => '');
recordFetchMetric({ url, source, ok: false, status, bytes: body.length, durationMs: Date.now() - started, error: `HTTP ${res.status}` });
throw new Error(`HTTP ${res.status}: ${body.slice(0, 200)}`);
}
const text = await res.text();
recordFetchMetric({ url, source, ok: true, status, bytes: text.length, durationMs: Date.now() - started });
if (!res.ok) {
const error = `HTTP ${res.status}`;
recordFetchMetric({ url, source, ok: false, status, bytes: text.length, durationMs: Date.now() - started, error });
metricRecorded = true;
throw new Error(`${error}: ${text.slice(0, 200)}`);
}
const trimmed = text.trim();
const contentType = res.headers.get('content-type') || '';
if (contentType.includes('text/html') || trimmed.startsWith('<!DOCTYPE html') || trimmed.startsWith('<html')) {
throw new Error(`Expected JSON but received HTML from ${new URL(url).host}`);
const error = `Expected JSON but received HTML from ${new URL(url).host}`;
recordFetchMetric({ url, source, ok: false, status, bytes: text.length, durationMs: Date.now() - started, error });
metricRecorded = true;
throw new Error(error);
}
recordFetchMetric({ url, source, ok: true, status, bytes: text.length, durationMs: Date.now() - started });
metricRecorded = true;
try { return JSON.parse(text); } catch { return { rawText: text.slice(0, 500) }; }
} catch (e) {
lastError = e;
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
if (!metricRecorded) {
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
}
// GDELT needs 5s between requests, others are fine with shorter delays
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
}
@@ -79,6 +87,7 @@ export async function safeFetchText(url, opts = {}) {
let lastError;
for (let i = 0; i <= retries; i++) {
const started = Date.now();
let metricRecorded = false;
try {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeout);
@@ -89,11 +98,14 @@ export async function safeFetchText(url, opts = {}) {
clearTimeout(timer);
const text = await res.text();
recordFetchMetric({ url, source, ok: res.ok, status: res.status, bytes: text.length, durationMs: Date.now() - started, error: res.ok ? null : `HTTP ${res.status}` });
metricRecorded = true;
if (!res.ok) throw new Error(`HTTP ${res.status}: ${text.slice(0, 200)}`);
return { text, status: res.status, bytes: text.length };
} catch (e) {
lastError = e;
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
if (!metricRecorded) {
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
}
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
}
}

View File

@@ -23,8 +23,12 @@ export default {
refreshIntervalMinutes: intEnv('REFRESH_INTERVAL_MINUTES', 15),
autoOpenBrowser: boolEnv('AUTO_OPEN_BROWSER', false),
staleDataMaxAgeMinutes: intEnv('STALE_DATA_MAX_AGE_MINUTES', 60),
staleAlertCooldownMinutes: intEnv('STALE_ALERT_COOLDOWN_MINUTES', 60),
dashboardUrl: process.env.DASHBOARD_URL || null,
sweepToken: process.env.SWEEP_TOKEN || null,
terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', true),
terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', !!process.env.SWEEP_TOKEN || process.env.NODE_ENV !== 'production'),
terminalActionRateLimitWindowMs: intEnv('TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS', 60_000),
terminalActionRateLimitMax: intEnv('TERMINAL_ACTION_RATE_LIMIT_MAX', 10),
llm: {
provider: process.env.LLM_PROVIDER || null, // anthropic | openai | gemini | codex | openrouter | minimax | mistral | ollama | grok

View File

@@ -83,16 +83,48 @@ const geoKeywords = {
'IMF':[38.9,-77],'World Bank':[38.9,-77],'UN':[40.7,-74],
};
function geoTagText(text) {
function escapeRegex(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
function geoKeywordRegex(keyword) {
const flags = keyword.length <= 3 && keyword === keyword.toUpperCase() ? 'u' : 'iu';
return new RegExp(`(^|[^\\p{L}\\p{N}])${escapeRegex(keyword)}(?=$|[^\\p{L}\\p{N}])`, flags);
}
const geoKeywordEntries = Object.entries(geoKeywords)
.sort((a, b) => b[0].length - a[0].length)
.map(([keyword, coords]) => ({ keyword, coords, pattern: geoKeywordRegex(keyword) }));
export function geoTagText(text) {
if (!text) return null;
for (const [keyword, [lat, lon]] of Object.entries(geoKeywords)) {
if (text.includes(keyword)) {
for (const { keyword, coords, pattern } of geoKeywordEntries) {
if (pattern.test(text)) {
const [lat, lon] = coords;
return { lat, lon, region: keyword };
}
}
return null;
}
function stableHash(value) {
let hash = 2166136261;
for (let i = 0; i < value.length; i++) {
hash ^= value.charCodeAt(i);
hash = Math.imul(hash, 16777619);
}
return hash >>> 0;
}
export function stableGeoJitter(key, axis) {
const bucket = stableHash(`${axis}:${key}`) / 0xffffffff;
return (bucket - 0.5) * 2;
}
function newsGeoKey(item) {
return `${item.source || ''}|${item.title || ''}|${item.date || ''}|${item.url || ''}`;
}
function sanitizeExternalUrl(raw) {
if (!raw) return undefined;
try {
@@ -235,8 +267,8 @@ export async function fetchAllNews() {
source: item.source,
date: item.date,
url: item.url,
lat: geo.lat + (Math.random() - 0.5) * 2,
lon: geo.lon + (Math.random() - 0.5) * 2,
lat: geo.lat + stableGeoJitter(newsGeoKey(item), 'lat'),
lon: geo.lon + stableGeoJitter(newsGeoKey(item), 'lon'),
region: geo.region
});
}

File diff suppressed because one or more lines are too long

View File

@@ -16,4 +16,5 @@ Source docs:
- [Telegram](telegram.md)
- [FIRMS](firms.md)
- [Maritime](maritime.md)
- [ADS-B](adsb.md)
- [Reddit](reddit.md)

View File

@@ -2,8 +2,11 @@
Provides conflict events, fatalities, event types, and locations.
- Auth: `ACLED_EMAIL` and `ACLED_PASSWORD`.
- Auth: `ACLED_EMAIL` and `ACLED_PASSWORD`. `ACLED_USER` or `ACLED_USERNAME` may be used as aliases for `ACLED_EMAIL`.
- Flow: OAuth password grant is tried first, then cookie session fallback.
- Failure modes: missing credentials, terms/profile not completed, expired token, account missing API access.
- Behavior: missing or rejected credentials produce degraded source health with the ACLED error text.
- Test: set credentials, run `node apis/sources/acled.mjs`, then check `/api/health`.
- Failure modes are classified as `no_credentials`, `auth_failed`, `access_denied`, or `api_failed`.
- Behavior: missing, rejected, or unauthorized credentials produce degraded source health with a concise operator message.
- Secret handling: debug output never prints bearer tokens, cookies, or the configured password.
- Test: run `node --test test/acled-source.test.mjs`; with real credentials, run `node apis/sources/acled.mjs`, then check `/api/health`.
`access_denied` normally means the login worked but the account cannot read API data. Check that ACLED terms are accepted, required profile fields are complete, and API access is enabled for the account.

24
docs/sources/adsb.md Normal file
View File

@@ -0,0 +1,24 @@
# ADS-B Source
ADS-B Exchange support is optional and intended for unfiltered aircraft and military-flight awareness.
- Source module: `apis/sources/adsb.mjs`
- Preferred provider: ADS-B Exchange via RapidAPI
- Credentials: `ADSB_API_KEY` or `RAPIDAPI_KEY`
- Runtime status without credentials: `disabled`
- Runtime status when providers fail: `degraded`
- Runtime status with usable aircraft payloads: `live`
The source does not treat a missing key or unavailable public feed as normal live data. `/api/health` and `/api/metrics` surface the degraded source state through the sweep source summary.
Known failure modes:
- Missing `ADSB_API_KEY` / `RAPIDAPI_KEY`: source is disabled with operator guidance.
- RapidAPI rejects or rate-limits the request: source is degraded and records provider failure detail.
- Public feed is blocked, rate-limited, or changes shape: source remains degraded instead of returning stale-looking data.
Register for the provider documented in the README, then set:
```env
ADSB_API_KEY=<rapidapi-key>
```

View File

@@ -2,6 +2,9 @@
import { existsSync, mkdirSync, writeFileSync } from 'fs';
import { dirname, join } from 'path';
import { createHash } from 'crypto';
const PREDICTION_STATES = new Set(['open', 'monitoring', 'observed', 'expired_unverified', 'invalidated']);
export class IntelligenceStore {
constructor(dbPath) {
@@ -30,15 +33,24 @@ export class IntelligenceStore {
);
CREATE TABLE IF NOT EXISTS predictions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
stable_id TEXT UNIQUE,
created_at TEXT NOT NULL,
updated_at TEXT,
title TEXT NOT NULL,
type TEXT,
hypothesis TEXT,
evidence_json TEXT,
confidence TEXT,
horizon TEXT,
outcome_state TEXT DEFAULT 'open',
outcome_json TEXT,
last_evaluated_at TEXT,
source TEXT,
payload_json TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS entities (
id INTEGER PRIMARY KEY AUTOINCREMENT,
stable_id TEXT UNIQUE,
first_seen TEXT NOT NULL,
last_seen TEXT NOT NULL,
name TEXT NOT NULL,
@@ -46,7 +58,21 @@ export class IntelligenceStore {
count INTEGER DEFAULT 1,
UNIQUE(name, kind)
);
CREATE TABLE IF NOT EXISTS events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
stable_id TEXT NOT NULL UNIQUE,
first_seen TEXT NOT NULL,
last_seen TEXT NOT NULL,
kind TEXT NOT NULL,
name TEXT NOT NULL,
region TEXT,
severity TEXT,
source TEXT,
evidence_json TEXT NOT NULL,
count INTEGER DEFAULT 1
);
`);
this._migrate();
this.available = true;
} catch (err) {
this.available = false;
@@ -71,24 +97,141 @@ export class IntelligenceStore {
delta?.summary?.direction || null,
JSON.stringify({ meta, delta: delta?.summary || null }),
);
for (const idea of data.ideas || []) {
this.db.prepare(`INSERT INTO predictions (created_at, title, type, confidence, source, payload_json)
VALUES (?, ?, ?, ?, ?, ?)`).run(
timestamp,
idea.title || 'Untitled idea',
idea.type || null,
idea.confidence || null,
idea.source || data.ideasSource || null,
JSON.stringify(idea),
);
}
this._recordEntities(data, timestamp);
this._recordEvents(data, delta, timestamp);
this.evaluatePredictions(data, timestamp);
this._recordPredictions(data, timestamp);
}
status() {
return { available: this.available, path: this.dbPath, reason: this.reason };
}
queryMemory({ q = '', limit = 25 } = {}) {
if (!this.available || !this.db) return { available: false, reason: this.reason, results: [] };
const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
const term = String(q || '').trim();
const like = `%${term}%`;
const where = term
? 'WHERE name LIKE ? OR region LIKE ? OR source LIKE ? OR kind LIKE ?'
: '';
const params = term ? [like, like, like, like, safeLimit] : [safeLimit];
const events = this.db.prepare(`
SELECT stable_id, first_seen, last_seen, kind, name, region, severity, source, count, evidence_json
FROM events
${where}
ORDER BY last_seen DESC
LIMIT ?
`).all(...params).map(row => ({ ...row, evidence: parseJson(row.evidence_json, {}) }));
return { available: true, q: term, results: events };
}
listPredictions({ state = null, limit = 25 } = {}) {
if (!this.available || !this.db) return { available: false, reason: this.reason, predictions: [] };
const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
const normalizedState = state && PREDICTION_STATES.has(String(state)) ? String(state) : null;
const rows = normalizedState
? this.db.prepare(`SELECT * FROM predictions WHERE outcome_state = ? ORDER BY created_at DESC LIMIT ?`).all(normalizedState, safeLimit)
: this.db.prepare(`SELECT * FROM predictions ORDER BY created_at DESC LIMIT ?`).all(safeLimit);
return {
available: true,
predictions: rows.map(row => ({
stable_id: row.stable_id,
created_at: row.created_at,
updated_at: row.updated_at,
title: row.title,
type: row.type,
hypothesis: row.hypothesis,
confidence: row.confidence,
horizon: row.horizon,
outcome_state: row.outcome_state,
last_evaluated_at: row.last_evaluated_at,
source: row.source,
evidence: parseJson(row.evidence_json, []),
outcome: parseJson(row.outcome_json, null),
})),
};
}
evaluatePredictions(data, timestamp = new Date().toISOString()) {
if (!this.available || !this.db) return;
const rows = this.db.prepare(`
SELECT id, created_at, title, type, horizon, outcome_state, payload_json
FROM predictions
WHERE outcome_state IN ('open', 'monitoring')
ORDER BY created_at ASC
LIMIT 200
`).all();
for (const row of rows) {
const payload = parseJson(row.payload_json, {});
const evaluation = evaluatePredictionAgainstSweep(row, payload, data, timestamp);
this.db.prepare(`UPDATE predictions
SET outcome_state = ?, outcome_json = ?, last_evaluated_at = ?, updated_at = ?
WHERE id = ?`).run(
evaluation.state,
JSON.stringify(evaluation),
timestamp,
timestamp,
row.id,
);
}
}
_migrate() {
const columns = {
predictions: [
['stable_id', 'TEXT'],
['updated_at', 'TEXT'],
['hypothesis', 'TEXT'],
['evidence_json', 'TEXT'],
['horizon', 'TEXT'],
['outcome_state', "TEXT DEFAULT 'open'"],
['outcome_json', 'TEXT'],
['last_evaluated_at', 'TEXT'],
],
entities: [
['stable_id', 'TEXT'],
],
};
for (const [table, defs] of Object.entries(columns)) {
for (const [name, type] of defs) {
try { this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${name} ${type}`); } catch { }
}
}
try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_predictions_stable_id ON predictions(stable_id)`); } catch { }
try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_entities_stable_id ON entities(stable_id)`); } catch { }
}
_recordPredictions(data, timestamp) {
for (const idea of data.ideas || []) {
const title = idea.title || 'Untitled idea';
const stableId = stableId('prediction', title, idea.type || '', idea.ticker || '', idea.horizon || '');
const evidence = Array.isArray(idea.signals) ? idea.signals : [];
this.db.prepare(`INSERT INTO predictions (
stable_id, created_at, updated_at, title, type, hypothesis, evidence_json, confidence,
horizon, outcome_state, source, payload_json
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, ?)
ON CONFLICT(stable_id) DO UPDATE SET
updated_at=excluded.updated_at,
confidence=excluded.confidence,
evidence_json=excluded.evidence_json,
payload_json=excluded.payload_json`).run(
stableId,
timestamp,
timestamp,
title,
idea.type || null,
idea.rationale || idea.text || title,
JSON.stringify(evidence),
idea.confidence || null,
idea.horizon || null,
idea.source || data.ideasSource || null,
JSON.stringify(idea),
);
}
}
_recordEntities(data, timestamp) {
const names = [];
for (const item of data.acled?.deadliestEvents || []) {
@@ -99,14 +242,154 @@ export class IntelligenceStore {
if (item.region) names.push([item.region, 'region']);
}
for (const [name, kind] of names.slice(0, 200)) {
this.db.prepare(`INSERT INTO entities (first_seen, last_seen, name, kind, count)
VALUES (?, ?, ?, ?, 1)
const cleanName = String(name).slice(0, 160);
this.db.prepare(`INSERT INTO entities (stable_id, first_seen, last_seen, name, kind, count)
VALUES (?, ?, ?, ?, ?, 1)
ON CONFLICT(name, kind) DO UPDATE SET last_seen=excluded.last_seen, count=count+1`).run(
stableId('entity', kind, cleanName),
timestamp,
timestamp,
String(name).slice(0, 160),
cleanName,
kind,
);
}
}
_recordEvents(data, delta, timestamp) {
const events = extractEvents(data, delta);
for (const event of events.slice(0, 300)) {
this.db.prepare(`INSERT INTO events (
stable_id, first_seen, last_seen, kind, name, region, severity, source, evidence_json, count
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
ON CONFLICT(stable_id) DO UPDATE SET
last_seen=excluded.last_seen,
severity=COALESCE(excluded.severity, severity),
evidence_json=excluded.evidence_json,
count=count+1`).run(
event.stable_id,
timestamp,
timestamp,
event.kind,
event.name,
event.region || null,
event.severity || null,
event.source || null,
JSON.stringify(event.evidence || {}),
);
}
}
}
function stableId(...parts) {
const input = parts.map(part => String(part || '').trim().toLowerCase()).join('|');
return createHash('sha256').update(input).digest('hex').slice(0, 24);
}
function parseJson(value, fallback) {
try { return value ? JSON.parse(value) : fallback; } catch { return fallback; }
}
function extractEvents(data, delta) {
const events = [];
const push = ({ kind, name, region, severity, source, evidence }) => {
if (!kind || !name) return;
events.push({
stable_id: stableId('event', kind, name, region || source || ''),
kind,
name: String(name).slice(0, 240),
region: region ? String(region).slice(0, 120) : null,
severity: severity || null,
source: source || null,
evidence: evidence || {},
});
};
for (const item of data.acled?.deadliestEvents || []) {
push({
kind: 'conflict',
name: item.event_type || item.sub_event_type || item.location || item.country,
region: item.country || item.location,
severity: Number(item.fatalities || 0) > 0 ? 'high' : 'medium',
source: 'ACLED',
evidence: item,
});
}
for (const item of data.tg?.urgent || []) {
push({
kind: 'osint',
name: (item.text || '').slice(0, 120),
region: item.region || 'OSINT',
severity: 'high',
source: item.channel || item.chat || 'telegram',
evidence: item,
});
}
for (const item of data.newsFeed || data.news || []) {
if (!item.urgent) continue;
push({
kind: 'news',
name: item.headline || item.title,
region: item.region,
severity: 'medium',
source: item.source,
evidence: item,
});
}
for (const signal of delta?.signals?.new || []) {
push({
kind: 'delta',
name: signal.label || signal.reason || signal.key,
region: signal.region,
severity: signal.severity || 'medium',
source: 'delta',
evidence: signal,
});
}
return events;
}
function evaluatePredictionAgainstSweep(row, payload, data, timestamp) {
const terms = [
row.title,
payload.ticker,
...(Array.isArray(payload.signals) ? payload.signals : []),
].filter(Boolean).map(v => String(v).toLowerCase());
const evidenceText = [
...(data.tSignals || []),
...(data.newsFeed || []).slice(0, 40).map(n => `${n.source || ''} ${n.headline || n.title || ''}`),
...(data.tg?.urgent || []).slice(0, 20).map(p => p.text || ''),
].join('\n').toLowerCase();
const matched = terms.filter(term => term.length >= 4 && evidenceText.includes(term.slice(0, 60)));
const expired = predictionExpired(row.created_at, row.horizon, timestamp);
const state = matched.length
? 'observed'
: expired
? 'expired_unverified'
: 'monitoring';
return {
state,
evaluated_at: timestamp,
matched_terms: matched.slice(0, 10),
expired,
reason: matched.length
? 'Current sweep contains matching evidence terms.'
: expired
? 'Prediction horizon elapsed without matching evidence.'
: 'Prediction remains open for future sweeps.',
};
}
function predictionExpired(createdAt, horizon, nowIso) {
const created = new Date(createdAt).getTime();
const now = new Date(nowIso).getTime();
if (!Number.isFinite(created) || !Number.isFinite(now)) return false;
const text = String(horizon || '').toLowerCase();
const days = text.includes('intraday') ? 1
: text.includes('day') ? 7
: text.includes('week') ? 45
: text.includes('month') ? 180
: text.includes('strategic') ? 365
: 30;
return now - created > days * 24 * 60 * 60 * 1000;
}

52
lib/stale-alerts.mjs Normal file
View File

@@ -0,0 +1,52 @@
const DEFAULT_COOLDOWN_MS = 60 * 60 * 1000;
export function shouldSendStaleAlert(health, state = {}, opts = {}) {
const now = opts.now ?? Date.now();
const cooldownMs = opts.cooldownMs ?? DEFAULT_COOLDOWN_MS;
if (!health?.stale) {
state.lastStaleAlertKey = null;
return { send: false, reason: 'not_stale' };
}
const key = [
health.lastSuccessfulSweep || 'never',
health.lastSweepError || 'no-error',
health.sourcesFailed || 0,
health.sourcesDegraded || 0,
].join('|');
if (state.lastStaleAlertKey === key && now - (state.lastStaleAlertAt || 0) < cooldownMs) {
return { send: false, reason: 'cooldown', key };
}
state.lastStaleAlertKey = key;
state.lastStaleAlertAt = now;
return { send: true, reason: 'stale', key };
}
export function formatStaleAlert(health, opts = {}) {
const dashboardUrl = opts.dashboardUrl || 'http://localhost:3117';
const context = opts.context || 'scheduled sweep';
const ageMinutes = health.dataAgeSeconds == null ? 'unknown' : Math.floor(health.dataAgeSeconds / 60);
const affected = (health.sourceHealth || [])
.filter(s => (s.status && s.status !== 'ok') || s.error)
.slice(0, 6)
.map(s => `- ${s.name || s.n || 'source'}: ${s.status || 'degraded'}${s.error ? ` (${String(s.error).slice(0, 100)})` : ''}`);
return [
'*CRUCIX STALE DATA ALERT*',
'',
`Context: ${context}`,
`Status: ${health.status || 'unknown'}`,
`Data age: ${ageMinutes} minutes`,
`Last successful sweep: ${health.lastSuccessfulSweep || 'never'}`,
`Last attempted sweep: ${health.lastSweep || 'never'}`,
`Last error: ${health.lastSweepError || 'none'}`,
`Sources: ${health.sourcesOk || 0} OK / ${health.sourcesDegraded || 0} degraded / ${health.sourcesFailed || 0} failed`,
'',
'*Affected sources*',
affected.length ? affected.join('\n') : '- No per-source errors available',
'',
`Dashboard: ${dashboardUrl}`,
].join('\n');
}

View File

@@ -12,7 +12,7 @@
"brief:save": "node apis/save-briefing.mjs",
"diag": "node diag.mjs",
"test": "npm run test:unit",
"test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs",
"test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs test/acled-source.test.mjs test/mojibake-text.test.mjs test/adsb.test.mjs test/dashboard-geotagging.test.mjs",
"compose:config": "docker compose config",
"clean": "node scripts/clean.mjs",
"fresh-start": "npm run clean && npm start"

View File

@@ -18,6 +18,7 @@ import { TelegramAlerter } from './lib/alerts/telegram.mjs';
import { DiscordAlerter } from './lib/alerts/discord.mjs';
import { getFetchMetrics } from './apis/utils/fetch.mjs';
import { IntelligenceStore } from './lib/intelligence-store.mjs';
import { formatStaleAlert, shouldSendStaleAlert } from './lib/stale-alerts.mjs';
import { evaluateScenarios } from './lib/scenarios.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -40,6 +41,8 @@ let sweepStartedAt = null; // Timestamp when current/last sweep started
let sweepInProgress = false;
const startTime = Date.now();
const sseClients = new Set();
const terminalActionBuckets = new Map();
const staleAlertState = {};
// === Delta/Memory ===
const memory = new MemoryManager(RUNS_DIR);
@@ -289,29 +292,67 @@ app.get('/api/metrics', (req, res) => {
});
});
app.post('/api/sweep', express.json(), (req, res) => {
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
triggerSweep(res);
app.get('/api/memory/search', (req, res) => {
const guard = authorizeTerminalAction(req, res, 'memory:search');
if (!guard.ok) return;
auditTerminalAction(req, 'memory:search', 'ok');
res.json(intelligenceStore.queryMemory({
q: req.query.q || '',
limit: req.query.limit || 25,
}));
});
app.post('/api/action', express.json(), async (req, res) => {
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
const action = String(req.body?.action || req.query.action || '').toLowerCase();
app.get('/api/memory/predictions', (req, res) => {
const guard = authorizeTerminalAction(req, res, 'memory:predictions');
if (!guard.ok) return;
auditTerminalAction(req, 'memory:predictions', 'ok');
res.json(intelligenceStore.listPredictions({
state: req.query.state || null,
limit: req.query.limit || 25,
}));
});
app.post('/api/sweep', express.json(), (req, res) => {
const guard = authorizeTerminalAction(req, res, 'sweep');
if (!guard.ok) return;
triggerSweepAction(req, res, 'sweep');
});
app.post('/api/action', express.json(), (req, res) => {
const action = String(req.body?.action || req.body?.command || '').trim().toLowerCase();
const guard = authorizeTerminalAction(req, res, action || 'unknown');
if (!guard.ok) return;
if (action === 'status') {
return res.json({ ok: true, action, health: buildHealth() });
auditTerminalAction(req, 'status', 'ok');
return res.json({ ok: true, action, status: 'ok', health: buildHealth() });
}
if (action === 'brief') {
if (!currentData) return res.status(503).json({ ok: false, action, error: 'No data yet — first sweep in progress' });
return res.json({ ok: true, action, text: buildBrief(currentData) });
if (!currentData) {
auditTerminalAction(req, 'brief', 'rejected', 'no_data');
return res.status(503).json({ ok: false, action, error: 'No data yet - first sweep in progress' });
}
auditTerminalAction(req, 'brief', 'ok');
const brief = buildBrief(currentData);
return res.json({ ok: true, action, status: 'ok', brief, text: brief });
}
if (action === 'sweep') {
return triggerSweep(res);
if (action === 'memory') {
auditTerminalAction(req, 'memory', 'ok');
return res.json({
ok: true,
action,
memory: intelligenceStore.status(),
recentEvents: intelligenceStore.queryMemory({ q: req.body?.q || '', limit: 8 }).results,
predictions: intelligenceStore.listPredictions({ limit: 8 }).predictions,
});
}
res.status(400).json({ ok: false, error: 'Unknown action', actions: ['status', 'brief', 'sweep'] });
if (action === 'sweep') return triggerSweepAction(req, res, 'action:sweep');
auditTerminalAction(req, action || 'unknown', 'rejected', 'unknown_action');
return res.status(400).json({ ok: false, error: 'Unknown action', allowed: ['status', 'brief', 'memory', 'sweep'], actions: ['status', 'brief', 'memory', 'sweep'] });
});
// API: available locales
@@ -342,26 +383,114 @@ function broadcast(data) {
}
}
function requestIp(req) {
return req.ip || req.socket?.remoteAddress || 'unknown';
}
function isLocalRequest(req) {
const remote = requestIp(req);
return remote === '::1'
|| remote === '127.0.0.1'
|| remote === '::ffff:127.0.0.1'
|| remote.startsWith('127.')
|| remote === 'localhost';
}
function sameOriginPost(req) {
const origin = req.get('origin');
if (!origin) return true;
try {
const originUrl = new URL(origin);
const host = req.get('host');
return host && originUrl.host === host;
} catch {
return false;
}
}
function actionToken(req) {
return req.get('x-crucix-token') || req.body?.token || null;
}
function auditTerminalAction(req, action, outcome, detail = null) {
const suffix = detail ? ` detail=${detail}` : '';
console.log(`[Crucix][audit] terminal_action action=${action || 'unknown'} outcome=${outcome} ip=${requestIp(req)}${suffix}`);
}
function rateLimitTerminalAction(req, action) {
const now = Date.now();
const windowMs = Math.max(1000, config.terminalActionRateLimitWindowMs || 60_000);
const max = Math.max(1, config.terminalActionRateLimitMax || 10);
const key = `${requestIp(req)}:${action}`;
const bucket = terminalActionBuckets.get(key);
if (!bucket || now > bucket.resetAt) {
terminalActionBuckets.set(key, { count: 1, resetAt: now + windowMs });
return { ok: true };
}
bucket.count += 1;
if (bucket.count > max) {
return { ok: false, retryAfterSeconds: Math.ceil((bucket.resetAt - now) / 1000) };
}
return { ok: true };
}
function authorizeTerminalAction(req, res, action) {
const rate = rateLimitTerminalAction(req, action);
if (!rate.ok) {
auditTerminalAction(req, action, 'rejected', 'rate_limited');
res.set('Retry-After', String(rate.retryAfterSeconds));
res.status(429).json({ error: 'Too many terminal actions', retryAfterSeconds: rate.retryAfterSeconds });
return { ok: false };
}
if (!sameOriginPost(req)) {
auditTerminalAction(req, action, 'rejected', 'csrf_origin');
res.status(403).json({ error: 'Origin mismatch' });
return { ok: false };
}
const local = isLocalRequest(req);
const token = actionToken(req);
if (!config.terminalActionsEnabled) {
auditTerminalAction(req, action, 'rejected', 'disabled');
res.status(403).json({ error: 'Terminal actions are disabled' });
return { ok: false };
}
if (config.sweepToken) {
if (token !== config.sweepToken) {
auditTerminalAction(req, action, 'rejected', 'invalid_token');
res.status(401).json({ error: 'Invalid terminal action token' });
return { ok: false };
}
return { ok: true };
}
if (!local) {
auditTerminalAction(req, action, 'rejected', 'missing_token');
res.status(403).json({ error: 'Terminal actions are local-only unless SWEEP_TOKEN is set' });
return { ok: false };
}
return { ok: true };
}
function triggerSweepAction(req, res, auditAction) {
if (sweepInProgress) {
auditTerminalAction(req, auditAction, 'rejected', 'already_running');
return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
}
auditTerminalAction(req, auditAction, 'accepted');
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
return res.status(202).json({ ok: true, status: 'accepted' });
}
function dataAgeMs() {
const ts = currentData?.meta?.timestamp || lastSuccessfulSweepTime || lastSweepTime;
const ms = ts ? Date.now() - new Date(ts).getTime() : null;
return Number.isFinite(ms) ? ms : null;
}
function canRunTerminalAction(req) {
const remote = req.ip || '';
const local = remote.includes('127.0.0.1') || remote === '::1' || remote === '::ffff:127.0.0.1';
const token = req.get('x-crucix-token') || req.query.token || req.body?.token;
if (config.sweepToken) return token === config.sweepToken;
return Boolean(config.terminalActionsEnabled || local);
}
function triggerSweep(res) {
if (sweepInProgress) return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
return res.status(202).json({ ok: true, status: 'accepted' });
}
function getLLMStatus() {
if (!config.llm.provider) return { state: 'disabled' };
if (!llmProvider) return { state: 'misconfigured', provider: config.llm.provider };
@@ -405,13 +534,39 @@ function buildHealth() {
llm: getLLMStatus(),
telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId),
discordEnabled: !!(config.discord?.botToken || config.discord?.webhookUrl),
terminalActionsEnabled: Boolean(config.terminalActionsEnabled || config.sweepToken),
terminalActionsEnabled: config.terminalActionsEnabled,
terminalActionsTokenRequired: !!config.sweepToken,
refreshIntervalMinutes: config.refreshIntervalMinutes,
language: currentLanguage,
memory: intelligenceStore.status(),
};
}
async function notifyIfDataStale(context = 'scheduled sweep') {
const health = buildHealth();
const decision = shouldSendStaleAlert(health, staleAlertState, {
cooldownMs: config.staleAlertCooldownMinutes * 60 * 1000,
});
if (!decision.send) return false;
const dashboardUrl = config.dashboardUrl || `http://localhost:${config.port}`;
const message = formatStaleAlert(health, { dashboardUrl, context });
const sends = [];
if (telegramAlerter.isConfigured) sends.push(telegramAlerter.sendMessage(message));
if (discordAlerter.isConfigured) sends.push(discordAlerter.sendAlert(message));
if (sends.length === 0) {
console.warn('[Crucix] Data is stale but no operator alert channel is configured');
return false;
}
const results = await Promise.allSettled(sends);
const sent = results.some(r => r.status === 'fulfilled' && (r.value === true || r.value?.ok === true));
if (sent) console.warn('[Crucix] Operator stale-data alert sent');
else console.warn('[Crucix] Operator stale-data alert attempted but no channel accepted it');
return sent;
}
function buildBrief(data) {
const verbosity = config.telegram.briefVerbosity || 'standard';
const delta = memory.getLastDelta();
@@ -562,6 +717,9 @@ async function runSweepCycle() {
broadcast({ type: 'sweep_error', error: err.message });
} finally {
sweepInProgress = false;
await notifyIfDataStale(lastSweepError ? 'failed sweep' : 'completed sweep').catch(err => {
console.error('[Crucix] Stale-data operator alert failed:', err.message);
});
}
}

View File

@@ -0,0 +1,95 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { authenticate, briefing, resetAcledSessionCache } from '../apis/sources/acled.mjs';
function jsonResponse(status, body, ok = status >= 200 && status < 300) {
return {
ok,
status,
headers: { getSetCookie: () => [] },
json: async () => body,
text: async () => JSON.stringify(body),
};
}
test('ACLED reports missing credentials without network access', async () => {
resetAcledSessionCache();
let calls = 0;
const data = await briefing({
env: {},
fetchImpl: async () => {
calls++;
throw new Error('unexpected network access');
},
});
assert.equal(calls, 0);
assert.equal(data.status, 'no_credentials');
assert.equal(data.error, 'missing_acled_credentials');
assert.deepEqual(data.missing, ['ACLED_EMAIL', 'ACLED_PASSWORD']);
});
test('ACLED accepts ACLED_USER as email alias and returns empty valid result', async () => {
resetAcledSessionCache();
const urls = [];
const data = await briefing({
env: { ACLED_USER: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
fetchImpl: async url => {
urls.push(String(url));
if (String(url).includes('/oauth/token')) {
return jsonResponse(200, { access_token: 'token' });
}
return jsonResponse(200, { status: 200, data: [] });
},
});
assert.equal(data.status, 'ok');
assert.equal(data.totalEvents, 0);
assert.ok(urls.some(url => url.includes('/oauth/token')));
assert.ok(urls.some(url => url.includes('/api/acled/read')));
});
test('ACLED classifies auth failure without exposing credentials', async () => {
resetAcledSessionCache();
const result = await authenticate({
env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'super-secret' },
fetchImpl: async url => {
if (String(url).includes('/oauth/token')) {
return jsonResponse(401, { error: 'invalid_grant' }, false);
}
return {
ok: false,
status: 403,
headers: { getSetCookie: () => [] },
text: async () => 'forbidden',
};
},
});
assert.equal(result.status, 'auth_failed');
assert.equal(result.error, 'acled_auth_failed');
assert.equal(result.diagnostics.length, 2);
assert.doesNotMatch(JSON.stringify(result), /super-secret/);
});
test('ACLED classifies data access denied distinctly from auth failure', async () => {
resetAcledSessionCache();
const data = await briefing({
env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
fetchImpl: async url => {
if (String(url).includes('/oauth/token')) {
return jsonResponse(200, { access_token: 'token' });
}
return {
ok: false,
status: 403,
headers: { getSetCookie: () => [] },
text: async () => 'terms not accepted',
};
},
});
assert.equal(data.status, 'access_denied');
assert.equal(data.error, 'acled_data_http_403');
assert.match(data.hint, /Accept ACLED terms/);
});

82
test/adsb.test.mjs Normal file
View File

@@ -0,0 +1,82 @@
import test from 'node:test';
import assert from 'node:assert/strict';
async function withFetch(mockFetch, fn) {
const originalFetch = globalThis.fetch;
const originalAdsbKey = process.env.ADSB_API_KEY;
const originalRapidKey = process.env.RAPIDAPI_KEY;
globalThis.fetch = mockFetch;
delete process.env.ADSB_API_KEY;
delete process.env.RAPIDAPI_KEY;
try {
return await fn();
} finally {
globalThis.fetch = originalFetch;
if (originalAdsbKey === undefined) delete process.env.ADSB_API_KEY;
else process.env.ADSB_API_KEY = originalAdsbKey;
if (originalRapidKey === undefined) delete process.env.RAPIDAPI_KEY;
else process.env.RAPIDAPI_KEY = originalRapidKey;
}
}
function jsonResponse(payload, ok = true, status = 200) {
return {
ok,
status,
headers: { get: () => 'application/json' },
text: async () => JSON.stringify(payload),
};
}
test('ADS-B reports disabled when no key is configured and public feed fails', async () => {
await withFetch(async () => jsonResponse({ error: 'blocked' }, false, 403), async () => {
const { briefing } = await import('../apis/sources/adsb.mjs');
const data = await briefing();
assert.equal(data.status, 'disabled');
assert.match(data.error, /ADSB_API_KEY|RAPIDAPI_KEY/);
assert.equal(data.militaryAircraft.length, 0);
});
});
test('ADS-B reports degraded when RapidAPI and public feed fail', async () => {
await withFetch(async () => jsonResponse({ error: 'unavailable' }, false, 503), async () => {
process.env.ADSB_API_KEY = 'test-key';
const { briefing } = await import('../apis/sources/adsb.mjs');
const data = await briefing();
assert.equal(data.status, 'degraded');
assert.match(data.error, /providers returned no usable/);
assert.equal(data.failures.length, 2);
});
});
test('ADS-B returns live RapidAPI military aircraft payloads', async () => {
await withFetch(async () => jsonResponse({
ac: [{
hex: 'AE1234',
flight: 'RCH123',
t: 'KC135',
lat: 50,
lon: 8,
mil: true,
}],
}), async () => {
process.env.ADSB_API_KEY = 'test-key';
const { briefing } = await import('../apis/sources/adsb.mjs');
const data = await briefing();
assert.equal(data.status, 'live');
assert.equal(data.provider, 'rapidapi');
assert.equal(data.totalMilitary, 1);
assert.equal(data.militaryAircraft[0].callsign, 'RCH123');
});
});
test('runSource treats disabled source status as degraded health', async () => {
const { runSource } = await import('../apis/briefing.mjs');
const result = await runSource('ADS-B', async () => ({ status: 'disabled', message: 'missing key' }));
assert.equal(result.status, 'degraded');
assert.equal(result.error, null);
});

View File

@@ -0,0 +1,47 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { geoTagText, stableGeoJitter } from '../dashboard/inject.mjs';
test('geoTagText matches headlines case-insensitively', () => {
assert.deepEqual(geoTagText('ukraine reports new air defense activity'), {
lat: 49,
lon: 32,
region: 'Ukraine',
});
assert.deepEqual(geoTagText('flooding disrupts são paulo transport'), {
lat: -23.5,
lon: -46.6,
region: 'São Paulo',
});
});
test('geoTagText prefers longer place names before broad countries', () => {
assert.deepEqual(geoTagText('New York markets react before wider US session'), {
lat: 40.7,
lon: -74,
region: 'New York',
});
});
test('geoTagText uses word boundaries to reduce false positives', () => {
assert.equal(geoTagText('A music festival announces its lineup'), null);
assert.equal(geoTagText('Officials discuss a new focus for aid'), null);
assert.deepEqual(geoTagText('US officials discuss a new aid package'), {
lat: 39,
lon: -98,
region: 'US',
});
});
test('stableGeoJitter is deterministic and bounded', () => {
const key = 'BBC|lower-case ukraine headline|Sun, 17 May 2026 12:00:00 GMT|https://example.test/a';
const latA = stableGeoJitter(key, 'lat');
const latB = stableGeoJitter(key, 'lat');
const lon = stableGeoJitter(key, 'lon');
assert.equal(latA, latB);
assert.notEqual(latA, lon);
assert.ok(latA >= -1 && latA <= 1);
assert.ok(lon >= -1 && lon <= 1);
});

View File

@@ -2,9 +2,11 @@ import test from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { safeFetch, safeFetchText, getFetchMetrics } from '../apis/utils/fetch.mjs';
import { formatStaleAlert, shouldSendStaleAlert } from '../lib/stale-alerts.mjs';
test('safeFetch reports HTML as degraded JSON response', async () => {
const originalFetch = globalThis.fetch;
const source = 'unit-html-once';
globalThis.fetch = async () => ({
ok: true,
status: 200,
@@ -12,9 +14,72 @@ test('safeFetch reports HTML as degraded JSON response', async () => {
text: async () => '<html>not json</html>',
});
try {
const data = await safeFetch('https://example.test/json', { retries: 0, source: 'unit' });
const data = await safeFetch('https://example.test/json', { retries: 0, source });
assert.match(data.error, /Expected JSON/);
assert.ok(getFetchMetrics().bySource.unit.requests >= 1);
const bucket = getFetchMetrics().bySource[source];
assert.equal(bucket.requests, 1);
assert.equal(bucket.ok, 0);
assert.equal(bucket.failed, 1);
assert.equal(bucket.lastStatus, 200);
} finally {
globalThis.fetch = originalFetch;
}
});
test('safeFetch records HTTP failure once with status and bytes', async () => {
const originalFetch = globalThis.fetch;
const source = 'unit-http-failure-once';
globalThis.fetch = async () => ({
ok: false,
status: 503,
headers: { get: () => 'application/json' },
text: async () => 'service unavailable',
});
try {
const data = await safeFetch('https://example.test/fail', { retries: 0, source });
assert.match(data.error, /HTTP 503/);
const bucket = getFetchMetrics().bySource[source];
assert.equal(bucket.requests, 1);
assert.equal(bucket.ok, 0);
assert.equal(bucket.failed, 1);
assert.equal(bucket.lastStatus, 503);
assert.equal(bucket.bytes, 'service unavailable'.length);
assert.match(bucket.lastError, /HTTP 503/);
} finally {
globalThis.fetch = originalFetch;
}
});
test('safeFetch retry metrics count one record per attempt', async () => {
const originalFetch = globalThis.fetch;
const source = 'unit-retry-attempts';
let calls = 0;
globalThis.fetch = async () => {
calls += 1;
if (calls === 1) {
return {
ok: false,
status: 502,
headers: { get: () => 'application/json' },
text: async () => 'bad gateway',
};
}
return {
ok: true,
status: 200,
headers: { get: () => 'application/json' },
text: async () => '{"ok":true}',
};
};
try {
const data = await safeFetch('https://example.test/retry', { retries: 1, source });
assert.equal(data.ok, true);
assert.equal(calls, 2);
const bucket = getFetchMetrics().bySource[source];
assert.equal(bucket.requests, 2);
assert.equal(bucket.ok, 1);
assert.equal(bucket.failed, 1);
assert.equal(bucket.lastStatus, 200);
} finally {
globalThis.fetch = originalFetch;
}
@@ -36,6 +101,121 @@ test('safeFetchText returns text and byte count', async () => {
}
});
test('intelligence store defines durable memory and prediction lifecycle tables', () => {
const store = readFileSync(new URL('../lib/intelligence-store.mjs', import.meta.url), 'utf8');
assert.match(store, /CREATE TABLE IF NOT EXISTS events/);
assert.match(store, /stable_id TEXT NOT NULL UNIQUE/);
assert.match(store, /hypothesis TEXT/);
assert.match(store, /evidence_json TEXT/);
assert.match(store, /outcome_state TEXT DEFAULT 'open'/);
assert.match(store, /evaluatePredictions/);
assert.match(store, /queryMemory/);
assert.match(store, /listPredictions/);
});
test('server exposes memory-backed query APIs and dashboard memory action', () => {
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
assert.match(server, /\/api\/memory\/search/);
assert.match(server, /\/api\/memory\/predictions/);
assert.match(server, /action === 'memory'/);
assert.match(html, /runTerminalAction\('memory'\)/);
});
test('terminal action endpoints avoid URL tokens and include hardening gates', () => {
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
assert.match(server, /app\.post\('\/api\/action'/);
assert.match(server, /app\.post\('\/api\/sweep'/);
assert.match(server, /x-crucix-token/);
assert.match(server, /sameOriginPost/);
assert.match(server, /rateLimitTerminalAction/);
assert.match(server, /auditTerminalAction/);
assert.doesNotMatch(server, /req\.query\.token/);
});
test('dashboard exposes token configuration flow without devtools edits', () => {
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
assert.match(html, /configureTerminalActionToken/);
assert.match(html, /crucix_sweep_token/);
assert.match(html, /x-crucix-token/);
assert.match(html, /SET TOKEN/);
});
test('server dashboard shell does not embed an operational snapshot', () => {
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
assert.match(html, /let D = createDashboardShellData\(\);/);
assert.doesNotMatch(html, /2026-04-03T16:18:10\.188Z/);
assert.doesNotMatch(html, /Trump announced new strikes on Iran/);
});
test('server dashboard fetches api data before initialization', () => {
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
const serverMode = html.indexOf('if (canProbeApi)');
const apiFetch = html.indexOf("fetch('/api/data')");
const firstInitAfterServerMode = html.indexOf('init();', serverMode);
assert.ok(serverMode > -1);
assert.ok(apiFetch > serverMode);
assert.ok(firstInitAfterServerMode > apiFetch);
});
test('stale alert is skipped for fresh health and resets active key', () => {
const state = { lastStaleAlertKey: 'old', lastStaleAlertAt: 100 };
const decision = shouldSendStaleAlert({ stale: false }, state, { now: 200 });
assert.equal(decision.send, false);
assert.equal(decision.reason, 'not_stale');
assert.equal(state.lastStaleAlertKey, null);
});
test('stale alert sends once and deduplicates during cooldown', () => {
const state = {};
const health = {
stale: true,
lastSuccessfulSweep: '2026-05-17T08:00:00.000Z',
lastSweepError: 'network timeout',
sourcesFailed: 2,
sourcesDegraded: 1,
};
const first = shouldSendStaleAlert(health, state, { now: 1_000, cooldownMs: 60_000 });
const second = shouldSendStaleAlert(health, state, { now: 2_000, cooldownMs: 60_000 });
assert.equal(first.send, true);
assert.equal(second.send, false);
assert.equal(second.reason, 'cooldown');
});
test('stale alert repeats after cooldown', () => {
const state = {};
const health = { stale: true, lastSuccessfulSweep: 'a', lastSweepError: 'timeout', sourcesFailed: 1 };
assert.equal(shouldSendStaleAlert(health, state, { now: 1_000, cooldownMs: 60_000 }).send, true);
assert.equal(shouldSendStaleAlert(health, state, { now: 62_000, cooldownMs: 60_000 }).send, true);
});
test('stale alert message includes operator context and affected sources', () => {
const message = formatStaleAlert({
status: 'stale',
stale: true,
dataAgeSeconds: 7200,
lastSuccessfulSweep: '2026-05-17T08:00:00.000Z',
lastSweep: '2026-05-17T10:00:00.000Z',
lastSweepError: 'GDELT timeout',
sourcesOk: 20,
sourcesDegraded: 3,
sourcesFailed: 2,
sourceHealth: [
{ name: 'GDELT', status: 'degraded', error: 'timeout' },
{ name: 'Reddit', status: 'no_credentials' },
],
}, { dashboardUrl: 'https://terminal.example.test', context: 'failed sweep' });
assert.match(message, /CRUCIX STALE DATA ALERT/);
assert.match(message, /Data age: 120 minutes/);
assert.match(message, /GDELT: degraded \(timeout\)/);
assert.match(message, /Dashboard: https:\/\/terminal\.example\.test/);
});
test('scenario watchlist feature is wired into sweep, briefing, and dashboard', () => {
const scenarios = readFileSync(new URL('../lib/scenarios.mjs', import.meta.url), 'utf8');
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');

View File

@@ -0,0 +1,65 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { readdirSync, readFileSync, statSync } from 'node:fs';
import { join } from 'node:path';
const TEXT_ROOTS = ['locales'];
const TEXT_FILES = [];
const EXTENSIONS = new Set(['.json', '.html', '.mjs']);
const MOJIBAKE_PATTERNS = [
{ name: 'latin1-accent', pattern: /\u00c3./g },
{ name: 'stray-cp1252-prefix', pattern: /\u00c2./g },
{ name: 'emoji-mojibake', pattern: /\u00f0\u0178/g },
{
name: 'punctuation-mojibake',
pattern: /\u00e2[\u0080-\u009f\u20ac\u0153\u2018\u2019\u201c\u201d\u2013\u2014\u2022\u2026\u201e\u2021\u02c6\u2030\u2039\u203a\u0152\u017d]/g,
},
{ name: 'variation-selector-mojibake', pattern: /\u00ef\u00b8/g },
{ name: 'ligature-mojibake', pattern: /\u00c5[\u0080-\u017f]/g },
{ name: 'replacement-character', pattern: /\ufffd/g },
];
function collectFiles(root) {
const out = [];
for (const entry of readdirSync(root, { withFileTypes: true })) {
const path = join(root, entry.name);
if (entry.isDirectory()) {
out.push(...collectFiles(path));
} else if (EXTENSIONS.has(path.slice(path.lastIndexOf('.')))) {
out.push(path);
}
}
return out;
}
function textFiles() {
const discovered = TEXT_ROOTS.flatMap(root => collectFiles(root));
const explicit = TEXT_FILES.filter(path => statSync(path, { throwIfNoEntry: false })?.isFile());
return [...new Set([...discovered, ...explicit])].sort();
}
test('locale JSON files are valid UTF-8 JSON', () => {
for (const file of collectFiles('locales')) {
assert.doesNotThrow(() => JSON.parse(readFileSync(file, 'utf8')), `${file} must parse as JSON`);
}
});
test('locale text does not contain known mojibake sequences', () => {
const failures = [];
for (const file of textFiles()) {
const text = readFileSync(file, 'utf8');
for (const { name, pattern } of MOJIBAKE_PATTERNS) {
for (const match of text.matchAll(pattern)) {
const start = Math.max(0, match.index - 30);
const end = Math.min(text.length, match.index + 50);
failures.push(`${file}: ${name}: ${JSON.stringify(text.slice(start, end))}`);
}
}
}
assert.deepEqual(failures, []);
});