33 Commits

Author SHA1 Message Date
c159c83a07 Merge pull request 'revert(ui): remove app-style dashboard redesign' (#44) from codex/revert-app-style-design into codex/production-intelligence-terminal
Some checks failed
Release Dry Run / release-dry-run (push) Successful in 14s
Codex Template Compliance / template-compliance (push) Successful in 6s
Build / test-and-image (push) Successful in 25s
Scheduled Security Scan / security-scan (push) Failing after 8s
Scheduled Repository Cleanup Check / cleanup-check (push) Successful in 9s
Scheduled Dependency Check / dependency-check (push) Failing after 12s
Reviewed-on: MrSphay/intelligence-terminal#44
2026-05-17 20:45:24 +00:00
MrSphay
a1d415e449 Revert "Merge pull request 'feat(ui): redesign dashboard in app-style shell' (#42) from codex/modrinth-app-redesign into codex/production-intelligence-terminal"
All checks were successful
Build / test-and-image (pull_request) Successful in 25s
Codex Template Compliance / template-compliance (pull_request) Successful in 4s
This reverts commit 9f2083a324, reversing
changes made to 1c2b48f588.
2026-05-17 22:35:12 +02:00
MrSphay
0f5f9c5f91 Revert "Merge pull request 'feat(ui): finalize app-style dashboard motion and QA' (#43) from codex/modrinth-app-finalization into codex/production-intelligence-terminal"
This reverts commit 096544f6e6, reversing
changes made to 9f2083a324.
2026-05-17 22:35:07 +02:00
096544f6e6 Merge pull request 'feat(ui): finalize app-style dashboard motion and QA' (#43) from codex/modrinth-app-finalization into codex/production-intelligence-terminal
All checks were successful
Release Dry Run / release-dry-run (push) Successful in 14s
Codex Template Compliance / template-compliance (push) Successful in 6s
Build / test-and-image (push) Successful in 26s
Reviewed-on: MrSphay/intelligence-terminal#43
2026-05-17 20:08:03 +00:00
MrSphay
5a3dbc6252 feat(ui): redesign dashboard in app-style shell
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m4s
2026-05-17 21:54:36 +02:00
9f2083a324 Merge pull request 'feat(ui): redesign dashboard in app-style shell' (#42) from codex/modrinth-app-redesign into codex/production-intelligence-terminal
All checks were successful
Release Dry Run / release-dry-run (push) Successful in 14s
Codex Template Compliance / template-compliance (push) Successful in 5s
Build / test-and-image (push) Successful in 24s
Reviewed-on: MrSphay/intelligence-terminal#42
2026-05-17 19:04:46 +00:00
dd08ecaf27 Merge branch 'codex/production-intelligence-terminal' into codex/modrinth-app-redesign
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 57s
2026-05-17 19:03:03 +00:00
MrSphay
bc354e7bc5 feat(ui): redesign dashboard in app-style shell
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 59s
2026-05-17 20:57:54 +02:00
1c2b48f588 Merge pull request 'fix: infer source fetch metrics' (#35) from codex/issue-22-source-fetch-instrumentation into codex/production-intelligence-terminal
All checks were successful
Release Dry Run / release-dry-run (push) Successful in 14s
Codex Template Compliance / template-compliance (push) Successful in 5s
Build / test-and-image (push) Successful in 27s
2026-05-17 18:53:45 +00:00
MrSphay
a590bf62c2 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-22-source-fetch-instrumentation
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 57s
# Conflicts:
#	test/fetch-utils.test.mjs
2026-05-17 20:52:27 +02:00
6a9918bc98 Merge pull request 'fix: keep sse streams alive behind proxies' (#34) from codex/issue-17-sse-heartbeat into codex/production-intelligence-terminal
All checks were successful
Build / test-and-image (push) Successful in 25s
Release Dry Run / release-dry-run (push) Successful in 13s
Codex Template Compliance / template-compliance (push) Successful in 5s
2026-05-17 18:51:04 +00:00
MrSphay
4448f5931b Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-17-sse-heartbeat
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 4s
Build / test-and-image (pull_request) Successful in 57s
# Conflicts:
#	.env.example
#	README.md
#	crucix.config.mjs
#	test/fetch-utils.test.mjs
2026-05-17 20:49:14 +02:00
9b15913049 Merge pull request 'feat: extend memory prediction loop' (#32) from codex/issue-4-memory-prediction-loop into codex/production-intelligence-terminal
All checks were successful
Build / test-and-image (push) Successful in 24s
Release Dry Run / release-dry-run (push) Successful in 14s
Codex Template Compliance / template-compliance (push) Successful in 5s
2026-05-17 18:47:39 +00:00
MrSphay
331175ae3c Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-4-memory-prediction-loop
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 57s
# Conflicts:
#	server.mjs
#	test/fetch-utils.test.mjs
2026-05-17 20:46:02 +02:00
e288881c41 Merge pull request 'fix: harden terminal action endpoints' (#25) from codex/issue-6-terminal-actions-hardening into codex/production-intelligence-terminal
All checks were successful
Release Dry Run / release-dry-run (push) Successful in 14s
Build / test-and-image (push) Successful in 27s
Codex Template Compliance / template-compliance (push) Successful in 4s
Reviewed-on: MrSphay/intelligence-terminal#25
2026-05-17 18:43:21 +00:00
MrSphay
e4834cd3cd Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-22-source-fetch-instrumentation
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 57s
# Conflicts:
#	test/fetch-utils.test.mjs
2026-05-17 20:40:44 +02:00
MrSphay
0fbd8640ca Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-17-sse-heartbeat
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 6s
Build / test-and-image (pull_request) Successful in 1m0s
# Conflicts:
#	test/fetch-utils.test.mjs
2026-05-17 20:40:10 +02:00
MrSphay
3069114ffd Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-4-memory-prediction-loop
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 6s
Build / test-and-image (pull_request) Successful in 1m3s
# Conflicts:
#	test/fetch-utils.test.mjs
2026-05-17 20:39:38 +02:00
MrSphay
09df127e06 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-6-terminal-actions-hardening
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m4s
# Conflicts:
#	test/fetch-utils.test.mjs
2026-05-17 20:39:04 +02:00
MrSphay
c102017b16 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-22-source-fetch-instrumentation
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 55s
# Conflicts:
#	README.md
#	test/fetch-utils.test.mjs
2026-05-17 20:37:21 +02:00
49176b42fd Merge pull request 'fix: remove embedded dashboard snapshot' (#20) from codex/issue-5-dashboard-shell into codex/production-intelligence-terminal
All checks were successful
Build / test-and-image (push) Successful in 25s
Release Dry Run / release-dry-run (push) Successful in 14s
Codex Template Compliance / template-compliance (push) Successful in 6s
Reviewed-on: MrSphay/intelligence-terminal#20
2026-05-17 18:35:58 +00:00
MrSphay
090e90ea70 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-4-memory-prediction-loop
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 55s
# Conflicts:
#	README.md
#	test/fetch-utils.test.mjs
2026-05-17 20:35:44 +02:00
e70801ae98 Merge branch 'codex/production-intelligence-terminal' into codex/issue-5-dashboard-shell
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 6s
Build / test-and-image (pull_request) Successful in 54s
2026-05-17 18:34:34 +00:00
703670e7a0 Merge pull request 'fix: report adsb unavailable state as degraded' (#11) from codex/issue-7-adsb-degraded into codex/production-intelligence-terminal
All checks were successful
Build / test-and-image (push) Successful in 28s
Release Dry Run / release-dry-run (push) Successful in 14s
Codex Template Compliance / template-compliance (push) Successful in 5s
Reviewed-on: MrSphay/intelligence-terminal#11
2026-05-17 18:33:51 +00:00
MrSphay
1423dca199 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-6-terminal-actions-hardening
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 56s
# Conflicts:
#	README.md
#	server.mjs
#	test/fetch-utils.test.mjs
2026-05-17 20:33:45 +02:00
MrSphay
5b013947b4 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-5-dashboard-shell
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 53s
# Conflicts:
#	test/fetch-utils.test.mjs
2026-05-17 20:32:04 +02:00
MrSphay
5113e341b2 Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-7-adsb-degraded
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m14s
# Conflicts:
#	package.json
2026-05-17 20:30:53 +02:00
d7f10bf545 merge: update adsb degraded branch
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 6s
Build / test-and-image (pull_request) Successful in 1m13s
2026-05-17 19:03:49 +02:00
MrSphay
2025ae09db fix: infer source fetch metrics
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 4s
Build / test-and-image (pull_request) Successful in 53s
2026-05-17 14:44:21 +02:00
MrSphay
6096a0ad03 fix: remove embedded dashboard snapshot
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 49s
2026-05-17 14:33:52 +02:00
MrSphay
267af03b22 feat: extend memory prediction loop
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 50s
2026-05-17 14:30:39 +02:00
MrSphay
d7df2e4aee fix: harden terminal action endpoints
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 49s
2026-05-17 14:19:28 +02:00
MrSphay
b2f604b120 fix: report adsb unavailable state as degraded
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 1m13s
2026-05-17 13:55:42 +02:00
15 changed files with 855 additions and 64 deletions

View File

@@ -11,6 +11,8 @@ DASHBOARD_URL=
TERMINAL_ACTIONS_ENABLED=true TERMINAL_ACTIONS_ENABLED=true
SWEEP_TOKEN= SWEEP_TOKEN=
SSE_HEARTBEAT_INTERVAL_MS=25000 SSE_HEARTBEAT_INTERVAL_MS=25000
TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
TERMINAL_ACTION_RATE_LIMIT_MAX=10
BRIEF_VERBOSITY=standard BRIEF_VERBOSITY=standard
# LLM layer # LLM layer

View File

@@ -135,6 +135,8 @@ DASHBOARD_URL=https://intelligence.example.internal
TERMINAL_ACTIONS_ENABLED=true TERMINAL_ACTIONS_ENABLED=true
SWEEP_TOKEN= SWEEP_TOKEN=
SSE_HEARTBEAT_INTERVAL_MS=25000 SSE_HEARTBEAT_INTERVAL_MS=25000
TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
TERMINAL_ACTION_RATE_LIMIT_MAX=10
BRIEF_VERBOSITY=standard BRIEF_VERBOSITY=standard
LLM_PROVIDER=openrouter LLM_PROVIDER=openrouter
@@ -186,9 +188,52 @@ LLM_MODEL=your-model
For Pangolin or another reverse proxy, forward HTTP traffic to `intelligence-terminal:3117` (or the `PORT` you set). Missing API keys do not crash sweeps; affected sources are reported as degraded in `/api/health`. For Pangolin or another reverse proxy, forward HTTP traffic to `intelligence-terminal:3117` (or the `PORT` you set). Missing API keys do not crash sweeps; affected sources are reported as degraded in `/api/health`.
#### Terminal Action Exposure
`POST /api/action` and `POST /api/sweep` can trigger operational actions such as manual sweeps. The dashboard has a **SET TOKEN** control that stores your `SWEEP_TOKEN` in browser local storage and sends it as the `x-crucix-token` header; do not put action tokens in URLs.
Recommended settings:
| Deployment | Settings |
| --- | --- |
| Private local machine | `NODE_ENV=development`, optional `SWEEP_TOKEN`, optional `TERMINAL_ACTIONS_ENABLED=true`. Localhost can run actions without a token for development. |
| Private LAN / Dockge | Set a strong `SWEEP_TOKEN`, keep `TERMINAL_ACTIONS_ENABLED=true`, expose only to trusted clients. |
| Pangolin-authenticated reverse proxy | Set a strong `SWEEP_TOKEN`, keep Pangolin auth in front, use the dashboard **SET TOKEN** flow once per browser. |
| Public internet | Do not expose Terminal Actions directly. If exposure is unavoidable, require `SWEEP_TOKEN`, keep proxy authentication enabled, lower `TERMINAL_ACTION_RATE_LIMIT_MAX`, and monitor server audit logs. |
Action endpoints reject cross-origin POST origins, apply a small in-memory per-IP rate limit, and write sanitized audit lines without logging the token.
When data remains stale past `STALE_DATA_MAX_AGE_MINUTES`, the server sends an operator alert through configured Telegram/Discord channels after failed or degraded sweep attempts. `STALE_ALERT_COOLDOWN_MINUTES` prevents repeated stale alerts from spamming every refresh interval. Set `DASHBOARD_URL` to the Pangolin/public URL you want included in those alerts. When data remains stale past `STALE_DATA_MAX_AGE_MINUTES`, the server sends an operator alert through configured Telegram/Discord channels after failed or degraded sweep attempts. `STALE_ALERT_COOLDOWN_MINUTES` prevents repeated stale alerts from spamming every refresh interval. Set `DASHBOARD_URL` to the Pangolin/public URL you want included in those alerts.
The dashboard Terminal Actions panel can trigger `status`, `sweep`, and `brief` through `/api/action`. Leave `TERMINAL_ACTIONS_ENABLED=true` for a private home-server deployment. For an internet-exposed deployment, set `SWEEP_TOKEN` and pass it through trusted automation, or set `TERMINAL_ACTIONS_ENABLED=false` to disable browser-triggered actions. If you protect actions with `SWEEP_TOKEN`, the browser can send it from `localStorage.crucix_sweep_token`. #### Memory And Prediction Loop
Crucix stores longitudinal memory in `runs/intelligence.db` when the current Node.js build exposes `node:sqlite`. If SQLite is unavailable, the file is created as a harmless placeholder and `/api/health` reports the memory store as unavailable instead of failing the sweep.
The memory layer persists:
| Table | Purpose |
| --- | --- |
| `runs` | Sweep timestamps, source health counts, and delta direction summaries. |
| `entities` | Stable entity IDs for recurring countries, regions, and locations. |
| `events` | Stable event IDs for conflict, OSINT, urgent news, and new delta signals across sweeps. |
| `predictions` | Trade/intelligence hypotheses with evidence, confidence, horizon, outcome state, and latest grading. |
Query endpoints:
```text
GET /api/memory/search?q=iran&limit=25
GET /api/memory/predictions?state=open&limit=25
```
Memory endpoints use the same operator authorization gate as Terminal Actions. The dashboard Terminal Actions panel includes a `Memory` action for a quick operator-facing view of recent events and prediction states.
Retention, backup, and privacy expectations:
- Treat `runs/intelligence.db` as operator data. It can contain source excerpts, headlines, generated hypotheses, and URLs from your configured feeds.
- Back up `runs/` with the rest of your Dockge volume if you want longitudinal learning to survive container replacement.
- Delete `runs/intelligence.db` to reset SQLite memory; the next sweep recreates the schema.
- Do not commit `runs/` or `.env`. API credentials stay in `.env`; memory stores derived observations, not secrets.
- If you expose the dashboard through a reverse proxy, protect Terminal Actions and memory queries behind your normal authentication boundary.
#### Reverse Proxy SSE #### Reverse Proxy SSE
@@ -204,6 +249,8 @@ Recommended proxy settings:
If you raise the heartbeat interval, keep it shorter than the lowest idle timeout in the proxy chain. If you raise the heartbeat interval, keep it shorter than the lowest idle timeout in the proxy chain.
`/api/metrics` includes network health grouped by host and source/provider. Source modules should use `safeFetch(url, { source: 'SourceName' })`; when omitted, the shared helper infers a stable provider bucket from the URL host instead of grouping normal source traffic under `unknown`. Raw fetch exceptions are documented in [Source Fetch Instrumentation](docs/source-fetch-instrumentation.md).
#### Scenario Watchlist #### Scenario Watchlist
Intelligence Terminal can track operator hypotheses across sweeps with a runtime scenario file at `runs/scenarios.json`. On first run, the server creates three disabled starter examples: Intelligence Terminal can track operator hypotheses across sweeps with a runtime scenario file at `runs/scenarios.json`. On first run, the server creates three disabled starter examples:

View File

@@ -59,7 +59,8 @@ export async function runSource(name, fn, ...args) {
}); });
const data = await Promise.race([dataPromise, timeoutPromise]); const data = await Promise.race([dataPromise, timeoutPromise]);
const hasError = Boolean(data?.error); const hasError = Boolean(data?.error);
const isDegraded = hasError || ['no_credentials', 'degraded', 'failed'].includes(data?.status); const degradedStatuses = ['no_credentials', 'no_key', 'disabled', 'degraded', 'failed', 'error'];
const isDegraded = hasError || degradedStatuses.includes(data?.status);
return { return {
name, name,
status: isDegraded ? 'degraded' : 'ok', status: isDegraded ? 'degraded' : 'ok',

View File

@@ -1,7 +1,8 @@
// ADS-B Exchange — Unfiltered Flight Tracking (including Military) // ADS-B Exchange — Unfiltered Flight Tracking (including Military)
// Unlike FlightRadar24/FlightAware, ADS-B Exchange does NOT filter military aircraft. // Unlike FlightRadar24/FlightAware, ADS-B Exchange does NOT filter military aircraft.
// Public feed access varies; RapidAPI tier available for programmatic use. // Public feed access varies; RapidAPI tier available for programmatic use.
// This module attempts the public endpoints and falls back to a documented stub. // This module reports explicit disabled/degraded state instead of making
// unavailable aircraft data look live.
import { safeFetch } from '../utils/fetch.mjs'; import { safeFetch } from '../utils/fetch.mjs';
@@ -140,6 +141,7 @@ async function fetchViaRapidApi(apiKey) {
// Get all military aircraft // Get all military aircraft
const data = await safeFetch(`${ENDPOINTS.rapidApi}/mil`, { const data = await safeFetch(`${ENDPOINTS.rapidApi}/mil`, {
timeout: 20000, timeout: 20000,
source: 'adsb-rapidapi',
headers: { headers: {
'X-RapidAPI-Key': apiKey, 'X-RapidAPI-Key': apiKey,
'X-RapidAPI-Host': 'adsbexchange-com1.p.rapidapi.com', 'X-RapidAPI-Host': 'adsbexchange-com1.p.rapidapi.com',
@@ -151,21 +153,26 @@ async function fetchViaRapidApi(apiKey) {
// Attempt to fetch from public feed // Attempt to fetch from public feed
async function fetchPublicFeed() { async function fetchPublicFeed() {
const data = await safeFetch(ENDPOINTS.publicFeed, { timeout: 15000 }); const data = await safeFetch(ENDPOINTS.publicFeed, { timeout: 15000, source: 'adsb-public' });
return data; return data;
} }
// Get military aircraft from available sources async function getMilitaryAircraftResult(apiKey) {
export async function getMilitaryAircraft(apiKey) { const failures = [];
// Try RapidAPI first if key available // Try RapidAPI first if key available
if (apiKey) { if (apiKey) {
const data = await fetchViaRapidApi(apiKey); const data = await fetchViaRapidApi(apiKey);
if (data && !data.error) { if (data && !data.error) {
const aircraft = data.ac || data.aircraft || []; const aircraft = data.ac || data.aircraft || [];
if (Array.isArray(aircraft)) { if (Array.isArray(aircraft)) {
return aircraft.map(classifyAircraft).filter(a => a.isMilitary); return {
provider: 'rapidapi',
aircraft: aircraft.map(classifyAircraft).filter(a => a.isMilitary),
};
} }
} }
failures.push({ provider: 'rapidapi', error: data?.error || 'RapidAPI returned an unsupported payload' });
} }
// Try public feed // Try public feed
@@ -173,11 +180,21 @@ export async function getMilitaryAircraft(apiKey) {
if (pubData && !pubData.error) { if (pubData && !pubData.error) {
const aircraft = pubData.ac || pubData.aircraft || pubData.states || []; const aircraft = pubData.ac || pubData.aircraft || pubData.states || [];
if (Array.isArray(aircraft)) { if (Array.isArray(aircraft)) {
return aircraft.map(classifyAircraft).filter(a => a.isMilitary); return {
provider: 'public-feed',
aircraft: aircraft.map(classifyAircraft).filter(a => a.isMilitary),
};
} }
} }
failures.push({ provider: 'public-feed', error: pubData?.error || 'Public feed returned an unsupported payload' });
return null; // all sources failed return { provider: null, aircraft: null, failures };
}
// Get military aircraft from available sources
export async function getMilitaryAircraft(apiKey) {
const result = await getMilitaryAircraftResult(apiKey);
return result.aircraft;
} }
// Get all aircraft in a geographic bounding box via RapidAPI // Get all aircraft in a geographic bounding box via RapidAPI
@@ -208,7 +225,8 @@ export async function getAircraftInArea(lat, lon, radiusNm = 250, apiKey) {
// Briefing — attempt to get military flight data, document what's available // Briefing — attempt to get military flight data, document what's available
export async function briefing() { export async function briefing() {
const apiKey = process.env.ADSB_API_KEY || process.env.RAPIDAPI_KEY || null; const apiKey = process.env.ADSB_API_KEY || process.env.RAPIDAPI_KEY || null;
const militaryAircraft = await getMilitaryAircraft(apiKey); const result = await getMilitaryAircraftResult(apiKey);
const militaryAircraft = result.aircraft;
// If we got data, analyze it // If we got data, analyze it
if (militaryAircraft && militaryAircraft.length > 0) { if (militaryAircraft && militaryAircraft.length > 0) {
@@ -255,6 +273,7 @@ export async function briefing() {
source: 'ADS-B Exchange', source: 'ADS-B Exchange',
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
status: 'live', status: 'live',
provider: result.provider,
totalMilitary: militaryAircraft.length, totalMilitary: militaryAircraft.length,
byCountry, byCountry,
categories: { categories: {
@@ -269,10 +288,18 @@ export async function briefing() {
} }
// No data available — return stub with integration documentation // No data available — return stub with integration documentation
const status = apiKey ? 'degraded' : 'disabled';
const error = apiKey
? 'ADS-B providers returned no usable aircraft data'
: 'ADSB_API_KEY or RAPIDAPI_KEY is not configured';
return { return {
source: 'ADS-B Exchange', source: 'ADS-B Exchange',
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
status: apiKey ? 'error' : 'no_key', status,
provider: result.provider,
error,
failures: result.failures,
militaryAircraft: [], militaryAircraft: [],
message: apiKey message: apiKey
? 'ADS-B Exchange API returned no data. The endpoint may be temporarily unavailable.' ? 'ADS-B Exchange API returned no data. The endpoint may be temporarily unavailable.'

View File

@@ -10,6 +10,44 @@ const fetchMetrics = {
recent: [], recent: [],
}; };
const SOURCE_BY_HOST = [
[/api\.bls\.gov$/i, 'BLS'],
[/api\.fred\.stlouisfed\.org$/i, 'FRED'],
[/api\.eia\.gov$/i, 'EIA'],
[/api\.gdeltproject\.org$/i, 'GDELT'],
[/api\.weather\.gov$/i, 'NOAA'],
[/api\.open-notify\.org$/i, 'OpenNotify'],
[/opensky-network\.org$/i, 'OpenSky'],
[/firms\.modaps\.eosdis\.nasa\.gov$/i, 'FIRMS'],
[/api\.acleddata\.com$/i, 'ACLED'],
[/api\.reliefweb\.int$/i, 'ReliefWeb'],
[/receiverbook\.de$/i, 'KiwiSDR'],
[/safecast\.org$/i, 'Safecast'],
[/api\.patentsview\.org$/i, 'PatentsView'],
[/api\.trade\.gov$/i, 'Comtrade'],
[/api\.usaspending\.gov$/i, 'USASpending'],
[/api\.telegram\.org$/i, 'Telegram'],
[/oauth\.reddit\.com$/i, 'Reddit'],
[/reddit\.com$/i, 'Reddit'],
[/api\.bsky\.app$/i, 'Bluesky'],
[/api\.yahoo\.com$/i, 'YahooFinance'],
[/query\d?\.finance\.yahoo\.com$/i, 'YahooFinance'],
[/api\.cloudflare\.com$/i, 'CloudflareRadar'],
[/api\.opensanctions\.org$/i, 'OpenSanctions'],
[/home\.treasury\.gov$/i, 'Treasury'],
[/fiscaldata\.treasury\.gov$/i, 'Treasury'],
[/who\.int$/i, 'WHO'],
];
export function inferFetchSource(url) {
let host = 'unknown';
try { host = new URL(url).host.toLowerCase(); } catch { return 'unknown'; }
for (const [pattern, source] of SOURCE_BY_HOST) {
if (pattern.test(host)) return source;
}
return host;
}
function metricBucket(map, key) { function metricBucket(map, key) {
if (!map[key]) map[key] = { requests: 0, ok: 0, failed: 0, bytes: 0, lastStatus: null, lastError: null, lastMs: 0 }; if (!map[key]) map[key] = { requests: 0, ok: 0, failed: 0, bytes: 0, lastStatus: null, lastError: null, lastMs: 0 };
return map[key]; return map[key];
@@ -38,7 +76,7 @@ export function getFetchMetrics() {
} }
export async function safeFetch(url, opts = {}) { export async function safeFetch(url, opts = {}) {
const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts; const { timeout = 15000, retries = 1, headers = {}, source = inferFetchSource(url) } = opts;
let lastError; let lastError;
for (let i = 0; i <= retries; i++) { for (let i = 0; i <= retries; i++) {
const started = Date.now(); const started = Date.now();
@@ -79,11 +117,11 @@ export async function safeFetch(url, opts = {}) {
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1))); if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
} }
} }
return { error: lastError?.message || 'Unknown error', source: url }; return { error: lastError?.message || 'Unknown error', source };
} }
export async function safeFetchText(url, opts = {}) { export async function safeFetchText(url, opts = {}) {
const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts; const { timeout = 15000, retries = 1, headers = {}, source = inferFetchSource(url) } = opts;
let lastError; let lastError;
for (let i = 0; i <= retries; i++) { for (let i = 0; i <= retries; i++) {
const started = Date.now(); const started = Date.now();

View File

@@ -26,7 +26,9 @@ export default {
staleAlertCooldownMinutes: intEnv('STALE_ALERT_COOLDOWN_MINUTES', 60), staleAlertCooldownMinutes: intEnv('STALE_ALERT_COOLDOWN_MINUTES', 60),
dashboardUrl: process.env.DASHBOARD_URL || null, dashboardUrl: process.env.DASHBOARD_URL || null,
sweepToken: process.env.SWEEP_TOKEN || null, sweepToken: process.env.SWEEP_TOKEN || null,
terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', true), terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', !!process.env.SWEEP_TOKEN || process.env.NODE_ENV !== 'production'),
terminalActionRateLimitWindowMs: intEnv('TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS', 60_000),
terminalActionRateLimitMax: intEnv('TERMINAL_ACTION_RATE_LIMIT_MAX', 10),
sseHeartbeatIntervalMs: intEnv('SSE_HEARTBEAT_INTERVAL_MS', 25000), sseHeartbeatIntervalMs: intEnv('SSE_HEARTBEAT_INTERVAL_MS', 25000),
llm: { llm: {

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,21 @@
# Source Fetch Instrumentation
`safeFetch()` and `safeFetchText()` attribute requests to `/api/metrics.fetch.bySource`.
Rules:
- Prefer passing an explicit `source` option from source modules when the call has a clear Crucix source name.
- If `source` is omitted, the shared helper infers a stable provider name from the request host.
- Unknown hosts fall back to the lowercase host instead of the old `unknown` bucket.
- Raw `fetch()` calls should be limited to cases where the shared helper cannot represent the protocol cleanly.
Current raw-fetch exceptions:
| Area | Reason |
| --- | --- |
| OAuth/session handshakes | Token exchange calls often need custom form bodies, credential headers, or status-specific diagnostics. |
| Bot and alert delivery | Telegram/Discord alert calls are outbound operator notifications, not intelligence source health. |
| LLM providers | Provider clients already track model/provider status separately from source fetch health. |
| Dashboard browser calls | Browser-side `/api/*` and asset fetches are UI behavior, not source provider health. |
When adding a new intelligence source, use `safeFetch(url, { source: 'SourceName' })` unless there is a documented exception.

View File

@@ -16,4 +16,5 @@ Source docs:
- [Telegram](telegram.md) - [Telegram](telegram.md)
- [FIRMS](firms.md) - [FIRMS](firms.md)
- [Maritime](maritime.md) - [Maritime](maritime.md)
- [ADS-B](adsb.md)
- [Reddit](reddit.md) - [Reddit](reddit.md)

24
docs/sources/adsb.md Normal file
View File

@@ -0,0 +1,24 @@
# ADS-B Source
ADS-B Exchange support is optional and intended for unfiltered aircraft and military-flight awareness.
- Source module: `apis/sources/adsb.mjs`
- Preferred provider: ADS-B Exchange via RapidAPI
- Credentials: `ADSB_API_KEY` or `RAPIDAPI_KEY`
- Runtime status without credentials: `disabled`
- Runtime status when providers fail: `degraded`
- Runtime status with usable aircraft payloads: `live`
The source does not treat a missing key or unavailable public feed as normal live data. `/api/health` and `/api/metrics` surface the degraded source state through the sweep source summary.
Known failure modes:
- Missing `ADSB_API_KEY` / `RAPIDAPI_KEY`: source is disabled with operator guidance.
- RapidAPI rejects or rate-limits the request: source is degraded and records provider failure detail.
- Public feed is blocked, rate-limited, or changes shape: source remains degraded instead of returning stale-looking data.
Register for the provider documented in the README, then set:
```env
ADSB_API_KEY=<rapidapi-key>
```

View File

@@ -2,6 +2,9 @@
import { existsSync, mkdirSync, writeFileSync } from 'fs'; import { existsSync, mkdirSync, writeFileSync } from 'fs';
import { dirname, join } from 'path'; import { dirname, join } from 'path';
import { createHash } from 'crypto';
const PREDICTION_STATES = new Set(['open', 'monitoring', 'observed', 'expired_unverified', 'invalidated']);
export class IntelligenceStore { export class IntelligenceStore {
constructor(dbPath) { constructor(dbPath) {
@@ -30,15 +33,24 @@ export class IntelligenceStore {
); );
CREATE TABLE IF NOT EXISTS predictions ( CREATE TABLE IF NOT EXISTS predictions (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
stable_id TEXT UNIQUE,
created_at TEXT NOT NULL, created_at TEXT NOT NULL,
updated_at TEXT,
title TEXT NOT NULL, title TEXT NOT NULL,
type TEXT, type TEXT,
hypothesis TEXT,
evidence_json TEXT,
confidence TEXT, confidence TEXT,
horizon TEXT,
outcome_state TEXT DEFAULT 'open',
outcome_json TEXT,
last_evaluated_at TEXT,
source TEXT, source TEXT,
payload_json TEXT NOT NULL payload_json TEXT NOT NULL
); );
CREATE TABLE IF NOT EXISTS entities ( CREATE TABLE IF NOT EXISTS entities (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
stable_id TEXT UNIQUE,
first_seen TEXT NOT NULL, first_seen TEXT NOT NULL,
last_seen TEXT NOT NULL, last_seen TEXT NOT NULL,
name TEXT NOT NULL, name TEXT NOT NULL,
@@ -46,7 +58,21 @@ export class IntelligenceStore {
count INTEGER DEFAULT 1, count INTEGER DEFAULT 1,
UNIQUE(name, kind) UNIQUE(name, kind)
); );
CREATE TABLE IF NOT EXISTS events (
id INTEGER PRIMARY KEY AUTOINCREMENT,
stable_id TEXT NOT NULL UNIQUE,
first_seen TEXT NOT NULL,
last_seen TEXT NOT NULL,
kind TEXT NOT NULL,
name TEXT NOT NULL,
region TEXT,
severity TEXT,
source TEXT,
evidence_json TEXT NOT NULL,
count INTEGER DEFAULT 1
);
`); `);
this._migrate();
this.available = true; this.available = true;
} catch (err) { } catch (err) {
this.available = false; this.available = false;
@@ -71,24 +97,141 @@ export class IntelligenceStore {
delta?.summary?.direction || null, delta?.summary?.direction || null,
JSON.stringify({ meta, delta: delta?.summary || null }), JSON.stringify({ meta, delta: delta?.summary || null }),
); );
for (const idea of data.ideas || []) {
this.db.prepare(`INSERT INTO predictions (created_at, title, type, confidence, source, payload_json)
VALUES (?, ?, ?, ?, ?, ?)`).run(
timestamp,
idea.title || 'Untitled idea',
idea.type || null,
idea.confidence || null,
idea.source || data.ideasSource || null,
JSON.stringify(idea),
);
}
this._recordEntities(data, timestamp); this._recordEntities(data, timestamp);
this._recordEvents(data, delta, timestamp);
this.evaluatePredictions(data, timestamp);
this._recordPredictions(data, timestamp);
} }
status() { status() {
return { available: this.available, path: this.dbPath, reason: this.reason }; return { available: this.available, path: this.dbPath, reason: this.reason };
} }
queryMemory({ q = '', limit = 25 } = {}) {
if (!this.available || !this.db) return { available: false, reason: this.reason, results: [] };
const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
const term = String(q || '').trim();
const like = `%${term}%`;
const where = term
? 'WHERE name LIKE ? OR region LIKE ? OR source LIKE ? OR kind LIKE ?'
: '';
const params = term ? [like, like, like, like, safeLimit] : [safeLimit];
const events = this.db.prepare(`
SELECT stable_id, first_seen, last_seen, kind, name, region, severity, source, count, evidence_json
FROM events
${where}
ORDER BY last_seen DESC
LIMIT ?
`).all(...params).map(row => ({ ...row, evidence: parseJson(row.evidence_json, {}) }));
return { available: true, q: term, results: events };
}
listPredictions({ state = null, limit = 25 } = {}) {
if (!this.available || !this.db) return { available: false, reason: this.reason, predictions: [] };
const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
const normalizedState = state && PREDICTION_STATES.has(String(state)) ? String(state) : null;
const rows = normalizedState
? this.db.prepare(`SELECT * FROM predictions WHERE outcome_state = ? ORDER BY created_at DESC LIMIT ?`).all(normalizedState, safeLimit)
: this.db.prepare(`SELECT * FROM predictions ORDER BY created_at DESC LIMIT ?`).all(safeLimit);
return {
available: true,
predictions: rows.map(row => ({
stable_id: row.stable_id,
created_at: row.created_at,
updated_at: row.updated_at,
title: row.title,
type: row.type,
hypothesis: row.hypothesis,
confidence: row.confidence,
horizon: row.horizon,
outcome_state: row.outcome_state,
last_evaluated_at: row.last_evaluated_at,
source: row.source,
evidence: parseJson(row.evidence_json, []),
outcome: parseJson(row.outcome_json, null),
})),
};
}
evaluatePredictions(data, timestamp = new Date().toISOString()) {
if (!this.available || !this.db) return;
const rows = this.db.prepare(`
SELECT id, created_at, title, type, horizon, outcome_state, payload_json
FROM predictions
WHERE outcome_state IN ('open', 'monitoring')
ORDER BY created_at ASC
LIMIT 200
`).all();
for (const row of rows) {
const payload = parseJson(row.payload_json, {});
const evaluation = evaluatePredictionAgainstSweep(row, payload, data, timestamp);
this.db.prepare(`UPDATE predictions
SET outcome_state = ?, outcome_json = ?, last_evaluated_at = ?, updated_at = ?
WHERE id = ?`).run(
evaluation.state,
JSON.stringify(evaluation),
timestamp,
timestamp,
row.id,
);
}
}
_migrate() {
const columns = {
predictions: [
['stable_id', 'TEXT'],
['updated_at', 'TEXT'],
['hypothesis', 'TEXT'],
['evidence_json', 'TEXT'],
['horizon', 'TEXT'],
['outcome_state', "TEXT DEFAULT 'open'"],
['outcome_json', 'TEXT'],
['last_evaluated_at', 'TEXT'],
],
entities: [
['stable_id', 'TEXT'],
],
};
for (const [table, defs] of Object.entries(columns)) {
for (const [name, type] of defs) {
try { this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${name} ${type}`); } catch { }
}
}
try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_predictions_stable_id ON predictions(stable_id)`); } catch { }
try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_entities_stable_id ON entities(stable_id)`); } catch { }
}
_recordPredictions(data, timestamp) {
for (const idea of data.ideas || []) {
const title = idea.title || 'Untitled idea';
const stableId = stableId('prediction', title, idea.type || '', idea.ticker || '', idea.horizon || '');
const evidence = Array.isArray(idea.signals) ? idea.signals : [];
this.db.prepare(`INSERT INTO predictions (
stable_id, created_at, updated_at, title, type, hypothesis, evidence_json, confidence,
horizon, outcome_state, source, payload_json
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, ?)
ON CONFLICT(stable_id) DO UPDATE SET
updated_at=excluded.updated_at,
confidence=excluded.confidence,
evidence_json=excluded.evidence_json,
payload_json=excluded.payload_json`).run(
stableId,
timestamp,
timestamp,
title,
idea.type || null,
idea.rationale || idea.text || title,
JSON.stringify(evidence),
idea.confidence || null,
idea.horizon || null,
idea.source || data.ideasSource || null,
JSON.stringify(idea),
);
}
}
_recordEntities(data, timestamp) { _recordEntities(data, timestamp) {
const names = []; const names = [];
for (const item of data.acled?.deadliestEvents || []) { for (const item of data.acled?.deadliestEvents || []) {
@@ -99,14 +242,154 @@ export class IntelligenceStore {
if (item.region) names.push([item.region, 'region']); if (item.region) names.push([item.region, 'region']);
} }
for (const [name, kind] of names.slice(0, 200)) { for (const [name, kind] of names.slice(0, 200)) {
this.db.prepare(`INSERT INTO entities (first_seen, last_seen, name, kind, count) const cleanName = String(name).slice(0, 160);
VALUES (?, ?, ?, ?, 1) this.db.prepare(`INSERT INTO entities (stable_id, first_seen, last_seen, name, kind, count)
VALUES (?, ?, ?, ?, ?, 1)
ON CONFLICT(name, kind) DO UPDATE SET last_seen=excluded.last_seen, count=count+1`).run( ON CONFLICT(name, kind) DO UPDATE SET last_seen=excluded.last_seen, count=count+1`).run(
stableId('entity', kind, cleanName),
timestamp, timestamp,
timestamp, timestamp,
String(name).slice(0, 160), cleanName,
kind, kind,
); );
} }
} }
_recordEvents(data, delta, timestamp) {
const events = extractEvents(data, delta);
for (const event of events.slice(0, 300)) {
this.db.prepare(`INSERT INTO events (
stable_id, first_seen, last_seen, kind, name, region, severity, source, evidence_json, count
)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
ON CONFLICT(stable_id) DO UPDATE SET
last_seen=excluded.last_seen,
severity=COALESCE(excluded.severity, severity),
evidence_json=excluded.evidence_json,
count=count+1`).run(
event.stable_id,
timestamp,
timestamp,
event.kind,
event.name,
event.region || null,
event.severity || null,
event.source || null,
JSON.stringify(event.evidence || {}),
);
}
}
}
function stableId(...parts) {
const input = parts.map(part => String(part || '').trim().toLowerCase()).join('|');
return createHash('sha256').update(input).digest('hex').slice(0, 24);
}
function parseJson(value, fallback) {
try { return value ? JSON.parse(value) : fallback; } catch { return fallback; }
}
function extractEvents(data, delta) {
const events = [];
const push = ({ kind, name, region, severity, source, evidence }) => {
if (!kind || !name) return;
events.push({
stable_id: stableId('event', kind, name, region || source || ''),
kind,
name: String(name).slice(0, 240),
region: region ? String(region).slice(0, 120) : null,
severity: severity || null,
source: source || null,
evidence: evidence || {},
});
};
for (const item of data.acled?.deadliestEvents || []) {
push({
kind: 'conflict',
name: item.event_type || item.sub_event_type || item.location || item.country,
region: item.country || item.location,
severity: Number(item.fatalities || 0) > 0 ? 'high' : 'medium',
source: 'ACLED',
evidence: item,
});
}
for (const item of data.tg?.urgent || []) {
push({
kind: 'osint',
name: (item.text || '').slice(0, 120),
region: item.region || 'OSINT',
severity: 'high',
source: item.channel || item.chat || 'telegram',
evidence: item,
});
}
for (const item of data.newsFeed || data.news || []) {
if (!item.urgent) continue;
push({
kind: 'news',
name: item.headline || item.title,
region: item.region,
severity: 'medium',
source: item.source,
evidence: item,
});
}
for (const signal of delta?.signals?.new || []) {
push({
kind: 'delta',
name: signal.label || signal.reason || signal.key,
region: signal.region,
severity: signal.severity || 'medium',
source: 'delta',
evidence: signal,
});
}
return events;
}
function evaluatePredictionAgainstSweep(row, payload, data, timestamp) {
const terms = [
row.title,
payload.ticker,
...(Array.isArray(payload.signals) ? payload.signals : []),
].filter(Boolean).map(v => String(v).toLowerCase());
const evidenceText = [
...(data.tSignals || []),
...(data.newsFeed || []).slice(0, 40).map(n => `${n.source || ''} ${n.headline || n.title || ''}`),
...(data.tg?.urgent || []).slice(0, 20).map(p => p.text || ''),
].join('\n').toLowerCase();
const matched = terms.filter(term => term.length >= 4 && evidenceText.includes(term.slice(0, 60)));
const expired = predictionExpired(row.created_at, row.horizon, timestamp);
const state = matched.length
? 'observed'
: expired
? 'expired_unverified'
: 'monitoring';
return {
state,
evaluated_at: timestamp,
matched_terms: matched.slice(0, 10),
expired,
reason: matched.length
? 'Current sweep contains matching evidence terms.'
: expired
? 'Prediction horizon elapsed without matching evidence.'
: 'Prediction remains open for future sweeps.',
};
}
function predictionExpired(createdAt, horizon, nowIso) {
const created = new Date(createdAt).getTime();
const now = new Date(nowIso).getTime();
if (!Number.isFinite(created) || !Number.isFinite(now)) return false;
const text = String(horizon || '').toLowerCase();
const days = text.includes('intraday') ? 1
: text.includes('day') ? 7
: text.includes('week') ? 45
: text.includes('month') ? 180
: text.includes('strategic') ? 365
: 30;
return now - created > days * 24 * 60 * 60 * 1000;
} }

View File

@@ -12,7 +12,7 @@
"brief:save": "node apis/save-briefing.mjs", "brief:save": "node apis/save-briefing.mjs",
"diag": "node diag.mjs", "diag": "node diag.mjs",
"test": "npm run test:unit", "test": "npm run test:unit",
"test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs test/acled-source.test.mjs test/mojibake-text.test.mjs test/dashboard-geotagging.test.mjs", "test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs test/acled-source.test.mjs test/mojibake-text.test.mjs test/adsb.test.mjs test/dashboard-geotagging.test.mjs",
"compose:config": "docker compose config", "compose:config": "docker compose config",
"clean": "node scripts/clean.mjs", "clean": "node scripts/clean.mjs",
"fresh-start": "npm run clean && npm start" "fresh-start": "npm run clean && npm start"

View File

@@ -41,6 +41,7 @@ let sweepStartedAt = null; // Timestamp when current/last sweep started
let sweepInProgress = false; let sweepInProgress = false;
const startTime = Date.now(); const startTime = Date.now();
const sseClients = new Set(); const sseClients = new Set();
const terminalActionBuckets = new Map();
const staleAlertState = {}; const staleAlertState = {};
// === Delta/Memory === // === Delta/Memory ===
@@ -291,29 +292,67 @@ app.get('/api/metrics', (req, res) => {
}); });
}); });
app.post('/api/sweep', express.json(), (req, res) => { app.get('/api/memory/search', (req, res) => {
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' }); const guard = authorizeTerminalAction(req, res, 'memory:search');
triggerSweep(res); if (!guard.ok) return;
auditTerminalAction(req, 'memory:search', 'ok');
res.json(intelligenceStore.queryMemory({
q: req.query.q || '',
limit: req.query.limit || 25,
}));
}); });
app.post('/api/action', express.json(), async (req, res) => { app.get('/api/memory/predictions', (req, res) => {
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' }); const guard = authorizeTerminalAction(req, res, 'memory:predictions');
const action = String(req.body?.action || req.query.action || '').toLowerCase(); if (!guard.ok) return;
auditTerminalAction(req, 'memory:predictions', 'ok');
res.json(intelligenceStore.listPredictions({
state: req.query.state || null,
limit: req.query.limit || 25,
}));
});
app.post('/api/sweep', express.json(), (req, res) => {
const guard = authorizeTerminalAction(req, res, 'sweep');
if (!guard.ok) return;
triggerSweepAction(req, res, 'sweep');
});
app.post('/api/action', express.json(), (req, res) => {
const action = String(req.body?.action || req.body?.command || '').trim().toLowerCase();
const guard = authorizeTerminalAction(req, res, action || 'unknown');
if (!guard.ok) return;
if (action === 'status') { if (action === 'status') {
return res.json({ ok: true, action, health: buildHealth() }); auditTerminalAction(req, 'status', 'ok');
return res.json({ ok: true, action, status: 'ok', health: buildHealth() });
} }
if (action === 'brief') { if (action === 'brief') {
if (!currentData) return res.status(503).json({ ok: false, action, error: 'No data yet — first sweep in progress' }); if (!currentData) {
return res.json({ ok: true, action, text: buildBrief(currentData) }); auditTerminalAction(req, 'brief', 'rejected', 'no_data');
return res.status(503).json({ ok: false, action, error: 'No data yet - first sweep in progress' });
}
auditTerminalAction(req, 'brief', 'ok');
const brief = buildBrief(currentData);
return res.json({ ok: true, action, status: 'ok', brief, text: brief });
} }
if (action === 'sweep') { if (action === 'memory') {
return triggerSweep(res); auditTerminalAction(req, 'memory', 'ok');
return res.json({
ok: true,
action,
memory: intelligenceStore.status(),
recentEvents: intelligenceStore.queryMemory({ q: req.body?.q || '', limit: 8 }).results,
predictions: intelligenceStore.listPredictions({ limit: 8 }).predictions,
});
} }
res.status(400).json({ ok: false, error: 'Unknown action', actions: ['status', 'brief', 'sweep'] }); if (action === 'sweep') return triggerSweepAction(req, res, 'action:sweep');
auditTerminalAction(req, action || 'unknown', 'rejected', 'unknown_action');
return res.status(400).json({ ok: false, error: 'Unknown action', allowed: ['status', 'brief', 'memory', 'sweep'], actions: ['status', 'brief', 'memory', 'sweep'] });
}); });
// API: available locales // API: available locales
@@ -358,26 +397,114 @@ function broadcast(data) {
} }
} }
function requestIp(req) {
return req.ip || req.socket?.remoteAddress || 'unknown';
}
function isLocalRequest(req) {
const remote = requestIp(req);
return remote === '::1'
|| remote === '127.0.0.1'
|| remote === '::ffff:127.0.0.1'
|| remote.startsWith('127.')
|| remote === 'localhost';
}
function sameOriginPost(req) {
const origin = req.get('origin');
if (!origin) return true;
try {
const originUrl = new URL(origin);
const host = req.get('host');
return host && originUrl.host === host;
} catch {
return false;
}
}
function actionToken(req) {
return req.get('x-crucix-token') || req.body?.token || null;
}
function auditTerminalAction(req, action, outcome, detail = null) {
const suffix = detail ? ` detail=${detail}` : '';
console.log(`[Crucix][audit] terminal_action action=${action || 'unknown'} outcome=${outcome} ip=${requestIp(req)}${suffix}`);
}
function rateLimitTerminalAction(req, action) {
const now = Date.now();
const windowMs = Math.max(1000, config.terminalActionRateLimitWindowMs || 60_000);
const max = Math.max(1, config.terminalActionRateLimitMax || 10);
const key = `${requestIp(req)}:${action}`;
const bucket = terminalActionBuckets.get(key);
if (!bucket || now > bucket.resetAt) {
terminalActionBuckets.set(key, { count: 1, resetAt: now + windowMs });
return { ok: true };
}
bucket.count += 1;
if (bucket.count > max) {
return { ok: false, retryAfterSeconds: Math.ceil((bucket.resetAt - now) / 1000) };
}
return { ok: true };
}
function authorizeTerminalAction(req, res, action) {
const rate = rateLimitTerminalAction(req, action);
if (!rate.ok) {
auditTerminalAction(req, action, 'rejected', 'rate_limited');
res.set('Retry-After', String(rate.retryAfterSeconds));
res.status(429).json({ error: 'Too many terminal actions', retryAfterSeconds: rate.retryAfterSeconds });
return { ok: false };
}
if (!sameOriginPost(req)) {
auditTerminalAction(req, action, 'rejected', 'csrf_origin');
res.status(403).json({ error: 'Origin mismatch' });
return { ok: false };
}
const local = isLocalRequest(req);
const token = actionToken(req);
if (!config.terminalActionsEnabled) {
auditTerminalAction(req, action, 'rejected', 'disabled');
res.status(403).json({ error: 'Terminal actions are disabled' });
return { ok: false };
}
if (config.sweepToken) {
if (token !== config.sweepToken) {
auditTerminalAction(req, action, 'rejected', 'invalid_token');
res.status(401).json({ error: 'Invalid terminal action token' });
return { ok: false };
}
return { ok: true };
}
if (!local) {
auditTerminalAction(req, action, 'rejected', 'missing_token');
res.status(403).json({ error: 'Terminal actions are local-only unless SWEEP_TOKEN is set' });
return { ok: false };
}
return { ok: true };
}
function triggerSweepAction(req, res, auditAction) {
if (sweepInProgress) {
auditTerminalAction(req, auditAction, 'rejected', 'already_running');
return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
}
auditTerminalAction(req, auditAction, 'accepted');
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
return res.status(202).json({ ok: true, status: 'accepted' });
}
function dataAgeMs() { function dataAgeMs() {
const ts = currentData?.meta?.timestamp || lastSuccessfulSweepTime || lastSweepTime; const ts = currentData?.meta?.timestamp || lastSuccessfulSweepTime || lastSweepTime;
const ms = ts ? Date.now() - new Date(ts).getTime() : null; const ms = ts ? Date.now() - new Date(ts).getTime() : null;
return Number.isFinite(ms) ? ms : null; return Number.isFinite(ms) ? ms : null;
} }
function canRunTerminalAction(req) {
const remote = req.ip || '';
const local = remote.includes('127.0.0.1') || remote === '::1' || remote === '::ffff:127.0.0.1';
const token = req.get('x-crucix-token') || req.query.token || req.body?.token;
if (config.sweepToken) return token === config.sweepToken;
return Boolean(config.terminalActionsEnabled || local);
}
function triggerSweep(res) {
if (sweepInProgress) return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
return res.status(202).json({ ok: true, status: 'accepted' });
}
function getLLMStatus() { function getLLMStatus() {
if (!config.llm.provider) return { state: 'disabled' }; if (!config.llm.provider) return { state: 'disabled' };
if (!llmProvider) return { state: 'misconfigured', provider: config.llm.provider }; if (!llmProvider) return { state: 'misconfigured', provider: config.llm.provider };
@@ -421,7 +548,8 @@ function buildHealth() {
llm: getLLMStatus(), llm: getLLMStatus(),
telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId), telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId),
discordEnabled: !!(config.discord?.botToken || config.discord?.webhookUrl), discordEnabled: !!(config.discord?.botToken || config.discord?.webhookUrl),
terminalActionsEnabled: Boolean(config.terminalActionsEnabled || config.sweepToken), terminalActionsEnabled: config.terminalActionsEnabled,
terminalActionsTokenRequired: !!config.sweepToken,
refreshIntervalMinutes: config.refreshIntervalMinutes, refreshIntervalMinutes: config.refreshIntervalMinutes,
language: currentLanguage, language: currentLanguage,
memory: intelligenceStore.status(), memory: intelligenceStore.status(),

82
test/adsb.test.mjs Normal file
View File

@@ -0,0 +1,82 @@
import test from 'node:test';
import assert from 'node:assert/strict';
async function withFetch(mockFetch, fn) {
const originalFetch = globalThis.fetch;
const originalAdsbKey = process.env.ADSB_API_KEY;
const originalRapidKey = process.env.RAPIDAPI_KEY;
globalThis.fetch = mockFetch;
delete process.env.ADSB_API_KEY;
delete process.env.RAPIDAPI_KEY;
try {
return await fn();
} finally {
globalThis.fetch = originalFetch;
if (originalAdsbKey === undefined) delete process.env.ADSB_API_KEY;
else process.env.ADSB_API_KEY = originalAdsbKey;
if (originalRapidKey === undefined) delete process.env.RAPIDAPI_KEY;
else process.env.RAPIDAPI_KEY = originalRapidKey;
}
}
function jsonResponse(payload, ok = true, status = 200) {
return {
ok,
status,
headers: { get: () => 'application/json' },
text: async () => JSON.stringify(payload),
};
}
test('ADS-B reports disabled when no key is configured and public feed fails', async () => {
await withFetch(async () => jsonResponse({ error: 'blocked' }, false, 403), async () => {
const { briefing } = await import('../apis/sources/adsb.mjs');
const data = await briefing();
assert.equal(data.status, 'disabled');
assert.match(data.error, /ADSB_API_KEY|RAPIDAPI_KEY/);
assert.equal(data.militaryAircraft.length, 0);
});
});
test('ADS-B reports degraded when RapidAPI and public feed fail', async () => {
await withFetch(async () => jsonResponse({ error: 'unavailable' }, false, 503), async () => {
process.env.ADSB_API_KEY = 'test-key';
const { briefing } = await import('../apis/sources/adsb.mjs');
const data = await briefing();
assert.equal(data.status, 'degraded');
assert.match(data.error, /providers returned no usable/);
assert.equal(data.failures.length, 2);
});
});
test('ADS-B returns live RapidAPI military aircraft payloads', async () => {
await withFetch(async () => jsonResponse({
ac: [{
hex: 'AE1234',
flight: 'RCH123',
t: 'KC135',
lat: 50,
lon: 8,
mil: true,
}],
}), async () => {
process.env.ADSB_API_KEY = 'test-key';
const { briefing } = await import('../apis/sources/adsb.mjs');
const data = await briefing();
assert.equal(data.status, 'live');
assert.equal(data.provider, 'rapidapi');
assert.equal(data.totalMilitary, 1);
assert.equal(data.militaryAircraft[0].callsign, 'RCH123');
});
});
test('runSource treats disabled source status as degraded health', async () => {
const { runSource } = await import('../apis/briefing.mjs');
const result = await runSource('ADS-B', async () => ({ status: 'disabled', message: 'missing key' }));
assert.equal(result.status, 'degraded');
assert.equal(result.error, null);
});

View File

@@ -1,7 +1,7 @@
import test from 'node:test'; import test from 'node:test';
import assert from 'node:assert/strict'; import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs'; import { readFileSync } from 'node:fs';
import { safeFetch, safeFetchText, getFetchMetrics } from '../apis/utils/fetch.mjs'; import { safeFetch, safeFetchText, getFetchMetrics, inferFetchSource } from '../apis/utils/fetch.mjs';
import { formatStaleAlert, shouldSendStaleAlert } from '../lib/stale-alerts.mjs'; import { formatStaleAlert, shouldSendStaleAlert } from '../lib/stale-alerts.mjs';
test('safeFetch reports HTML as degraded JSON response', async () => { test('safeFetch reports HTML as degraded JSON response', async () => {
@@ -101,6 +101,31 @@ test('safeFetchText returns text and byte count', async () => {
} }
}); });
test('safeFetch attributes unlabelled requests to a stable provider source', async () => {
const originalFetch = globalThis.fetch;
globalThis.fetch = async () => ({
ok: true,
status: 200,
headers: { get: () => 'application/json' },
text: async () => '{"observations":[]}',
});
try {
const data = await safeFetch('https://api.fred.stlouisfed.org/fred/series/observations?series_id=VIXCLS', { retries: 0 });
assert.deepEqual(data, { observations: [] });
const bucket = getFetchMetrics().bySource.FRED;
assert.ok(bucket.requests >= 1);
assert.equal(bucket.lastStatus, 200);
} finally {
globalThis.fetch = originalFetch;
}
});
test('inferFetchSource returns provider names and host fallback', () => {
assert.equal(inferFetchSource('https://api.bls.gov/publicAPI/v2/timeseries/data/CPI'), 'BLS');
assert.equal(inferFetchSource('https://query1.finance.yahoo.com/v8/finance/chart/%5EGSPC'), 'YahooFinance');
assert.equal(inferFetchSource('https://unknown.example.test/path'), 'unknown.example.test');
});
test('SSE endpoint sends reconnect guidance and clears heartbeat timer', () => { test('SSE endpoint sends reconnect guidance and clears heartbeat timer', () => {
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8'); const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
const config = readFileSync(new URL('../crucix.config.mjs', import.meta.url), 'utf8'); const config = readFileSync(new URL('../crucix.config.mjs', import.meta.url), 'utf8');
@@ -112,6 +137,64 @@ test('SSE endpoint sends reconnect guidance and clears heartbeat timer', () => {
assert.match(server, /X-Accel-Buffering/); assert.match(server, /X-Accel-Buffering/);
}); });
test('intelligence store defines durable memory and prediction lifecycle tables', () => {
const store = readFileSync(new URL('../lib/intelligence-store.mjs', import.meta.url), 'utf8');
assert.match(store, /CREATE TABLE IF NOT EXISTS events/);
assert.match(store, /stable_id TEXT NOT NULL UNIQUE/);
assert.match(store, /hypothesis TEXT/);
assert.match(store, /evidence_json TEXT/);
assert.match(store, /outcome_state TEXT DEFAULT 'open'/);
assert.match(store, /evaluatePredictions/);
assert.match(store, /queryMemory/);
assert.match(store, /listPredictions/);
});
test('server exposes memory-backed query APIs and dashboard memory action', () => {
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
assert.match(server, /\/api\/memory\/search/);
assert.match(server, /\/api\/memory\/predictions/);
assert.match(server, /action === 'memory'/);
assert.match(html, /runTerminalAction\('memory'\)/);
});
test('terminal action endpoints avoid URL tokens and include hardening gates', () => {
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
assert.match(server, /app\.post\('\/api\/action'/);
assert.match(server, /app\.post\('\/api\/sweep'/);
assert.match(server, /x-crucix-token/);
assert.match(server, /sameOriginPost/);
assert.match(server, /rateLimitTerminalAction/);
assert.match(server, /auditTerminalAction/);
assert.doesNotMatch(server, /req\.query\.token/);
});
test('dashboard exposes token configuration flow without devtools edits', () => {
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
assert.match(html, /configureTerminalActionToken/);
assert.match(html, /crucix_sweep_token/);
assert.match(html, /x-crucix-token/);
assert.match(html, /SET TOKEN/);
});
test('server dashboard shell does not embed an operational snapshot', () => {
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
assert.match(html, /let D = createDashboardShellData\(\);/);
assert.doesNotMatch(html, /2026-04-03T16:18:10\.188Z/);
assert.doesNotMatch(html, /Trump announced new strikes on Iran/);
});
test('server dashboard fetches api data before initialization', () => {
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
const serverMode = html.indexOf('if (canProbeApi)');
const apiFetch = html.indexOf("fetch('/api/data')");
const firstInitAfterServerMode = html.indexOf('init();', serverMode);
assert.ok(serverMode > -1);
assert.ok(apiFetch > serverMode);
assert.ok(firstInitAfterServerMode > apiFetch);
});
test('stale alert is skipped for fresh health and resets active key', () => { test('stale alert is skipped for fresh health and resets active key', () => {
const state = { lastStaleAlertKey: 'old', lastStaleAlertAt: 100 }; const state = { lastStaleAlertKey: 'old', lastStaleAlertAt: 100 };
const decision = shouldSendStaleAlert({ stale: false }, state, { now: 200 }); const decision = shouldSendStaleAlert({ stale: false }, state, { now: 200 });