Compare commits
43 Commits
533abb914c
...
codex/issu
| Author | SHA1 | Date | |
|---|---|---|---|
| 28c5e7955a | |||
| 5c4bf80eb0 | |||
| e0e408d1eb | |||
| c159c83a07 | |||
|
|
a1d415e449 | ||
|
|
0f5f9c5f91 | ||
| 096544f6e6 | |||
|
|
5a3dbc6252 | ||
| 9f2083a324 | |||
| dd08ecaf27 | |||
|
|
bc354e7bc5 | ||
| 1c2b48f588 | |||
|
|
a590bf62c2 | ||
| 6a9918bc98 | |||
|
|
4448f5931b | ||
| 9b15913049 | |||
|
|
331175ae3c | ||
| e288881c41 | |||
|
|
e4834cd3cd | ||
|
|
0fbd8640ca | ||
|
|
3069114ffd | ||
|
|
09df127e06 | ||
|
|
c102017b16 | ||
|
|
eefc1a4c77 | ||
| 49176b42fd | |||
|
|
090e90ea70 | ||
| e70801ae98 | |||
| 703670e7a0 | |||
|
|
1423dca199 | ||
|
|
5b013947b4 | ||
|
|
5113e341b2 | ||
| d625bffd4a | |||
| 776d200853 | |||
| b8f34d3d19 | |||
| d7f10bf545 | |||
| a6e1026aef | |||
|
|
2025ae09db | ||
|
|
446076cb84 | ||
|
|
6096a0ad03 | ||
|
|
267af03b22 | ||
|
|
d7df2e4aee | ||
| e574ad1c3d | |||
|
|
b2f604b120 |
@@ -18,7 +18,7 @@ Production-ready Crucix fork for Docker, Dockge, Pangolin, local OSINT sweeps, s
|
||||
- `npm run test:unit`
|
||||
- `npm test`
|
||||
- `docker compose config`
|
||||
- `docker build -t git.wilkensxl.de/mrsphay/intelligence-terminal:latest .`
|
||||
- `docker build -t git.wilkensxl.de/code-inc/intelligence-terminal:latest .`
|
||||
|
||||
Heavy install/build/audit/release work should run on Gitea Ubuntu runners where possible. Local work should stay limited to targeted verification and Docker checks required for this deployment.
|
||||
|
||||
|
||||
10
.env.example
10
.env.example
@@ -6,12 +6,17 @@ PORT=3117
|
||||
REFRESH_INTERVAL_MINUTES=15
|
||||
AUTO_OPEN_BROWSER=false
|
||||
STALE_DATA_MAX_AGE_MINUTES=60
|
||||
STALE_ALERT_COOLDOWN_MINUTES=60
|
||||
DASHBOARD_URL=
|
||||
TERMINAL_ACTIONS_ENABLED=true
|
||||
SWEEP_TOKEN=
|
||||
SSE_HEARTBEAT_INTERVAL_MS=25000
|
||||
TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
|
||||
TERMINAL_ACTION_RATE_LIMIT_MAX=10
|
||||
BRIEF_VERBOSITY=standard
|
||||
|
||||
# LLM layer
|
||||
# Providers: openrouter | openai-compatible | lmstudio | ollama | openai | anthropic | gemini | mistral | minimax | grok | codex
|
||||
# Providers: litellm | openrouter | openai-compatible | lmstudio | ollama | openai | anthropic | gemini | mistral | minimax | grok | codex
|
||||
LLM_PROVIDER=openrouter
|
||||
LLM_BASE_URL=https://openrouter.ai/api/v1
|
||||
LLM_API_KEY=
|
||||
@@ -19,10 +24,11 @@ LLM_MODEL=openrouter/free
|
||||
LLM_TEMPERATURE=0.2
|
||||
LLM_MAX_TOKENS=2000
|
||||
LLM_TIMEOUT_MS=90000
|
||||
OPENROUTER_SITE_URL=https://git.wilkensxl.de/MrSphay/intelligence-terminal
|
||||
OPENROUTER_SITE_URL=https://git.wilkensxl.de/Code-Inc/intelligence-terminal
|
||||
OPENROUTER_APP_NAME=Intelligence Terminal
|
||||
|
||||
# Local OpenAI-compatible examples
|
||||
# LiteLLM: LLM_PROVIDER=litellm, LLM_BASE_URL=https://llm.example.com/v1, LLM_API_KEY=your-proxy-key, LLM_MODEL=your-model-alias
|
||||
# LM Studio: LLM_PROVIDER=lmstudio, LLM_BASE_URL=http://host.docker.internal:1234/v1, LLM_MODEL=local-model
|
||||
# Ollama: LLM_PROVIDER=ollama, LLM_BASE_URL=http://host.docker.internal:11434, LLM_MODEL=llama3.1:8b
|
||||
# Generic: LLM_PROVIDER=openai-compatible, LLM_BASE_URL=http://host.docker.internal:8000/v1, LLM_MODEL=your-model
|
||||
|
||||
@@ -15,7 +15,7 @@ jobs:
|
||||
env:
|
||||
REGISTRY_HOST: git.wilkensxl.de
|
||||
REGISTRY_USERNAME: MrSphay
|
||||
REGISTRY_NAMESPACE: mrsphay
|
||||
REGISTRY_NAMESPACE: code-inc
|
||||
IMAGE_NAME: intelligence-terminal
|
||||
REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
|
||||
steps:
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
docker build -t "${image}:${build_tag}" .
|
||||
|
||||
- name: Publish Docker image
|
||||
if: ${{ env.REGISTRY_TOKEN != '' }}
|
||||
if: ${{ env.REGISTRY_TOKEN != '' && github.event_name == 'push' && github.ref == 'refs/heads/codex/production-intelligence-terminal' }}
|
||||
shell: bash
|
||||
run: |
|
||||
image="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${IMAGE_NAME}"
|
||||
|
||||
@@ -18,8 +18,8 @@ Intelligence Terminal is a Docker-first Crucix fork for home-server OSINT, marke
|
||||
- Unit tests: `npm run test:unit`
|
||||
- Full tests: `npm test`
|
||||
- Compose validation: `docker compose config`
|
||||
- Docker image: `docker build -t git.wilkensxl.de/mrsphay/intelligence-terminal:latest .`
|
||||
- Docker image: `docker build -t git.wilkensxl.de/code-inc/intelligence-terminal:latest .`
|
||||
|
||||
## Release Target
|
||||
|
||||
Push source to `https://git.wilkensxl.de/MrSphay/intelligence-terminal.git` and publish the Docker image to `git.wilkensxl.de/mrsphay/intelligence-terminal`.
|
||||
Push source to `https://git.wilkensxl.de/Code-Inc/intelligence-terminal.git` and publish the Docker image to `git.wilkensxl.de/code-inc/intelligence-terminal`.
|
||||
|
||||
105
README.md
105
README.md
@@ -4,8 +4,8 @@
|
||||
|
||||
**Self-hosted intelligence dashboard. 27 open sources. Docker-first. No telemetry.**
|
||||
|
||||
[](https://git.wilkensxl.de/MrSphay/intelligence-terminal)
|
||||
[](https://git.wilkensxl.de/MrSphay/-/packages/container/intelligence-terminal/latest)
|
||||
[](https://git.wilkensxl.de/Code-Inc/intelligence-terminal)
|
||||
[](https://git.wilkensxl.de/Code-Inc/-/packages/container/intelligence-terminal/latest)
|
||||
|
||||
[](#quick-start)
|
||||
[](LICENSE)
|
||||
@@ -32,7 +32,7 @@
|
||||
|
||||
> **Supported deployment:** private home-server or lab deployment through Docker, Dockge, Pangolin, or local Node.js.
|
||||
> Runtime data stays in your configured `runs/` volume and API keys are operator-owned.
|
||||
> **Source:** [git.wilkensxl.de/MrSphay/intelligence-terminal](https://git.wilkensxl.de/MrSphay/intelligence-terminal)
|
||||
> **Source:** [git.wilkensxl.de/Code-Inc/intelligence-terminal](https://git.wilkensxl.de/Code-Inc/intelligence-terminal)
|
||||
> Pull the image or clone the repository to run Intelligence Terminal on your own infrastructure.
|
||||
|
||||
Intelligence Terminal pulls satellite fire detection, flight tracking, radiation monitoring, satellite constellation tracking, economic indicators, live market prices, conflict data, sanctions lists, and social sentiment from 27 open-source intelligence feeds in parallel, every 15 minutes, and renders everything on a single self-contained dashboard.
|
||||
@@ -66,7 +66,7 @@ It was built for anyone who wants to understand what's actually happening in the
|
||||
|
||||
```bash
|
||||
# 1. Clone the repo
|
||||
git clone https://git.wilkensxl.de/MrSphay/intelligence-terminal.git
|
||||
git clone https://git.wilkensxl.de/Code-Inc/intelligence-terminal.git
|
||||
cd intelligence-terminal
|
||||
|
||||
# 2. Install dependencies (just Express)
|
||||
@@ -92,7 +92,7 @@ The dashboard opens automatically at `http://localhost:3117` and immediately beg
|
||||
### Docker
|
||||
|
||||
```bash
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
docker pull git.wilkensxl.de/code-inc/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
Dashboard at `http://localhost:3117`. Sweep data persists in `./runs/` via volume mount. The container disables browser auto-open by default, exposes `/api/health` and `/api/metrics`, and is suitable for Dockge/Pangolin.
|
||||
@@ -102,7 +102,7 @@ Dashboard at `http://localhost:3117`. Sweep data persists in `./runs/` via volum
|
||||
```yaml
|
||||
services:
|
||||
intelligence-terminal:
|
||||
image: git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
image: git.wilkensxl.de/code-inc/intelligence-terminal:latest
|
||||
container_name: intelligence-terminal
|
||||
env_file:
|
||||
- path: .env
|
||||
@@ -130,8 +130,13 @@ PORT=3117
|
||||
REFRESH_INTERVAL_MINUTES=15
|
||||
AUTO_OPEN_BROWSER=false
|
||||
STALE_DATA_MAX_AGE_MINUTES=60
|
||||
STALE_ALERT_COOLDOWN_MINUTES=60
|
||||
DASHBOARD_URL=https://intelligence.example.internal
|
||||
TERMINAL_ACTIONS_ENABLED=true
|
||||
SWEEP_TOKEN=
|
||||
SSE_HEARTBEAT_INTERVAL_MS=25000
|
||||
TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS=60000
|
||||
TERMINAL_ACTION_RATE_LIMIT_MAX=10
|
||||
BRIEF_VERBOSITY=standard
|
||||
|
||||
LLM_PROVIDER=openrouter
|
||||
@@ -141,7 +146,7 @@ LLM_MODEL=openrouter/free
|
||||
LLM_TEMPERATURE=0.2
|
||||
LLM_MAX_TOKENS=2000
|
||||
LLM_TIMEOUT_MS=90000
|
||||
OPENROUTER_SITE_URL=https://git.wilkensxl.de/MrSphay/intelligence-terminal
|
||||
OPENROUTER_SITE_URL=https://git.wilkensxl.de/Code-Inc/intelligence-terminal
|
||||
OPENROUTER_APP_NAME=Intelligence Terminal
|
||||
|
||||
FRED_API_KEY=
|
||||
@@ -164,6 +169,12 @@ DISCORD_WEBHOOK_URL=
|
||||
Local LLM examples:
|
||||
|
||||
```env
|
||||
# LiteLLM proxy (the URL must include the OpenAI-compatible /v1 path)
|
||||
LLM_PROVIDER=litellm
|
||||
LLM_BASE_URL=https://llm.example.com/v1
|
||||
LLM_API_KEY=your-litellm-api-key
|
||||
LLM_MODEL=your-model-alias
|
||||
|
||||
# LM Studio
|
||||
LLM_PROVIDER=lmstudio
|
||||
LLM_BASE_URL=http://host.docker.internal:1234/v1
|
||||
@@ -183,7 +194,68 @@ LLM_MODEL=your-model
|
||||
|
||||
For Pangolin or another reverse proxy, forward HTTP traffic to `intelligence-terminal:3117` (or the `PORT` you set). Missing API keys do not crash sweeps; affected sources are reported as degraded in `/api/health`.
|
||||
|
||||
The dashboard Terminal Actions panel can trigger `status`, `sweep`, and `brief` through `/api/action`. Leave `TERMINAL_ACTIONS_ENABLED=true` for a private home-server deployment. For an internet-exposed deployment, set `SWEEP_TOKEN` and pass it through trusted automation, or set `TERMINAL_ACTIONS_ENABLED=false` to disable browser-triggered actions. If you protect actions with `SWEEP_TOKEN`, the browser can send it from `localStorage.crucix_sweep_token`.
|
||||
#### Terminal Action Exposure
|
||||
|
||||
`POST /api/action` and `POST /api/sweep` can trigger operational actions such as manual sweeps. The dashboard has a **SET TOKEN** control that stores your `SWEEP_TOKEN` in browser local storage and sends it as the `x-crucix-token` header; do not put action tokens in URLs.
|
||||
|
||||
Recommended settings:
|
||||
|
||||
| Deployment | Settings |
|
||||
| --- | --- |
|
||||
| Private local machine | `NODE_ENV=development`, optional `SWEEP_TOKEN`, optional `TERMINAL_ACTIONS_ENABLED=true`. Localhost can run actions without a token for development. |
|
||||
| Private LAN / Dockge | Set a strong `SWEEP_TOKEN`, keep `TERMINAL_ACTIONS_ENABLED=true`, expose only to trusted clients. |
|
||||
| Pangolin-authenticated reverse proxy | Set a strong `SWEEP_TOKEN`, keep Pangolin auth in front, use the dashboard **SET TOKEN** flow once per browser. |
|
||||
| Public internet | Do not expose Terminal Actions directly. If exposure is unavoidable, require `SWEEP_TOKEN`, keep proxy authentication enabled, lower `TERMINAL_ACTION_RATE_LIMIT_MAX`, and monitor server audit logs. |
|
||||
|
||||
Action endpoints reject cross-origin POST origins, apply a small in-memory per-IP rate limit, and write sanitized audit lines without logging the token.
|
||||
|
||||
When data remains stale past `STALE_DATA_MAX_AGE_MINUTES`, the server sends an operator alert through configured Telegram/Discord channels after failed or degraded sweep attempts. `STALE_ALERT_COOLDOWN_MINUTES` prevents repeated stale alerts from spamming every refresh interval. Set `DASHBOARD_URL` to the Pangolin/public URL you want included in those alerts.
|
||||
|
||||
#### Memory And Prediction Loop
|
||||
|
||||
Crucix stores longitudinal memory in `runs/intelligence.db` when the current Node.js build exposes `node:sqlite`. If SQLite is unavailable, the file is created as a harmless placeholder and `/api/health` reports the memory store as unavailable instead of failing the sweep.
|
||||
|
||||
The memory layer persists:
|
||||
|
||||
| Table | Purpose |
|
||||
| --- | --- |
|
||||
| `runs` | Sweep timestamps, source health counts, and delta direction summaries. |
|
||||
| `entities` | Stable entity IDs for recurring countries, regions, and locations. |
|
||||
| `events` | Stable event IDs for conflict, OSINT, urgent news, and new delta signals across sweeps. |
|
||||
| `predictions` | Trade/intelligence hypotheses with evidence, confidence, horizon, outcome state, and latest grading. |
|
||||
|
||||
Query endpoints:
|
||||
|
||||
```text
|
||||
GET /api/memory/search?q=iran&limit=25
|
||||
GET /api/memory/predictions?state=open&limit=25
|
||||
```
|
||||
|
||||
Memory endpoints use the same operator authorization gate as Terminal Actions. The dashboard Terminal Actions panel includes a `Memory` action for a quick operator-facing view of recent events and prediction states.
|
||||
|
||||
Retention, backup, and privacy expectations:
|
||||
|
||||
- Treat `runs/intelligence.db` as operator data. It can contain source excerpts, headlines, generated hypotheses, and URLs from your configured feeds.
|
||||
- Back up `runs/` with the rest of your Dockge volume if you want longitudinal learning to survive container replacement.
|
||||
- Delete `runs/intelligence.db` to reset SQLite memory; the next sweep recreates the schema.
|
||||
- Do not commit `runs/` or `.env`. API credentials stay in `.env`; memory stores derived observations, not secrets.
|
||||
- If you expose the dashboard through a reverse proxy, protect Terminal Actions and memory queries behind your normal authentication boundary.
|
||||
|
||||
#### Reverse Proxy SSE
|
||||
|
||||
The dashboard receives live sweep updates from `GET /events` using Server-Sent Events. The server sends `retry: 10000` reconnect guidance and lightweight heartbeat comments every `SSE_HEARTBEAT_INTERVAL_MS` milliseconds so reverse proxies do not close an otherwise idle stream between 15-minute sweeps.
|
||||
|
||||
Recommended proxy settings:
|
||||
|
||||
| Proxy | Setting |
|
||||
| --- | --- |
|
||||
| Pangolin / Traefik-style frontends | Keep response streaming enabled and set idle timeouts above `SSE_HEARTBEAT_INTERVAL_MS`. |
|
||||
| Nginx | Disable proxy buffering for `/events`, keep `proxy_read_timeout` above the heartbeat interval, and preserve `Connection: keep-alive`. |
|
||||
| Cloudflare-style proxies | Keep the heartbeat below common idle cutoffs; the default 25s is intentionally conservative. |
|
||||
|
||||
If you raise the heartbeat interval, keep it shorter than the lowest idle timeout in the proxy chain.
|
||||
|
||||
`/api/metrics` includes network health grouped by host and source/provider. Source modules should use `safeFetch(url, { source: 'SourceName' })`; when omitted, the shared helper infers a stable provider bucket from the URL host instead of grouping normal source traffic under `unknown`. Raw fetch exceptions are documented in [Source Fetch Instrumentation](docs/source-fetch-instrumentation.md).
|
||||
|
||||
#### Scenario Watchlist
|
||||
|
||||
@@ -222,10 +294,10 @@ Scenario states are `dormant`, `watching`, `building`, and `confirmed`. The dash
|
||||
|
||||
```bash
|
||||
docker login git.wilkensxl.de -u MrSphay
|
||||
docker build -t git.wilkensxl.de/mrsphay/intelligence-terminal:latest .
|
||||
docker tag git.wilkensxl.de/mrsphay/intelligence-terminal:latest git.wilkensxl.de/mrsphay/intelligence-terminal:20260516
|
||||
docker push git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
docker push git.wilkensxl.de/mrsphay/intelligence-terminal:20260516
|
||||
docker build -t git.wilkensxl.de/code-inc/intelligence-terminal:latest .
|
||||
docker tag git.wilkensxl.de/code-inc/intelligence-terminal:latest git.wilkensxl.de/code-inc/intelligence-terminal:YYYYMMDD
|
||||
docker push git.wilkensxl.de/code-inc/intelligence-terminal:latest
|
||||
docker push git.wilkensxl.de/code-inc/intelligence-terminal:YYYYMMDD
|
||||
```
|
||||
|
||||
Gitea Actions publishes the same image automatically when the repository secret `REGISTRY_TOKEN` is set with package read/write permissions. The workflow tags images as `latest`, the commit SHA, and a UTC `YYYYMMDD` release tag.
|
||||
@@ -314,7 +386,7 @@ Alerts are delivered as rich embeds with color-coded sidebars: red for FLASH, ye
|
||||
Connect cloud or local OpenAI-compatible LLM providers for enhanced analysis:
|
||||
- **AI trade ideas** — quantitative analyst producing 5-8 actionable ideas citing specific data
|
||||
- **Smarter alert evaluation** — LLM classifies signals into FLASH/PRIORITY/ROUTINE tiers with cross-domain correlation and confidence scoring
|
||||
- Providers: OpenRouter, OpenAI-compatible APIs, LM Studio, Ollama, OpenAI, Anthropic Claude, Google Gemini, OpenAI Codex, MiniMax, Mistral, Grok
|
||||
- Providers: LiteLLM, OpenRouter, OpenAI-compatible APIs, LM Studio, Ollama, OpenAI, Anthropic Claude, Google Gemini, OpenAI Codex, MiniMax, Mistral, Grok
|
||||
- Graceful fallback — when LLM is unavailable, a rule-based engine takes over alert evaluation. LLM failures never crash the sweep cycle.
|
||||
|
||||
Primary env keys:
|
||||
@@ -556,7 +628,10 @@ All settings are in `.env` with sensible defaults:
|
||||
|----------|---------|-------------|
|
||||
| `PORT` | `3117` | Dashboard server port |
|
||||
| `REFRESH_INTERVAL_MINUTES` | `15` | Auto-refresh interval |
|
||||
| `LLM_PROVIDER` | disabled | `anthropic`, `openai`, `gemini`, `codex`, `openrouter`, `minimax`, `mistral`, or `grok` |
|
||||
| `STALE_DATA_MAX_AGE_MINUTES` | `60` | Data age threshold for stale health state |
|
||||
| `STALE_ALERT_COOLDOWN_MINUTES` | `60` | Minimum time between repeated operator stale-data alerts |
|
||||
| `DASHBOARD_URL` | local URL | Dashboard URL included in operator alerts |
|
||||
| `LLM_PROVIDER` | disabled | `litellm`, `openrouter`, `openai-compatible`, `lmstudio`, `ollama`, `anthropic`, `openai`, `gemini`, `codex`, `minimax`, `mistral`, or `grok` |
|
||||
| `LLM_API_KEY` | — | API key (not needed for codex) |
|
||||
| `LLM_MODEL` | per-provider default | Override model selection |
|
||||
| `TELEGRAM_BOT_TOKEN` | disabled | For Telegram alerts + bot commands |
|
||||
@@ -682,7 +757,7 @@ For contribution guidelines, review expectations, and source-add rules, see `CON
|
||||
|
||||
For bugs, feature requests, and integration ideas, use the Gitea issue tracker so discussion stays visible and actionable:
|
||||
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/issues
|
||||
|
||||
## Upstream And License
|
||||
|
||||
|
||||
@@ -59,7 +59,8 @@ export async function runSource(name, fn, ...args) {
|
||||
});
|
||||
const data = await Promise.race([dataPromise, timeoutPromise]);
|
||||
const hasError = Boolean(data?.error);
|
||||
const isDegraded = hasError || ['no_credentials', 'degraded', 'failed'].includes(data?.status);
|
||||
const degradedStatuses = ['no_credentials', 'no_key', 'disabled', 'degraded', 'failed', 'error'];
|
||||
const isDegraded = hasError || degradedStatuses.includes(data?.status);
|
||||
return {
|
||||
name,
|
||||
status: isDegraded ? 'degraded' : 'ok',
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
// ADS-B Exchange — Unfiltered Flight Tracking (including Military)
|
||||
// Unlike FlightRadar24/FlightAware, ADS-B Exchange does NOT filter military aircraft.
|
||||
// Public feed access varies; RapidAPI tier available for programmatic use.
|
||||
// This module attempts the public endpoints and falls back to a documented stub.
|
||||
// This module reports explicit disabled/degraded state instead of making
|
||||
// unavailable aircraft data look live.
|
||||
|
||||
import { safeFetch } from '../utils/fetch.mjs';
|
||||
|
||||
@@ -140,6 +141,7 @@ async function fetchViaRapidApi(apiKey) {
|
||||
// Get all military aircraft
|
||||
const data = await safeFetch(`${ENDPOINTS.rapidApi}/mil`, {
|
||||
timeout: 20000,
|
||||
source: 'adsb-rapidapi',
|
||||
headers: {
|
||||
'X-RapidAPI-Key': apiKey,
|
||||
'X-RapidAPI-Host': 'adsbexchange-com1.p.rapidapi.com',
|
||||
@@ -151,21 +153,26 @@ async function fetchViaRapidApi(apiKey) {
|
||||
|
||||
// Attempt to fetch from public feed
|
||||
async function fetchPublicFeed() {
|
||||
const data = await safeFetch(ENDPOINTS.publicFeed, { timeout: 15000 });
|
||||
const data = await safeFetch(ENDPOINTS.publicFeed, { timeout: 15000, source: 'adsb-public' });
|
||||
return data;
|
||||
}
|
||||
|
||||
// Get military aircraft from available sources
|
||||
export async function getMilitaryAircraft(apiKey) {
|
||||
async function getMilitaryAircraftResult(apiKey) {
|
||||
const failures = [];
|
||||
|
||||
// Try RapidAPI first if key available
|
||||
if (apiKey) {
|
||||
const data = await fetchViaRapidApi(apiKey);
|
||||
if (data && !data.error) {
|
||||
const aircraft = data.ac || data.aircraft || [];
|
||||
if (Array.isArray(aircraft)) {
|
||||
return aircraft.map(classifyAircraft).filter(a => a.isMilitary);
|
||||
return {
|
||||
provider: 'rapidapi',
|
||||
aircraft: aircraft.map(classifyAircraft).filter(a => a.isMilitary),
|
||||
};
|
||||
}
|
||||
}
|
||||
failures.push({ provider: 'rapidapi', error: data?.error || 'RapidAPI returned an unsupported payload' });
|
||||
}
|
||||
|
||||
// Try public feed
|
||||
@@ -173,11 +180,21 @@ export async function getMilitaryAircraft(apiKey) {
|
||||
if (pubData && !pubData.error) {
|
||||
const aircraft = pubData.ac || pubData.aircraft || pubData.states || [];
|
||||
if (Array.isArray(aircraft)) {
|
||||
return aircraft.map(classifyAircraft).filter(a => a.isMilitary);
|
||||
return {
|
||||
provider: 'public-feed',
|
||||
aircraft: aircraft.map(classifyAircraft).filter(a => a.isMilitary),
|
||||
};
|
||||
}
|
||||
}
|
||||
failures.push({ provider: 'public-feed', error: pubData?.error || 'Public feed returned an unsupported payload' });
|
||||
|
||||
return null; // all sources failed
|
||||
return { provider: null, aircraft: null, failures };
|
||||
}
|
||||
|
||||
// Get military aircraft from available sources
|
||||
export async function getMilitaryAircraft(apiKey) {
|
||||
const result = await getMilitaryAircraftResult(apiKey);
|
||||
return result.aircraft;
|
||||
}
|
||||
|
||||
// Get all aircraft in a geographic bounding box via RapidAPI
|
||||
@@ -208,7 +225,8 @@ export async function getAircraftInArea(lat, lon, radiusNm = 250, apiKey) {
|
||||
// Briefing — attempt to get military flight data, document what's available
|
||||
export async function briefing() {
|
||||
const apiKey = process.env.ADSB_API_KEY || process.env.RAPIDAPI_KEY || null;
|
||||
const militaryAircraft = await getMilitaryAircraft(apiKey);
|
||||
const result = await getMilitaryAircraftResult(apiKey);
|
||||
const militaryAircraft = result.aircraft;
|
||||
|
||||
// If we got data, analyze it
|
||||
if (militaryAircraft && militaryAircraft.length > 0) {
|
||||
@@ -255,6 +273,7 @@ export async function briefing() {
|
||||
source: 'ADS-B Exchange',
|
||||
timestamp: new Date().toISOString(),
|
||||
status: 'live',
|
||||
provider: result.provider,
|
||||
totalMilitary: militaryAircraft.length,
|
||||
byCountry,
|
||||
categories: {
|
||||
@@ -269,10 +288,18 @@ export async function briefing() {
|
||||
}
|
||||
|
||||
// No data available — return stub with integration documentation
|
||||
const status = apiKey ? 'degraded' : 'disabled';
|
||||
const error = apiKey
|
||||
? 'ADS-B providers returned no usable aircraft data'
|
||||
: 'ADSB_API_KEY or RAPIDAPI_KEY is not configured';
|
||||
|
||||
return {
|
||||
source: 'ADS-B Exchange',
|
||||
timestamp: new Date().toISOString(),
|
||||
status: apiKey ? 'error' : 'no_key',
|
||||
status,
|
||||
provider: result.provider,
|
||||
error,
|
||||
failures: result.failures,
|
||||
militaryAircraft: [],
|
||||
message: apiKey
|
||||
? 'ADS-B Exchange API returned no data. The endpoint may be temporarily unavailable.'
|
||||
|
||||
@@ -10,6 +10,44 @@ const fetchMetrics = {
|
||||
recent: [],
|
||||
};
|
||||
|
||||
const SOURCE_BY_HOST = [
|
||||
[/api\.bls\.gov$/i, 'BLS'],
|
||||
[/api\.fred\.stlouisfed\.org$/i, 'FRED'],
|
||||
[/api\.eia\.gov$/i, 'EIA'],
|
||||
[/api\.gdeltproject\.org$/i, 'GDELT'],
|
||||
[/api\.weather\.gov$/i, 'NOAA'],
|
||||
[/api\.open-notify\.org$/i, 'OpenNotify'],
|
||||
[/opensky-network\.org$/i, 'OpenSky'],
|
||||
[/firms\.modaps\.eosdis\.nasa\.gov$/i, 'FIRMS'],
|
||||
[/api\.acleddata\.com$/i, 'ACLED'],
|
||||
[/api\.reliefweb\.int$/i, 'ReliefWeb'],
|
||||
[/receiverbook\.de$/i, 'KiwiSDR'],
|
||||
[/safecast\.org$/i, 'Safecast'],
|
||||
[/api\.patentsview\.org$/i, 'PatentsView'],
|
||||
[/api\.trade\.gov$/i, 'Comtrade'],
|
||||
[/api\.usaspending\.gov$/i, 'USASpending'],
|
||||
[/api\.telegram\.org$/i, 'Telegram'],
|
||||
[/oauth\.reddit\.com$/i, 'Reddit'],
|
||||
[/reddit\.com$/i, 'Reddit'],
|
||||
[/api\.bsky\.app$/i, 'Bluesky'],
|
||||
[/api\.yahoo\.com$/i, 'YahooFinance'],
|
||||
[/query\d?\.finance\.yahoo\.com$/i, 'YahooFinance'],
|
||||
[/api\.cloudflare\.com$/i, 'CloudflareRadar'],
|
||||
[/api\.opensanctions\.org$/i, 'OpenSanctions'],
|
||||
[/home\.treasury\.gov$/i, 'Treasury'],
|
||||
[/fiscaldata\.treasury\.gov$/i, 'Treasury'],
|
||||
[/who\.int$/i, 'WHO'],
|
||||
];
|
||||
|
||||
export function inferFetchSource(url) {
|
||||
let host = 'unknown';
|
||||
try { host = new URL(url).host.toLowerCase(); } catch { return 'unknown'; }
|
||||
for (const [pattern, source] of SOURCE_BY_HOST) {
|
||||
if (pattern.test(host)) return source;
|
||||
}
|
||||
return host;
|
||||
}
|
||||
|
||||
function metricBucket(map, key) {
|
||||
if (!map[key]) map[key] = { requests: 0, ok: 0, failed: 0, bytes: 0, lastStatus: null, lastError: null, lastMs: 0 };
|
||||
return map[key];
|
||||
@@ -38,7 +76,7 @@ export function getFetchMetrics() {
|
||||
}
|
||||
|
||||
export async function safeFetch(url, opts = {}) {
|
||||
const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts;
|
||||
const { timeout = 15000, retries = 1, headers = {}, source = inferFetchSource(url) } = opts;
|
||||
let lastError;
|
||||
for (let i = 0; i <= retries; i++) {
|
||||
const started = Date.now();
|
||||
@@ -79,11 +117,11 @@ export async function safeFetch(url, opts = {}) {
|
||||
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
|
||||
}
|
||||
}
|
||||
return { error: lastError?.message || 'Unknown error', source: url };
|
||||
return { error: lastError?.message || 'Unknown error', source };
|
||||
}
|
||||
|
||||
export async function safeFetchText(url, opts = {}) {
|
||||
const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts;
|
||||
const { timeout = 15000, retries = 1, headers = {}, source = inferFetchSource(url) } = opts;
|
||||
let lastError;
|
||||
for (let i = 0; i <= retries; i++) {
|
||||
const started = Date.now();
|
||||
|
||||
@@ -23,18 +23,23 @@ export default {
|
||||
refreshIntervalMinutes: intEnv('REFRESH_INTERVAL_MINUTES', 15),
|
||||
autoOpenBrowser: boolEnv('AUTO_OPEN_BROWSER', false),
|
||||
staleDataMaxAgeMinutes: intEnv('STALE_DATA_MAX_AGE_MINUTES', 60),
|
||||
staleAlertCooldownMinutes: intEnv('STALE_ALERT_COOLDOWN_MINUTES', 60),
|
||||
dashboardUrl: process.env.DASHBOARD_URL || null,
|
||||
sweepToken: process.env.SWEEP_TOKEN || null,
|
||||
terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', true),
|
||||
terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', !!process.env.SWEEP_TOKEN || process.env.NODE_ENV !== 'production'),
|
||||
terminalActionRateLimitWindowMs: intEnv('TERMINAL_ACTION_RATE_LIMIT_WINDOW_MS', 60_000),
|
||||
terminalActionRateLimitMax: intEnv('TERMINAL_ACTION_RATE_LIMIT_MAX', 10),
|
||||
sseHeartbeatIntervalMs: intEnv('SSE_HEARTBEAT_INTERVAL_MS', 25000),
|
||||
|
||||
llm: {
|
||||
provider: process.env.LLM_PROVIDER || null, // anthropic | openai | gemini | codex | openrouter | minimax | mistral | ollama | grok
|
||||
provider: process.env.LLM_PROVIDER || null, // litellm | openrouter | openai-compatible | lmstudio | ollama | other supported providers
|
||||
apiKey: process.env.LLM_API_KEY || null,
|
||||
model: process.env.LLM_MODEL || null,
|
||||
baseUrl: process.env.LLM_BASE_URL || process.env.OPENAI_BASE_URL || process.env.OLLAMA_BASE_URL || null,
|
||||
temperature: floatEnv('LLM_TEMPERATURE', 0.2),
|
||||
maxTokens: intEnv('LLM_MAX_TOKENS', 2000),
|
||||
timeoutMs: intEnv('LLM_TIMEOUT_MS', 90000),
|
||||
openRouterSiteUrl: process.env.OPENROUTER_SITE_URL || 'https://git.wilkensxl.de/MrSphay/intelligence-terminal',
|
||||
openRouterSiteUrl: process.env.OPENROUTER_SITE_URL || 'https://git.wilkensxl.de/Code-Inc/intelligence-terminal',
|
||||
openRouterAppName: process.env.OPENROUTER_APP_NAME || 'Intelligence Terminal',
|
||||
},
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
intelligence-terminal:
|
||||
image: git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
image: git.wilkensxl.de/code-inc/intelligence-terminal:latest
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
|
||||
@@ -1,6 +1,18 @@
|
||||
# Agent Handoff
|
||||
|
||||
Last updated: 2026-05-17
|
||||
Last updated: 2026-07-03
|
||||
|
||||
## Latest Completed Work
|
||||
|
||||
- Canonical repository: `https://git.wilkensxl.de/Code-Inc/intelligence-terminal`
|
||||
- LiteLLM implementation merge: `5c4bf80eb0c19bd59080f5432a2a344798d7a3ce`
|
||||
- Merged PR: `#48 feat: add LiteLLM provider and publish Code-Inc image`
|
||||
- Completed issue: `#47 Add first-class LiteLLM provider and publish updated image`
|
||||
- LiteLLM is implemented through the OpenAI-compatible `/chat/completions` API and requires `LLM_BASE_URL`, `LLM_API_KEY`, and `LLM_MODEL`.
|
||||
- The build workflow now targets `git.wilkensxl.de/code-inc/intelligence-terminal` and publishes only from the production branch, not from pull requests.
|
||||
- Gitea Actions runs 231-235 passed for the PR and production merge, including unit tests, Compose validation, Docker build, release dry-run, and template compliance.
|
||||
- The first `code-inc` registry publication was verified through the Gitea Package API on 2026-07-03.
|
||||
- Related maintenance: issue #21 tracks the failing security scan, #45 tracks the dependency workflow, and #46 tracks remaining namespace/handoff cleanup.
|
||||
|
||||
## Repository State
|
||||
|
||||
@@ -15,7 +27,7 @@ C:\Users\MrSphay\Documents\Codex\Crucix\intelligence-terminal
|
||||
Remotes:
|
||||
|
||||
```text
|
||||
origin https://git.wilkensxl.de/MrSphay/intelligence-terminal.git
|
||||
origin https://git.wilkensxl.de/Code-Inc/intelligence-terminal.git
|
||||
upstream https://github.com/calesthio/Crucix.git
|
||||
```
|
||||
|
||||
@@ -25,23 +37,22 @@ Current branch tip:
|
||||
Run `git rev-parse HEAD` after clone/pull. This handoff was updated by the `docs: sync issue tracker and handoff` commit after the implementation commit below.
|
||||
```
|
||||
|
||||
Latest implementation commit before issue-sync documentation:
|
||||
Production baseline before the current LiteLLM work:
|
||||
|
||||
```text
|
||||
53470cc701ec322080a89d220aef449b25850590
|
||||
c159c83a0768486c8c6f445b458b760dba4ba385
|
||||
```
|
||||
|
||||
Both pushed branches currently point to this commit:
|
||||
The default production branch points to this commit before the current PR:
|
||||
|
||||
```text
|
||||
origin/codex/production-intelligence-terminal
|
||||
origin/main
|
||||
```
|
||||
|
||||
Gitea repository:
|
||||
|
||||
```text
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal
|
||||
```
|
||||
|
||||
Default branch observed through the Gitea API:
|
||||
@@ -79,7 +90,7 @@ Rules applied from the kit:
|
||||
- `docker-compose.yml` uses the Gitea Registry image by default:
|
||||
|
||||
```text
|
||||
git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
git.wilkensxl.de/code-inc/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
### API And Health
|
||||
@@ -226,31 +237,40 @@ README includes:
|
||||
|
||||
## Registry And Images
|
||||
|
||||
Registry image:
|
||||
Production registry image:
|
||||
|
||||
```text
|
||||
git.wilkensxl.de/mrsphay/intelligence-terminal
|
||||
git.wilkensxl.de/code-inc/intelligence-terminal
|
||||
```
|
||||
|
||||
Verified package tags through Gitea API:
|
||||
The legacy `mrsphay` package remains available. Verified `code-inc` tags from the LiteLLM production merge:
|
||||
|
||||
```text
|
||||
latest
|
||||
20260517
|
||||
e933586b220656a2858d2215b934b22d1f08a908
|
||||
53470cc701ec322080a89d220aef449b25850590
|
||||
20260703
|
||||
5c4bf80eb0c19bd59080f5432a2a344798d7a3ce
|
||||
```
|
||||
|
||||
Successful pull test:
|
||||
Operator pull command:
|
||||
|
||||
```bash
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
docker pull git.wilkensxl.de/code-inc/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
Observed digest:
|
||||
Published manifest digest:
|
||||
|
||||
```text
|
||||
sha256:780a41413921bd9a676461eca1cd1372591f523be4b7c9513d9bc085cbe7922d
|
||||
sha256:5e29483ebfd9baae368673adc790789f02aed2d5d5d3a550fe55a4b71b5b62dd
|
||||
```
|
||||
|
||||
Relevant successful runner executions:
|
||||
|
||||
```text
|
||||
PR build: https://git.wilkensxl.de/Code-Inc/intelligence-terminal/actions/runs/231
|
||||
PR template check: https://git.wilkensxl.de/Code-Inc/intelligence-terminal/actions/runs/232
|
||||
Production publish: https://git.wilkensxl.de/Code-Inc/intelligence-terminal/actions/runs/233
|
||||
Release dry-run: https://git.wilkensxl.de/Code-Inc/intelligence-terminal/actions/runs/234
|
||||
Template compliance: https://git.wilkensxl.de/Code-Inc/intelligence-terminal/actions/runs/235
|
||||
```
|
||||
|
||||
## Gitea Actions
|
||||
@@ -280,12 +300,12 @@ template-compliance.yml on codex/production-intelligence-terminal: success
|
||||
Relevant run URLs:
|
||||
|
||||
```text
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/23
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/24
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/25
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/26
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/27
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/28
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/actions/runs/23
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/actions/runs/24
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/actions/runs/25
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/actions/runs/26
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/actions/runs/27
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/actions/runs/28
|
||||
```
|
||||
|
||||
Repository secret expected by the registry publish workflow:
|
||||
@@ -315,28 +335,28 @@ The following Gitea issues were created for real remaining work:
|
||||
|
||||
```text
|
||||
#1 Reddit source must stop unauthenticated .json scraping
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/1
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/issues/1
|
||||
|
||||
#2 Send operator alerts when dashboard data remains stale
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/2
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/issues/2
|
||||
|
||||
#3 ACLED credentialed integration needs regression test and diagnostics
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/3
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/issues/3
|
||||
|
||||
#4 Complete memory and prediction loop beyond Phase-1 SQLite
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/4
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/issues/4
|
||||
|
||||
#5 Remove old inline dashboard snapshot from production builds
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/5
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/issues/5
|
||||
|
||||
#6 Harden Terminal Actions for public reverse-proxy deployments
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/6
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/issues/6
|
||||
|
||||
#7 Replace ADS-B stub with real disabled/degraded source handling
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/7
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/issues/7
|
||||
|
||||
#8 Clean inherited public-demo and upstream marketing references
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/8
|
||||
https://git.wilkensxl.de/Code-Inc/intelligence-terminal/issues/8
|
||||
```
|
||||
|
||||
## Verification Already Performed
|
||||
@@ -366,7 +386,7 @@ Audit result:
|
||||
0 high vulnerabilities
|
||||
```
|
||||
|
||||
Docker build and smoke test were performed locally earlier:
|
||||
Docker build and smoke test were performed locally earlier against the now-legacy `mrsphay` image namespace. Current deployments must use the `code-inc` image documented above:
|
||||
|
||||
```bash
|
||||
docker build -t git.wilkensxl.de/mrsphay/intelligence-terminal:latest .
|
||||
@@ -420,7 +440,7 @@ origin/main
|
||||
1. Clone the Gitea repository:
|
||||
|
||||
```bash
|
||||
git clone https://git.wilkensxl.de/MrSphay/intelligence-terminal.git
|
||||
git clone https://git.wilkensxl.de/Code-Inc/intelligence-terminal.git
|
||||
cd intelligence-terminal
|
||||
git checkout codex/production-intelligence-terminal
|
||||
```
|
||||
@@ -460,7 +480,7 @@ if ($env:GITEA_TOKEN) { "GITEA_TOKEN=set" } else { "GITEA_TOKEN=missing" }
|
||||
```bash
|
||||
npm run test:unit
|
||||
docker compose --env-file .env.example config
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
docker pull git.wilkensxl.de/code-inc/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
6. Start with Dockge/Pangolin using the README compose example and a `.env` based on `.env.example`.
|
||||
@@ -479,11 +499,11 @@ docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
For deployment:
|
||||
|
||||
```bash
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
docker pull git.wilkensxl.de/code-inc/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
For a pinned deployment:
|
||||
|
||||
```bash
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:20260517
|
||||
docker pull git.wilkensxl.de/code-inc/intelligence-terminal:20260703
|
||||
```
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
1. Confirm `.env.example`, README compose sample, and registry image name match.
|
||||
2. Run `npm run test:unit`.
|
||||
3. Run `docker compose config`.
|
||||
4. Build `git.wilkensxl.de/mrsphay/intelligence-terminal:latest`.
|
||||
4. Build `git.wilkensxl.de/code-inc/intelligence-terminal:latest`.
|
||||
5. Start the image and verify `/api/health`.
|
||||
6. Push branch to Gitea.
|
||||
7. Push `latest` and a dated image tag to the Gitea Registry.
|
||||
|
||||
21
docs/source-fetch-instrumentation.md
Normal file
21
docs/source-fetch-instrumentation.md
Normal file
@@ -0,0 +1,21 @@
|
||||
# Source Fetch Instrumentation
|
||||
|
||||
`safeFetch()` and `safeFetchText()` attribute requests to `/api/metrics.fetch.bySource`.
|
||||
|
||||
Rules:
|
||||
|
||||
- Prefer passing an explicit `source` option from source modules when the call has a clear Crucix source name.
|
||||
- If `source` is omitted, the shared helper infers a stable provider name from the request host.
|
||||
- Unknown hosts fall back to the lowercase host instead of the old `unknown` bucket.
|
||||
- Raw `fetch()` calls should be limited to cases where the shared helper cannot represent the protocol cleanly.
|
||||
|
||||
Current raw-fetch exceptions:
|
||||
|
||||
| Area | Reason |
|
||||
| --- | --- |
|
||||
| OAuth/session handshakes | Token exchange calls often need custom form bodies, credential headers, or status-specific diagnostics. |
|
||||
| Bot and alert delivery | Telegram/Discord alert calls are outbound operator notifications, not intelligence source health. |
|
||||
| LLM providers | Provider clients already track model/provider status separately from source fetch health. |
|
||||
| Dashboard browser calls | Browser-side `/api/*` and asset fetches are UI behavior, not source provider health. |
|
||||
|
||||
When adding a new intelligence source, use `safeFetch(url, { source: 'SourceName' })` unless there is a documented exception.
|
||||
@@ -16,4 +16,5 @@ Source docs:
|
||||
- [Telegram](telegram.md)
|
||||
- [FIRMS](firms.md)
|
||||
- [Maritime](maritime.md)
|
||||
- [ADS-B](adsb.md)
|
||||
- [Reddit](reddit.md)
|
||||
|
||||
24
docs/sources/adsb.md
Normal file
24
docs/sources/adsb.md
Normal file
@@ -0,0 +1,24 @@
|
||||
# ADS-B Source
|
||||
|
||||
ADS-B Exchange support is optional and intended for unfiltered aircraft and military-flight awareness.
|
||||
|
||||
- Source module: `apis/sources/adsb.mjs`
|
||||
- Preferred provider: ADS-B Exchange via RapidAPI
|
||||
- Credentials: `ADSB_API_KEY` or `RAPIDAPI_KEY`
|
||||
- Runtime status without credentials: `disabled`
|
||||
- Runtime status when providers fail: `degraded`
|
||||
- Runtime status with usable aircraft payloads: `live`
|
||||
|
||||
The source does not treat a missing key or unavailable public feed as normal live data. `/api/health` and `/api/metrics` surface the degraded source state through the sweep source summary.
|
||||
|
||||
Known failure modes:
|
||||
|
||||
- Missing `ADSB_API_KEY` / `RAPIDAPI_KEY`: source is disabled with operator guidance.
|
||||
- RapidAPI rejects or rate-limits the request: source is degraded and records provider failure detail.
|
||||
- Public feed is blocked, rate-limited, or changes shape: source remains degraded instead of returning stale-looking data.
|
||||
|
||||
Register for the provider documented in the README, then set:
|
||||
|
||||
```env
|
||||
ADSB_API_KEY=<rapidapi-key>
|
||||
```
|
||||
@@ -2,6 +2,9 @@
|
||||
|
||||
import { existsSync, mkdirSync, writeFileSync } from 'fs';
|
||||
import { dirname, join } from 'path';
|
||||
import { createHash } from 'crypto';
|
||||
|
||||
const PREDICTION_STATES = new Set(['open', 'monitoring', 'observed', 'expired_unverified', 'invalidated']);
|
||||
|
||||
export class IntelligenceStore {
|
||||
constructor(dbPath) {
|
||||
@@ -30,15 +33,24 @@ export class IntelligenceStore {
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS predictions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
stable_id TEXT UNIQUE,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT,
|
||||
title TEXT NOT NULL,
|
||||
type TEXT,
|
||||
hypothesis TEXT,
|
||||
evidence_json TEXT,
|
||||
confidence TEXT,
|
||||
horizon TEXT,
|
||||
outcome_state TEXT DEFAULT 'open',
|
||||
outcome_json TEXT,
|
||||
last_evaluated_at TEXT,
|
||||
source TEXT,
|
||||
payload_json TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS entities (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
stable_id TEXT UNIQUE,
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
@@ -46,7 +58,21 @@ export class IntelligenceStore {
|
||||
count INTEGER DEFAULT 1,
|
||||
UNIQUE(name, kind)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
stable_id TEXT NOT NULL UNIQUE,
|
||||
first_seen TEXT NOT NULL,
|
||||
last_seen TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
region TEXT,
|
||||
severity TEXT,
|
||||
source TEXT,
|
||||
evidence_json TEXT NOT NULL,
|
||||
count INTEGER DEFAULT 1
|
||||
);
|
||||
`);
|
||||
this._migrate();
|
||||
this.available = true;
|
||||
} catch (err) {
|
||||
this.available = false;
|
||||
@@ -71,24 +97,141 @@ export class IntelligenceStore {
|
||||
delta?.summary?.direction || null,
|
||||
JSON.stringify({ meta, delta: delta?.summary || null }),
|
||||
);
|
||||
for (const idea of data.ideas || []) {
|
||||
this.db.prepare(`INSERT INTO predictions (created_at, title, type, confidence, source, payload_json)
|
||||
VALUES (?, ?, ?, ?, ?, ?)`).run(
|
||||
timestamp,
|
||||
idea.title || 'Untitled idea',
|
||||
idea.type || null,
|
||||
idea.confidence || null,
|
||||
idea.source || data.ideasSource || null,
|
||||
JSON.stringify(idea),
|
||||
);
|
||||
}
|
||||
this._recordEntities(data, timestamp);
|
||||
this._recordEvents(data, delta, timestamp);
|
||||
this.evaluatePredictions(data, timestamp);
|
||||
this._recordPredictions(data, timestamp);
|
||||
}
|
||||
|
||||
status() {
|
||||
return { available: this.available, path: this.dbPath, reason: this.reason };
|
||||
}
|
||||
|
||||
queryMemory({ q = '', limit = 25 } = {}) {
|
||||
if (!this.available || !this.db) return { available: false, reason: this.reason, results: [] };
|
||||
const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
|
||||
const term = String(q || '').trim();
|
||||
const like = `%${term}%`;
|
||||
const where = term
|
||||
? 'WHERE name LIKE ? OR region LIKE ? OR source LIKE ? OR kind LIKE ?'
|
||||
: '';
|
||||
const params = term ? [like, like, like, like, safeLimit] : [safeLimit];
|
||||
const events = this.db.prepare(`
|
||||
SELECT stable_id, first_seen, last_seen, kind, name, region, severity, source, count, evidence_json
|
||||
FROM events
|
||||
${where}
|
||||
ORDER BY last_seen DESC
|
||||
LIMIT ?
|
||||
`).all(...params).map(row => ({ ...row, evidence: parseJson(row.evidence_json, {}) }));
|
||||
return { available: true, q: term, results: events };
|
||||
}
|
||||
|
||||
listPredictions({ state = null, limit = 25 } = {}) {
|
||||
if (!this.available || !this.db) return { available: false, reason: this.reason, predictions: [] };
|
||||
const safeLimit = Math.max(1, Math.min(100, Number(limit) || 25));
|
||||
const normalizedState = state && PREDICTION_STATES.has(String(state)) ? String(state) : null;
|
||||
const rows = normalizedState
|
||||
? this.db.prepare(`SELECT * FROM predictions WHERE outcome_state = ? ORDER BY created_at DESC LIMIT ?`).all(normalizedState, safeLimit)
|
||||
: this.db.prepare(`SELECT * FROM predictions ORDER BY created_at DESC LIMIT ?`).all(safeLimit);
|
||||
return {
|
||||
available: true,
|
||||
predictions: rows.map(row => ({
|
||||
stable_id: row.stable_id,
|
||||
created_at: row.created_at,
|
||||
updated_at: row.updated_at,
|
||||
title: row.title,
|
||||
type: row.type,
|
||||
hypothesis: row.hypothesis,
|
||||
confidence: row.confidence,
|
||||
horizon: row.horizon,
|
||||
outcome_state: row.outcome_state,
|
||||
last_evaluated_at: row.last_evaluated_at,
|
||||
source: row.source,
|
||||
evidence: parseJson(row.evidence_json, []),
|
||||
outcome: parseJson(row.outcome_json, null),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
evaluatePredictions(data, timestamp = new Date().toISOString()) {
|
||||
if (!this.available || !this.db) return;
|
||||
const rows = this.db.prepare(`
|
||||
SELECT id, created_at, title, type, horizon, outcome_state, payload_json
|
||||
FROM predictions
|
||||
WHERE outcome_state IN ('open', 'monitoring')
|
||||
ORDER BY created_at ASC
|
||||
LIMIT 200
|
||||
`).all();
|
||||
for (const row of rows) {
|
||||
const payload = parseJson(row.payload_json, {});
|
||||
const evaluation = evaluatePredictionAgainstSweep(row, payload, data, timestamp);
|
||||
this.db.prepare(`UPDATE predictions
|
||||
SET outcome_state = ?, outcome_json = ?, last_evaluated_at = ?, updated_at = ?
|
||||
WHERE id = ?`).run(
|
||||
evaluation.state,
|
||||
JSON.stringify(evaluation),
|
||||
timestamp,
|
||||
timestamp,
|
||||
row.id,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
_migrate() {
|
||||
const columns = {
|
||||
predictions: [
|
||||
['stable_id', 'TEXT'],
|
||||
['updated_at', 'TEXT'],
|
||||
['hypothesis', 'TEXT'],
|
||||
['evidence_json', 'TEXT'],
|
||||
['horizon', 'TEXT'],
|
||||
['outcome_state', "TEXT DEFAULT 'open'"],
|
||||
['outcome_json', 'TEXT'],
|
||||
['last_evaluated_at', 'TEXT'],
|
||||
],
|
||||
entities: [
|
||||
['stable_id', 'TEXT'],
|
||||
],
|
||||
};
|
||||
for (const [table, defs] of Object.entries(columns)) {
|
||||
for (const [name, type] of defs) {
|
||||
try { this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${name} ${type}`); } catch { }
|
||||
}
|
||||
}
|
||||
try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_predictions_stable_id ON predictions(stable_id)`); } catch { }
|
||||
try { this.db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_entities_stable_id ON entities(stable_id)`); } catch { }
|
||||
}
|
||||
|
||||
_recordPredictions(data, timestamp) {
|
||||
for (const idea of data.ideas || []) {
|
||||
const title = idea.title || 'Untitled idea';
|
||||
const stableId = stableId('prediction', title, idea.type || '', idea.ticker || '', idea.horizon || '');
|
||||
const evidence = Array.isArray(idea.signals) ? idea.signals : [];
|
||||
this.db.prepare(`INSERT INTO predictions (
|
||||
stable_id, created_at, updated_at, title, type, hypothesis, evidence_json, confidence,
|
||||
horizon, outcome_state, source, payload_json
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'open', ?, ?)
|
||||
ON CONFLICT(stable_id) DO UPDATE SET
|
||||
updated_at=excluded.updated_at,
|
||||
confidence=excluded.confidence,
|
||||
evidence_json=excluded.evidence_json,
|
||||
payload_json=excluded.payload_json`).run(
|
||||
stableId,
|
||||
timestamp,
|
||||
timestamp,
|
||||
title,
|
||||
idea.type || null,
|
||||
idea.rationale || idea.text || title,
|
||||
JSON.stringify(evidence),
|
||||
idea.confidence || null,
|
||||
idea.horizon || null,
|
||||
idea.source || data.ideasSource || null,
|
||||
JSON.stringify(idea),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
_recordEntities(data, timestamp) {
|
||||
const names = [];
|
||||
for (const item of data.acled?.deadliestEvents || []) {
|
||||
@@ -99,14 +242,154 @@ export class IntelligenceStore {
|
||||
if (item.region) names.push([item.region, 'region']);
|
||||
}
|
||||
for (const [name, kind] of names.slice(0, 200)) {
|
||||
this.db.prepare(`INSERT INTO entities (first_seen, last_seen, name, kind, count)
|
||||
VALUES (?, ?, ?, ?, 1)
|
||||
const cleanName = String(name).slice(0, 160);
|
||||
this.db.prepare(`INSERT INTO entities (stable_id, first_seen, last_seen, name, kind, count)
|
||||
VALUES (?, ?, ?, ?, ?, 1)
|
||||
ON CONFLICT(name, kind) DO UPDATE SET last_seen=excluded.last_seen, count=count+1`).run(
|
||||
stableId('entity', kind, cleanName),
|
||||
timestamp,
|
||||
timestamp,
|
||||
String(name).slice(0, 160),
|
||||
cleanName,
|
||||
kind,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
_recordEvents(data, delta, timestamp) {
|
||||
const events = extractEvents(data, delta);
|
||||
for (const event of events.slice(0, 300)) {
|
||||
this.db.prepare(`INSERT INTO events (
|
||||
stable_id, first_seen, last_seen, kind, name, region, severity, source, evidence_json, count
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1)
|
||||
ON CONFLICT(stable_id) DO UPDATE SET
|
||||
last_seen=excluded.last_seen,
|
||||
severity=COALESCE(excluded.severity, severity),
|
||||
evidence_json=excluded.evidence_json,
|
||||
count=count+1`).run(
|
||||
event.stable_id,
|
||||
timestamp,
|
||||
timestamp,
|
||||
event.kind,
|
||||
event.name,
|
||||
event.region || null,
|
||||
event.severity || null,
|
||||
event.source || null,
|
||||
JSON.stringify(event.evidence || {}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function stableId(...parts) {
|
||||
const input = parts.map(part => String(part || '').trim().toLowerCase()).join('|');
|
||||
return createHash('sha256').update(input).digest('hex').slice(0, 24);
|
||||
}
|
||||
|
||||
function parseJson(value, fallback) {
|
||||
try { return value ? JSON.parse(value) : fallback; } catch { return fallback; }
|
||||
}
|
||||
|
||||
function extractEvents(data, delta) {
|
||||
const events = [];
|
||||
const push = ({ kind, name, region, severity, source, evidence }) => {
|
||||
if (!kind || !name) return;
|
||||
events.push({
|
||||
stable_id: stableId('event', kind, name, region || source || ''),
|
||||
kind,
|
||||
name: String(name).slice(0, 240),
|
||||
region: region ? String(region).slice(0, 120) : null,
|
||||
severity: severity || null,
|
||||
source: source || null,
|
||||
evidence: evidence || {},
|
||||
});
|
||||
};
|
||||
|
||||
for (const item of data.acled?.deadliestEvents || []) {
|
||||
push({
|
||||
kind: 'conflict',
|
||||
name: item.event_type || item.sub_event_type || item.location || item.country,
|
||||
region: item.country || item.location,
|
||||
severity: Number(item.fatalities || 0) > 0 ? 'high' : 'medium',
|
||||
source: 'ACLED',
|
||||
evidence: item,
|
||||
});
|
||||
}
|
||||
for (const item of data.tg?.urgent || []) {
|
||||
push({
|
||||
kind: 'osint',
|
||||
name: (item.text || '').slice(0, 120),
|
||||
region: item.region || 'OSINT',
|
||||
severity: 'high',
|
||||
source: item.channel || item.chat || 'telegram',
|
||||
evidence: item,
|
||||
});
|
||||
}
|
||||
for (const item of data.newsFeed || data.news || []) {
|
||||
if (!item.urgent) continue;
|
||||
push({
|
||||
kind: 'news',
|
||||
name: item.headline || item.title,
|
||||
region: item.region,
|
||||
severity: 'medium',
|
||||
source: item.source,
|
||||
evidence: item,
|
||||
});
|
||||
}
|
||||
for (const signal of delta?.signals?.new || []) {
|
||||
push({
|
||||
kind: 'delta',
|
||||
name: signal.label || signal.reason || signal.key,
|
||||
region: signal.region,
|
||||
severity: signal.severity || 'medium',
|
||||
source: 'delta',
|
||||
evidence: signal,
|
||||
});
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
function evaluatePredictionAgainstSweep(row, payload, data, timestamp) {
|
||||
const terms = [
|
||||
row.title,
|
||||
payload.ticker,
|
||||
...(Array.isArray(payload.signals) ? payload.signals : []),
|
||||
].filter(Boolean).map(v => String(v).toLowerCase());
|
||||
const evidenceText = [
|
||||
...(data.tSignals || []),
|
||||
...(data.newsFeed || []).slice(0, 40).map(n => `${n.source || ''} ${n.headline || n.title || ''}`),
|
||||
...(data.tg?.urgent || []).slice(0, 20).map(p => p.text || ''),
|
||||
].join('\n').toLowerCase();
|
||||
const matched = terms.filter(term => term.length >= 4 && evidenceText.includes(term.slice(0, 60)));
|
||||
const expired = predictionExpired(row.created_at, row.horizon, timestamp);
|
||||
const state = matched.length
|
||||
? 'observed'
|
||||
: expired
|
||||
? 'expired_unverified'
|
||||
: 'monitoring';
|
||||
return {
|
||||
state,
|
||||
evaluated_at: timestamp,
|
||||
matched_terms: matched.slice(0, 10),
|
||||
expired,
|
||||
reason: matched.length
|
||||
? 'Current sweep contains matching evidence terms.'
|
||||
: expired
|
||||
? 'Prediction horizon elapsed without matching evidence.'
|
||||
: 'Prediction remains open for future sweeps.',
|
||||
};
|
||||
}
|
||||
|
||||
function predictionExpired(createdAt, horizon, nowIso) {
|
||||
const created = new Date(createdAt).getTime();
|
||||
const now = new Date(nowIso).getTime();
|
||||
if (!Number.isFinite(created) || !Number.isFinite(now)) return false;
|
||||
const text = String(horizon || '').toLowerCase();
|
||||
const days = text.includes('intraday') ? 1
|
||||
: text.includes('day') ? 7
|
||||
: text.includes('week') ? 45
|
||||
: text.includes('month') ? 180
|
||||
: text.includes('strategic') ? 365
|
||||
: 30;
|
||||
return now - created > days * 24 * 60 * 60 * 1000;
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import { MistralProvider } from "./mistral.mjs";
|
||||
import { OllamaProvider } from "./ollama.mjs";
|
||||
import { GrokProvider } from "./grok.mjs";
|
||||
import { OpenAICompatibleProvider } from "./openai-compatible.mjs";
|
||||
import { LiteLLMProvider } from "./litellm.mjs";
|
||||
|
||||
export { LLMProvider } from "./provider.mjs";
|
||||
export { AnthropicProvider } from "./anthropic.mjs";
|
||||
@@ -22,6 +23,7 @@ export { MistralProvider } from "./mistral.mjs";
|
||||
export { OllamaProvider } from "./ollama.mjs";
|
||||
export { GrokProvider } from "./grok.mjs";
|
||||
export { OpenAICompatibleProvider } from "./openai-compatible.mjs";
|
||||
export { LiteLLMProvider } from "./litellm.mjs";
|
||||
|
||||
/**
|
||||
* Create an LLM provider based on config.
|
||||
@@ -66,6 +68,9 @@ export function createLLMProvider(llmConfig) {
|
||||
model: model || 'local-model',
|
||||
requiresApiKey: false,
|
||||
});
|
||||
case "litellm":
|
||||
case "lite-llm":
|
||||
return new LiteLLMProvider(common);
|
||||
case "openrouter":
|
||||
return new OpenRouterProvider(common);
|
||||
case "gemini":
|
||||
|
||||
32
lib/llm/litellm.mjs
Normal file
32
lib/llm/litellm.mjs
Normal file
@@ -0,0 +1,32 @@
|
||||
// LiteLLM proxy provider using the OpenAI-compatible chat completions API.
|
||||
|
||||
import { OpenAICompatibleProvider } from './openai-compatible.mjs';
|
||||
|
||||
export class LiteLLMProvider extends OpenAICompatibleProvider {
|
||||
constructor(config = {}) {
|
||||
const baseUrl = config.baseUrl?.replace(/\/+$/, '') || null;
|
||||
const model = config.model || null;
|
||||
|
||||
super({
|
||||
...config,
|
||||
name: 'litellm',
|
||||
baseUrl: baseUrl || 'http://localhost:4000/v1',
|
||||
model: model || 'unconfigured',
|
||||
requiresApiKey: true,
|
||||
});
|
||||
|
||||
this.baseUrl = baseUrl;
|
||||
this.model = model;
|
||||
}
|
||||
|
||||
get isConfigured() {
|
||||
return Boolean(this.baseUrl && this.apiKey && this.model);
|
||||
}
|
||||
|
||||
get status() {
|
||||
if (!this.baseUrl) return { state: 'misconfigured', reason: 'LLM_BASE_URL is required for LiteLLM' };
|
||||
if (!this.apiKey) return { state: 'misconfigured', reason: 'LLM_API_KEY is required for LiteLLM' };
|
||||
if (!this.model) return { state: 'misconfigured', reason: 'LLM_MODEL is required for LiteLLM' };
|
||||
return { state: 'configured', provider: this.name, model: this.model, baseUrl: this.baseUrl };
|
||||
}
|
||||
}
|
||||
52
lib/stale-alerts.mjs
Normal file
52
lib/stale-alerts.mjs
Normal file
@@ -0,0 +1,52 @@
|
||||
const DEFAULT_COOLDOWN_MS = 60 * 60 * 1000;
|
||||
|
||||
export function shouldSendStaleAlert(health, state = {}, opts = {}) {
|
||||
const now = opts.now ?? Date.now();
|
||||
const cooldownMs = opts.cooldownMs ?? DEFAULT_COOLDOWN_MS;
|
||||
if (!health?.stale) {
|
||||
state.lastStaleAlertKey = null;
|
||||
return { send: false, reason: 'not_stale' };
|
||||
}
|
||||
|
||||
const key = [
|
||||
health.lastSuccessfulSweep || 'never',
|
||||
health.lastSweepError || 'no-error',
|
||||
health.sourcesFailed || 0,
|
||||
health.sourcesDegraded || 0,
|
||||
].join('|');
|
||||
|
||||
if (state.lastStaleAlertKey === key && now - (state.lastStaleAlertAt || 0) < cooldownMs) {
|
||||
return { send: false, reason: 'cooldown', key };
|
||||
}
|
||||
|
||||
state.lastStaleAlertKey = key;
|
||||
state.lastStaleAlertAt = now;
|
||||
return { send: true, reason: 'stale', key };
|
||||
}
|
||||
|
||||
export function formatStaleAlert(health, opts = {}) {
|
||||
const dashboardUrl = opts.dashboardUrl || 'http://localhost:3117';
|
||||
const context = opts.context || 'scheduled sweep';
|
||||
const ageMinutes = health.dataAgeSeconds == null ? 'unknown' : Math.floor(health.dataAgeSeconds / 60);
|
||||
const affected = (health.sourceHealth || [])
|
||||
.filter(s => (s.status && s.status !== 'ok') || s.error)
|
||||
.slice(0, 6)
|
||||
.map(s => `- ${s.name || s.n || 'source'}: ${s.status || 'degraded'}${s.error ? ` (${String(s.error).slice(0, 100)})` : ''}`);
|
||||
|
||||
return [
|
||||
'*CRUCIX STALE DATA ALERT*',
|
||||
'',
|
||||
`Context: ${context}`,
|
||||
`Status: ${health.status || 'unknown'}`,
|
||||
`Data age: ${ageMinutes} minutes`,
|
||||
`Last successful sweep: ${health.lastSuccessfulSweep || 'never'}`,
|
||||
`Last attempted sweep: ${health.lastSweep || 'never'}`,
|
||||
`Last error: ${health.lastSweepError || 'none'}`,
|
||||
`Sources: ${health.sourcesOk || 0} OK / ${health.sourcesDegraded || 0} degraded / ${health.sourcesFailed || 0} failed`,
|
||||
'',
|
||||
'*Affected sources*',
|
||||
affected.length ? affected.join('\n') : '- No per-source errors available',
|
||||
'',
|
||||
`Dashboard: ${dashboardUrl}`,
|
||||
].join('\n');
|
||||
}
|
||||
@@ -12,7 +12,7 @@
|
||||
"brief:save": "node apis/save-briefing.mjs",
|
||||
"diag": "node diag.mjs",
|
||||
"test": "npm run test:unit",
|
||||
"test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs test/acled-source.test.mjs test/mojibake-text.test.mjs test/dashboard-geotagging.test.mjs",
|
||||
"test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/llm-litellm.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs test/acled-source.test.mjs test/mojibake-text.test.mjs test/adsb.test.mjs test/dashboard-geotagging.test.mjs",
|
||||
"compose:config": "docker compose config",
|
||||
"clean": "node scripts/clean.mjs",
|
||||
"fresh-start": "npm run clean && npm start"
|
||||
|
||||
228
server.mjs
228
server.mjs
@@ -18,6 +18,7 @@ import { TelegramAlerter } from './lib/alerts/telegram.mjs';
|
||||
import { DiscordAlerter } from './lib/alerts/discord.mjs';
|
||||
import { getFetchMetrics } from './apis/utils/fetch.mjs';
|
||||
import { IntelligenceStore } from './lib/intelligence-store.mjs';
|
||||
import { formatStaleAlert, shouldSendStaleAlert } from './lib/stale-alerts.mjs';
|
||||
import { evaluateScenarios } from './lib/scenarios.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
@@ -40,6 +41,8 @@ let sweepStartedAt = null; // Timestamp when current/last sweep started
|
||||
let sweepInProgress = false;
|
||||
const startTime = Date.now();
|
||||
const sseClients = new Set();
|
||||
const terminalActionBuckets = new Map();
|
||||
const staleAlertState = {};
|
||||
|
||||
// === Delta/Memory ===
|
||||
const memory = new MemoryManager(RUNS_DIR);
|
||||
@@ -289,29 +292,67 @@ app.get('/api/metrics', (req, res) => {
|
||||
});
|
||||
});
|
||||
|
||||
app.post('/api/sweep', express.json(), (req, res) => {
|
||||
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
|
||||
triggerSweep(res);
|
||||
app.get('/api/memory/search', (req, res) => {
|
||||
const guard = authorizeTerminalAction(req, res, 'memory:search');
|
||||
if (!guard.ok) return;
|
||||
auditTerminalAction(req, 'memory:search', 'ok');
|
||||
res.json(intelligenceStore.queryMemory({
|
||||
q: req.query.q || '',
|
||||
limit: req.query.limit || 25,
|
||||
}));
|
||||
});
|
||||
|
||||
app.post('/api/action', express.json(), async (req, res) => {
|
||||
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
|
||||
const action = String(req.body?.action || req.query.action || '').toLowerCase();
|
||||
app.get('/api/memory/predictions', (req, res) => {
|
||||
const guard = authorizeTerminalAction(req, res, 'memory:predictions');
|
||||
if (!guard.ok) return;
|
||||
auditTerminalAction(req, 'memory:predictions', 'ok');
|
||||
res.json(intelligenceStore.listPredictions({
|
||||
state: req.query.state || null,
|
||||
limit: req.query.limit || 25,
|
||||
}));
|
||||
});
|
||||
|
||||
app.post('/api/sweep', express.json(), (req, res) => {
|
||||
const guard = authorizeTerminalAction(req, res, 'sweep');
|
||||
if (!guard.ok) return;
|
||||
triggerSweepAction(req, res, 'sweep');
|
||||
});
|
||||
|
||||
app.post('/api/action', express.json(), (req, res) => {
|
||||
const action = String(req.body?.action || req.body?.command || '').trim().toLowerCase();
|
||||
const guard = authorizeTerminalAction(req, res, action || 'unknown');
|
||||
if (!guard.ok) return;
|
||||
|
||||
if (action === 'status') {
|
||||
return res.json({ ok: true, action, health: buildHealth() });
|
||||
auditTerminalAction(req, 'status', 'ok');
|
||||
return res.json({ ok: true, action, status: 'ok', health: buildHealth() });
|
||||
}
|
||||
|
||||
if (action === 'brief') {
|
||||
if (!currentData) return res.status(503).json({ ok: false, action, error: 'No data yet — first sweep in progress' });
|
||||
return res.json({ ok: true, action, text: buildBrief(currentData) });
|
||||
if (!currentData) {
|
||||
auditTerminalAction(req, 'brief', 'rejected', 'no_data');
|
||||
return res.status(503).json({ ok: false, action, error: 'No data yet - first sweep in progress' });
|
||||
}
|
||||
auditTerminalAction(req, 'brief', 'ok');
|
||||
const brief = buildBrief(currentData);
|
||||
return res.json({ ok: true, action, status: 'ok', brief, text: brief });
|
||||
}
|
||||
|
||||
if (action === 'sweep') {
|
||||
return triggerSweep(res);
|
||||
if (action === 'memory') {
|
||||
auditTerminalAction(req, 'memory', 'ok');
|
||||
return res.json({
|
||||
ok: true,
|
||||
action,
|
||||
memory: intelligenceStore.status(),
|
||||
recentEvents: intelligenceStore.queryMemory({ q: req.body?.q || '', limit: 8 }).results,
|
||||
predictions: intelligenceStore.listPredictions({ limit: 8 }).predictions,
|
||||
});
|
||||
}
|
||||
|
||||
res.status(400).json({ ok: false, error: 'Unknown action', actions: ['status', 'brief', 'sweep'] });
|
||||
if (action === 'sweep') return triggerSweepAction(req, res, 'action:sweep');
|
||||
|
||||
auditTerminalAction(req, action || 'unknown', 'rejected', 'unknown_action');
|
||||
return res.status(400).json({ ok: false, error: 'Unknown action', allowed: ['status', 'brief', 'memory', 'sweep'], actions: ['status', 'brief', 'memory', 'sweep'] });
|
||||
});
|
||||
|
||||
// API: available locales
|
||||
@@ -329,10 +370,24 @@ app.get('/events', (req, res) => {
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'X-Accel-Buffering': 'no',
|
||||
});
|
||||
res.write('retry: 10000\n');
|
||||
res.write('data: {"type":"connected"}\n\n');
|
||||
const heartbeatMs = Math.max(5000, config.sseHeartbeatIntervalMs || 25000);
|
||||
const heartbeat = setInterval(() => {
|
||||
try {
|
||||
res.write(`: heartbeat ${new Date().toISOString()}\n\n`);
|
||||
} catch {
|
||||
clearInterval(heartbeat);
|
||||
sseClients.delete(res);
|
||||
}
|
||||
}, heartbeatMs);
|
||||
sseClients.add(res);
|
||||
req.on('close', () => sseClients.delete(res));
|
||||
req.on('close', () => {
|
||||
clearInterval(heartbeat);
|
||||
sseClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
function broadcast(data) {
|
||||
@@ -342,26 +397,114 @@ function broadcast(data) {
|
||||
}
|
||||
}
|
||||
|
||||
function requestIp(req) {
|
||||
return req.ip || req.socket?.remoteAddress || 'unknown';
|
||||
}
|
||||
|
||||
function isLocalRequest(req) {
|
||||
const remote = requestIp(req);
|
||||
return remote === '::1'
|
||||
|| remote === '127.0.0.1'
|
||||
|| remote === '::ffff:127.0.0.1'
|
||||
|| remote.startsWith('127.')
|
||||
|| remote === 'localhost';
|
||||
}
|
||||
|
||||
function sameOriginPost(req) {
|
||||
const origin = req.get('origin');
|
||||
if (!origin) return true;
|
||||
try {
|
||||
const originUrl = new URL(origin);
|
||||
const host = req.get('host');
|
||||
return host && originUrl.host === host;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function actionToken(req) {
|
||||
return req.get('x-crucix-token') || req.body?.token || null;
|
||||
}
|
||||
|
||||
function auditTerminalAction(req, action, outcome, detail = null) {
|
||||
const suffix = detail ? ` detail=${detail}` : '';
|
||||
console.log(`[Crucix][audit] terminal_action action=${action || 'unknown'} outcome=${outcome} ip=${requestIp(req)}${suffix}`);
|
||||
}
|
||||
|
||||
function rateLimitTerminalAction(req, action) {
|
||||
const now = Date.now();
|
||||
const windowMs = Math.max(1000, config.terminalActionRateLimitWindowMs || 60_000);
|
||||
const max = Math.max(1, config.terminalActionRateLimitMax || 10);
|
||||
const key = `${requestIp(req)}:${action}`;
|
||||
const bucket = terminalActionBuckets.get(key);
|
||||
if (!bucket || now > bucket.resetAt) {
|
||||
terminalActionBuckets.set(key, { count: 1, resetAt: now + windowMs });
|
||||
return { ok: true };
|
||||
}
|
||||
bucket.count += 1;
|
||||
if (bucket.count > max) {
|
||||
return { ok: false, retryAfterSeconds: Math.ceil((bucket.resetAt - now) / 1000) };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
function authorizeTerminalAction(req, res, action) {
|
||||
const rate = rateLimitTerminalAction(req, action);
|
||||
if (!rate.ok) {
|
||||
auditTerminalAction(req, action, 'rejected', 'rate_limited');
|
||||
res.set('Retry-After', String(rate.retryAfterSeconds));
|
||||
res.status(429).json({ error: 'Too many terminal actions', retryAfterSeconds: rate.retryAfterSeconds });
|
||||
return { ok: false };
|
||||
}
|
||||
|
||||
if (!sameOriginPost(req)) {
|
||||
auditTerminalAction(req, action, 'rejected', 'csrf_origin');
|
||||
res.status(403).json({ error: 'Origin mismatch' });
|
||||
return { ok: false };
|
||||
}
|
||||
|
||||
const local = isLocalRequest(req);
|
||||
const token = actionToken(req);
|
||||
if (!config.terminalActionsEnabled) {
|
||||
auditTerminalAction(req, action, 'rejected', 'disabled');
|
||||
res.status(403).json({ error: 'Terminal actions are disabled' });
|
||||
return { ok: false };
|
||||
}
|
||||
|
||||
if (config.sweepToken) {
|
||||
if (token !== config.sweepToken) {
|
||||
auditTerminalAction(req, action, 'rejected', 'invalid_token');
|
||||
res.status(401).json({ error: 'Invalid terminal action token' });
|
||||
return { ok: false };
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
if (!local) {
|
||||
auditTerminalAction(req, action, 'rejected', 'missing_token');
|
||||
res.status(403).json({ error: 'Terminal actions are local-only unless SWEEP_TOKEN is set' });
|
||||
return { ok: false };
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
}
|
||||
|
||||
function triggerSweepAction(req, res, auditAction) {
|
||||
if (sweepInProgress) {
|
||||
auditTerminalAction(req, auditAction, 'rejected', 'already_running');
|
||||
return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
|
||||
}
|
||||
auditTerminalAction(req, auditAction, 'accepted');
|
||||
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
|
||||
return res.status(202).json({ ok: true, status: 'accepted' });
|
||||
}
|
||||
|
||||
function dataAgeMs() {
|
||||
const ts = currentData?.meta?.timestamp || lastSuccessfulSweepTime || lastSweepTime;
|
||||
const ms = ts ? Date.now() - new Date(ts).getTime() : null;
|
||||
return Number.isFinite(ms) ? ms : null;
|
||||
}
|
||||
|
||||
function canRunTerminalAction(req) {
|
||||
const remote = req.ip || '';
|
||||
const local = remote.includes('127.0.0.1') || remote === '::1' || remote === '::ffff:127.0.0.1';
|
||||
const token = req.get('x-crucix-token') || req.query.token || req.body?.token;
|
||||
if (config.sweepToken) return token === config.sweepToken;
|
||||
return Boolean(config.terminalActionsEnabled || local);
|
||||
}
|
||||
|
||||
function triggerSweep(res) {
|
||||
if (sweepInProgress) return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
|
||||
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
|
||||
return res.status(202).json({ ok: true, status: 'accepted' });
|
||||
}
|
||||
|
||||
function getLLMStatus() {
|
||||
if (!config.llm.provider) return { state: 'disabled' };
|
||||
if (!llmProvider) return { state: 'misconfigured', provider: config.llm.provider };
|
||||
@@ -405,13 +548,39 @@ function buildHealth() {
|
||||
llm: getLLMStatus(),
|
||||
telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId),
|
||||
discordEnabled: !!(config.discord?.botToken || config.discord?.webhookUrl),
|
||||
terminalActionsEnabled: Boolean(config.terminalActionsEnabled || config.sweepToken),
|
||||
terminalActionsEnabled: config.terminalActionsEnabled,
|
||||
terminalActionsTokenRequired: !!config.sweepToken,
|
||||
refreshIntervalMinutes: config.refreshIntervalMinutes,
|
||||
language: currentLanguage,
|
||||
memory: intelligenceStore.status(),
|
||||
};
|
||||
}
|
||||
|
||||
async function notifyIfDataStale(context = 'scheduled sweep') {
|
||||
const health = buildHealth();
|
||||
const decision = shouldSendStaleAlert(health, staleAlertState, {
|
||||
cooldownMs: config.staleAlertCooldownMinutes * 60 * 1000,
|
||||
});
|
||||
if (!decision.send) return false;
|
||||
|
||||
const dashboardUrl = config.dashboardUrl || `http://localhost:${config.port}`;
|
||||
const message = formatStaleAlert(health, { dashboardUrl, context });
|
||||
const sends = [];
|
||||
if (telegramAlerter.isConfigured) sends.push(telegramAlerter.sendMessage(message));
|
||||
if (discordAlerter.isConfigured) sends.push(discordAlerter.sendAlert(message));
|
||||
|
||||
if (sends.length === 0) {
|
||||
console.warn('[Crucix] Data is stale but no operator alert channel is configured');
|
||||
return false;
|
||||
}
|
||||
|
||||
const results = await Promise.allSettled(sends);
|
||||
const sent = results.some(r => r.status === 'fulfilled' && (r.value === true || r.value?.ok === true));
|
||||
if (sent) console.warn('[Crucix] Operator stale-data alert sent');
|
||||
else console.warn('[Crucix] Operator stale-data alert attempted but no channel accepted it');
|
||||
return sent;
|
||||
}
|
||||
|
||||
function buildBrief(data) {
|
||||
const verbosity = config.telegram.briefVerbosity || 'standard';
|
||||
const delta = memory.getLastDelta();
|
||||
@@ -562,6 +731,9 @@ async function runSweepCycle() {
|
||||
broadcast({ type: 'sweep_error', error: err.message });
|
||||
} finally {
|
||||
sweepInProgress = false;
|
||||
await notifyIfDataStale(lastSweepError ? 'failed sweep' : 'completed sweep').catch(err => {
|
||||
console.error('[Crucix] Stale-data operator alert failed:', err.message);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
82
test/adsb.test.mjs
Normal file
82
test/adsb.test.mjs
Normal file
@@ -0,0 +1,82 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
async function withFetch(mockFetch, fn) {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const originalAdsbKey = process.env.ADSB_API_KEY;
|
||||
const originalRapidKey = process.env.RAPIDAPI_KEY;
|
||||
globalThis.fetch = mockFetch;
|
||||
delete process.env.ADSB_API_KEY;
|
||||
delete process.env.RAPIDAPI_KEY;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
if (originalAdsbKey === undefined) delete process.env.ADSB_API_KEY;
|
||||
else process.env.ADSB_API_KEY = originalAdsbKey;
|
||||
if (originalRapidKey === undefined) delete process.env.RAPIDAPI_KEY;
|
||||
else process.env.RAPIDAPI_KEY = originalRapidKey;
|
||||
}
|
||||
}
|
||||
|
||||
function jsonResponse(payload, ok = true, status = 200) {
|
||||
return {
|
||||
ok,
|
||||
status,
|
||||
headers: { get: () => 'application/json' },
|
||||
text: async () => JSON.stringify(payload),
|
||||
};
|
||||
}
|
||||
|
||||
test('ADS-B reports disabled when no key is configured and public feed fails', async () => {
|
||||
await withFetch(async () => jsonResponse({ error: 'blocked' }, false, 403), async () => {
|
||||
const { briefing } = await import('../apis/sources/adsb.mjs');
|
||||
const data = await briefing();
|
||||
|
||||
assert.equal(data.status, 'disabled');
|
||||
assert.match(data.error, /ADSB_API_KEY|RAPIDAPI_KEY/);
|
||||
assert.equal(data.militaryAircraft.length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
test('ADS-B reports degraded when RapidAPI and public feed fail', async () => {
|
||||
await withFetch(async () => jsonResponse({ error: 'unavailable' }, false, 503), async () => {
|
||||
process.env.ADSB_API_KEY = 'test-key';
|
||||
const { briefing } = await import('../apis/sources/adsb.mjs');
|
||||
const data = await briefing();
|
||||
|
||||
assert.equal(data.status, 'degraded');
|
||||
assert.match(data.error, /providers returned no usable/);
|
||||
assert.equal(data.failures.length, 2);
|
||||
});
|
||||
});
|
||||
|
||||
test('ADS-B returns live RapidAPI military aircraft payloads', async () => {
|
||||
await withFetch(async () => jsonResponse({
|
||||
ac: [{
|
||||
hex: 'AE1234',
|
||||
flight: 'RCH123',
|
||||
t: 'KC135',
|
||||
lat: 50,
|
||||
lon: 8,
|
||||
mil: true,
|
||||
}],
|
||||
}), async () => {
|
||||
process.env.ADSB_API_KEY = 'test-key';
|
||||
const { briefing } = await import('../apis/sources/adsb.mjs');
|
||||
const data = await briefing();
|
||||
|
||||
assert.equal(data.status, 'live');
|
||||
assert.equal(data.provider, 'rapidapi');
|
||||
assert.equal(data.totalMilitary, 1);
|
||||
assert.equal(data.militaryAircraft[0].callsign, 'RCH123');
|
||||
});
|
||||
});
|
||||
|
||||
test('runSource treats disabled source status as degraded health', async () => {
|
||||
const { runSource } = await import('../apis/briefing.mjs');
|
||||
const result = await runSource('ADS-B', async () => ({ status: 'disabled', message: 'missing key' }));
|
||||
|
||||
assert.equal(result.status, 'degraded');
|
||||
assert.equal(result.error, null);
|
||||
});
|
||||
@@ -1,7 +1,8 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { safeFetch, safeFetchText, getFetchMetrics } from '../apis/utils/fetch.mjs';
|
||||
import { safeFetch, safeFetchText, getFetchMetrics, inferFetchSource } from '../apis/utils/fetch.mjs';
|
||||
import { formatStaleAlert, shouldSendStaleAlert } from '../lib/stale-alerts.mjs';
|
||||
|
||||
test('safeFetch reports HTML as degraded JSON response', async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
@@ -100,6 +101,157 @@ test('safeFetchText returns text and byte count', async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('safeFetch attributes unlabelled requests to a stable provider source', async () => {
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = async () => ({
|
||||
ok: true,
|
||||
status: 200,
|
||||
headers: { get: () => 'application/json' },
|
||||
text: async () => '{"observations":[]}',
|
||||
});
|
||||
try {
|
||||
const data = await safeFetch('https://api.fred.stlouisfed.org/fred/series/observations?series_id=VIXCLS', { retries: 0 });
|
||||
assert.deepEqual(data, { observations: [] });
|
||||
const bucket = getFetchMetrics().bySource.FRED;
|
||||
assert.ok(bucket.requests >= 1);
|
||||
assert.equal(bucket.lastStatus, 200);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
test('inferFetchSource returns provider names and host fallback', () => {
|
||||
assert.equal(inferFetchSource('https://api.bls.gov/publicAPI/v2/timeseries/data/CPI'), 'BLS');
|
||||
assert.equal(inferFetchSource('https://query1.finance.yahoo.com/v8/finance/chart/%5EGSPC'), 'YahooFinance');
|
||||
assert.equal(inferFetchSource('https://unknown.example.test/path'), 'unknown.example.test');
|
||||
});
|
||||
|
||||
test('SSE endpoint sends reconnect guidance and clears heartbeat timer', () => {
|
||||
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
|
||||
const config = readFileSync(new URL('../crucix.config.mjs', import.meta.url), 'utf8');
|
||||
assert.match(config, /sseHeartbeatIntervalMs/);
|
||||
assert.match(server, /retry: 10000\\n/);
|
||||
assert.match(server, /setInterval\(\(\) =>/);
|
||||
assert.match(server, /: heartbeat/);
|
||||
assert.match(server, /clearInterval\(heartbeat\)/);
|
||||
assert.match(server, /X-Accel-Buffering/);
|
||||
});
|
||||
|
||||
test('intelligence store defines durable memory and prediction lifecycle tables', () => {
|
||||
const store = readFileSync(new URL('../lib/intelligence-store.mjs', import.meta.url), 'utf8');
|
||||
assert.match(store, /CREATE TABLE IF NOT EXISTS events/);
|
||||
assert.match(store, /stable_id TEXT NOT NULL UNIQUE/);
|
||||
assert.match(store, /hypothesis TEXT/);
|
||||
assert.match(store, /evidence_json TEXT/);
|
||||
assert.match(store, /outcome_state TEXT DEFAULT 'open'/);
|
||||
assert.match(store, /evaluatePredictions/);
|
||||
assert.match(store, /queryMemory/);
|
||||
assert.match(store, /listPredictions/);
|
||||
});
|
||||
|
||||
test('server exposes memory-backed query APIs and dashboard memory action', () => {
|
||||
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
|
||||
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
|
||||
assert.match(server, /\/api\/memory\/search/);
|
||||
assert.match(server, /\/api\/memory\/predictions/);
|
||||
assert.match(server, /action === 'memory'/);
|
||||
assert.match(html, /runTerminalAction\('memory'\)/);
|
||||
});
|
||||
|
||||
test('terminal action endpoints avoid URL tokens and include hardening gates', () => {
|
||||
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
|
||||
assert.match(server, /app\.post\('\/api\/action'/);
|
||||
assert.match(server, /app\.post\('\/api\/sweep'/);
|
||||
assert.match(server, /x-crucix-token/);
|
||||
assert.match(server, /sameOriginPost/);
|
||||
assert.match(server, /rateLimitTerminalAction/);
|
||||
assert.match(server, /auditTerminalAction/);
|
||||
assert.doesNotMatch(server, /req\.query\.token/);
|
||||
});
|
||||
|
||||
test('dashboard exposes token configuration flow without devtools edits', () => {
|
||||
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
|
||||
assert.match(html, /configureTerminalActionToken/);
|
||||
assert.match(html, /crucix_sweep_token/);
|
||||
assert.match(html, /x-crucix-token/);
|
||||
assert.match(html, /SET TOKEN/);
|
||||
});
|
||||
|
||||
test('server dashboard shell does not embed an operational snapshot', () => {
|
||||
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
|
||||
assert.match(html, /let D = createDashboardShellData\(\);/);
|
||||
assert.doesNotMatch(html, /2026-04-03T16:18:10\.188Z/);
|
||||
assert.doesNotMatch(html, /Trump announced new strikes on Iran/);
|
||||
});
|
||||
|
||||
test('server dashboard fetches api data before initialization', () => {
|
||||
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
|
||||
const serverMode = html.indexOf('if (canProbeApi)');
|
||||
const apiFetch = html.indexOf("fetch('/api/data')");
|
||||
const firstInitAfterServerMode = html.indexOf('init();', serverMode);
|
||||
|
||||
assert.ok(serverMode > -1);
|
||||
assert.ok(apiFetch > serverMode);
|
||||
assert.ok(firstInitAfterServerMode > apiFetch);
|
||||
});
|
||||
|
||||
test('stale alert is skipped for fresh health and resets active key', () => {
|
||||
const state = { lastStaleAlertKey: 'old', lastStaleAlertAt: 100 };
|
||||
const decision = shouldSendStaleAlert({ stale: false }, state, { now: 200 });
|
||||
assert.equal(decision.send, false);
|
||||
assert.equal(decision.reason, 'not_stale');
|
||||
assert.equal(state.lastStaleAlertKey, null);
|
||||
});
|
||||
|
||||
test('stale alert sends once and deduplicates during cooldown', () => {
|
||||
const state = {};
|
||||
const health = {
|
||||
stale: true,
|
||||
lastSuccessfulSweep: '2026-05-17T08:00:00.000Z',
|
||||
lastSweepError: 'network timeout',
|
||||
sourcesFailed: 2,
|
||||
sourcesDegraded: 1,
|
||||
};
|
||||
|
||||
const first = shouldSendStaleAlert(health, state, { now: 1_000, cooldownMs: 60_000 });
|
||||
const second = shouldSendStaleAlert(health, state, { now: 2_000, cooldownMs: 60_000 });
|
||||
|
||||
assert.equal(first.send, true);
|
||||
assert.equal(second.send, false);
|
||||
assert.equal(second.reason, 'cooldown');
|
||||
});
|
||||
|
||||
test('stale alert repeats after cooldown', () => {
|
||||
const state = {};
|
||||
const health = { stale: true, lastSuccessfulSweep: 'a', lastSweepError: 'timeout', sourcesFailed: 1 };
|
||||
|
||||
assert.equal(shouldSendStaleAlert(health, state, { now: 1_000, cooldownMs: 60_000 }).send, true);
|
||||
assert.equal(shouldSendStaleAlert(health, state, { now: 62_000, cooldownMs: 60_000 }).send, true);
|
||||
});
|
||||
|
||||
test('stale alert message includes operator context and affected sources', () => {
|
||||
const message = formatStaleAlert({
|
||||
status: 'stale',
|
||||
stale: true,
|
||||
dataAgeSeconds: 7200,
|
||||
lastSuccessfulSweep: '2026-05-17T08:00:00.000Z',
|
||||
lastSweep: '2026-05-17T10:00:00.000Z',
|
||||
lastSweepError: 'GDELT timeout',
|
||||
sourcesOk: 20,
|
||||
sourcesDegraded: 3,
|
||||
sourcesFailed: 2,
|
||||
sourceHealth: [
|
||||
{ name: 'GDELT', status: 'degraded', error: 'timeout' },
|
||||
{ name: 'Reddit', status: 'no_credentials' },
|
||||
],
|
||||
}, { dashboardUrl: 'https://terminal.example.test', context: 'failed sweep' });
|
||||
|
||||
assert.match(message, /CRUCIX STALE DATA ALERT/);
|
||||
assert.match(message, /Data age: 120 minutes/);
|
||||
assert.match(message, /GDELT: degraded \(timeout\)/);
|
||||
assert.match(message, /Dashboard: https:\/\/terminal\.example\.test/);
|
||||
});
|
||||
|
||||
test('scenario watchlist feature is wired into sweep, briefing, and dashboard', () => {
|
||||
const scenarios = readFileSync(new URL('../lib/scenarios.mjs', import.meta.url), 'utf8');
|
||||
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
|
||||
|
||||
76
test/llm-litellm.test.mjs
Normal file
76
test/llm-litellm.test.mjs
Normal file
@@ -0,0 +1,76 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { LiteLLMProvider } from '../lib/llm/litellm.mjs';
|
||||
import { createLLMProvider } from '../lib/llm/index.mjs';
|
||||
|
||||
test('factory creates a configured LiteLLM provider', () => {
|
||||
const provider = createLLMProvider({
|
||||
provider: 'litellm',
|
||||
baseUrl: 'https://llm.example.test/v1/',
|
||||
apiKey: 'proxy-key',
|
||||
model: 'private-model',
|
||||
});
|
||||
|
||||
assert.ok(provider instanceof LiteLLMProvider);
|
||||
assert.equal(provider.baseUrl, 'https://llm.example.test/v1');
|
||||
assert.equal(provider.isConfigured, true);
|
||||
assert.deepEqual(provider.status, {
|
||||
state: 'configured',
|
||||
provider: 'litellm',
|
||||
model: 'private-model',
|
||||
baseUrl: 'https://llm.example.test/v1',
|
||||
});
|
||||
});
|
||||
|
||||
test('LiteLLM requires base URL, API key, and model', () => {
|
||||
const missingBaseUrl = new LiteLLMProvider({ apiKey: 'key', model: 'model' });
|
||||
assert.equal(missingBaseUrl.isConfigured, false);
|
||||
assert.equal(missingBaseUrl.status.reason, 'LLM_BASE_URL is required for LiteLLM');
|
||||
|
||||
const missingKey = new LiteLLMProvider({ baseUrl: 'https://llm.example.test/v1', model: 'model' });
|
||||
assert.equal(missingKey.status.reason, 'LLM_API_KEY is required for LiteLLM');
|
||||
|
||||
const missingModel = new LiteLLMProvider({ baseUrl: 'https://llm.example.test/v1', apiKey: 'key' });
|
||||
assert.equal(missingModel.status.reason, 'LLM_MODEL is required for LiteLLM');
|
||||
});
|
||||
|
||||
test('LiteLLM sends bearer-authenticated OpenAI-compatible requests', async () => {
|
||||
const provider = new LiteLLMProvider({
|
||||
baseUrl: 'https://llm.example.test/v1',
|
||||
apiKey: 'proxy-key',
|
||||
model: 'private-model',
|
||||
temperature: 0.15,
|
||||
maxTokens: 512,
|
||||
});
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
globalThis.fetch = async (url, options) => {
|
||||
assert.equal(url, 'https://llm.example.test/v1/chat/completions');
|
||||
assert.equal(options.headers.Authorization, 'Bearer proxy-key');
|
||||
assert.deepEqual(JSON.parse(options.body), {
|
||||
model: 'private-model',
|
||||
temperature: 0.15,
|
||||
messages: [
|
||||
{ role: 'system', content: 'system' },
|
||||
{ role: 'user', content: 'user' },
|
||||
],
|
||||
max_tokens: 512,
|
||||
});
|
||||
return {
|
||||
ok: true,
|
||||
json: async () => ({
|
||||
choices: [{ message: { content: 'response' } }],
|
||||
usage: { prompt_tokens: 7, completion_tokens: 11 },
|
||||
model: 'private-model',
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await provider.complete('system', 'user');
|
||||
assert.equal(result.text, 'response');
|
||||
assert.deepEqual(result.usage, { inputTokens: 7, outputTokens: 11 });
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user