MrSphay/intelligence-terminal
1
0
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/codex/production-intelligence-terminal' into codex/issue-4-memory-prediction-loop
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Details
Build / test-and-image (pull_request) Successful in 55s
Details

Browse Source 
# Conflicts:
#	README.md
#	test/fetch-utils.test.mjs
This commit is contained in:
MrSphay
2026-05-17 20:35:44 +02:00
parent 267af03b22 d625bffd4a
commit 090e90ea70
27 changed files with 1195 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.env.example
View File

@@ -1,4 +1,4 @@
# Intelligence Terminal / Crucix configuration # Intelligence Terminal configuration
# Copy to .env. Keep comments on separate lines; Docker env_file treats inline comments as values. # Copy to .env. Keep comments on separate lines; Docker env_file treats inline comments as values.
# Server # Server
@@ -6,6 +6,8 @@ PORT=3117
REFRESH_INTERVAL_MINUTES=15 REFRESH_INTERVAL_MINUTES=15
AUTO_OPEN_BROWSER=false AUTO_OPEN_BROWSER=false
STALE_DATA_MAX_AGE_MINUTES=60 STALE_DATA_MAX_AGE_MINUTES=60
STALE_ALERT_COOLDOWN_MINUTES=60
DASHBOARD_URL=
TERMINAL_ACTIONS_ENABLED=true TERMINAL_ACTIONS_ENABLED=true
SWEEP_TOKEN= SWEEP_TOKEN=
BRIEF_VERBOSITY=standard BRIEF_VERBOSITY=standard
@@ -36,6 +38,8 @@ ACLED_EMAIL=
ACLED_PASSWORD= ACLED_PASSWORD=
CLOUDFLARE_API_TOKEN= CLOUDFLARE_API_TOKEN=
BLS_API_KEY= BLS_API_KEY=
REDDIT_CLIENT_ID=
REDDIT_CLIENT_SECRET=
# Telegram bot and alerts # Telegram bot and alerts
TELEGRAM_BOT_TOKEN= TELEGRAM_BOT_TOKEN=

2
.github/CODEOWNERS vendored
View File

@@ -1 +1 @@
* @calesthio * @MrSphay

2
.github/ISSUE_TEMPLATE/config.yml vendored
View File

@@ -1,5 +1,5 @@
blank_issues_enabled: true blank_issues_enabled: true
contact_links: contact_links:
- name: Security report - name: Security report
url: mailto:celesthioailabs@gmail.com url: https://git.wilkensxl.de/MrSphay/intelligence-terminal
about: Report security issues privately instead of opening a public issue. about: Report security issues privately instead of opening a public issue.

68
.github/workflows/docker-publish.yml vendored
View File

@@ -1,68 +0,0 @@
name: Build & Publish Docker Image
on:
push:
branches: [master]
tags: ['v*']
pull_request:
branches: [master]
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
jobs:
build-and-push:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to GHCR
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Log in to Docker Hub
if: github.event_name != 'pull_request' && vars.DOCKERHUB_ENABLED == 'true'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@v5
with:
images: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
${{ vars.DOCKERHUB_ENABLED == 'true' && format('{0}/{1}', secrets.DOCKERHUB_USERNAME, 'crucix') || '' }}
tags: |
type=raw,value=latest,enable={{is_default_branch}}
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix=
- name: Build and push
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max

4
CONTRIBUTING.md
View File

@@ -1,6 +1,6 @@
# Contributing to Crucix # Contributing to Intelligence Terminal
Crucix moves quickly, but review bandwidth is limited. The easiest way to get a change merged is to keep it small, well-scoped, and aligned with the project's direction. Intelligence Terminal moves quickly, but review bandwidth is limited. The easiest way to get a change merged is to keep it small, well-scoped, and aligned with the project's private home-server deployment direction.
## What Contributions Are Most Helpful ## What Contributions Are Most Helpful

122
README.md
View File

@@ -1,13 +1,11 @@
<div align="center"> <div align="center">
# Crucix # Intelligence Terminal
**Your own intelligence terminal. 27 sources. One command. Zero cloud.** **Self-hosted intelligence dashboard. 27 open sources. Docker-first. No telemetry.**
## [Visit The Live Site: crucix.live](https://www.crucix.live/) [![Gitea Repository](https://img.shields.io/badge/source-Gitea-609926?style=for-the-badge&logo=gitea&logoColor=white)](https://git.wilkensxl.de/MrSphay/intelligence-terminal)
[![Docker Image](https://img.shields.io/badge/image-Gitea%20Registry-0b1220?style=for-the-badge&logo=docker&logoColor=white)](https://git.wilkensxl.de/MrSphay/-/packages/container/intelligence-terminal/latest)
[![Live Website](https://img.shields.io/badge/live-crucix.live-00d4ff?style=for-the-badge)](https://www.crucix.live/)
[![Open Demo](https://img.shields.io/badge/open-live%20dashboard-0b1220?style=for-the-badge&logo=googlechrome&logoColor=white)](https://www.crucix.live/)
[![Node.js 22+](https://img.shields.io/badge/node-22%2B-brightgreen)](#quick-start) [![Node.js 22+](https://img.shields.io/badge/node-22%2B-brightgreen)](#quick-start)
[![License: AGPL v3](https://img.shields.io/badge/license-AGPLv3-blue.svg)](LICENSE) [![License: AGPL v3](https://img.shields.io/badge/license-AGPLv3-blue.svg)](LICENSE)
@@ -15,12 +13,7 @@
[![Sources](https://img.shields.io/badge/OSINT%20sources-27-cyan)](#data-sources-27) [![Sources](https://img.shields.io/badge/OSINT%20sources-27-cyan)](#data-sources-27)
[![Docker](https://img.shields.io/badge/docker-ready-blue?logo=docker)](#docker) [![Docker](https://img.shields.io/badge/docker-ready-blue?logo=docker)](#docker)
**Enter The Signal Network** ![Intelligence Terminal Dashboard](docs/dashboard.png)
[![Signal Wire](https://img.shields.io/badge/Signal%20Wire-%40crucixmonitor-111111?style=for-the-badge&logo=x&logoColor=white)](https://x.com/crucixmonitor)
[![Ops Room](https://img.shields.io/badge/Ops%20Room-Discord-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/ChVy7SF4)
![Crucix Dashboard](docs/dashboard.png)
<details> <details>
<summary>More screenshots</summary> <summary>More screenshots</summary>
@@ -37,22 +30,24 @@
</div> </div>
> **Live website:** [https://www.crucix.live/](https://www.crucix.live/) > **Supported deployment:** private home-server or lab deployment through Docker, Dockge, Pangolin, or local Node.js.
> Explore the public demo first, then clone the repo to run Crucix locally. > Runtime data stays in your configured `runs/` volume and API keys are operator-owned.
> **Source:** [git.wilkensxl.de/MrSphay/intelligence-terminal](https://git.wilkensxl.de/MrSphay/intelligence-terminal)
> Pull the image or clone the repository to run Intelligence Terminal on your own infrastructure.
Crucix pulls satellite fire detection, flight tracking, radiation monitoring, satellite constellation tracking, economic indicators, live market prices, conflict data, sanctions lists, and social sentiment from 27 open-source intelligence feeds — in parallel, every 15 minutes — and renders everything on a single self-contained Jarvis-style dashboard. Intelligence Terminal pulls satellite fire detection, flight tracking, radiation monitoring, satellite constellation tracking, economic indicators, live market prices, conflict data, sanctions lists, and social sentiment from 27 open-source intelligence feeds in parallel, every 15 minutes, and renders everything on a single self-contained dashboard.
Hook it up to an LLM and it becomes a **two-way intelligence assistant** — pushing multi-tier alerts to Telegram and Discord when something meaningful changes, responding to commands like `/brief` and `/sweep` from your phone, and generating actionable trade ideas grounded in real cross-domain data. Your own analyst that watches the world while you sleep. Hook it up to an LLM and it becomes a **two-way intelligence assistant**: pushing multi-tier alerts to Telegram and Discord when something meaningful changes, responding to commands like `/brief` and `/sweep` from your phone, and generating trade ideas grounded in real cross-domain data.
Try the live demo first at [https://www.crucix.live/](https://www.crucix.live/), then clone the repo when you want the full local stack. Run it locally with Node.js or pull the published Docker image for a home-server deployment.
No cloud. No telemetry. No subscriptions. Just `node server.mjs` and you're running. No cloud. No telemetry. No subscriptions. Just `node server.mjs` and you're running.
## Token / Asset Warning ## Token / Asset Warning
> [!WARNING] > [!WARNING]
> **Crucix has not launched any official token, coin, NFT, airdrop, presale, or other blockchain-based asset.** > **Intelligence Terminal has not launched any official token, coin, NFT, airdrop, presale, or other blockchain-based asset.**
> Any token or digital asset using the Crucix name, logo, or branding is not affiliated with or endorsed by Crucix. > Any token or digital asset using the Intelligence Terminal or Crucix name, logo, or branding is not affiliated with or endorsed by this project.
> Do not buy it, promote it, connect a wallet to claim it, sign transactions, or send funds based on third-party posts, DMs, or websites. > Do not buy it, promote it, connect a wallet to claim it, sign transactions, or send funds based on third-party posts, DMs, or websites.
--- ---
@@ -61,7 +56,7 @@ No cloud. No telemetry. No subscriptions. Just `node server.mjs` and you're runn
Most of the world's real-time intelligence — satellite imagery, radiation levels, conflict events, economic indicators, flight tracking, maritime activity — is publicly available. It's just scattered across dozens of government APIs, research institutions, and open data feeds that nobody has time to check individually. Most of the world's real-time intelligence — satellite imagery, radiation levels, conflict events, economic indicators, flight tracking, maritime activity — is publicly available. It's just scattered across dozens of government APIs, research institutions, and open data feeds that nobody has time to check individually.
Crucix brings it all into one place. Not behind a paywall, not locked in an enterprise platform, not requiring a security clearance. Just open data, aggregated and cross-correlated on your own machine, updated every 15 minutes. Intelligence Terminal brings it all into one place. Not behind a paywall, not locked in an enterprise platform, not requiring a security clearance. Just open data, aggregated and cross-correlated on your own machine, updated every 15 minutes.
It was built for anyone who wants to understand what's actually happening in the world right now — researchers, journalists, traders, OSINT analysts, or just curious people who believe access to information shouldn't depend on your budget. It was built for anyone who wants to understand what's actually happening in the world right now — researchers, journalists, traders, OSINT analysts, or just curious people who believe access to information shouldn't depend on your budget.
@@ -71,8 +66,8 @@ It was built for anyone who wants to understand what's actually happening in the
```bash ```bash
# 1. Clone the repo # 1. Clone the repo
git clone https://github.com/calesthio/Crucix.git git clone https://git.wilkensxl.de/MrSphay/intelligence-terminal.git
cd Crucix cd intelligence-terminal
# 2. Install dependencies (just Express) # 2. Install dependencies (just Express)
npm install npm install
@@ -135,6 +130,8 @@ PORT=3117
REFRESH_INTERVAL_MINUTES=15 REFRESH_INTERVAL_MINUTES=15
AUTO_OPEN_BROWSER=false AUTO_OPEN_BROWSER=false
STALE_DATA_MAX_AGE_MINUTES=60 STALE_DATA_MAX_AGE_MINUTES=60
STALE_ALERT_COOLDOWN_MINUTES=60
DASHBOARD_URL=https://intelligence.example.internal
TERMINAL_ACTIONS_ENABLED=true TERMINAL_ACTIONS_ENABLED=true
SWEEP_TOKEN= SWEEP_TOKEN=
BRIEF_VERBOSITY=standard BRIEF_VERBOSITY=standard
@@ -188,6 +185,8 @@ LLM_MODEL=your-model
For Pangolin or another reverse proxy, forward HTTP traffic to `intelligence-terminal:3117` (or the `PORT` you set). Missing API keys do not crash sweeps; affected sources are reported as degraded in `/api/health`. For Pangolin or another reverse proxy, forward HTTP traffic to `intelligence-terminal:3117` (or the `PORT` you set). Missing API keys do not crash sweeps; affected sources are reported as degraded in `/api/health`.
When data remains stale past `STALE_DATA_MAX_AGE_MINUTES`, the server sends an operator alert through configured Telegram/Discord channels after failed or degraded sweep attempts. `STALE_ALERT_COOLDOWN_MINUTES` prevents repeated stale alerts from spamming every refresh interval. Set `DASHBOARD_URL` to the Pangolin/public URL you want included in those alerts.
The dashboard Terminal Actions panel can trigger `status`, `sweep`, and `brief` through `/api/action`. Leave `TERMINAL_ACTIONS_ENABLED=true` for a private home-server deployment. For an internet-exposed deployment, set `SWEEP_TOKEN` and pass it through trusted automation, or set `TERMINAL_ACTIONS_ENABLED=false` to disable browser-triggered actions. If you protect actions with `SWEEP_TOKEN`, the browser can send it from `localStorage.crucix_sweep_token`. The dashboard Terminal Actions panel can trigger `status`, `sweep`, and `brief` through `/api/action`. Leave `TERMINAL_ACTIONS_ENABLED=true` for a private home-server deployment. For an internet-exposed deployment, set `SWEEP_TOKEN` and pass it through trusted automation, or set `TERMINAL_ACTIONS_ENABLED=false` to disable browser-triggered actions. If you protect actions with `SWEEP_TOKEN`, the browser can send it from `localStorage.crucix_sweep_token`.
#### Memory And Prediction Loop #### Memory And Prediction Loop
@@ -220,6 +219,39 @@ Retention, backup, and privacy expectations:
- Do not commit `runs/` or `.env`. API credentials stay in `.env`; memory stores derived observations, not secrets. - Do not commit `runs/` or `.env`. API credentials stay in `.env`; memory stores derived observations, not secrets.
- If you expose the dashboard through a reverse proxy, protect Terminal Actions and memory queries behind your normal authentication boundary. - If you expose the dashboard through a reverse proxy, protect Terminal Actions and memory queries behind your normal authentication boundary.
#### Scenario Watchlist
Intelligence Terminal can track operator hypotheses across sweeps with a runtime scenario file at `runs/scenarios.json`. On first run, the server creates three disabled starter examples:
- Middle East energy shock
- Macro stress spillover
- Regional escalation risk
Enable or add scenarios by editing `runs/scenarios.json`:
```json
{
"version": 1,
"scenarios": [
{
"id": "middle-east-energy-shock",
"enabled": true,
"name": "Middle East energy shock",
"description": "Energy supply risk building from regional conflict.",
"regions": ["Middle East", "Iran", "Strait of Hormuz"],
"categories": ["osint", "energy", "maritime"],
"keywords": ["missile", "strike", "hormuz", "oil"],
"thresholds": { "watching": 2, "building": 4, "confirmed": 7 },
"invalidation": "WTI normalizes and urgent regional signals fade."
}
]
}
```
Malformed scenario config degrades safely: sweeps continue and the dashboard shows the watchlist as a config issue. Scenario state is persisted in `runs/scenario-state.json`; delete that file to reset state transitions without deleting definitions.
Scenario states are `dormant`, `watching`, `building`, and `confirmed`. The dashboard shows active scenario state, confidence, score, and recent trigger time. Briefings include a `Scenario Watchlist` section when one or more scenarios change state.
#### Build And Publish Your Gitea Image #### Build And Publish Your Gitea Image
```bash ```bash
@@ -278,7 +310,7 @@ The server runs a sweep cycle every 15 minutes (configurable). Each cycle:
6. Pushes update to all connected browsers via SSE 6. Pushes update to all connected browsers via SSE
### Telegram Bot (Two-Way) ### Telegram Bot (Two-Way)
Crucix doubles as an interactive Telegram bot. Beyond sending alerts, it responds to commands directly from your chat: Intelligence Terminal doubles as an interactive Telegram bot. Beyond sending alerts, it responds to commands directly from your chat:
| Command | What It Does | | Command | What It Does |
|---------|-------------| |---------|-------------|
@@ -295,7 +327,7 @@ This requires `TELEGRAM_BOT_TOKEN` and `TELEGRAM_CHAT_ID` in `.env`. The bot pol
### Discord Bot (Two-Way) ### Discord Bot (Two-Way)
Crucix also supports Discord as a full-featured bot with slash commands and rich embed alerts. It mirrors the Telegram bot's capabilities with Discord-native formatting. Intelligence Terminal also supports Discord as a full-featured bot with slash commands and rich embed alerts. It mirrors the Telegram bot's capabilities with Discord-native formatting.
| Command | What It Does | | Command | What It Does |
|---------|-------------| |---------|-------------|
@@ -310,7 +342,7 @@ Alerts are delivered as rich embeds with color-coded sidebars: red for FLASH, ye
**Webhook fallback:** If you don't want to run a full bot, set `DISCORD_WEBHOOK_URL` instead. This enables one-way alerts (no slash commands) with zero dependencies — no `discord.js` needed. **Webhook fallback:** If you don't want to run a full bot, set `DISCORD_WEBHOOK_URL` instead. This enables one-way alerts (no slash commands) with zero dependencies — no `discord.js` needed.
**Optional dependency:** The full bot requires `discord.js`. Install it with `npm install discord.js`. If it's not installed, Crucix automatically falls back to webhook-only mode. **Optional dependency:** The full bot requires `discord.js`. Install it with `npm install discord.js`. If it's not installed, Intelligence Terminal automatically falls back to webhook-only mode.
### Optional LLM Layer ### Optional LLM Layer
Connect cloud or local OpenAI-compatible LLM providers for enhanced analysis: Connect cloud or local OpenAI-compatible LLM providers for enhanced analysis:
@@ -357,9 +389,12 @@ These three unlock the most valuable economic and satellite data. Each takes abo
| Key | Source | How to Get | | Key | Source | How to Get |
|-----|--------|------------| |-----|--------|------------|
| `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2 | | `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2. `ACLED_USER` / `ACLED_USERNAME` are accepted as email aliases |
| `AISSTREAM_API_KEY` | Maritime AIS vessel tracking | [aisstream.io](https://aisstream.io/) — free | | `AISSTREAM_API_KEY` | Maritime AIS vessel tracking | [aisstream.io](https://aisstream.io/) — free |
| `ADSB_API_KEY` | Unfiltered flight tracking | [RapidAPI](https://rapidapi.com/adsbexchange/api/adsbexchange-com1) — ~$10/mo | | `ADSB_API_KEY` | Unfiltered flight tracking | [RapidAPI](https://rapidapi.com/adsbexchange/api/adsbexchange-com1) — ~$10/mo |
| `REDDIT_CLIENT_ID` + `REDDIT_CLIENT_SECRET` | Reddit social sentiment | [reddit.com/prefs/apps](https://www.reddit.com/prefs/apps/) — create a script app |
Reddit is OAuth-only in this fork. If the Reddit credentials are missing or rejected, the Reddit source is reported as degraded and no unauthenticated `reddit.com/.../hot.json` fallback is used.
### LLM Provider (optional, for AI-enhanced ideas) ### LLM Provider (optional, for AI-enhanced ideas)
@@ -408,14 +443,14 @@ Alerts work with or without an LLM on both Telegram and Discord. With an LLM con
### Without Any Keys ### Without Any Keys
Crucix still works with zero API keys. 18+ sources require no authentication at all. Sources that need keys return structured errors and the rest of the sweep continues normally. Intelligence Terminal still works with zero API keys. 18+ sources require no authentication at all. Sources that need keys return structured errors and the rest of the sweep continues normally.
--- ---
## Architecture ## Architecture
``` ```
crucix/ intelligence-terminal/
├── server.mjs # Express dev server (SSE, auto-refresh, LLM, bot commands) ├── server.mjs # Express dev server (SSE, auto-refresh, LLM, bot commands)
├── crucix.config.mjs # Configuration with env var overrides + delta thresholds ├── crucix.config.mjs # Configuration with env var overrides + delta thresholds
├── diag.mjs # Diagnostic script — run if server fails to start ├── diag.mjs # Diagnostic script — run if server fails to start
@@ -555,6 +590,9 @@ All settings are in `.env` with sensible defaults:
|----------|---------|-------------| |----------|---------|-------------|
| `PORT` | `3117` | Dashboard server port | | `PORT` | `3117` | Dashboard server port |
| `REFRESH_INTERVAL_MINUTES` | `15` | Auto-refresh interval | | `REFRESH_INTERVAL_MINUTES` | `15` | Auto-refresh interval |
| `STALE_DATA_MAX_AGE_MINUTES` | `60` | Data age threshold for stale health state |
| `STALE_ALERT_COOLDOWN_MINUTES` | `60` | Minimum time between repeated operator stale-data alerts |
| `DASHBOARD_URL` | local URL | Dashboard URL included in operator alerts |
| `LLM_PROVIDER` | disabled | `anthropic`, `openai`, `gemini`, `codex`, `openrouter`, `minimax`, `mistral`, or `grok` | | `LLM_PROVIDER` | disabled | `anthropic`, `openai`, `gemini`, `codex`, `openrouter`, `minimax`, `mistral`, or `grok` |
| `LLM_API_KEY` | — | API key (not needed for codex) | | `LLM_API_KEY` | — | API key (not needed for codex) |
| `LLM_MODEL` | per-provider default | Override model selection | | `LLM_MODEL` | per-provider default | Override model selection |
@@ -604,7 +642,7 @@ This tests every import one by one, checks your Node.js version, and verifies po
**3. Check if port 3117 is already in use:** **3. Check if port 3117 is already in use:**
A previous Crucix instance may still be running in the background. A previous Intelligence Terminal instance may still be running in the background.
```powershell ```powershell
# Windows PowerShell # Windows PowerShell
@@ -626,7 +664,7 @@ Then try starting again. You can also change the port by setting `PORT=3118` in
```bash ```bash
node --version node --version
``` ```
Crucix requires Node.js 22 or later. If you have an older version, download the latest LTS from [nodejs.org](https://nodejs.org/). Intelligence Terminal requires Node.js 22 or later. If you have an older version, download the latest LTS from [nodejs.org](https://nodejs.org/).
### Dashboard shows empty panels after first start ### Dashboard shows empty panels after first start
@@ -636,11 +674,11 @@ This is normal — the first sweep takes 30–60 seconds to query all 27 sources
Expected behavior. Sources that require API keys will return structured errors if the key isn't set. The rest of the sweep continues normally. Check the Source Integrity section in the dashboard (or the server logs) to see which sources failed and why. The 3 most impactful free keys to add are `FRED_API_KEY`, `FIRMS_MAP_KEY`, and `EIA_API_KEY`. Expected behavior. Sources that require API keys will return structured errors if the key isn't set. The rest of the sweep continues normally. Check the Source Integrity section in the dashboard (or the server logs) to see which sources failed and why. The 3 most impactful free keys to add are `FRED_API_KEY`, `FIRMS_MAP_KEY`, and `EIA_API_KEY`.
OpenSky can also return `HTTP 429` when its public hotspots are queried too aggressively. Crucix does not try to evade that limit. Instead, it surfaces the throttle/error in source health and preserves the most recent non-empty air traffic snapshot from `runs/` so the dashboard flight layer does not suddenly go blank on a throttled sweep. OpenSky can also return `HTTP 429` when its public hotspots are queried too aggressively. Intelligence Terminal does not try to evade that limit. Instead, it surfaces the throttle/error in source health and preserves the most recent non-empty air traffic snapshot from `runs/` so the dashboard flight layer does not suddenly go blank on a throttled sweep.
### Telegram bot not responding to commands ### Telegram bot not responding to commands
Make sure both `TELEGRAM_BOT_TOKEN` and `TELEGRAM_CHAT_ID` are set in `.env`. The bot only responds to messages from the configured chat ID (security measure). You should see `[Crucix] Telegram alerts enabled` and `[Crucix] Bot command polling started` in the server logs on startup. If not, double-check your token with `curl https://api.telegram.org/bot<YOUR_TOKEN>/getMe`. Make sure both `TELEGRAM_BOT_TOKEN` and `TELEGRAM_CHAT_ID` are set in `.env`. The bot only responds to messages from the configured chat ID (security measure). You should see Telegram alert and bot polling startup lines in the server logs. If not, double-check your token with `curl https://api.telegram.org/bot<YOUR_TOKEN>/getMe`.
### Discord bot not responding to slash commands ### Discord bot not responding to slash commands
@@ -671,29 +709,21 @@ To update them: run the dashboard, wait for a sweep to complete, then use your b
## Contributing ## Contributing
Found a bug? Want to add a 28th source? PRs welcome. Each source is a standalone module in `apis/sources/` — just export a `briefing()` function that returns structured data and add it to the orchestrator in `apis/briefing.mjs`. Found a bug? Want to add a 28th source? PRs welcome. Each source is a standalone module in `apis/sources/` - just export a `briefing()` function that returns structured data and add it to the orchestrator in `apis/briefing.mjs`.
If you find this useful, a star helps others find it too. If you find this useful, a star on the Gitea repository helps other operators find it too.
For contribution guidelines, review expectations, and source-add rules, see `CONTRIBUTING.md`. For security reports, see `SECURITY.md`. For contribution guidelines, review expectations, and source-add rules, see `CONTRIBUTING.md`. For security reports, see `SECURITY.md`.
## Contact ## Contact
For partnerships, integrations, or other non-issue inquiries, you can reach me at `celesthioailabs@gmail.com`. For bugs, feature requests, and integration ideas, use the Gitea issue tracker so discussion stays visible and actionable:
For bugs and feature requests, please use GitHub Issues so discussion stays visible and actionable. https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues
--- ## Upstream And License
## Star History Intelligence Terminal is an AGPL-3.0-only Crucix fork focused on Docker-first home-server operation, source health transparency, Gitea Registry delivery, and operator-owned deployments. Upstream project credit remains with the original Crucix project.
<a href="https://www.star-history.com/?repos=calesthio%2FCrucix&type=date&legend=top-left">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/image?repos=calesthio/Crucix&type=date&theme=dark&legend=top-left" />
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/image?repos=calesthio/Crucix&type=date&legend=top-left" />
<img alt="Star History Chart" src="https://api.star-history.com/image?repos=calesthio/Crucix&type=date&legend=top-left" />
</picture>
</a>
--- ---

6
SECURITY.md
View File

@@ -2,13 +2,13 @@
## Reporting a Vulnerability ## Reporting a Vulnerability
If you discover a security issue in Crucix, please report it privately instead of opening a public GitHub issue. If you discover a security issue in Intelligence Terminal, please report it privately instead of opening a public issue.
Email: `celesthioailabs@gmail.com` Use the private security contact configured for this Gitea repository or contact the repository owner directly.
Use a subject line like: Use a subject line like:
`[Crucix Security] short description` `[Intelligence Terminal Security] short description`
Please include: Please include:

244
apis/sources/acled.mjs
View File

@@ -1,9 +1,9 @@
// ACLED — Armed Conflict Location & Event Data // ACLED - Armed Conflict Location & Event Data.
// Auth strategy (tries in order): // Auth strategy (tries in order):
// 1. Cookie-based session: POST /user/login?_format=json → session cookie // 1. OAuth Bearer token: POST /oauth/token -> Authorization header
// 2. OAuth Bearer token: POST /oauth/token → Authorization header // 2. Cookie-based session: POST /user/login?_format=json -> session cookie
// Set ACLED_EMAIL and ACLED_PASSWORD in .env (your myACLED login credentials). // Set ACLED_EMAIL and ACLED_PASSWORD in .env. ACLED_USER or ACLED_USERNAME are
// Data endpoint: GET https://acleddata.com/api/acled/read // accepted as aliases for ACLED_EMAIL.
import { daysAgo } from '../utils/fetch.mjs'; import { daysAgo } from '../utils/fetch.mjs';
import '../utils/env.mjs'; import '../utils/env.mjs';
@@ -12,124 +12,135 @@ const LOGIN_URL = 'https://acleddata.com/user/login?_format=json';
const TOKEN_URL = 'https://acleddata.com/oauth/token'; const TOKEN_URL = 'https://acleddata.com/oauth/token';
const API_BASE = 'https://acleddata.com/api/acled/read'; const API_BASE = 'https://acleddata.com/api/acled/read';
// Session cache
let sessionCache = { cookies: null, token: null, method: null, expires: 0 }; let sessionCache = { cookies: null, token: null, method: null, expires: 0 };
// Strategy 1: Cookie-based session login (mirrors browser login) export function resetAcledSessionCache() {
async function loginCookie(email, password) { sessionCache = { cookies: null, token: null, method: null, expires: 0 };
}
export function getAcledConfig(env = process.env) {
const email = env.ACLED_EMAIL || env.ACLED_USER || env.ACLED_USERNAME || '';
const password = env.ACLED_PASSWORD || '';
const missing = [];
if (!email) missing.push('ACLED_EMAIL');
if (!password) missing.push('ACLED_PASSWORD');
return { email, password, configured: missing.length === 0, missing };
}
function acledError(status, error, message, extra = {}) {
return { status, error, message, ...extra };
}
function safeText(value, max = 200) {
return String(value || '').replace(/Bearer\s+[A-Za-z0-9._-]+/gi, 'Bearer [redacted]').slice(0, max);
}
async function fetchWithTimeout(fetchImpl, url, init, timeoutMs) {
const controller = new AbortController(); const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 15000); const timer = setTimeout(() => controller.abort(), timeoutMs);
try { try {
const res = await fetch(LOGIN_URL, { return await fetchImpl(url, { ...init, signal: controller.signal });
} finally {
clearTimeout(timer);
}
}
export async function loginCookie(email, password, opts = {}) {
const fetchImpl = opts.fetchImpl || globalThis.fetch;
try {
const res = await fetchWithTimeout(fetchImpl, LOGIN_URL, {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ name: email, pass: password }), body: JSON.stringify({ name: email, pass: password }),
redirect: 'manual', redirect: 'manual',
signal: controller.signal, }, 15000);
});
clearTimeout(timer);
// Collect Set-Cookie headers
const setCookies = res.headers.getSetCookie?.() || []; const setCookies = res.headers.getSetCookie?.() || [];
const cookieStr = setCookies.map(c => c.split(';')[0]).join('; '); const cookieStr = setCookies.map(c => c.split(';')[0]).join('; ');
if (res.ok && cookieStr) { if ((res.ok || (res.status >= 300 && res.status < 400)) && cookieStr) {
return { cookies: cookieStr }; return { ok: true, cookies: cookieStr };
}
// Some Drupal sites return 303 redirect on successful login — cookies still set
if (res.status >= 300 && res.status < 400 && cookieStr) {
return { cookies: cookieStr };
} }
const errText = await res.text().catch(() => ''); const errText = await res.text().catch(() => '');
return { error: `Cookie login failed (HTTP ${res.status}): ${errText.slice(0, 200)}` }; return acledError('auth_failed', `acled_cookie_http_${res.status}`, `ACLED cookie login failed with HTTP ${res.status}`, {
detail: safeText(errText),
});
} catch (e) { } catch (e) {
clearTimeout(timer); return acledError('auth_failed', 'acled_cookie_request_failed', `ACLED cookie login error: ${e.message}`);
const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : '';
return { error: `Cookie login error: ${e.message}${cause}` };
} }
} }
// Strategy 2: OAuth2 password grant export async function loginOAuth(email, password, opts = {}) {
async function loginOAuth(email, password) { const fetchImpl = opts.fetchImpl || globalThis.fetch;
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 15000);
try { try {
const body = new URLSearchParams({ const body = new URLSearchParams({
username: email, username: email,
password: password, password,
grant_type: 'password', grant_type: 'password',
client_id: 'acled', client_id: 'acled',
}); });
const res = await fetch(TOKEN_URL, { const res = await fetchWithTimeout(fetchImpl, TOKEN_URL, {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: body.toString(), body: body.toString(),
signal: controller.signal, }, 15000);
});
clearTimeout(timer);
if (!res.ok) { if (!res.ok) {
const errText = await res.text().catch(() => ''); const errText = await res.text().catch(() => '');
return { error: `OAuth failed (HTTP ${res.status}): ${errText.slice(0, 200)}` }; return acledError('auth_failed', `acled_oauth_http_${res.status}`, `ACLED OAuth failed with HTTP ${res.status}`, {
detail: safeText(errText),
});
} }
const data = await res.json(); const data = await res.json();
if (!data.access_token) { if (!data.access_token) {
return { error: `OAuth response missing access_token: ${JSON.stringify(data).slice(0, 200)}` }; return acledError('auth_failed', 'acled_oauth_missing_access_token', 'ACLED OAuth response did not include access_token');
} }
return { token: data.access_token }; return { ok: true, token: data.access_token };
} catch (e) { } catch (e) {
clearTimeout(timer); return acledError('auth_failed', 'acled_oauth_request_failed', `ACLED OAuth error: ${e.message}`);
const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : '';
return { error: `OAuth error: ${e.message}${cause}` };
} }
} }
// Try both auth strategies export async function authenticate(opts = {}) {
async function authenticate() { const env = opts.env || process.env;
const email = process.env.ACLED_EMAIL; const fetchImpl = opts.fetchImpl || globalThis.fetch;
const password = process.env.ACLED_PASSWORD; const config = getAcledConfig(env);
if (!email || !password) { if (!config.configured) {
return { error: 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.' }; return acledError('no_credentials', 'missing_acled_credentials', 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.', {
missing: config.missing,
});
} }
// Return cached session if still valid
if (sessionCache.method && Date.now() < sessionCache.expires) { if (sessionCache.method && Date.now() < sessionCache.expires) {
return sessionCache; return sessionCache;
} }
const errors = []; const diagnostics = [];
const debug = process.argv.includes('--debug'); const oauthResult = await loginOAuth(config.email, config.password, { fetchImpl });
if (oauthResult.ok) {
// Try OAuth first (official programmatic method per ACLED docs)
const oauthResult = await loginOAuth(email, password);
if (oauthResult.token) {
if (debug) console.error(`[ACLED DEBUG] OAuth OK — token: ${oauthResult.token.slice(0, 20)}...`);
sessionCache = { cookies: null, token: oauthResult.token, method: 'oauth', expires: Date.now() + 23 * 60 * 60 * 1000 }; sessionCache = { cookies: null, token: oauthResult.token, method: 'oauth', expires: Date.now() + 23 * 60 * 60 * 1000 };
return sessionCache; return sessionCache;
} }
errors.push(`OAuth: ${oauthResult.error}`); diagnostics.push({ method: 'oauth', status: oauthResult.status, error: oauthResult.error, message: oauthResult.message });
if (debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`); if (opts.debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
// Fall back to cookie-based session const cookieResult = await loginCookie(config.email, config.password, { fetchImpl });
const cookieResult = await loginCookie(email, password); if (cookieResult.ok) {
if (cookieResult.cookies) {
if (debug) console.error(`[ACLED DEBUG] Cookie OK — cookies: ${cookieResult.cookies.slice(0, 80)}...`);
sessionCache = { cookies: cookieResult.cookies, token: null, method: 'cookie', expires: Date.now() + 12 * 60 * 60 * 1000 }; sessionCache = { cookies: cookieResult.cookies, token: null, method: 'cookie', expires: Date.now() + 12 * 60 * 60 * 1000 };
return sessionCache; return sessionCache;
} }
errors.push(`Cookie: ${cookieResult.error}`); diagnostics.push({ method: 'cookie', status: cookieResult.status, error: cookieResult.error, message: cookieResult.message });
if (opts.debug) console.error(`[ACLED DEBUG] Cookie login failed: ${cookieResult.error}`);
return { error: `All ACLED auth methods failed.\n${errors.join('\n')}` }; return acledError('auth_failed', 'acled_auth_failed', 'All ACLED auth methods failed.', { diagnostics });
} }
// Build headers based on auth method
function authHeaders(session) { function authHeaders(session) {
const headers = { 'User-Agent': 'Crucix/1.0', 'Content-Type': 'application/json' }; const headers = { 'User-Agent': 'Crucix/2.0', 'Content-Type': 'application/json' };
if (session.method === 'cookie' && session.cookies) { if (session.method === 'cookie' && session.cookies) {
headers['Cookie'] = session.cookies; headers['Cookie'] = session.cookies;
} else if (session.method === 'oauth' && session.token) { } else if (session.method === 'oauth' && session.token) {
@@ -138,7 +149,6 @@ function authHeaders(session) {
return headers; return headers;
} }
// Event type constants
export const EVENT_TYPES = [ export const EVENT_TYPES = [
'Battles', 'Battles',
'Explosions/Remote violence', 'Explosions/Remote violence',
@@ -148,7 +158,6 @@ export const EVENT_TYPES = [
'Strategic developments', 'Strategic developments',
]; ];
// Query conflict events with flexible filters
export async function getEvents(opts = {}) { export async function getEvents(opts = {}) {
const { const {
limit = 500, limit = 500,
@@ -157,10 +166,13 @@ export async function getEvents(opts = {}) {
eventType, eventType,
country, country,
region, region,
env = process.env,
fetchImpl = globalThis.fetch,
debug = process.argv.includes('--debug'),
} = opts; } = opts;
const session = await authenticate(); const session = await authenticate({ env, fetchImpl, debug });
if (session.error) return { error: session.error }; if (session.error) return session;
const params = new URLSearchParams({ _format: 'json', limit: String(limit) }); const params = new URLSearchParams({ _format: 'json', limit: String(limit) });
if (eventDateStart && eventDateEnd) { if (eventDateStart && eventDateEnd) {
@@ -171,59 +183,43 @@ export async function getEvents(opts = {}) {
if (country) params.set('country', country); if (country) params.set('country', country);
if (region) params.set('region', String(region)); if (region) params.set('region', String(region));
const debug = process.argv.includes('--debug');
try { try {
const url = `${API_BASE}?${params}`; const url = `${API_BASE}?${params}`;
const hdrs = authHeaders(session); if (debug) console.error(`[ACLED DEBUG] Data request: GET ${url}`);
if (debug) { const res = await fetchWithTimeout(fetchImpl, url, { headers: authHeaders(session) }, 25000);
console.error(`[ACLED DEBUG] Data request: GET ${url}`);
console.error(`[ACLED DEBUG] Headers: ${JSON.stringify(hdrs)}`);
}
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 25000);
const res = await fetch(url, {
headers: hdrs,
signal: controller.signal,
});
clearTimeout(timer);
if (debug) console.error(`[ACLED DEBUG] Data response: HTTP ${res.status}`); if (debug) console.error(`[ACLED DEBUG] Data response: HTTP ${res.status}`);
if (!res.ok) { if (!res.ok) {
const errText = await res.text().catch(() => ''); const errText = await res.text().catch(() => '');
if (debug) console.error(`[ACLED DEBUG] Error body: ${errText.slice(0, 500)}`);
if (res.status === 401 || res.status === 403) { if (res.status === 401 || res.status === 403) {
// Clear cache and report
sessionCache = { cookies: null, token: null, method: null, expires: 0 }; sessionCache = { cookies: null, token: null, method: null, expires: 0 };
const hint = res.status === 403 return acledError('access_denied', `acled_data_http_${res.status}`, `ACLED data access denied with HTTP ${res.status}`, {
? '\n→ Fix: Log in at https://acleddata.com/user/login, then:\n' authMethod: session.method,
+ ' 1. Accept Terms of Use (profile → Terms of Use button → check the box)\n' detail: safeText(errText, 300),
+ ' 2. Complete all required profile fields\n' hint: 'Accept ACLED terms, complete profile fields, and confirm API access for the account.',
+ ' 3. Ensure your account has the "API" access group\n' });
+ ' Contact access@acleddata.com if issues persist.'
: '';
return { error: `ACLED data access denied (HTTP ${res.status}, auth method: ${session.method}). Response: ${errText.slice(0, 300)}${hint}` };
} }
return { error: `HTTP ${res.status}: ${errText.slice(0, 200)}` }; return acledError('api_failed', `acled_data_http_${res.status}`, `ACLED data request failed with HTTP ${res.status}`, {
detail: safeText(errText),
});
} }
const data = await res.json(); const data = await res.json();
// ACLED may return a 200 with an error status in the body
if (data?.status && data.status !== 200) { if (data?.status && data.status !== 200) {
return { error: `ACLED API error: status ${data.status} — ${data.message || 'Unknown error'}` }; return acledError('api_failed', `acled_api_status_${data.status}`, `ACLED API returned status ${data.status}`, {
detail: safeText(data.message),
});
} }
return data; return data;
} catch (e) { } catch (e) {
if (e.name === 'AbortError') { if (e.name === 'AbortError') {
return { error: 'ACLED data request timed out (25s)' }; return acledError('api_failed', 'acled_data_timeout', 'ACLED data request timed out after 25s');
} }
const rootCause = e.cause ? `${e.cause.message || e.cause.code || e.cause}` : ''; return acledError('api_failed', 'acled_data_request_failed', `ACLED data error: ${e.message}`);
return { error: `ACLED data error: ${e.message}${rootCause ? ' → ' + rootCause : ''}` };
} }
} }
// Summarize events by a given field
function groupBy(events, field) { function groupBy(events, field) {
const map = {}; const map = {};
for (const e of events) { for (const e of events) {
@@ -235,33 +231,47 @@ function groupBy(events, field) {
return map; return map;
} }
// Briefing — last 7 days of global conflict events export async function briefing(opts = {}) {
export async function briefing() { const env = opts.env || process.env;
if (!process.env.ACLED_EMAIL || !process.env.ACLED_PASSWORD) { const fetchImpl = opts.fetchImpl || globalThis.fetch;
const config = getAcledConfig(env);
if (!config.configured) {
return { return {
source: 'ACLED', source: 'ACLED',
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
status: 'no_credentials', status: 'no_credentials',
error: 'missing_acled_credentials',
missing: config.missing,
message: 'Set ACLED_EMAIL and ACLED_PASSWORD in .env. Register at https://acleddata.com/user/register', message: 'Set ACLED_EMAIL and ACLED_PASSWORD in .env. Register at https://acleddata.com/user/register',
}; };
} }
const start = daysAgo(7); const start = daysAgo(7);
const end = daysAgo(0); const end = daysAgo(0);
const data = await getEvents({ const data = await getEvents({
eventDateStart: start, eventDateStart: start,
eventDateEnd: end, eventDateEnd: end,
limit: 2000, limit: 2000,
env,
fetchImpl,
debug: opts.debug,
}); });
if (data?.error) { if (data?.error) {
return { source: 'ACLED', timestamp: new Date().toISOString(), error: data.error }; return {
source: 'ACLED',
timestamp: new Date().toISOString(),
status: data.status || 'api_failed',
error: data.error,
message: data.message,
detail: data.detail,
hint: data.hint,
diagnostics: data.diagnostics,
};
} }
let events = data?.data || []; let events = data?.data || [];
// Enrich all events with numeric lat/lon
events = events.map(e => ({ events = events.map(e => ({
...e, ...e,
lat: parseFloat(e.latitude) || null, lat: parseFloat(e.latitude) || null,
@@ -272,10 +282,9 @@ export async function briefing() {
(sum, e) => sum + (parseInt(e.fatalities, 10) || 0), 0 (sum, e) => sum + (parseInt(e.fatalities, 10) || 0), 0
); );
const byRegion = groupBy(events, 'region'); const byRegion = groupBy(events, 'region');
const byType = groupBy(events, 'event_type'); const byType = groupBy(events, 'event_type');
const byCountry = groupBy(events, 'country'); const byCountry = groupBy(events, 'country');
const topCountries = Object.entries(byCountry) const topCountries = Object.entries(byCountry)
.sort((a, b) => b[1].count - a[1].count) .sort((a, b) => b[1].count - a[1].count)
.slice(0, 10) .slice(0, 10)
@@ -286,20 +295,21 @@ export async function briefing() {
.sort((a, b) => (parseInt(b.fatalities, 10) || 0) - (parseInt(a.fatalities, 10) || 0)) .sort((a, b) => (parseInt(b.fatalities, 10) || 0) - (parseInt(a.fatalities, 10) || 0))
.slice(0, 15) .slice(0, 15)
.map(e => ({ .map(e => ({
date: e.event_date, date: e.event_date,
type: e.event_type, type: e.event_type,
subType: e.sub_event_type, subType: e.sub_event_type,
country: e.country, country: e.country,
location: e.location, location: e.location,
fatalities: parseInt(e.fatalities, 10) || 0, fatalities: parseInt(e.fatalities, 10) || 0,
lat: parseFloat(e.latitude) || null, lat: parseFloat(e.latitude) || null,
lon: parseFloat(e.longitude) || null, lon: parseFloat(e.longitude) || null,
notes: e.notes?.slice(0, 200), notes: e.notes?.slice(0, 200),
})); }));
return { return {
source: 'ACLED', source: 'ACLED',
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
status: 'ok',
period: { start, end }, period: { start, end },
totalEvents: events.length, totalEvents: events.length,
totalFatalities, totalFatalities,

139
apis/sources/reddit.mjs
View File

@@ -1,14 +1,15 @@
// Reddit — social sentiment intelligence // Reddit social sentiment intelligence.
// Reddit now requires OAuth for API access (public JSON API returns 403). // Reddit API access requires OAuth. Runtime sweeps intentionally do not use
// Gracefully degrades when not authenticated. // unauthenticated reddit.com .json scraping because it is unreliable and not
// To enable: register an app at https://www.reddit.com/prefs/apps/ and set // acceptable for production operation.
// REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET in .env
import { safeFetch } from '../utils/fetch.mjs'; import { safeFetch } from '../utils/fetch.mjs';
import '../utils/env.mjs'; import '../utils/env.mjs';
function delay(ms) { return new Promise(r => setTimeout(r, ms)); } function delay(ms) { return new Promise(r => setTimeout(r, ms)); }
const USER_AGENT = 'Crucix/2.0 intelligence-engine';
const SUBREDDITS = [ const SUBREDDITS = [
'worldnews', 'worldnews',
'geopolitics', 'geopolitics',
@@ -17,48 +18,95 @@ const SUBREDDITS = [
'commodities', 'commodities',
]; ];
// Get OAuth token using client credentials flow (application-only) export function getRedditConfig(env = process.env) {
async function getToken() { const clientId = env.REDDIT_CLIENT_ID || '';
const clientId = process.env.REDDIT_CLIENT_ID; const clientSecret = env.REDDIT_CLIENT_SECRET || '';
const clientSecret = process.env.REDDIT_CLIENT_SECRET; const missing = [];
if (!clientId || !clientSecret) return null; if (!clientId) missing.push('REDDIT_CLIENT_ID');
if (!clientSecret) missing.push('REDDIT_CLIENT_SECRET');
return {
clientId,
clientSecret,
configured: missing.length === 0,
missing,
};
}
function credentialsMessage(missing) {
return `Reddit requires OAuth. Register a script app at https://www.reddit.com/prefs/apps/ and set ${missing.join(' and ')} in .env`;
}
export async function getToken({ env = process.env, fetchImpl = globalThis.fetch } = {}) {
const config = getRedditConfig(env);
if (!config.configured) {
return {
ok: false,
status: 'no_credentials',
missing: config.missing,
error: 'missing_reddit_oauth_credentials',
message: credentialsMessage(config.missing),
};
}
try { try {
const auth = Buffer.from(`${clientId}:${clientSecret}`).toString('base64'); const auth = Buffer.from(`${config.clientId}:${config.clientSecret}`).toString('base64');
const res = await fetch('https://www.reddit.com/api/v1/access_token', { const res = await fetchImpl('https://www.reddit.com/api/v1/access_token', {
method: 'POST', method: 'POST',
headers: { headers: {
'Authorization': `Basic ${auth}`, 'Authorization': `Basic ${auth}`,
'Content-Type': 'application/x-www-form-urlencoded', 'Content-Type': 'application/x-www-form-urlencoded',
'User-Agent': 'Crucix/1.0 intelligence-engine', 'User-Agent': USER_AGENT,
}, },
body: 'grant_type=client_credentials', body: 'grant_type=client_credentials',
}); });
if (!res.ok) return null; if (!res.ok) {
const body = await res.text().catch(() => '');
return {
ok: false,
status: 'auth_failed',
error: `reddit_oauth_http_${res.status}`,
message: `Reddit OAuth token request failed with HTTP ${res.status}`,
detail: body.slice(0, 200),
};
}
const data = await res.json(); const data = await res.json();
return data.access_token || null; if (!data.access_token) {
} catch { return {
return null; ok: false,
status: 'auth_failed',
error: 'reddit_oauth_missing_access_token',
message: 'Reddit OAuth token response did not include an access token',
};
}
return { ok: true, status: 'ok', token: data.access_token };
} catch (e) {
return {
ok: false,
status: 'auth_failed',
error: 'reddit_oauth_request_failed',
message: e.message,
};
} }
} }
// Fetch hot posts — tries OAuth first, then falls back to public endpoint
export async function getHot(subreddit, opts = {}) { export async function getHot(subreddit, opts = {}) {
const { limit = 10, token = null } = opts; const { limit = 10, token = null } = opts;
if (token) { if (!token) {
// Use OAuth endpoint return {
return safeFetch(`https://oauth.reddit.com/r/${subreddit}/hot?limit=${limit}&raw_json=1`, { status: 'no_credentials',
headers: { error: 'reddit_oauth_required',
'Authorization': `Bearer ${token}`, message: 'Reddit source requires OAuth; unauthenticated reddit.com .json scraping is disabled',
'User-Agent': 'Crucix/1.0 intelligence-engine', };
},
});
} }
// Try public endpoint (may 403) return safeFetch(`https://oauth.reddit.com/r/${subreddit}/hot?limit=${limit}&raw_json=1`, {
return safeFetch(`https://www.reddit.com/r/${subreddit}/hot.json?limit=${limit}&raw_json=1`, { source: 'Reddit',
headers: { 'User-Agent': 'Crucix/1.0 intelligence-engine' }, headers: {
'Authorization': `Bearer ${token}`,
'User-Agent': USER_AGENT,
},
}); });
} }
@@ -74,29 +122,46 @@ function compactPost(child) {
}; };
} }
export async function briefing() { export async function briefing(opts = {}) {
const token = await getToken(); const {
env = process.env,
subreddits = SUBREDDITS,
delayMs = 1000,
fetchImpl = globalThis.fetch,
} = opts;
const tokenResult = await getToken({ env, fetchImpl });
if (!token && !process.env.REDDIT_CLIENT_ID) { if (!tokenResult.ok) {
return { return {
source: 'Reddit', source: 'Reddit',
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
status: 'no_key', status: tokenResult.status,
message: 'Reddit requires OAuth. Register at https://www.reddit.com/prefs/apps/ (script type), set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET in .env', error: tokenResult.error,
message: tokenResult.message,
missing: tokenResult.missing || [],
}; };
} }
const subredditResults = {}; const subredditResults = {};
for (const sub of SUBREDDITS) { const errors = [];
const result = await getHot(sub, { limit: 10, token }); for (const sub of subreddits) {
const result = await getHot(sub, { limit: 10, token: tokenResult.token });
if (result?.error) {
errors.push({ subreddit: sub, error: result.error });
subredditResults[sub] = [];
if (delayMs > 0) await delay(delayMs);
continue;
}
const children = result?.data?.children || []; const children = result?.data?.children || [];
subredditResults[sub] = children.map(compactPost).filter(Boolean); subredditResults[sub] = children.map(compactPost).filter(Boolean);
await delay(token ? 1000 : 2000); if (delayMs > 0) await delay(delayMs);
} }
return { return {
source: 'Reddit', source: 'Reddit',
timestamp: new Date().toISOString(), timestamp: new Date().toISOString(),
status: errors.length > 0 ? 'degraded' : 'ok',
...(errors.length > 0 ? { error: 'reddit_subreddit_fetch_failed', errors } : {}),
subreddits: subredditResults, subreddits: subredditResults,
}; };
} }

30
apis/utils/fetch.mjs
View File

@@ -42,6 +42,7 @@ export async function safeFetch(url, opts = {}) {
let lastError; let lastError;
for (let i = 0; i <= retries; i++) { for (let i = 0; i <= retries; i++) {
const started = Date.now(); const started = Date.now();
let metricRecorded = false;
try { try {
const controller = new AbortController(); const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeout); const timer = setTimeout(() => controller.abort(), timeout);
@@ -51,22 +52,29 @@ export async function safeFetch(url, opts = {}) {
}); });
clearTimeout(timer); clearTimeout(timer);
const status = res.status; const status = res.status;
if (!res.ok) {
const body = await res.text().catch(() => '');
recordFetchMetric({ url, source, ok: false, status, bytes: body.length, durationMs: Date.now() - started, error: `HTTP ${res.status}` });
throw new Error(`HTTP ${res.status}: ${body.slice(0, 200)}`);
}
const text = await res.text(); const text = await res.text();
recordFetchMetric({ url, source, ok: true, status, bytes: text.length, durationMs: Date.now() - started }); if (!res.ok) {
const error = `HTTP ${res.status}`;
recordFetchMetric({ url, source, ok: false, status, bytes: text.length, durationMs: Date.now() - started, error });
metricRecorded = true;
throw new Error(`${error}: ${text.slice(0, 200)}`);
}
const trimmed = text.trim(); const trimmed = text.trim();
const contentType = res.headers.get('content-type') || ''; const contentType = res.headers.get('content-type') || '';
if (contentType.includes('text/html') || trimmed.startsWith('<!DOCTYPE html') || trimmed.startsWith('<html')) { if (contentType.includes('text/html') || trimmed.startsWith('<!DOCTYPE html') || trimmed.startsWith('<html')) {
throw new Error(`Expected JSON but received HTML from ${new URL(url).host}`); const error = `Expected JSON but received HTML from ${new URL(url).host}`;
recordFetchMetric({ url, source, ok: false, status, bytes: text.length, durationMs: Date.now() - started, error });
metricRecorded = true;
throw new Error(error);
} }
recordFetchMetric({ url, source, ok: true, status, bytes: text.length, durationMs: Date.now() - started });
metricRecorded = true;
try { return JSON.parse(text); } catch { return { rawText: text.slice(0, 500) }; } try { return JSON.parse(text); } catch { return { rawText: text.slice(0, 500) }; }
} catch (e) { } catch (e) {
lastError = e; lastError = e;
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message }); if (!metricRecorded) {
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
}
// GDELT needs 5s between requests, others are fine with shorter delays // GDELT needs 5s between requests, others are fine with shorter delays
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1))); if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
} }
@@ -79,6 +87,7 @@ export async function safeFetchText(url, opts = {}) {
let lastError; let lastError;
for (let i = 0; i <= retries; i++) { for (let i = 0; i <= retries; i++) {
const started = Date.now(); const started = Date.now();
let metricRecorded = false;
try { try {
const controller = new AbortController(); const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeout); const timer = setTimeout(() => controller.abort(), timeout);
@@ -89,11 +98,14 @@ export async function safeFetchText(url, opts = {}) {
clearTimeout(timer); clearTimeout(timer);
const text = await res.text(); const text = await res.text();
recordFetchMetric({ url, source, ok: res.ok, status: res.status, bytes: text.length, durationMs: Date.now() - started, error: res.ok ? null : `HTTP ${res.status}` }); recordFetchMetric({ url, source, ok: res.ok, status: res.status, bytes: text.length, durationMs: Date.now() - started, error: res.ok ? null : `HTTP ${res.status}` });
metricRecorded = true;
if (!res.ok) throw new Error(`HTTP ${res.status}: ${text.slice(0, 200)}`); if (!res.ok) throw new Error(`HTTP ${res.status}: ${text.slice(0, 200)}`);
return { text, status: res.status, bytes: text.length }; return { text, status: res.status, bytes: text.length };
} catch (e) { } catch (e) {
lastError = e; lastError = e;
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message }); if (!metricRecorded) {
recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message });
}
if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1))); if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1)));
} }
} }

2
crucix.config.mjs
View File

@@ -23,6 +23,8 @@ export default {
refreshIntervalMinutes: intEnv('REFRESH_INTERVAL_MINUTES', 15), refreshIntervalMinutes: intEnv('REFRESH_INTERVAL_MINUTES', 15),
autoOpenBrowser: boolEnv('AUTO_OPEN_BROWSER', false), autoOpenBrowser: boolEnv('AUTO_OPEN_BROWSER', false),
staleDataMaxAgeMinutes: intEnv('STALE_DATA_MAX_AGE_MINUTES', 60), staleDataMaxAgeMinutes: intEnv('STALE_DATA_MAX_AGE_MINUTES', 60),
staleAlertCooldownMinutes: intEnv('STALE_ALERT_COOLDOWN_MINUTES', 60),
dashboardUrl: process.env.DASHBOARD_URL || null,
sweepToken: process.env.SWEEP_TOKEN || null, sweepToken: process.env.SWEEP_TOKEN || null,
terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', true), terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', true),

42
dashboard/inject.mjs
View File

@@ -83,16 +83,48 @@ const geoKeywords = {
'IMF':[38.9,-77],'World Bank':[38.9,-77],'UN':[40.7,-74], 'IMF':[38.9,-77],'World Bank':[38.9,-77],'UN':[40.7,-74],
}; };
function geoTagText(text) { function escapeRegex(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
function geoKeywordRegex(keyword) {
const flags = keyword.length <= 3 && keyword === keyword.toUpperCase() ? 'u' : 'iu';
return new RegExp(`(^|[^\\p{L}\\p{N}])${escapeRegex(keyword)}(?=$|[^\\p{L}\\p{N}])`, flags);
}
const geoKeywordEntries = Object.entries(geoKeywords)
.sort((a, b) => b[0].length - a[0].length)
.map(([keyword, coords]) => ({ keyword, coords, pattern: geoKeywordRegex(keyword) }));
export function geoTagText(text) {
if (!text) return null; if (!text) return null;
for (const [keyword, [lat, lon]] of Object.entries(geoKeywords)) { for (const { keyword, coords, pattern } of geoKeywordEntries) {
if (text.includes(keyword)) { if (pattern.test(text)) {
const [lat, lon] = coords;
return { lat, lon, region: keyword }; return { lat, lon, region: keyword };
} }
} }
return null; return null;
} }
function stableHash(value) {
let hash = 2166136261;
for (let i = 0; i < value.length; i++) {
hash ^= value.charCodeAt(i);
hash = Math.imul(hash, 16777619);
}
return hash >>> 0;
}
export function stableGeoJitter(key, axis) {
const bucket = stableHash(`${axis}:${key}`) / 0xffffffff;
return (bucket - 0.5) * 2;
}
function newsGeoKey(item) {
return `${item.source || ''}|${item.title || ''}|${item.date || ''}|${item.url || ''}`;
}
function sanitizeExternalUrl(raw) { function sanitizeExternalUrl(raw) {
if (!raw) return undefined; if (!raw) return undefined;
try { try {
@@ -235,8 +267,8 @@ export async function fetchAllNews() {
source: item.source, source: item.source,
date: item.date, date: item.date,
url: item.url, url: item.url,
lat: geo.lat + (Math.random() - 0.5) * 2, lat: geo.lat + stableGeoJitter(newsGeoKey(item), 'lat'),
lon: geo.lon + (Math.random() - 0.5) * 2, lon: geo.lon + stableGeoJitter(newsGeoKey(item), 'lon'),
region: geo.region region: geo.region
}); });
} }

12
dashboard/public/jarvis.html
View File

@@ -1663,6 +1663,14 @@ function renderRight(){
deltaRows.push(`<div class="delta-row"><span class="delta-badge down">&#9660;</span><span class="delta-label">${s.label||s.key}</span><span class="delta-val">${s.from}→${s.to} (${val})</span></div>`); deltaRows.push(`<div class="delta-row"><span class="delta-badge down">&#9660;</span><span class="delta-label">${s.label||s.key}</span><span class="delta-val">${s.from}→${s.to} (${val})</span></div>`);
} }
const deltaHtml = hasDelta ? deltaRows.join('') : `<div style="padding:12px;text-align:center;color:var(--dim);font-family:var(--mono);font-size:10px">${t('delta.noChanges','No changes since last sweep')}</div>`; const deltaHtml = hasDelta ? deltaRows.join('') : `<div style="padding:12px;text-align:center;color:var(--dim);font-family:var(--mono);font-size:10px">${t('delta.noChanges','No changes since last sweep')}</div>`;
const scenarioItems = (D.scenarios?.items || []).filter(s => s.enabled || s.state !== 'dormant').slice(0,4);
const scenarioHtml = scenarioItems.length ? scenarioItems.map(s => `
<div class="signal-row">
<strong>${s.name} <span class="delta-badge ${s.changed?'new':''}">${(s.state||'dormant').toUpperCase()}</span></strong>
<p>${s.description || ''}</p>
<div class="layer-sub">${s.confidence || 0}% confidence · score ${s.score || 0}${s.lastTriggerTime ? ' · ' + getAge(s.lastTriggerTime) : ''}</div>
</div>
`).join('') : `<div style="padding:12px;text-align:center;color:var(--dim);font-family:var(--mono);font-size:10px">No active scenario watchlist items</div>`;
document.getElementById('rightRail').innerHTML=` document.getElementById('rightRail').innerHTML=`
<div class="g-panel right-actions"> <div class="g-panel right-actions">
@@ -1679,6 +1687,10 @@ function renderRight(){
<div class="sec-head"><h3>${t('panels.crossSourceSignals','Cross-Source Signals')}</h3><span class="badge">${t('badges.worldview','WORLDVIEW')}</span></div> <div class="sec-head"><h3>${t('panels.crossSourceSignals','Cross-Source Signals')}</h3><span class="badge">${t('badges.worldview','WORLDVIEW')}</span></div>
${signals} ${signals}
</div> </div>
<div class="g-panel right-scenarios">
<div class="sec-head"><h3>Scenario Watchlist</h3><span class="badge">${D.scenarios?.available===false?'CONFIG':'LIVE'}</span></div>
${scenarioHtml}
</div>
${mobile ? '' : buildOsintPanel('right-osint', 260)} ${mobile ? '' : buildOsintPanel('right-osint', 260)}
<div class="g-panel right-core"> <div class="g-panel right-core">
<div class="sec-head"><h3>${t('panels.signalCore','Signal Core')}</h3><span class="badge">${t('badges.hotMetrics','HOT METRICS')}</span></div> <div class="sec-head"><h3>${t('panels.signalCore','Signal Core')}</h3><span class="badge">${t('badges.hotMetrics','HOT METRICS')}</span></div>

1
docs/sources/README.md
View File

@@ -16,3 +16,4 @@ Source docs:
- [Telegram](telegram.md) - [Telegram](telegram.md)
- [FIRMS](firms.md) - [FIRMS](firms.md)
- [Maritime](maritime.md) - [Maritime](maritime.md)
- [Reddit](reddit.md)

11
docs/sources/acled.md
View File

@@ -2,8 +2,11 @@
Provides conflict events, fatalities, event types, and locations. Provides conflict events, fatalities, event types, and locations.
- Auth: `ACLED_EMAIL` and `ACLED_PASSWORD`. - Auth: `ACLED_EMAIL` and `ACLED_PASSWORD`. `ACLED_USER` or `ACLED_USERNAME` may be used as aliases for `ACLED_EMAIL`.
- Flow: OAuth password grant is tried first, then cookie session fallback. - Flow: OAuth password grant is tried first, then cookie session fallback.
- Failure modes: missing credentials, terms/profile not completed, expired token, account missing API access. - Failure modes are classified as `no_credentials`, `auth_failed`, `access_denied`, or `api_failed`.
- Behavior: missing or rejected credentials produce degraded source health with the ACLED error text. - Behavior: missing, rejected, or unauthorized credentials produce degraded source health with a concise operator message.
- Test: set credentials, run `node apis/sources/acled.mjs`, then check `/api/health`. - Secret handling: debug output never prints bearer tokens, cookies, or the configured password.
- Test: run `node --test test/acled-source.test.mjs`; with real credentials, run `node apis/sources/acled.mjs`, then check `/api/health`.
`access_denied` normally means the login worked but the account cannot read API data. Check that ACLED terms are accepted, required profile fields are complete, and API access is enabled for the account.

2
docs/sources/opensky.md
View File

@@ -6,4 +6,4 @@ Provides public aircraft state data for regional air-activity hotspots.
- Failure modes: timeouts, `HTTP 429`, and empty regions. - Failure modes: timeouts, `HTTP 429`, and empty regions.
- Behavior: source health is marked degraded on API errors. The dashboard may use the most recent non-empty air snapshot from `runs/` and marks it in `airMeta.fallback`. - Behavior: source health is marked degraded on API errors. The dashboard may use the most recent non-empty air snapshot from `runs/` and marks it in `airMeta.fallback`.
- Test: start a sweep and inspect `/api/health` plus `airMeta` from `/api/data`. - Test: start a sweep and inspect `/api/health` plus `airMeta` from `/api/data`.
- Operator note: Crucix does not bypass OpenSky throttles. Increase `REFRESH_INTERVAL_MINUTES` if the source degrades repeatedly. - Operator note: Intelligence Terminal does not bypass OpenSky throttles. Increase `REFRESH_INTERVAL_MINUTES` if the source degrades repeatedly.

33
docs/sources/reddit.md Normal file
View File

@@ -0,0 +1,33 @@
# Reddit Source
Reddit is used as a social sentiment input for selected geopolitical and market subreddits.
## Configuration
Create a Reddit script app at:
```text
https://www.reddit.com/prefs/apps/
```
Then set:
```env
REDDIT_CLIENT_ID=
REDDIT_CLIENT_SECRET=
```
## Runtime Behavior
- The source uses the OAuth client credentials flow and then reads `https://oauth.reddit.com`.
- Unauthenticated `reddit.com/.../hot.json` scraping is intentionally disabled.
- Missing credentials return `status: no_credentials` and are surfaced as source degradation.
- OAuth failures return `status: auth_failed` without logging or returning the client secret.
- Subreddit fetch failures return `status: degraded` with per-subreddit errors.
## Test
```bash
node apis/sources/reddit.mjs
npm run test:unit
```

212
lib/scenarios.mjs Normal file
View File

@@ -0,0 +1,212 @@
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs';
import { join } from 'path';
const DEFAULT_SCENARIOS = [
{
id: 'middle-east-energy-shock',
enabled: false,
name: 'Middle East energy shock',
description: 'Energy supply risk building from Middle East conflict or chokepoint pressure.',
regions: ['Middle East', 'Iran', 'Israel', 'Strait of Hormuz'],
categories: ['osint', 'energy', 'maritime'],
keywords: ['missile', 'strike', 'hormuz', 'oil', 'energy', 'blockade'],
thresholds: { watching: 2, building: 4, confirmed: 7 },
invalidation: 'WTI normalizes and regional urgent signals fade for several sweeps.',
},
{
id: 'macro-stress-spillover',
enabled: false,
name: 'Macro stress spillover',
description: 'Market stress spreads from volatility into credit, rates, or commodities.',
regions: ['US', 'Global'],
categories: ['macro', 'markets'],
keywords: ['vix', 'spread', 'credit', 'yield', 'inflation', 'gold'],
thresholds: { watching: 2, building: 4, confirmed: 6 },
invalidation: 'VIX and credit stress both normalize while source health remains stable.',
},
{
id: 'regional-escalation-risk',
enabled: false,
name: 'Regional escalation risk',
description: 'Local conflict signals broaden across adjacent regions or source categories.',
regions: ['Ukraine', 'Taiwan', 'Africa', 'Middle East'],
categories: ['conflict', 'thermal', 'osint', 'air'],
keywords: ['mobilization', 'intercept', 'drone', 'ballistic', 'fatalities', 'border'],
thresholds: { watching: 2, building: 5, confirmed: 8 },
invalidation: 'No fresh cross-source escalation signals appear inside the configured horizon.',
},
];
export function evaluateScenarios(data, delta, runsDir) {
const loaded = loadScenarioDefinitions(runsDir);
if (!loaded.ok) {
return { available: false, error: loaded.error, items: [], changed: [] };
}
const statePath = join(runsDir, 'scenario-state.json');
const previous = readJson(statePath, {});
const evaluatedAt = data.meta?.timestamp || new Date().toISOString();
const corpus = buildCorpus(data, delta);
const items = loaded.scenarios.map(def => evaluateScenario(def, corpus, previous[def.id], evaluatedAt));
const changed = items.filter(item => item.changed);
writeJson(statePath, Object.fromEntries(items.map(item => [item.id, {
state: item.state,
score: item.score,
confidence: item.confidence,
lastTriggerTime: item.lastTriggerTime,
updatedAt: evaluatedAt,
}])));
return {
available: true,
path: loaded.path,
items,
changed,
};
}
export function loadScenarioDefinitions(runsDir) {
const path = join(runsDir, 'scenarios.json');
try {
if (!existsSync(runsDir)) mkdirSync(runsDir, { recursive: true });
if (!existsSync(path)) {
writeJson(path, {
version: 1,
scenarios: DEFAULT_SCENARIOS,
});
}
const raw = JSON.parse(readFileSync(path, 'utf8'));
if (!raw || !Array.isArray(raw.scenarios)) throw new Error('scenarios must be an array');
const scenarios = raw.scenarios
.map(normalizeScenario)
.filter(Boolean);
return { ok: true, path, scenarios };
} catch (err) {
return { ok: false, path, error: err.message };
}
}
function normalizeScenario(input) {
if (!input || typeof input !== 'object') return null;
const id = String(input.id || input.name || '').trim().toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '');
const name = String(input.name || input.id || '').trim();
if (!id || !name) return null;
const thresholds = input.thresholds || {};
return {
id,
enabled: input.enabled === true,
name,
description: String(input.description || ''),
regions: arrayOfStrings(input.regions),
categories: arrayOfStrings(input.categories),
keywords: arrayOfStrings(input.keywords).map(s => s.toLowerCase()),
thresholds: {
watching: Number(thresholds.watching || 2),
building: Number(thresholds.building || 4),
confirmed: Number(thresholds.confirmed || 7),
},
invalidation: String(input.invalidation || ''),
};
}
function evaluateScenario(def, corpus, previous, evaluatedAt) {
if (!def.enabled) {
return {
...publicScenario(def),
state: 'dormant',
score: 0,
confidence: 0,
evidence: [],
changed: previous?.state && previous.state !== 'dormant',
lastTriggerTime: previous?.lastTriggerTime || null,
};
}
const evidence = [];
let score = 0;
for (const keyword of def.keywords) {
const hit = corpus.entries.find(entry => entry.text.includes(keyword));
if (hit) {
score += 1;
evidence.push({ type: 'keyword', label: keyword, source: hit.source, text: hit.original.slice(0, 180) });
}
}
for (const region of def.regions) {
const needle = region.toLowerCase();
const hit = corpus.entries.find(entry => entry.text.includes(needle));
if (hit) {
score += 1;
evidence.push({ type: 'region', label: region, source: hit.source, text: hit.original.slice(0, 180) });
}
}
for (const category of def.categories) {
if (corpus.categories.has(category.toLowerCase())) {
score += 1;
evidence.push({ type: 'category', label: category, source: 'sweep', text: `${category} category active` });
}
}
const state = score >= def.thresholds.confirmed ? 'confirmed'
: score >= def.thresholds.building ? 'building'
: score >= def.thresholds.watching ? 'watching'
: 'dormant';
const confidence = Math.min(100, Math.round((score / Math.max(1, def.thresholds.confirmed)) * 100));
const changed = previous?.state ? previous.state !== state : state !== 'dormant';
return {
...publicScenario(def),
state,
score,
confidence,
evidence: evidence.slice(0, 6),
changed,
lastTriggerTime: state === 'dormant' ? (previous?.lastTriggerTime || null) : evaluatedAt,
};
}
function publicScenario(def) {
return {
id: def.id,
name: def.name,
description: def.description,
enabled: def.enabled,
invalidation: def.invalidation,
};
}
function buildCorpus(data, delta) {
const entries = [];
const categories = new Set();
const push = (source, text, category) => {
if (!text) return;
entries.push({ source, original: String(text), text: String(text).toLowerCase() });
if (category) categories.add(category);
};
for (const signal of data.tSignals || []) push('thermal', signal, 'thermal');
for (const post of data.tg?.urgent || []) push(post.channel || 'telegram', post.text, 'osint');
for (const item of data.newsFeed || []) push(item.source || 'news', item.headline || item.title, 'news');
for (const item of data.news || []) push(item.source || 'news', item.headline || item.title, 'news');
for (const item of data.acled?.deadliestEvents || []) push('ACLED', `${item.country || ''} ${item.location || ''} ${item.event_type || ''} ${item.fatalities || ''}`, 'conflict');
for (const item of data.air || []) push('OpenSky', `${item.region} ${item.total} aircraft`, 'air');
for (const item of data.chokepoints || []) push('Maritime', `${item.label} ${item.note}`, 'maritime');
if (data.energy?.wti || data.energy?.brent) push('energy', `WTI ${data.energy.wti} Brent ${data.energy.brent}`, 'energy');
if (data.markets?.vix || data.fred?.some(f => f.id === 'VIXCLS')) push('markets', 'VIX volatility market stress', 'markets');
if (delta?.summary) push('delta', `${delta.summary.direction} ${delta.summary.totalChanges} changes ${delta.summary.criticalChanges} critical`, 'delta');
for (const signal of delta?.signals?.new || []) push('delta', signal.label || signal.reason || signal.key, 'delta');
for (const signal of delta?.signals?.escalated || []) push('delta', signal.label || signal.reason || signal.key, 'delta');
return { entries, categories };
}
function arrayOfStrings(value) {
return Array.isArray(value) ? value.map(v => String(v).trim()).filter(Boolean) : [];
}
function readJson(path, fallback) {
try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return fallback; }
}
function writeJson(path, value) {
writeFileSync(path, JSON.stringify(value, null, 2));
}

52
lib/stale-alerts.mjs Normal file
View File

@@ -0,0 +1,52 @@
const DEFAULT_COOLDOWN_MS = 60 * 60 * 1000;
export function shouldSendStaleAlert(health, state = {}, opts = {}) {
const now = opts.now ?? Date.now();
const cooldownMs = opts.cooldownMs ?? DEFAULT_COOLDOWN_MS;
if (!health?.stale) {
state.lastStaleAlertKey = null;
return { send: false, reason: 'not_stale' };
}
const key = [
health.lastSuccessfulSweep || 'never',
health.lastSweepError || 'no-error',
health.sourcesFailed || 0,
health.sourcesDegraded || 0,
].join('|');
if (state.lastStaleAlertKey === key && now - (state.lastStaleAlertAt || 0) < cooldownMs) {
return { send: false, reason: 'cooldown', key };
}
state.lastStaleAlertKey = key;
state.lastStaleAlertAt = now;
return { send: true, reason: 'stale', key };
}
export function formatStaleAlert(health, opts = {}) {
const dashboardUrl = opts.dashboardUrl || 'http://localhost:3117';
const context = opts.context || 'scheduled sweep';
const ageMinutes = health.dataAgeSeconds == null ? 'unknown' : Math.floor(health.dataAgeSeconds / 60);
const affected = (health.sourceHealth || [])
.filter(s => (s.status && s.status !== 'ok') || s.error)
.slice(0, 6)
.map(s => `- ${s.name || s.n || 'source'}: ${s.status || 'degraded'}${s.error ? ` (${String(s.error).slice(0, 100)})` : ''}`);
return [
'*CRUCIX STALE DATA ALERT*',
'',
`Context: ${context}`,
`Status: ${health.status || 'unknown'}`,
`Data age: ${ageMinutes} minutes`,
`Last successful sweep: ${health.lastSuccessfulSweep || 'never'}`,
`Last attempted sweep: ${health.lastSweep || 'never'}`,
`Last error: ${health.lastSweepError || 'none'}`,
`Sources: ${health.sourcesOk || 0} OK / ${health.sourcesDegraded || 0} degraded / ${health.sourcesFailed || 0} failed`,
'',
'*Affected sources*',
affected.length ? affected.join('\n') : '- No per-source errors available',
'',
`Dashboard: ${dashboardUrl}`,
].join('\n');
}

4
package-lock.json generated
View File

@@ -1,11 +1,11 @@
{ {
"name": "crucix", "name": "intelligence-terminal",
"version": "2.0.0", "version": "2.0.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "crucix", "name": "intelligence-terminal",
"version": "2.0.0", "version": "2.0.0",
"license": "AGPL-3.0-only", "license": "AGPL-3.0-only",
"dependencies": { "dependencies": {

8
package.json
View File

@@ -1,7 +1,7 @@
{ {
"name": "crucix", "name": "intelligence-terminal",
"version": "2.0.0", "version": "2.0.0",
"description": "Local intelligence engine - 27 OSINT sources, live dashboard, public demo at crucix.live, auto-refresh, optional LLM layer.", "description": "Docker-first local intelligence terminal with 27 OSINT sources, live dashboard, source health, auto-refresh, and optional LLM layer.",
"type": "module", "type": "module",
"scripts": { "scripts": {
"start": "node server.mjs", "start": "node server.mjs",
@@ -12,7 +12,7 @@
"brief:save": "node apis/save-briefing.mjs", "brief:save": "node apis/save-briefing.mjs",
"diag": "node diag.mjs", "diag": "node diag.mjs",
"test": "npm run test:unit", "test": "npm run test:unit",
"test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs", "test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs test/reddit-source.test.mjs test/acled-source.test.mjs test/mojibake-text.test.mjs test/dashboard-geotagging.test.mjs",
"compose:config": "docker compose config", "compose:config": "docker compose config",
"clean": "node scripts/clean.mjs", "clean": "node scripts/clean.mjs",
"fresh-start": "npm run clean && npm start" "fresh-start": "npm run clean && npm start"
@@ -23,7 +23,7 @@
"dashboard", "dashboard",
"geopolitical" "geopolitical"
], ],
"author": "Crucix", "author": "Intelligence Terminal contributors",
"license": "AGPL-3.0-only", "license": "AGPL-3.0-only",
"engines": { "engines": {
"node": ">=22", "node": ">=22",

39
server.mjs
View File

@@ -18,6 +18,8 @@ import { TelegramAlerter } from './lib/alerts/telegram.mjs';
import { DiscordAlerter } from './lib/alerts/discord.mjs'; import { DiscordAlerter } from './lib/alerts/discord.mjs';
import { getFetchMetrics } from './apis/utils/fetch.mjs'; import { getFetchMetrics } from './apis/utils/fetch.mjs';
import { IntelligenceStore } from './lib/intelligence-store.mjs'; import { IntelligenceStore } from './lib/intelligence-store.mjs';
import { formatStaleAlert, shouldSendStaleAlert } from './lib/stale-alerts.mjs';
import { evaluateScenarios } from './lib/scenarios.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url)); const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT = __dirname; const ROOT = __dirname;
@@ -39,6 +41,7 @@ let sweepStartedAt = null; // Timestamp when current/last sweep started
let sweepInProgress = false; let sweepInProgress = false;
const startTime = Date.now(); const startTime = Date.now();
const sseClients = new Set(); const sseClients = new Set();
const staleAlertState = {};
// === Delta/Memory === // === Delta/Memory ===
const memory = new MemoryManager(RUNS_DIR); const memory = new MemoryManager(RUNS_DIR);
@@ -437,6 +440,31 @@ function buildHealth() {
}; };
} }
async function notifyIfDataStale(context = 'scheduled sweep') {
const health = buildHealth();
const decision = shouldSendStaleAlert(health, staleAlertState, {
cooldownMs: config.staleAlertCooldownMinutes * 60 * 1000,
});
if (!decision.send) return false;
const dashboardUrl = config.dashboardUrl || `http://localhost:${config.port}`;
const message = formatStaleAlert(health, { dashboardUrl, context });
const sends = [];
if (telegramAlerter.isConfigured) sends.push(telegramAlerter.sendMessage(message));
if (discordAlerter.isConfigured) sends.push(discordAlerter.sendAlert(message));
if (sends.length === 0) {
console.warn('[Crucix] Data is stale but no operator alert channel is configured');
return false;
}
const results = await Promise.allSettled(sends);
const sent = results.some(r => r.status === 'fulfilled' && (r.value === true || r.value?.ok === true));
if (sent) console.warn('[Crucix] Operator stale-data alert sent');
else console.warn('[Crucix] Operator stale-data alert attempted but no channel accepted it');
return sent;
}
function buildBrief(data) { function buildBrief(data) {
const verbosity = config.telegram.briefVerbosity || 'standard'; const verbosity = config.telegram.briefVerbosity || 'standard';
const delta = memory.getLastDelta(); const delta = memory.getLastDelta();
@@ -473,6 +501,13 @@ function buildBrief(data) {
lines.push('', '*Why This Matters*'); lines.push('', '*Why This Matters*');
for (const idea of ideas) lines.push(`- ${idea.title}: ${(idea.rationale || idea.text || '').slice(0, 140)}`); for (const idea of ideas) lines.push(`- ${idea.title}: ${(idea.rationale || idea.text || '').slice(0, 140)}`);
} }
const scenarioChanges = data.scenarios?.changed || [];
if (scenarioChanges.length) {
lines.push('', '*Scenario Watchlist*');
for (const scenario of scenarioChanges.slice(0, 4)) {
lines.push(`- ${scenario.name}: ${scenario.state.toUpperCase()} (${scenario.confidence}% confidence)`);
}
}
lines.push('', '*What To Do Next*', '- Open the dashboard, verify the evidence links, and compare source health before acting.'); lines.push('', '*What To Do Next*', '- Open the dashboard, verify the evidence links, and compare source health before acting.');
return lines.join('\n'); return lines.join('\n');
} }
@@ -519,6 +554,7 @@ async function runSweepCycle() {
// 4. Delta computation + memory // 4. Delta computation + memory
const delta = memory.addRun(synthesized); const delta = memory.addRun(synthesized);
synthesized.delta = delta; synthesized.delta = delta;
synthesized.scenarios = evaluateScenarios(synthesized, delta, RUNS_DIR);
// 5. LLM-powered trade ideas (LLM-only feature) — isolated so failures don't kill sweep // 5. LLM-powered trade ideas (LLM-only feature) — isolated so failures don't kill sweep
if (llmProvider?.isConfigured) { if (llmProvider?.isConfigured) {
@@ -579,6 +615,9 @@ async function runSweepCycle() {
broadcast({ type: 'sweep_error', error: err.message }); broadcast({ type: 'sweep_error', error: err.message });
} finally { } finally {
sweepInProgress = false; sweepInProgress = false;
await notifyIfDataStale(lastSweepError ? 'failed sweep' : 'completed sweep').catch(err => {
console.error('[Crucix] Stale-data operator alert failed:', err.message);
});
} }
} }

95
test/acled-source.test.mjs Normal file
View File

@@ -0,0 +1,95 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { authenticate, briefing, resetAcledSessionCache } from '../apis/sources/acled.mjs';
function jsonResponse(status, body, ok = status >= 200 && status < 300) {
return {
ok,
status,
headers: { getSetCookie: () => [] },
json: async () => body,
text: async () => JSON.stringify(body),
};
}
test('ACLED reports missing credentials without network access', async () => {
resetAcledSessionCache();
let calls = 0;
const data = await briefing({
env: {},
fetchImpl: async () => {
calls++;
throw new Error('unexpected network access');
},
});
assert.equal(calls, 0);
assert.equal(data.status, 'no_credentials');
assert.equal(data.error, 'missing_acled_credentials');
assert.deepEqual(data.missing, ['ACLED_EMAIL', 'ACLED_PASSWORD']);
});
test('ACLED accepts ACLED_USER as email alias and returns empty valid result', async () => {
resetAcledSessionCache();
const urls = [];
const data = await briefing({
env: { ACLED_USER: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
fetchImpl: async url => {
urls.push(String(url));
if (String(url).includes('/oauth/token')) {
return jsonResponse(200, { access_token: 'token' });
}
return jsonResponse(200, { status: 200, data: [] });
},
});
assert.equal(data.status, 'ok');
assert.equal(data.totalEvents, 0);
assert.ok(urls.some(url => url.includes('/oauth/token')));
assert.ok(urls.some(url => url.includes('/api/acled/read')));
});
test('ACLED classifies auth failure without exposing credentials', async () => {
resetAcledSessionCache();
const result = await authenticate({
env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'super-secret' },
fetchImpl: async url => {
if (String(url).includes('/oauth/token')) {
return jsonResponse(401, { error: 'invalid_grant' }, false);
}
return {
ok: false,
status: 403,
headers: { getSetCookie: () => [] },
text: async () => 'forbidden',
};
},
});
assert.equal(result.status, 'auth_failed');
assert.equal(result.error, 'acled_auth_failed');
assert.equal(result.diagnostics.length, 2);
assert.doesNotMatch(JSON.stringify(result), /super-secret/);
});
test('ACLED classifies data access denied distinctly from auth failure', async () => {
resetAcledSessionCache();
const data = await briefing({
env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
fetchImpl: async url => {
if (String(url).includes('/oauth/token')) {
return jsonResponse(200, { access_token: 'token' });
}
return {
ok: false,
status: 403,
headers: { getSetCookie: () => [] },
text: async () => 'terms not accepted',
};
},
});
assert.equal(data.status, 'access_denied');
assert.equal(data.error, 'acled_data_http_403');
assert.match(data.hint, /Accept ACLED terms/);
});

47
test/dashboard-geotagging.test.mjs Normal file
View File

@@ -0,0 +1,47 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { geoTagText, stableGeoJitter } from '../dashboard/inject.mjs';
test('geoTagText matches headlines case-insensitively', () => {
assert.deepEqual(geoTagText('ukraine reports new air defense activity'), {
lat: 49,
lon: 32,
region: 'Ukraine',
});
assert.deepEqual(geoTagText('flooding disrupts são paulo transport'), {
lat: -23.5,
lon: -46.6,
region: 'São Paulo',
});
});
test('geoTagText prefers longer place names before broad countries', () => {
assert.deepEqual(geoTagText('New York markets react before wider US session'), {
lat: 40.7,
lon: -74,
region: 'New York',
});
});
test('geoTagText uses word boundaries to reduce false positives', () => {
assert.equal(geoTagText('A music festival announces its lineup'), null);
assert.equal(geoTagText('Officials discuss a new focus for aid'), null);
assert.deepEqual(geoTagText('US officials discuss a new aid package'), {
lat: 39,
lon: -98,
region: 'US',
});
});
test('stableGeoJitter is deterministic and bounded', () => {
const key = 'BBC|lower-case ukraine headline|Sun, 17 May 2026 12:00:00 GMT|https://example.test/a';
const latA = stableGeoJitter(key, 'lat');
const latB = stableGeoJitter(key, 'lat');
const lon = stableGeoJitter(key, 'lon');
assert.equal(latA, latB);
assert.notEqual(latA, lon);
assert.ok(latA >= -1 && latA <= 1);
assert.ok(lon >= -1 && lon <= 1);
});

141
test/fetch-utils.test.mjs
View File

@@ -2,9 +2,11 @@ import test from 'node:test';
import assert from 'node:assert/strict'; import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs'; import { readFileSync } from 'node:fs';
import { safeFetch, safeFetchText, getFetchMetrics } from '../apis/utils/fetch.mjs'; import { safeFetch, safeFetchText, getFetchMetrics } from '../apis/utils/fetch.mjs';
import { formatStaleAlert, shouldSendStaleAlert } from '../lib/stale-alerts.mjs';
test('safeFetch reports HTML as degraded JSON response', async () => { test('safeFetch reports HTML as degraded JSON response', async () => {
const originalFetch = globalThis.fetch; const originalFetch = globalThis.fetch;
const source = 'unit-html-once';
globalThis.fetch = async () => ({ globalThis.fetch = async () => ({
ok: true, ok: true,
status: 200, status: 200,
@@ -12,9 +14,72 @@ test('safeFetch reports HTML as degraded JSON response', async () => {
text: async () => '<html>not json</html>', text: async () => '<html>not json</html>',
}); });
try { try {
const data = await safeFetch('https://example.test/json', { retries: 0, source: 'unit' }); const data = await safeFetch('https://example.test/json', { retries: 0, source });
assert.match(data.error, /Expected JSON/); assert.match(data.error, /Expected JSON/);
assert.ok(getFetchMetrics().bySource.unit.requests >= 1); const bucket = getFetchMetrics().bySource[source];
assert.equal(bucket.requests, 1);
assert.equal(bucket.ok, 0);
assert.equal(bucket.failed, 1);
assert.equal(bucket.lastStatus, 200);
} finally {
globalThis.fetch = originalFetch;
}
});
test('safeFetch records HTTP failure once with status and bytes', async () => {
const originalFetch = globalThis.fetch;
const source = 'unit-http-failure-once';
globalThis.fetch = async () => ({
ok: false,
status: 503,
headers: { get: () => 'application/json' },
text: async () => 'service unavailable',
});
try {
const data = await safeFetch('https://example.test/fail', { retries: 0, source });
assert.match(data.error, /HTTP 503/);
const bucket = getFetchMetrics().bySource[source];
assert.equal(bucket.requests, 1);
assert.equal(bucket.ok, 0);
assert.equal(bucket.failed, 1);
assert.equal(bucket.lastStatus, 503);
assert.equal(bucket.bytes, 'service unavailable'.length);
assert.match(bucket.lastError, /HTTP 503/);
} finally {
globalThis.fetch = originalFetch;
}
});
test('safeFetch retry metrics count one record per attempt', async () => {
const originalFetch = globalThis.fetch;
const source = 'unit-retry-attempts';
let calls = 0;
globalThis.fetch = async () => {
calls += 1;
if (calls === 1) {
return {
ok: false,
status: 502,
headers: { get: () => 'application/json' },
text: async () => 'bad gateway',
};
}
return {
ok: true,
status: 200,
headers: { get: () => 'application/json' },
text: async () => '{"ok":true}',
};
};
try {
const data = await safeFetch('https://example.test/retry', { retries: 1, source });
assert.equal(data.ok, true);
assert.equal(calls, 2);
const bucket = getFetchMetrics().bySource[source];
assert.equal(bucket.requests, 2);
assert.equal(bucket.ok, 1);
assert.equal(bucket.failed, 1);
assert.equal(bucket.lastStatus, 200);
} finally { } finally {
globalThis.fetch = originalFetch; globalThis.fetch = originalFetch;
} }
@@ -56,3 +121,75 @@ test('server exposes memory-backed query APIs and dashboard memory action', () =
assert.match(server, /action === 'memory'/); assert.match(server, /action === 'memory'/);
assert.match(html, /runTerminalAction\('memory'\)/); assert.match(html, /runTerminalAction\('memory'\)/);
}); });
test('stale alert is skipped for fresh health and resets active key', () => {
const state = { lastStaleAlertKey: 'old', lastStaleAlertAt: 100 };
const decision = shouldSendStaleAlert({ stale: false }, state, { now: 200 });
assert.equal(decision.send, false);
assert.equal(decision.reason, 'not_stale');
assert.equal(state.lastStaleAlertKey, null);
});
test('stale alert sends once and deduplicates during cooldown', () => {
const state = {};
const health = {
stale: true,
lastSuccessfulSweep: '2026-05-17T08:00:00.000Z',
lastSweepError: 'network timeout',
sourcesFailed: 2,
sourcesDegraded: 1,
};
const first = shouldSendStaleAlert(health, state, { now: 1_000, cooldownMs: 60_000 });
const second = shouldSendStaleAlert(health, state, { now: 2_000, cooldownMs: 60_000 });
assert.equal(first.send, true);
assert.equal(second.send, false);
assert.equal(second.reason, 'cooldown');
});
test('stale alert repeats after cooldown', () => {
const state = {};
const health = { stale: true, lastSuccessfulSweep: 'a', lastSweepError: 'timeout', sourcesFailed: 1 };
assert.equal(shouldSendStaleAlert(health, state, { now: 1_000, cooldownMs: 60_000 }).send, true);
assert.equal(shouldSendStaleAlert(health, state, { now: 62_000, cooldownMs: 60_000 }).send, true);
});
test('stale alert message includes operator context and affected sources', () => {
const message = formatStaleAlert({
status: 'stale',
stale: true,
dataAgeSeconds: 7200,
lastSuccessfulSweep: '2026-05-17T08:00:00.000Z',
lastSweep: '2026-05-17T10:00:00.000Z',
lastSweepError: 'GDELT timeout',
sourcesOk: 20,
sourcesDegraded: 3,
sourcesFailed: 2,
sourceHealth: [
{ name: 'GDELT', status: 'degraded', error: 'timeout' },
{ name: 'Reddit', status: 'no_credentials' },
],
}, { dashboardUrl: 'https://terminal.example.test', context: 'failed sweep' });
assert.match(message, /CRUCIX STALE DATA ALERT/);
assert.match(message, /Data age: 120 minutes/);
assert.match(message, /GDELT: degraded \(timeout\)/);
assert.match(message, /Dashboard: https:\/\/terminal\.example\.test/);
});
test('scenario watchlist feature is wired into sweep, briefing, and dashboard', () => {
const scenarios = readFileSync(new URL('../lib/scenarios.mjs', import.meta.url), 'utf8');
const server = readFileSync(new URL('../server.mjs', import.meta.url), 'utf8');
const html = readFileSync(new URL('../dashboard/public/jarvis.html', import.meta.url), 'utf8');
const readme = readFileSync(new URL('../README.md', import.meta.url), 'utf8');
assert.match(scenarios, /DEFAULT_SCENARIOS/);
assert.match(scenarios, /runsDir, 'scenarios\.json'/);
assert.match(scenarios, /scenario-state\.json/);
assert.match(scenarios, /watching.*building.*confirmed/s);
assert.match(server, /evaluateScenarios\(synthesized, delta, RUNS_DIR\)/);
assert.match(server, /\*Scenario Watchlist\*/);
assert.match(html, /Scenario Watchlist/);
assert.match(readme, /runs\/scenarios\.json/);
});

65
test/mojibake-text.test.mjs Normal file
View File

@@ -0,0 +1,65 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { readdirSync, readFileSync, statSync } from 'node:fs';
import { join } from 'node:path';
const TEXT_ROOTS = ['locales'];
const TEXT_FILES = [];
const EXTENSIONS = new Set(['.json', '.html', '.mjs']);
const MOJIBAKE_PATTERNS = [
{ name: 'latin1-accent', pattern: /\u00c3./g },
{ name: 'stray-cp1252-prefix', pattern: /\u00c2./g },
{ name: 'emoji-mojibake', pattern: /\u00f0\u0178/g },
{
name: 'punctuation-mojibake',
pattern: /\u00e2[\u0080-\u009f\u20ac\u0153\u2018\u2019\u201c\u201d\u2013\u2014\u2022\u2026\u201e\u2021\u02c6\u2030\u2039\u203a\u0152\u017d]/g,
},
{ name: 'variation-selector-mojibake', pattern: /\u00ef\u00b8/g },
{ name: 'ligature-mojibake', pattern: /\u00c5[\u0080-\u017f]/g },
{ name: 'replacement-character', pattern: /\ufffd/g },
];
function collectFiles(root) {
const out = [];
for (const entry of readdirSync(root, { withFileTypes: true })) {
const path = join(root, entry.name);
if (entry.isDirectory()) {
out.push(...collectFiles(path));
} else if (EXTENSIONS.has(path.slice(path.lastIndexOf('.')))) {
out.push(path);
}
}
return out;
}
function textFiles() {
const discovered = TEXT_ROOTS.flatMap(root => collectFiles(root));
const explicit = TEXT_FILES.filter(path => statSync(path, { throwIfNoEntry: false })?.isFile());
return [...new Set([...discovered, ...explicit])].sort();
}
test('locale JSON files are valid UTF-8 JSON', () => {
for (const file of collectFiles('locales')) {
assert.doesNotThrow(() => JSON.parse(readFileSync(file, 'utf8')), `${file} must parse as JSON`);
}
});
test('locale text does not contain known mojibake sequences', () => {
const failures = [];
for (const file of textFiles()) {
const text = readFileSync(file, 'utf8');
for (const { name, pattern } of MOJIBAKE_PATTERNS) {
for (const match of text.matchAll(pattern)) {
const start = Math.max(0, match.index - 30);
const end = Math.min(text.length, match.index + 50);
failures.push(`${file}: ${name}: ${JSON.stringify(text.slice(start, end))}`);
}
}
}
assert.deepEqual(failures, []);
});

109
test/reddit-source.test.mjs Normal file
View File

@@ -0,0 +1,109 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import { briefing, getHot, getRedditConfig, getToken } from '../apis/sources/reddit.mjs';
test('Reddit reports missing OAuth credentials without network access', async () => {
let calls = 0;
const data = await briefing({
env: {},
delayMs: 0,
fetchImpl: async () => {
calls++;
throw new Error('unexpected network access');
},
});
assert.equal(calls, 0);
assert.equal(data.status, 'no_credentials');
assert.equal(data.error, 'missing_reddit_oauth_credentials');
assert.deepEqual(data.missing, ['REDDIT_CLIENT_ID', 'REDDIT_CLIENT_SECRET']);
});
test('Reddit hot posts require OAuth token and never use public JSON fallback', async () => {
const originalFetch = globalThis.fetch;
let calledUrl = null;
globalThis.fetch = async url => {
calledUrl = url;
throw new Error('unexpected public fallback');
};
try {
const data = await getHot('worldnews');
assert.equal(calledUrl, null);
assert.equal(data.status, 'no_credentials');
assert.equal(data.error, 'reddit_oauth_required');
} finally {
globalThis.fetch = originalFetch;
}
});
test('Reddit classifies OAuth HTTP failure without exposing secrets', async () => {
const result = await getToken({
env: { REDDIT_CLIENT_ID: 'client-id', REDDIT_CLIENT_SECRET: 'client-secret' },
fetchImpl: async () => ({
ok: false,
status: 401,
text: async () => 'invalid client',
}),
});
assert.equal(result.ok, false);
assert.equal(result.status, 'auth_failed');
assert.equal(result.error, 'reddit_oauth_http_401');
assert.doesNotMatch(JSON.stringify(result), /client-secret/);
});
test('Reddit fetches hot posts through oauth.reddit.com when configured', async () => {
const originalFetch = globalThis.fetch;
const urls = [];
globalThis.fetch = async url => {
urls.push(String(url));
if (String(url).includes('/api/v1/access_token')) {
return {
ok: true,
status: 200,
json: async () => ({ access_token: 'test-token' }),
};
}
return {
ok: true,
status: 200,
headers: { get: () => 'application/json' },
text: async () => JSON.stringify({
data: {
children: [
{
data: {
title: 'Market stress headline',
score: 42,
num_comments: 7,
url: 'https://example.test/post',
created_utc: 1700000000,
},
},
],
},
}),
};
};
try {
const data = await briefing({
env: { REDDIT_CLIENT_ID: 'client-id', REDDIT_CLIENT_SECRET: 'client-secret' },
subreddits: ['worldnews'],
delayMs: 0,
});
assert.equal(data.status, 'ok');
assert.equal(data.subreddits.worldnews[0].title, 'Market stress headline');
assert.ok(urls.some(url => url === 'https://www.reddit.com/api/v1/access_token'));
assert.ok(urls.some(url => url.startsWith('https://oauth.reddit.com/r/worldnews/hot')));
assert.equal(urls.some(url => url.includes('hot.json')), false);
} finally {
globalThis.fetch = originalFetch;
}
});
test('Reddit config reports partial credential state', () => {
assert.deepEqual(getRedditConfig({ REDDIT_CLIENT_ID: 'id' }).missing, ['REDDIT_CLIENT_SECRET']);
});