commit ef2c6470fb64ecd3cddb1339454532162e33cfaf Author: calesthio Date: Thu Mar 12 23:45:46 2026 -0700 Initial release — Crucix Intelligence Engine v2.0.0 26-source OSINT intelligence engine with live Jarvis dashboard, auto-refresh via SSE, optional LLM layer (4 providers), delta/memory system, and Telegram breaking news alerts. Co-Authored-By: Claude Opus 4.6 diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..0ee0732 --- /dev/null +++ b/.env.example @@ -0,0 +1,29 @@ +# ============================================ +# Crucix Intelligence Engine — Configuration +# ============================================ +# Copy this file to .env and fill in your keys. +# Keys are optional — sources without keys degrade gracefully. + +# === OSINT Source API Keys === +FRED_API_KEY= # Federal Reserve Economic Data (free: fred.stlouisfed.org/docs/api) +FIRMS_MAP_KEY= # NASA FIRMS fire data (free: firms.modaps.eosdis.nasa.gov/api) +EIA_API_KEY= # Energy Information Administration (free: api.eia.gov/register) +AISSTREAM_API_KEY= # Maritime AIS data (aisstream.io) +ACLED_EMAIL= # Armed Conflict Location & Event Data (acleddata.com/user/register) +ACLED_PASSWORD= # OAuth2 password grant (API keys deprecated Sept 2025) + +# === Server Configuration === +PORT=3117 # Dashboard server port +REFRESH_INTERVAL_MINUTES=15 # Auto-refresh interval (minutes) + +# === LLM Layer (optional) === +# Enables AI-enhanced trade ideas and breaking news Telegram alerts. +# Provider options: anthropic | openai | gemini | codex +LLM_PROVIDER= +LLM_API_KEY= # Not needed for codex (uses ~/.codex/auth.json) +LLM_MODEL= # Defaults: claude-sonnet-4-20250514 / gpt-4o / gemini-2.0-flash / gpt-5.2-codex + +# === Telegram Alerts (optional, requires LLM) === +# Create a bot via @BotFather, get chat ID via @userinfobot +TELEGRAM_BOT_TOKEN= +TELEGRAM_CHAT_ID= diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..311e4a4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,32 @@ +# Dependencies +node_modules/ + +# Environment (contains API keys) +.env +apis/.env + +# Runtime data +runs/ +output/ + +# IDE / Editor +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS files +.DS_Store +Thumbs.db +desktop.ini + +# Claude Code +.claude/ + +# Playwright +.playwright-cli/ + +# Logs +*.log +npm-debug.log* diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..59c29f1 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,186 @@ +# Crucix — Claude Code Project Instructions + +## What This Is + +Crucix is a local intelligence engine that aggregates 25 OSINT data sources in parallel and produces structured JSON. Claude's job is to synthesize that raw data into two outputs: a written intelligence briefing and a visual Jarvis-style dashboard. + +## Project Layout + +``` +Crucix/ +├── apis/ +│ ├── briefing.mjs # Master orchestrator — runs all 25 sources +│ ├── BRIEFING_PROMPT.md # Intelligence synthesis protocol (READ THIS) +│ ├── BRIEFING_TEMPLATE.md # Output template for written briefings +│ └── sources/ # Individual source modules +├── dashboard/ +│ ├── public/jarvis.html # Self-contained Jarvis HUD dashboard +│ └── inject.mjs # Data synthesis + injection script +├── runs/ +│ └── latest.json # Most recent sweep output +└── CLAUDE.md # You are here +``` + +## Trigger Phrases + +When the user says any of the following (or similar): +- "brief me" +- "what's going on" +- "what's the latest" +- "time for my brief" +- "what's happening in the world" +- "run a sweep" +- "update the dashboard" + +Execute the **Full Briefing Flow** below. + +## Full Briefing Flow + +### Step 1: Run the Crucix Sweep + +```bash +cd C:/Users/ishan/Documents/Crucix && node apis/briefing.mjs > runs/latest.json 2>&1 +``` + +This runs all 25 OSINT sources in parallel (~30-60 seconds). Output goes to `runs/latest.json`. If a timestamped backup is desired: + +```bash +cp runs/latest.json runs/briefing_$(date -u +%Y-%m-%dT%H-%M-%SZ).json +``` + +### Step 2: Gather Live Market Data + +Use the Alpaca MCP tools to pull real-time context: + +- **Broad indexes**: Get latest quotes/snapshots for SPY, QQQ, DIA, IWM +- **Rates proxies**: TLT, HYG, LQD +- **Commodities**: GLD, SLV, USO, UNG +- **Crypto**: BTC/USD, ETH/USD +- **VIX**: Get CBOE VIX latest + +This supplements the FRED/EIA/BLS data in `latest.json` with live market prices. + +### Step 3: Search for Breaking Developments + +Use web search to check for breaking news in the last 6 hours: +- Geopolitical escalation or de-escalation +- Central bank actions or statements +- Major economic data releases +- Conflict developments +- Health emergencies +- Sanctions or policy shifts + +### Step 4: Read the Briefing Protocol + +Read `apis/BRIEFING_PROMPT.md` for the full intelligence synthesis protocol. Read `apis/BRIEFING_TEMPLATE.md` for the output structure. + +Key principles: +- **Leverage first** — always lead with what the user can act on +- **Cross-correlate** — connect signals across conflict, economic, health, and market domains +- **Strong view** — form an opinion backed by evidence, not hedged filler +- **8 sections**: Leverageable Ideas → Executive Thesis → Situation Awareness → Pattern Recognition → Historical Parallels → Market Implications → Decision Board → Source Integrity + +### Step 5: Write the Intelligence Briefing + +Synthesize all inputs (Crucix sweep + Alpaca live data + web search) into a briefing following `BRIEFING_TEMPLATE.md`. Write it as markdown. + +Save the briefing to: +``` +runs/briefing_YYYY-MM-DDTHH-MM-SSZ.md +``` + +### Step 6: Generate the Jarvis Dashboard + +After the briefing is written, update the visual dashboard: + +```bash +cd C:/Users/ishan/Documents/Crucix && node dashboard/inject.mjs +``` + +This script: +1. Reads `runs/latest.json` +2. Fetches RSS news from BBC, NYT, Al Jazeera and geo-tags them +3. Generates signal-based Leverageable Ideas from cross-source correlation +4. Synthesizes the raw data into a compact format (~18KB) +5. Injects it into `dashboard/public/jarvis.html` replacing the data placeholder +6. Filters non-English Telegram posts (Cyrillic detection) +7. **Auto-opens the dashboard in the user's default browser** + +The dashboard is a self-contained HTML file — no server needed. It opens automatically after injection. + +### Step 7: Confirm to User + +Tell the user: +1. Briefing is ready (share key highlights from Leverageable Ideas and Executive Thesis) +2. Dashboard has been updated and auto-opened in their browser (if they already had it open, they should refresh) +3. Note any source failures or degraded data quality + +## Dashboard Architecture + +The Jarvis HUD (`dashboard/public/jarvis.html`) is a single self-contained file: +- **CDN dependencies**: GSAP (animations), D3.js + topojson (world map) +- **Visual style**: Glassmorphism, cyan-on-dark, IBM Plex Mono + Space Grotesk +- **Boot sequence**: Cinematic 3-4 second reveal with spinning logo ring +- **Layout**: 3-column grid + - Left rail: Threat Mesh layers, Nuclear Watch, Risk Gauges + - Center: D3 world map with 7 marker types + region filters + lower macro grid + - Right rail: English-only OSINT stream + WHO alerts + Signal Core metrics + +### Map Marker Types +- Green circles: Air traffic hotspots (OpenSky) +- Red circles: Thermal/fire detections (FIRMS) +- Cyan dots: SDR receivers in conflict zones (KiwiSDR) +- Yellow circles: Nuclear monitoring sites (Safecast) +- Purple diamonds: Maritime chokepoints +- Orange circles: OSINT events (Telegram urgent) +- Green circles (small): WHO health alerts +- Light blue broadcast icons: Geolocated world news (RSS) — click for article popup + +### Region Filters +World, Americas, Europe, Middle East, Asia Pacific, Africa — with smooth D3 zoom transitions. + +## Data Synthesis (inject.mjs) + +The inject script maps raw `latest.json` fields to what the HTML expects: + +| HTML property | Raw source | Key fields | +|---|---|---| +| `D.air` | OpenSky.hotspots | total, noCallsign, highAlt, region | +| `D.thermal` | FIRMS.hotspots | det, night, hc, fires[{lat,lon,frp}] | +| `D.chokepoints` | Maritime.chokepoints | label, note, lat, lon | +| `D.nuke` | Safecast.sites | site, anom, cpm, n | +| `D.sdr` | KiwiSDR | total, online, zones[{region,count,receivers}] | +| `D.tg` | Telegram | posts, urgent[], topPosts[] (English only) | +| `D.who` | WHO.diseaseOutbreakNews | title, date, summary | +| `D.fred` | FRED.indicators | id, value, momChange, etc. | +| `D.energy` | EIA | wti, brent, natgas, crudeStocks, wtiRecent[] | +| `D.bls` | BLS.indicators | id, value, momChange, momChangePct | +| `D.treasury` | Treasury.debt[0] | totalDebt | +| `D.gscpi` | GSCPI.latest | value, interpretation | +| `D.defense` | USAspending.recentDefenseContracts | recipient, amount, desc | +| `D.health` | All sources | name, error status | + +## If Only Dashboard Update is Requested + +If the user just says "update the dashboard" or "refresh the dashboard" (without wanting a full briefing): + +1. Run the sweep: `node apis/briefing.mjs > runs/latest.json 2>&1` +2. Inject data: `node dashboard/inject.mjs` +3. Confirm dashboard is updated + +## If Only Briefing is Requested + +If the user just wants the written briefing without the dashboard: + +1. Run the sweep +2. Gather Alpaca + web context +3. Read and follow BRIEFING_PROMPT.md +4. Write the briefing following BRIEFING_TEMPLATE.md +5. Share the briefing + +## Source Notes + +- **25 sources**: GDELT, OpenSky, FIRMS, Maritime, Safecast, ACLED, ReliefWeb, WHO, OFAC, OpenSanctions, ADS-B, FRED, Treasury, BLS, EIA, GSCPI, USAspending, Comtrade, NOAA, EPA, Patents, Bluesky, Reddit, Telegram, KiwiSDR +- Zero npm dependencies, pure ESM, Node 22+ +- Some sources may return errors (ACLED rate limits, GDELT empty results) — note this in Source Integrity +- Telegram posts filtered for English (Cyrillic detection) — Russian-language posts are skipped in both briefing and dashboard diff --git a/README.md b/README.md new file mode 100644 index 0000000..2b66afe --- /dev/null +++ b/README.md @@ -0,0 +1,268 @@ +# Crucix + +**Local intelligence engine. 26 OSINT sources. One command. Zero cloud dependency.** + +Crucix aggregates open-source intelligence from 26 data sources in parallel — satellite fire detection, flight tracking, radiation monitoring, economic indicators, live market prices, conflict data, sanctions lists, social sentiment, and more — and renders it as a real-time Jarvis-style dashboard that auto-refreshes every 15 minutes. + +Everything runs on your machine. No telemetry, no SaaS, no subscriptions required for core functionality. + +--- + +## Quick Start + +```bash +# 1. Clone the repo +git clone https://github.com/YOUR_USERNAME/crucix.git +cd crucix + +# 2. Install dependencies (just Express) +npm install + +# 3. Copy env template and add your API keys (see below) +cp .env.example .env + +# 4. Start the dashboard +npm run dev +``` + +The dashboard opens automatically at `http://localhost:3117`, runs the first intelligence sweep, and auto-refreshes every 15 minutes via SSE (Server-Sent Events). No manual page refresh needed. + +**Requirements:** Node.js 22+ (uses native `fetch`, top-level `await`, ESM) + +--- + +## What You Get + +### Live Dashboard +A self-contained Jarvis-style HUD with: +- **D3 world map** with 7 marker types (fire detections, air traffic, radiation sites, maritime chokepoints, SDR receivers, OSINT events, health alerts, geolocated news) +- **Region filters** (World, Americas, Europe, Middle East, Asia Pacific, Africa) with smooth zoom transitions +- **Live market data** — indexes, crypto, energy, commodities via Yahoo Finance (no API key needed) +- **Risk gauges** — VIX, high-yield spread, supply chain pressure index +- **OSINT feed** — English-language posts from 12 Telegram intelligence channels +- **News ticker** — merged RSS + GDELT headlines + Telegram posts, auto-scrolling +- **Nuclear watch** — real-time radiation readings from Safecast + EPA RadNet +- **Leverageable ideas** — AI-generated trade ideas (with LLM) or signal-correlated ideas (without) + +### Auto-Refresh +The server runs a sweep cycle every 15 minutes (configurable). Each cycle: +1. Queries all 26 sources in parallel (~30s) +2. Synthesizes raw data into dashboard format +3. Computes delta from previous run (what changed, escalated, de-escalated) +4. Generates LLM trade ideas (if configured) +5. Evaluates Telegram breaking news alerts (if configured) +6. Pushes update to all connected browsers via SSE + +### Optional LLM Layer +Connect any of 4 LLM providers for enhanced analysis: +- **AI trade ideas** — quantitative analyst producing 5-8 actionable ideas citing specific data +- **Breaking news alerts** — Telegram notifications when critical signals emerge +- Providers: Anthropic Claude, OpenAI, Google Gemini, OpenAI Codex (ChatGPT subscription) +- Graceful fallback — LLM failures never crash the sweep cycle + +--- + +## API Keys Setup + +Copy `.env.example` to `.env` at the project root: + +```bash +cp .env.example .env +``` + +### Required for Best Results (all free) + +| Key | Source | How to Get | +|-----|--------|------------| +| `FRED_API_KEY` | Federal Reserve Economic Data | [fred.stlouisfed.org](https://fred.stlouisfed.org/docs/api/api_key.html) — instant, free | +| `FIRMS_MAP_KEY` | NASA FIRMS (satellite fire data) | [firms.modaps.eosdis.nasa.gov](https://firms.modaps.eosdis.nasa.gov/api/area/) — instant, free | +| `EIA_API_KEY` | US Energy Information Administration | [api.eia.gov](https://www.eia.gov/opendata/register.php) — instant, free | + +These three unlock the most valuable economic and satellite data. Each takes about 60 seconds to register. + +### Optional (enable additional sources) + +| Key | Source | How to Get | +|-----|--------|------------| +| `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2 | +| `AISSTREAM_API_KEY` | Maritime AIS vessel tracking | [aisstream.io](https://aisstream.io/) — free | +| `ADSB_API_KEY` | Unfiltered flight tracking | [RapidAPI](https://rapidapi.com/adsbexchange/api/adsbexchange-com1) — ~$10/mo | + +### LLM Provider (optional, for AI-enhanced ideas) + +Set `LLM_PROVIDER` to one of: `anthropic`, `openai`, `gemini`, `codex` + +| Provider | Key Required | Default Model | +|----------|-------------|---------------| +| `anthropic` | `LLM_API_KEY` | claude-sonnet-4-20250514 | +| `openai` | `LLM_API_KEY` | gpt-4o | +| `gemini` | `LLM_API_KEY` | gemini-2.0-flash | +| `codex` | None (uses `~/.codex/auth.json`) | gpt-5.2-codex | + +For Codex, run `npx @openai/codex login` to authenticate via your ChatGPT subscription. + +### Telegram Alerts (optional, requires LLM) + +| Key | How to Get | +|-----|------------| +| `TELEGRAM_BOT_TOKEN` | Create via [@BotFather](https://t.me/BotFather) on Telegram | +| `TELEGRAM_CHAT_ID` | Get via [@userinfobot](https://t.me/userinfobot) | + +### Without Any Keys + +Crucix still works with zero API keys. 18+ sources require no authentication at all. Sources that need keys return structured errors and the rest of the sweep continues normally. + +--- + +## Architecture + +``` +crucix/ +├── server.mjs # Express dev server (SSE, auto-refresh, LLM orchestration) +├── crucix.config.mjs # Configuration with env var overrides +├── .env.example # All documented env vars +├── package.json # Single dependency: express +│ +├── apis/ +│ ├── briefing.mjs # Master orchestrator — runs all 26 sources in parallel +│ ├── save-briefing.mjs # CLI: save timestamped + latest.json +│ ├── BRIEFING_PROMPT.md # Intelligence synthesis protocol +│ ├── BRIEFING_TEMPLATE.md # Briefing output structure +│ ├── utils/ +│ │ ├── fetch.mjs # safeFetch() — timeout, retries, abort, auto-JSON +│ │ └── env.mjs # .env loader (no dotenv dependency) +│ └── sources/ # 26 self-contained source modules +│ ├── gdelt.mjs # Each exports briefing() → structured data +│ ├── fred.mjs # Can run standalone: node apis/sources/fred.mjs +│ ├── yfinance.mjs # Yahoo Finance — free live market data +│ └── ... # 23 more +│ +├── dashboard/ +│ ├── inject.mjs # Data synthesis + standalone HTML injection +│ └── public/ +│ └── jarvis.html # Self-contained Jarvis HUD +│ +├── lib/ +│ ├── llm/ # LLM abstraction (4 providers, raw fetch, no SDKs) +│ │ ├── provider.mjs # Base class +│ │ ├── anthropic.mjs # Claude +│ │ ├── openai.mjs # GPT +│ │ ├── gemini.mjs # Gemini +│ │ ├── codex.mjs # Codex (ChatGPT subscription) +│ │ ├── ideas.mjs # LLM-powered trade idea generation +│ │ └── index.mjs # Factory: createLLMProvider() +│ ├── delta/ # Change tracking between sweeps +│ │ ├── engine.mjs # Delta computation (new/escalated/de-escalated/removed) +│ │ ├── memory.mjs # Hot memory (3 runs) + cold storage (daily archives) +│ │ └── index.mjs # Re-exports +│ └── alerts/ +│ └── telegram.mjs # Breaking news alerts via Telegram +│ +└── runs/ # Runtime data (gitignored) + ├── latest.json # Most recent sweep output + └── memory/ # Delta memory (hot + cold storage) +``` + +### Design Principles +- **Pure ESM** — every file is `.mjs` with explicit imports +- **Minimal dependencies** — Express is the only runtime dependency. LLM providers use raw `fetch()`, no SDKs. +- **Parallel execution** — `Promise.allSettled()` fires all 26 sources simultaneously +- **Graceful degradation** — missing keys produce errors, not crashes. LLM failures don't kill sweeps. +- **Each source is standalone** — run `node apis/sources/gdelt.mjs` to test any source independently +- **Self-contained dashboard** — the HTML file works with or without the server + +--- + +## Data Sources (26) + +### Tier 1: Core OSINT & Geopolitical (11) + +| Source | What It Tracks | Auth | +|--------|---------------|------| +| **GDELT** | Global news events, conflict mapping (100+ languages) | None | +| **OpenSky** | Real-time ADS-B flight tracking across 6 hotspot regions | None | +| **NASA FIRMS** | Satellite fire/thermal anomaly detection (3hr latency) | Free key | +| **Maritime/AIS** | Vessel tracking, dark ships, sanctions evasion | Free key | +| **Safecast** | Citizen-science radiation monitoring near 6 nuclear sites | None | +| **ACLED** | Armed conflict events: battles, explosions, protests | Free (OAuth2) | +| **ReliefWeb** | UN humanitarian crisis tracking | None | +| **WHO** | Disease outbreaks and health emergencies | None | +| **OFAC** | US Treasury sanctions (SDN list) | None | +| **OpenSanctions** | Aggregated global sanctions (30+ sources) | Partial | +| **ADS-B Exchange** | Unfiltered flight tracking including military | Paid | + +### Tier 2: Economic & Financial (7) + +| Source | What It Tracks | Auth | +|--------|---------------|------| +| **FRED** | 22 key indicators: yield curve, CPI, VIX, fed funds, M2 | Free key | +| **US Treasury** | National debt, yields, fiscal data | None | +| **BLS** | CPI, unemployment, nonfarm payrolls, PPI | None | +| **EIA** | WTI/Brent crude, natural gas, inventories | Free key | +| **GSCPI** | NY Fed Global Supply Chain Pressure Index | None | +| **USAspending** | Federal spending and defense contracts | None | +| **UN Comtrade** | Strategic commodity trade flows between major powers | None | + +### Tier 3: Weather, Environment, Tech, Social, SIGINT (7) + +| Source | What It Tracks | Auth | +|--------|---------------|------| +| **NOAA/NWS** | Active US weather alerts | None | +| **EPA RadNet** | US government radiation monitoring | None | +| **USPTO Patents** | Patent filings in 7 strategic tech areas | None | +| **Bluesky** | Social sentiment on geopolitical/market topics | None | +| **Reddit** | Social sentiment from key subreddits | OAuth | +| **Telegram** | 12 curated OSINT/conflict channels (web scraping) | None | +| **KiwiSDR** | Global HF radio receiver network (~600 receivers) | None | + +### Tier 4: Live Market Data (1) + +| Source | What It Tracks | Auth | +|--------|---------------|------| +| **Yahoo Finance** | Real-time prices: SPY, QQQ, BTC, Gold, WTI, VIX + 9 more | None | + +--- + +## npm Scripts + +| Script | Command | Description | +|--------|---------|-------------| +| `npm run dev` | `node server.mjs` | Start dashboard with auto-refresh | +| `npm run sweep` | `node apis/briefing.mjs` | Run a single sweep, output JSON to stdout | +| `npm run inject` | `node dashboard/inject.mjs` | Inject latest data into static HTML | +| `npm run brief:save` | `node apis/save-briefing.mjs` | Run sweep + save timestamped JSON | + +--- + +## Configuration + +All settings are in `.env` with sensible defaults: + +| Variable | Default | Description | +|----------|---------|-------------| +| `PORT` | `3117` | Dashboard server port | +| `REFRESH_INTERVAL_MINUTES` | `15` | Auto-refresh interval | +| `LLM_PROVIDER` | disabled | `anthropic`, `openai`, `gemini`, or `codex` | +| `LLM_API_KEY` | — | API key (not needed for codex) | +| `LLM_MODEL` | per-provider default | Override model selection | +| `TELEGRAM_BOT_TOKEN` | disabled | For breaking news alerts | +| `TELEGRAM_CHAT_ID` | — | Your Telegram chat ID | + +--- + +## API Endpoints + +When running `npm run dev`: + +| Endpoint | Description | +|----------|-------------| +| `GET /` | Jarvis HUD dashboard | +| `GET /api/data` | Current synthesized intelligence data (JSON) | +| `GET /api/health` | Server status, uptime, source count, LLM status | +| `GET /events` | SSE stream for live push updates | + +--- + +## License + +MIT diff --git a/apis/BRIEFING_PROMPT.md b/apis/BRIEFING_PROMPT.md new file mode 100644 index 0000000..9a0a1c6 --- /dev/null +++ b/apis/BRIEFING_PROMPT.md @@ -0,0 +1,220 @@ +# Crucix Intelligence Briefing Protocol + +When the user says "brief me", "what's the latest", "what's going on", or asks for a world update, the goal is to answer one question first: + +**How can the user leverage this information?** + +The briefing is not a neutral recap. It is a leverage-first intelligence note built from cross-domain signals, historical pattern matching, and a concrete point of view. + +## What the analyst must do + +- detect regime shifts early +- connect hard data and weak signals +- distinguish what matters from noise +- form a coherent worldview +- map that worldview into positioning, hedging, and watchlists + +The user wants signal, judgment, and utility. + +## Step 1: Gather Inputs + +Run the full Crucix sweep: + +```bash +cd C:/Users/ishan/Documents/Crucix && node apis/briefing.mjs 2>&1 +``` + +Also gather: + +- live market context via Alpaca MCP for broad indexes, rates proxies, commodities, metals, and crypto +- breaking developments from the last 6 hours via web search +- official statements, policy moves, or confirmed reports that materially change the read + +## Step 2: Think Before Writing + +Before drafting, answer these questions internally: + +1. What changed? +2. Which signals are confirmed by multiple sources? +3. What regime is emerging? +4. What is likely to happen next if this continues? +5. What can the user do with that information now? + +Do not overweight noisy social sources. Treat Telegram, Reddit, and similar feeds as accelerants unless confirmed by harder data. + +## Step 3: Use the Standard Output Order + +Always structure the briefing in this order: + +1. Leverageable Ideas +2. Executive Thesis +3. Situation Awareness +4. Pattern Recognition +5. Historical Parallels +6. Market and Asset Implications +7. Decision Board +8. Source Integrity + +## Section Requirements + +### 1. Leverageable Ideas + +Start here. This is the most important section. + +Provide 3-5 specific ideas. Each idea must include: + +- thesis +- instrument, sector, geography, or behavior +- why now +- time horizon: days, weeks, or months +- catalyst(s) to watch +- invalidation criteria +- confidence: High, Medium, or Low + +Examples: + +- "Accumulate gold over the next 1-3 months if conflict-energy stress continues to broaden." +- "Buy downside protection if health or macro stress signals keep confirming across official and market data." + +Bad output: + +- "Watch metals" +- "Keep an eye on volatility" + +Good output: + +- "Gold remains the cleanest hedge against war-driven inflation stress; accumulate on consolidation with a 1-3 month horizon." + +### 2. Executive Thesis + +State the worldview clearly: + +- the 1-3 most important things happening +- the regime you believe is forming +- the single most important implication for the user + +Write this as a strong view, not hedged filler. + +### 3. Situation Awareness + +Identify the top 3-5 global developments right now. + +For each: + +- what happened +- who is involved +- why it matters +- what changes because of it + +Categories: + +- CONFLICT +- ECONOMIC +- HEALTH +- CLIMATE +- TECHNOLOGY +- POLICY + +### 4. Pattern Recognition + +This is the core of Crucix. + +Cross-correlate across sources and surface non-obvious patterns such as: + +- conflict plus energy plus inflation +- macro weakness plus market stress +- health signals plus travel or sentiment shifts +- sanctions plus logistics or trade anomalies +- weather plus shipping plus supply chain disruption + +For each major pattern, state: + +- evidence +- why it matters +- whether it is strengthening, stable, or fading +- what would invalidate the interpretation + +### 5. Historical Parallels + +Ask: what does this rhyme with? + +Useful comparisons may include: + +- early 2020 health-risk buildup +- 2007-2008 financial deterioration +- 2021-2022 inflation and commodity shock +- pre-invasion 2022 Europe escalation +- prior oil, metals, or volatility regimes + +For each parallel: + +- what matched +- what is different this time +- what happened next historically +- where the current setup sits in that sequence + +### 6. Market and Asset Implications + +Translate the worldview into consequences for: + +- equities +- bonds and rates +- commodities +- gold and silver +- oil and gas +- crypto +- sectors, countries, or themes likely to outperform or underperform + +Be explicit on direction when the evidence supports it. + +### 7. Decision Board + +Close with a concise action board: + +- best long +- best hedge +- best watchlist item +- biggest unresolved question +- what to monitor in the next 24-72 hours + +### 8. Source Integrity + +Briefly state: + +- which sources returned meaningful data +- which were degraded, stale, missing, or stubbed +- where the thesis relies on hard data versus softer signals + +## Quality Bar + +The briefing should read like a private note from a sharp global macro and intelligence analyst: + +- early +- synthetic +- opinionated +- evidence-backed +- useful for action + +Avoid: + +- generic recaps +- long raw-data summaries +- false precision +- unsupported conviction +- laundry lists without a thesis + +## Handling Uncertainty + +If the evidence is mixed: + +- give the base case +- give the upside or escalation case +- give the downside or de-escalation case + +If confidence is low, still provide the best current interpretation and explain what confirmation is needed next. + +## Remember + +- The product is valuable when it spots a shift before the crowd. +- The user wants a worldview they can use. +- Always start with leverage. diff --git a/apis/BRIEFING_TEMPLATE.md b/apis/BRIEFING_TEMPLATE.md new file mode 100644 index 0000000..4379cbc --- /dev/null +++ b/apis/BRIEFING_TEMPLATE.md @@ -0,0 +1,106 @@ +# Crucix Briefing Template + +Use this output shape for every intelligence briefing. + +## 1. Leverageable Ideas + +### Idea 1 +- Thesis: +- Exposure: +- Why now: +- Time horizon: +- Catalysts: +- Invalidation: +- Confidence: + +### Idea 2 +- Thesis: +- Exposure: +- Why now: +- Time horizon: +- Catalysts: +- Invalidation: +- Confidence: + +### Idea 3 +- Thesis: +- Exposure: +- Why now: +- Time horizon: +- Catalysts: +- Invalidation: +- Confidence: + +## 2. Executive Thesis + +- Regime forming: +- What matters most: +- Main implication for the user: + +## 3. Situation Awareness + +### Event 1 +- Category: +- What happened: +- Why it matters: +- What changes: + +### Event 2 +- Category: +- What happened: +- Why it matters: +- What changes: + +### Event 3 +- Category: +- What happened: +- Why it matters: +- What changes: + +## 4. Pattern Recognition + +### Pattern 1 +- Evidence: +- Interpretation: +- Direction: +- Invalidation: + +### Pattern 2 +- Evidence: +- Interpretation: +- Direction: +- Invalidation: + +## 5. Historical Parallels + +### Parallel 1 +- Analog: +- What matches: +- What is different: +- What happened next: +- Current position in sequence: + +## 6. Market and Asset Implications + +- Equities: +- Bonds and rates: +- Commodities: +- Gold and silver: +- Oil and gas: +- Crypto: +- Sector and country effects: + +## 7. Decision Board + +- Best long: +- Best hedge: +- Best watchlist item: +- Biggest unresolved question: +- Monitor in the next 24-72 hours: + +## 8. Source Integrity + +- Strong sources this run: +- Weak or degraded sources: +- Hard-data core: +- Soft-signal support: diff --git a/apis/briefing.mjs b/apis/briefing.mjs new file mode 100644 index 0000000..64a110c --- /dev/null +++ b/apis/briefing.mjs @@ -0,0 +1,124 @@ +#!/usr/bin/env node + +// Crucix Master Orchestrator — runs all intelligence sources in parallel +// Outputs structured JSON for Claude to synthesize into actionable briefing + +import './utils/env.mjs'; // Load API keys from .env +import { pathToFileURL } from 'node:url'; + +// === Tier 1: Core OSINT & Geopolitical === +import { briefing as gdelt } from './sources/gdelt.mjs'; +import { briefing as opensky } from './sources/opensky.mjs'; +import { briefing as firms } from './sources/firms.mjs'; +import { briefing as ships } from './sources/ships.mjs'; +import { briefing as safecast } from './sources/safecast.mjs'; +import { briefing as acled } from './sources/acled.mjs'; +import { briefing as reliefweb } from './sources/reliefweb.mjs'; +import { briefing as who } from './sources/who.mjs'; +import { briefing as ofac } from './sources/ofac.mjs'; +import { briefing as opensanctions } from './sources/opensanctions.mjs'; +import { briefing as adsb } from './sources/adsb.mjs'; + +// === Tier 2: Economic & Financial === +import { briefing as fred } from './sources/fred.mjs'; +import { briefing as treasury } from './sources/treasury.mjs'; +import { briefing as bls } from './sources/bls.mjs'; +import { briefing as eia } from './sources/eia.mjs'; +import { briefing as gscpi } from './sources/gscpi.mjs'; +import { briefing as usaspending } from './sources/usaspending.mjs'; +import { briefing as comtrade } from './sources/comtrade.mjs'; + +// === Tier 3: Weather, Environment, Technology, Social === +import { briefing as noaa } from './sources/noaa.mjs'; +import { briefing as epa } from './sources/epa.mjs'; +import { briefing as patents } from './sources/patents.mjs'; +import { briefing as bluesky } from './sources/bluesky.mjs'; +import { briefing as reddit } from './sources/reddit.mjs'; +import { briefing as telegram } from './sources/telegram.mjs'; +import { briefing as kiwisdr } from './sources/kiwisdr.mjs'; + +// === Tier 4: Live Market Data === +import { briefing as yfinance } from './sources/yfinance.mjs'; + +export async function runSource(name, fn, ...args) { + const start = Date.now(); + try { + const data = await fn(...args); + return { name, status: 'ok', durationMs: Date.now() - start, data }; + } catch (e) { + return { name, status: 'error', durationMs: Date.now() - start, error: e.message }; + } +} + +export async function fullBriefing() { + console.error('[Crucix] Starting intelligence sweep — 26 sources...'); + const start = Date.now(); + + const results = await Promise.allSettled([ + // Tier 1: Core OSINT & Geopolitical + runSource('GDELT', gdelt), + runSource('OpenSky', opensky), + runSource('FIRMS', firms), + runSource('Maritime', ships), + runSource('Safecast', safecast), + runSource('ACLED', acled), + runSource('ReliefWeb', reliefweb), + runSource('WHO', who), + runSource('OFAC', ofac), + runSource('OpenSanctions', opensanctions), + runSource('ADS-B', adsb), + + // Tier 2: Economic & Financial + runSource('FRED', fred, process.env.FRED_API_KEY), + runSource('Treasury', treasury), + runSource('BLS', bls, process.env.BLS_API_KEY), + runSource('EIA', eia, process.env.EIA_API_KEY), + runSource('GSCPI', gscpi), + runSource('USAspending', usaspending), + runSource('Comtrade', comtrade), + + // Tier 3: Weather, Environment, Technology, Social + runSource('NOAA', noaa), + runSource('EPA', epa), + runSource('Patents', patents), + runSource('Bluesky', bluesky), + runSource('Reddit', reddit), + runSource('Telegram', telegram), + runSource('KiwiSDR', kiwisdr), + + // Tier 4: Live Market Data + runSource('YFinance', yfinance), + ]); + + const sources = results.map(r => r.status === 'fulfilled' ? r.value : { status: 'failed', error: r.reason?.message }); + const totalMs = Date.now() - start; + + const output = { + crucix: { + version: '2.0.0', + timestamp: new Date().toISOString(), + totalDurationMs: totalMs, + sourcesQueried: sources.length, + sourcesOk: sources.filter(s => s.status === 'ok').length, + sourcesFailed: sources.filter(s => s.status !== 'ok').length, + }, + sources: Object.fromEntries( + sources.filter(s => s.status === 'ok').map(s => [s.name, s.data]) + ), + errors: sources.filter(s => s.status !== 'ok').map(s => ({ name: s.name, error: s.error })), + timing: Object.fromEntries( + sources.map(s => [s.name, { status: s.status, ms: s.durationMs }]) + ), + }; + + console.error(`[Crucix] Sweep complete in ${totalMs}ms — ${output.crucix.sourcesOk}/${sources.length} sources returned data`); + return output; +} + +// Run and output when executed directly +const entryHref = process.argv[1] ? pathToFileURL(process.argv[1]).href : null; + +if (entryHref && import.meta.url === entryHref) { + const data = await fullBriefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/save-briefing.mjs b/apis/save-briefing.mjs new file mode 100644 index 0000000..f031fe8 --- /dev/null +++ b/apis/save-briefing.mjs @@ -0,0 +1,24 @@ +#!/usr/bin/env node + +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { fullBriefing } from './briefing.mjs'; + +function formatTimestamp(date = new Date()) { + return date.toISOString().replace(/[:]/g, '-').replace(/\.\d{3}Z$/, 'Z'); +} + +const runsDir = join(process.cwd(), 'runs'); +await mkdir(runsDir, { recursive: true }); + +const data = await fullBriefing(); +const json = JSON.stringify(data, null, 2); +const timestamp = formatTimestamp(new Date(data.crucix.timestamp)); +const runFile = join(runsDir, `briefing_${timestamp}.json`); +const latestFile = join(runsDir, 'latest.json'); + +await writeFile(runFile, json, 'utf8'); +await writeFile(latestFile, json, 'utf8'); + +console.error(`[Crucix] Saved UTF-8 briefing to ${runFile}`); +console.log(json); diff --git a/apis/sources/acled.mjs b/apis/sources/acled.mjs new file mode 100644 index 0000000..38dbea4 --- /dev/null +++ b/apis/sources/acled.mjs @@ -0,0 +1,316 @@ +// ACLED — Armed Conflict Location & Event Data +// Auth strategy (tries in order): +// 1. Cookie-based session: POST /user/login?_format=json → session cookie +// 2. OAuth Bearer token: POST /oauth/token → Authorization header +// Set ACLED_EMAIL and ACLED_PASSWORD in .env (your myACLED login credentials). +// Data endpoint: GET https://acleddata.com/api/acled/read + +import { daysAgo } from '../utils/fetch.mjs'; +import '../utils/env.mjs'; + +const LOGIN_URL = 'https://acleddata.com/user/login?_format=json'; +const TOKEN_URL = 'https://acleddata.com/oauth/token'; +const API_BASE = 'https://acleddata.com/api/acled/read'; + +// Session cache +let sessionCache = { cookies: null, token: null, method: null, expires: 0 }; + +// Strategy 1: Cookie-based session login (mirrors browser login) +async function loginCookie(email, password) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 15000); + try { + const res = await fetch(LOGIN_URL, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: email, pass: password }), + redirect: 'manual', + signal: controller.signal, + }); + clearTimeout(timer); + + // Collect Set-Cookie headers + const setCookies = res.headers.getSetCookie?.() || []; + const cookieStr = setCookies.map(c => c.split(';')[0]).join('; '); + + if (res.ok && cookieStr) { + return { cookies: cookieStr }; + } + + // Some Drupal sites return 303 redirect on successful login — cookies still set + if (res.status >= 300 && res.status < 400 && cookieStr) { + return { cookies: cookieStr }; + } + + const errText = await res.text().catch(() => ''); + return { error: `Cookie login failed (HTTP ${res.status}): ${errText.slice(0, 200)}` }; + } catch (e) { + clearTimeout(timer); + const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : ''; + return { error: `Cookie login error: ${e.message}${cause}` }; + } +} + +// Strategy 2: OAuth2 password grant +async function loginOAuth(email, password) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 15000); + try { + const body = new URLSearchParams({ + username: email, + password: password, + grant_type: 'password', + client_id: 'acled', + }); + + const res = await fetch(TOKEN_URL, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: body.toString(), + signal: controller.signal, + }); + clearTimeout(timer); + + if (!res.ok) { + const errText = await res.text().catch(() => ''); + return { error: `OAuth failed (HTTP ${res.status}): ${errText.slice(0, 200)}` }; + } + + const data = await res.json(); + if (!data.access_token) { + return { error: `OAuth response missing access_token: ${JSON.stringify(data).slice(0, 200)}` }; + } + + return { token: data.access_token }; + } catch (e) { + clearTimeout(timer); + const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : ''; + return { error: `OAuth error: ${e.message}${cause}` }; + } +} + +// Try both auth strategies +async function authenticate() { + const email = process.env.ACLED_EMAIL; + const password = process.env.ACLED_PASSWORD; + if (!email || !password) { + return { error: 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.' }; + } + + // Return cached session if still valid + if (sessionCache.method && Date.now() < sessionCache.expires) { + return sessionCache; + } + + const errors = []; + const debug = process.argv.includes('--debug'); + + // Try OAuth first (official programmatic method per ACLED docs) + const oauthResult = await loginOAuth(email, password); + if (oauthResult.token) { + if (debug) console.error(`[ACLED DEBUG] OAuth OK — token: ${oauthResult.token.slice(0, 20)}...`); + sessionCache = { cookies: null, token: oauthResult.token, method: 'oauth', expires: Date.now() + 23 * 60 * 60 * 1000 }; + return sessionCache; + } + errors.push(`OAuth: ${oauthResult.error}`); + if (debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`); + + // Fall back to cookie-based session + const cookieResult = await loginCookie(email, password); + if (cookieResult.cookies) { + if (debug) console.error(`[ACLED DEBUG] Cookie OK — cookies: ${cookieResult.cookies.slice(0, 80)}...`); + sessionCache = { cookies: cookieResult.cookies, token: null, method: 'cookie', expires: Date.now() + 12 * 60 * 60 * 1000 }; + return sessionCache; + } + errors.push(`Cookie: ${cookieResult.error}`); + + return { error: `All ACLED auth methods failed.\n${errors.join('\n')}` }; +} + +// Build headers based on auth method +function authHeaders(session) { + const headers = { 'User-Agent': 'Crucix/1.0', 'Content-Type': 'application/json' }; + if (session.method === 'cookie' && session.cookies) { + headers['Cookie'] = session.cookies; + } else if (session.method === 'oauth' && session.token) { + headers['Authorization'] = `Bearer ${session.token}`; + } + return headers; +} + +// Event type constants +export const EVENT_TYPES = [ + 'Battles', + 'Explosions/Remote violence', + 'Violence against civilians', + 'Protests', + 'Riots', + 'Strategic developments', +]; + +// Query conflict events with flexible filters +export async function getEvents(opts = {}) { + const { + limit = 500, + eventDateStart, + eventDateEnd, + eventType, + country, + region, + } = opts; + + const session = await authenticate(); + if (session.error) return { error: session.error }; + + const params = new URLSearchParams({ _format: 'json', limit: String(limit) }); + if (eventDateStart && eventDateEnd) { + params.set('event_date', `${eventDateStart}|${eventDateEnd}`); + params.set('event_date_where', 'BETWEEN'); + } + if (eventType) params.set('event_type', eventType); + if (country) params.set('country', country); + if (region) params.set('region', String(region)); + + const debug = process.argv.includes('--debug'); + try { + const url = `${API_BASE}?${params}`; + const hdrs = authHeaders(session); + if (debug) { + console.error(`[ACLED DEBUG] Data request: GET ${url}`); + console.error(`[ACLED DEBUG] Headers: ${JSON.stringify(hdrs)}`); + } + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 25000); + const res = await fetch(url, { + headers: hdrs, + signal: controller.signal, + }); + clearTimeout(timer); + if (debug) console.error(`[ACLED DEBUG] Data response: HTTP ${res.status}`); + + if (!res.ok) { + const errText = await res.text().catch(() => ''); + if (debug) console.error(`[ACLED DEBUG] Error body: ${errText.slice(0, 500)}`); + if (res.status === 401 || res.status === 403) { + // Clear cache and report + sessionCache = { cookies: null, token: null, method: null, expires: 0 }; + const hint = res.status === 403 + ? '\n→ Fix: Log in at https://acleddata.com/user/login, then:\n' + + ' 1. Accept Terms of Use (profile → Terms of Use button → check the box)\n' + + ' 2. Complete all required profile fields\n' + + ' 3. Ensure your account has the "API" access group\n' + + ' Contact access@acleddata.com if issues persist.' + : ''; + return { error: `ACLED data access denied (HTTP ${res.status}, auth method: ${session.method}). Response: ${errText.slice(0, 300)}${hint}` }; + } + return { error: `HTTP ${res.status}: ${errText.slice(0, 200)}` }; + } + + const data = await res.json(); + + // ACLED may return a 200 with an error status in the body + if (data?.status && data.status !== 200) { + return { error: `ACLED API error: status ${data.status} — ${data.message || 'Unknown error'}` }; + } + + return data; + } catch (e) { + if (e.name === 'AbortError') { + return { error: 'ACLED data request timed out (25s)' }; + } + const rootCause = e.cause ? `${e.cause.message || e.cause.code || e.cause}` : ''; + return { error: `ACLED data error: ${e.message}${rootCause ? ' → ' + rootCause : ''}` }; + } +} + +// Summarize events by a given field +function groupBy(events, field) { + const map = {}; + for (const e of events) { + const key = e[field] || 'Unknown'; + if (!map[key]) map[key] = { count: 0, fatalities: 0 }; + map[key].count += 1; + map[key].fatalities += parseInt(e.fatalities, 10) || 0; + } + return map; +} + +// Briefing — last 7 days of global conflict events +export async function briefing() { + if (!process.env.ACLED_EMAIL || !process.env.ACLED_PASSWORD) { + return { + source: 'ACLED', + timestamp: new Date().toISOString(), + status: 'no_credentials', + message: 'Set ACLED_EMAIL and ACLED_PASSWORD in .env. Register at https://acleddata.com/user/register', + }; + } + + const start = daysAgo(7); + const end = daysAgo(0); + + const data = await getEvents({ + eventDateStart: start, + eventDateEnd: end, + limit: 2000, + }); + + if (data?.error) { + return { source: 'ACLED', timestamp: new Date().toISOString(), error: data.error }; + } + + let events = data?.data || []; + + // Enrich all events with numeric lat/lon + events = events.map(e => ({ + ...e, + lat: parseFloat(e.latitude) || null, + lon: parseFloat(e.longitude) || null, + })); + + const totalFatalities = events.reduce( + (sum, e) => sum + (parseInt(e.fatalities, 10) || 0), 0 + ); + + const byRegion = groupBy(events, 'region'); + const byType = groupBy(events, 'event_type'); + const byCountry = groupBy(events, 'country'); + + const topCountries = Object.entries(byCountry) + .sort((a, b) => b[1].count - a[1].count) + .slice(0, 10) + .reduce((obj, [k, v]) => { obj[k] = v; return obj; }, {}); + + const deadliestEvents = events + .filter(e => parseInt(e.fatalities, 10) > 0) + .sort((a, b) => (parseInt(b.fatalities, 10) || 0) - (parseInt(a.fatalities, 10) || 0)) + .slice(0, 15) + .map(e => ({ + date: e.event_date, + type: e.event_type, + subType: e.sub_event_type, + country: e.country, + location: e.location, + fatalities: parseInt(e.fatalities, 10) || 0, + lat: parseFloat(e.latitude) || null, + lon: parseFloat(e.longitude) || null, + notes: e.notes?.slice(0, 200), + })); + + return { + source: 'ACLED', + timestamp: new Date().toISOString(), + period: { start, end }, + totalEvents: events.length, + totalFatalities, + byRegion, + byType, + topCountries, + deadliestEvents, + }; +} + +if (process.argv[1]?.endsWith('acled.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/adsb.mjs b/apis/sources/adsb.mjs new file mode 100644 index 0000000..b954d75 --- /dev/null +++ b/apis/sources/adsb.mjs @@ -0,0 +1,302 @@ +// ADS-B Exchange — Unfiltered Flight Tracking (including Military) +// Unlike FlightRadar24/FlightAware, ADS-B Exchange does NOT filter military aircraft. +// Public feed access varies; RapidAPI tier available for programmatic use. +// This module attempts the public endpoints and falls back to a documented stub. + +import { safeFetch } from '../utils/fetch.mjs'; + +// Known endpoints (availability may change) +const ENDPOINTS = { + // v2 API via RapidAPI (requires ADSB_API_KEY) + rapidApi: 'https://adsbexchange-com1.p.rapidapi.com/v2', + // Public globe feed (may be rate-limited or blocked for automated access) + publicFeed: 'https://globe.adsbexchange.com/data/aircraft.json', + // Alternative: aircraft within bounding box + publicTrace: 'https://globe.adsbexchange.com/data/traces', +}; + +// Known military aircraft types and ICAO type designators +const MILITARY_TYPES = { + // Reconnaissance / ISR + 'RC135': 'RC-135 Rivet Joint (SIGINT)', + 'E3CF': 'E-3 Sentry AWACS', + 'E3TF': 'E-3 Sentry AWACS', + 'E6B': 'E-6B Mercury (TACAMO)', + 'EP3': 'EP-3 Aries (SIGINT)', + 'P8': 'P-8 Poseidon (Maritime Patrol)', + 'P8A': 'P-8A Poseidon', + 'RQ4': 'RQ-4 Global Hawk (UAV)', + 'RQ4B': 'RQ-4B Global Hawk', + 'U2': 'U-2 Dragon Lady', + 'MQ9': 'MQ-9 Reaper (UAV)', + 'MQ1': 'MQ-1 Predator (UAV)', + 'E8': 'E-8 JSTARS', + // Tankers + 'KC135': 'KC-135 Stratotanker', + 'KC10': 'KC-10 Extender', + 'KC46': 'KC-46 Pegasus', + // Bombers + 'B52': 'B-52 Stratofortress', + 'B1': 'B-1B Lancer', + 'B2': 'B-2 Spirit', + // Transport / Special + 'C17': 'C-17 Globemaster III', + 'C5': 'C-5 Galaxy', + 'C130': 'C-130 Hercules', + 'VC25': 'VC-25 (Air Force One)', + 'E4B': 'E-4B Nightwatch (Doomsday Plane)', + 'C32': 'C-32 (Air Force Two)', + 'C40': 'C-40 Clipper', +}; + +// Known military ICAO hex ranges (partial — US military allocations) +const MIL_HEX_RANGES = [ + { start: 0xADF7C8, end: 0xAFFFFF, country: 'US Military' }, + { start: 0xAE0000, end: 0xAFFFFF, country: 'US Military (alt)' }, + { start: 0x43C000, end: 0x43CFFF, country: 'UK Military' }, + { start: 0x3F0000, end: 0x3FFFFF, country: 'France Military' }, + { start: 0x3CC000, end: 0x3CFFFF, country: 'Germany Military' }, +]; + +// Interesting callsign patterns that suggest military/government flights +const MIL_CALLSIGN_PATTERNS = [ + /^RCH/, // US AMC (Air Mobility Command) — strategic airlift + /^REACH/, // US AMC alternate + /^DUKE/, // Often military special ops + /^IRON/, // US military + /^JAKE/, // Military + /^NAVY/, // US Navy + /^TOPCAT/, // E-6B Mercury + /^DARKST/, // Dark Star / classified + /^GORDO/, // USAF + /^BISON/, // B-52 + /^DEATH/, // B-1B + /^DOOM/, // E-4B + /^SAM/, // Special Air Mission (VIP) + /^EXEC/, // Executive transport + /^PCSF/, // Chinese military + /^CHN/, // Chinese military + /^RF/, // Russian Air Force + /^RFF/, // Russian Air Force +]; + +// Check if an ICAO hex code falls in known military ranges +function isMilitaryHex(hex) { + if (!hex) return false; + const num = parseInt(hex, 16); + if (isNaN(num)) return false; + return MIL_HEX_RANGES.find(r => num >= r.start && num <= r.end) || null; +} + +// Check if a callsign matches military patterns +function isMilitaryCallsign(callsign) { + if (!callsign) return false; + const cs = callsign.trim().toUpperCase(); + return MIL_CALLSIGN_PATTERNS.some(p => p.test(cs)); +} + +// Check if aircraft type is a known military type +function isMilitaryType(typeCode) { + if (!typeCode) return false; + const tc = typeCode.toUpperCase().replace(/[^A-Z0-9]/g, ''); + return MILITARY_TYPES[tc] || null; +} + +// Classify an aircraft from ADS-B data +function classifyAircraft(ac) { + const hex = ac.hex || ac.icao || ac.icao24 || null; + const callsign = ac.flight || ac.callsign || ac.call || ''; + const type = ac.t || ac.type || ac.typecode || ''; + const mil = ac.mil || ac.military || false; + + const milHex = isMilitaryHex(hex); + const milCall = isMilitaryCallsign(callsign); + const milType = isMilitaryType(type); + + const isMilitary = !!(mil || milHex || milCall || milType); + + return { + hex, + callsign: callsign.trim(), + type, + typeDescription: milType || null, + latitude: ac.lat || ac.latitude || null, + longitude: ac.lon || ac.longitude || null, + altitude: ac.alt_baro || ac.alt_geom || ac.altitude || null, + speed: ac.gs || ac.speed || null, + heading: ac.track || ac.heading || null, + squawk: ac.squawk || null, + isMilitary, + militaryMatch: milHex?.country || (milCall ? 'callsign pattern' : null) || (milType ? 'type match' : null), + registration: ac.r || ac.registration || null, + seen: ac.seen || ac.last_contact || null, + }; +} + +// Attempt to fetch from RapidAPI (requires ADSB_API_KEY) +async function fetchViaRapidApi(apiKey) { + if (!apiKey) return null; + + // Get all military aircraft + const data = await safeFetch(`${ENDPOINTS.rapidApi}/mil`, { + timeout: 20000, + headers: { + 'X-RapidAPI-Key': apiKey, + 'X-RapidAPI-Host': 'adsbexchange-com1.p.rapidapi.com', + }, + }); + + return data; +} + +// Attempt to fetch from public feed +async function fetchPublicFeed() { + const data = await safeFetch(ENDPOINTS.publicFeed, { timeout: 15000 }); + return data; +} + +// Get military aircraft from available sources +export async function getMilitaryAircraft(apiKey) { + // Try RapidAPI first if key available + if (apiKey) { + const data = await fetchViaRapidApi(apiKey); + if (data && !data.error) { + const aircraft = data.ac || data.aircraft || []; + if (Array.isArray(aircraft)) { + return aircraft.map(classifyAircraft).filter(a => a.isMilitary); + } + } + } + + // Try public feed + const pubData = await fetchPublicFeed(); + if (pubData && !pubData.error) { + const aircraft = pubData.ac || pubData.aircraft || pubData.states || []; + if (Array.isArray(aircraft)) { + return aircraft.map(classifyAircraft).filter(a => a.isMilitary); + } + } + + return null; // all sources failed +} + +// Get all aircraft in a geographic bounding box via RapidAPI +export async function getAircraftInArea(lat, lon, radiusNm = 250, apiKey) { + if (!apiKey) { + return { error: 'ADSB_API_KEY required for area search', hint: 'Set ADSB_API_KEY (RapidAPI key)' }; + } + + const data = await safeFetch( + `${ENDPOINTS.rapidApi}/lat/${lat}/lon/${lon}/dist/${radiusNm}/`, + { + timeout: 20000, + headers: { + 'X-RapidAPI-Key': apiKey, + 'X-RapidAPI-Host': 'adsbexchange-com1.p.rapidapi.com', + }, + } + ); + + if (data && !data.error) { + const aircraft = data.ac || data.aircraft || []; + if (Array.isArray(aircraft)) return aircraft.map(classifyAircraft); + } + + return data; +} + +// Briefing — attempt to get military flight data, document what's available +export async function briefing() { + const apiKey = process.env.ADSB_API_KEY || process.env.RAPIDAPI_KEY || null; + const militaryAircraft = await getMilitaryAircraft(apiKey); + + // If we got data, analyze it + if (militaryAircraft && militaryAircraft.length > 0) { + // Group by military match type + const byCountry = {}; + const reconAircraft = []; + const bombers = []; + const tankers = []; + const vipTransport = []; + + for (const ac of militaryAircraft) { + const country = ac.militaryMatch || 'Unknown'; + byCountry[country] = (byCountry[country] || 0) + 1; + + const desc = (ac.typeDescription || '').toLowerCase(); + if (desc.includes('sigint') || desc.includes('awacs') || desc.includes('patrol') || + desc.includes('global hawk') || desc.includes('dragon lady') || desc.includes('jstars')) { + reconAircraft.push(ac); + } else if (desc.includes('stratofortress') || desc.includes('lancer') || desc.includes('spirit')) { + bombers.push(ac); + } else if (desc.includes('tanker') || desc.includes('extender') || desc.includes('pegasus')) { + tankers.push(ac); + } else if (desc.includes('air force one') || desc.includes('nightwatch') || + desc.includes('air force two') || desc.includes('special air')) { + vipTransport.push(ac); + } + } + + const signals = []; + if (reconAircraft.length > 5) { + signals.push(`HIGH ISR ACTIVITY: ${reconAircraft.length} reconnaissance/surveillance aircraft airborne`); + } + if (bombers.length > 0) { + signals.push(`BOMBERS AIRBORNE: ${bombers.length} strategic bombers detected`); + } + if (tankers.length > 8) { + signals.push(`ELEVATED TANKER OPS: ${tankers.length} aerial refueling aircraft active (possible surge)`); + } + if (vipTransport.length > 0) { + signals.push(`VIP AIRCRAFT: ${vipTransport.length} VIP/continuity-of-government aircraft airborne`); + } + + return { + source: 'ADS-B Exchange', + timestamp: new Date().toISOString(), + status: 'live', + totalMilitary: militaryAircraft.length, + byCountry, + categories: { + reconnaissance: reconAircraft.slice(0, 20), + bombers: bombers.slice(0, 10), + tankers: tankers.slice(0, 10), + vipTransport: vipTransport.slice(0, 5), + }, + militaryAircraft: militaryAircraft.slice(0, 50), // cap for briefing size + signals: signals.length > 0 ? signals : ['Military flight activity within normal patterns'], + }; + } + + // No data available — return stub with integration documentation + return { + source: 'ADS-B Exchange', + timestamp: new Date().toISOString(), + status: apiKey ? 'error' : 'no_key', + militaryAircraft: [], + message: apiKey + ? 'ADS-B Exchange API returned no data. The endpoint may be temporarily unavailable.' + : 'No ADS-B Exchange API key configured. Set ADSB_API_KEY for military flight tracking.', + signals: ['ADS-B data unavailable — cannot assess military flight activity'], + integrationGuide: { + step1: 'Sign up at https://rapidapi.com/adsbexchange/api/adsbexchange-com1', + step2: 'Subscribe to the free tier (500 requests/month)', + step3: 'Set ADSB_API_KEY= in .env', + features: [ + 'Unfiltered military aircraft tracking (unlike FlightRadar24)', + 'Real-time position, altitude, speed, heading', + 'ICAO hex code identification for military registrations', + 'Geographic area search within radius', + 'Dedicated /mil endpoint for military-only feed', + ], + }, + complementarySource: 'OpenSky (opensky.mjs) provides partial military coverage for free', + knownMilitaryTypes: MILITARY_TYPES, + }; +} + +// Run standalone +if (process.argv[1]?.endsWith('adsb.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/bls.mjs b/apis/sources/bls.mjs new file mode 100644 index 0000000..85195fb --- /dev/null +++ b/apis/sources/bls.mjs @@ -0,0 +1,162 @@ +// BLS — Bureau of Labor Statistics +// CPI, unemployment, nonfarm payrolls, PPI. No auth required (v1 API). +// v2 with registration key supports more requests; v1 is rate-limited but functional. + +import { safeFetch, daysAgo } from '../utils/fetch.mjs'; + +const V1_BASE = 'https://api.bls.gov/publicAPI/v1/timeseries/data/'; +const V2_BASE = 'https://api.bls.gov/publicAPI/v2/timeseries/data/'; + +// Key economic series +const SERIES = { + 'CUUR0000SA0': 'CPI-U All Items', + 'CUUR0000SA0L1E': 'CPI-U Core (ex Food & Energy)', + 'LNS14000000': 'Unemployment Rate', + 'CES0000000001': 'Nonfarm Payrolls (thousands)', + 'WPUFD49104': 'PPI Final Demand', +}; + +// Fetch a single series via GET (v1, no key needed) +export async function getSeriesV1(seriesId) { + return safeFetch(`${V1_BASE}/${seriesId}`); +} + +// Fetch one or more series via POST (v2 if key available, v1 otherwise) +export async function getSeries(seriesIds, opts = {}) { + const { startYear, endYear, apiKey } = opts; + const now = new Date(); + const start = startYear || String(now.getFullYear() - 1); + const end = endYear || String(now.getFullYear()); + + const base = apiKey ? V2_BASE : V1_BASE; + const payload = { + seriesid: Array.isArray(seriesIds) ? seriesIds : [seriesIds], + startyear: start, + endyear: end, + }; + if (apiKey) payload.registrationkey = apiKey; + + try { + const res = await fetch(base, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + }); + return await res.json(); + } catch (e) { + return { error: e.message }; + } +} + +// Extract the latest observation from a BLS series response +function latestFromSeries(seriesData) { + if (!seriesData?.data?.length) return null; + // BLS returns data sorted by year desc, period desc + // Filter out unavailable values (BLS uses "-" for missing data) + const valid = seriesData.data.filter(d => d.value !== '-' && d.value !== '.'); + if (!valid.length) return null; + const sorted = [...valid].sort((a, b) => { + const ya = parseInt(a.year), yb = parseInt(b.year); + if (ya !== yb) return yb - ya; + // period is M01..M12 or M13 (annual avg) or Q01..Q05 + return b.period.localeCompare(a.period); + }); + return sorted[0]; +} + +// Get the two most recent observations to compute month-over-month change +function momChange(seriesData) { + if (!seriesData?.data?.length || seriesData.data.length < 2) return null; + const sorted = [...seriesData.data] + .filter(d => d.period.startsWith('M') && d.period !== 'M13' && d.value !== '-' && d.value !== '.') + .sort((a, b) => { + const ya = parseInt(a.year), yb = parseInt(b.year); + if (ya !== yb) return yb - ya; + return b.period.localeCompare(a.period); + }); + if (sorted.length < 2) return null; + const curr = parseFloat(sorted[0].value); + const prev = parseFloat(sorted[1].value); + if (isNaN(curr) || isNaN(prev) || prev === 0) return null; + return { + current: curr, + previous: prev, + change: +(curr - prev).toFixed(4), + changePct: +(((curr - prev) / prev) * 100).toFixed(4), + currentPeriod: `${sorted[0].year}-${sorted[0].period}`, + previousPeriod: `${sorted[1].year}-${sorted[1].period}`, + }; +} + +// Briefing — pull latest CPI, unemployment, payrolls +export async function briefing(apiKey) { + const seriesIds = Object.keys(SERIES); + const resp = await getSeries(seriesIds, { apiKey }); + + if (resp.error) { + return { source: 'BLS', error: resp.error, timestamp: new Date().toISOString() }; + } + + if (resp.status !== 'REQUEST_SUCCEEDED' || !resp.Results?.series?.length) { + return { + source: 'BLS', + error: resp.message?.[0] || 'BLS API returned no data', + rawStatus: resp.status, + timestamp: new Date().toISOString(), + }; + } + + const indicators = []; + const signals = []; + + for (const s of resp.Results.series) { + const id = s.seriesID; + const label = SERIES[id] || id; + const latest = latestFromSeries(s); + const mom = momChange(s); + + if (!latest) { + indicators.push({ id, label, value: null, date: null }); + continue; + } + + const value = parseFloat(latest.value); + const period = `${latest.year}-${latest.period}`; + + indicators.push({ + id, + label, + value, + period, + date: latest.year + '-' + latest.period.replace('M', '').padStart(2, '0'), + momChange: mom ? mom.change : null, + momChangePct: mom ? mom.changePct : null, + }); + + // Generate signals + if (id === 'LNS14000000' && value > 5.0) { + signals.push(`Unemployment elevated at ${value}%`); + } + if (id === 'CUUR0000SA0' && mom && mom.changePct > 0.4) { + signals.push(`CPI-U MoM jump: ${mom.changePct}% (${mom.previousPeriod} -> ${mom.currentPeriod})`); + } + if (id === 'CUUR0000SA0L1E' && mom && mom.changePct > 0.3) { + signals.push(`Core CPI MoM rising: ${mom.changePct}%`); + } + if (id === 'CES0000000001' && mom && mom.change < -50) { + signals.push(`Nonfarm payrolls dropped by ${Math.abs(mom.change)}K`); + } + } + + return { + source: 'BLS', + timestamp: new Date().toISOString(), + indicators, + signals, + }; +} + +if (process.argv[1]?.endsWith('bls.mjs')) { + const data = await briefing(process.env.BLS_API_KEY); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/bluesky.mjs b/apis/sources/bluesky.mjs new file mode 100644 index 0000000..31059dc --- /dev/null +++ b/apis/sources/bluesky.mjs @@ -0,0 +1,77 @@ +// Bluesky — AT Protocol social intelligence +// No auth required for public search. Real-time social sentiment on geopolitical/market topics. +// Public API: app.bsky.feed.searchPosts (full-text search, sorted by latest) + +import { safeFetch } from '../utils/fetch.mjs'; + +const BASE = 'https://public.api.bsky.app/xrpc'; + +// Rate-limit-safe delay +function delay(ms) { return new Promise(r => setTimeout(r, ms)); } + +// Search public posts by query string +export async function searchPosts(query, opts = {}) { + const { limit = 25, sort = 'latest' } = opts; + const params = new URLSearchParams({ + q: query, + limit: String(limit), + sort, + }); + return safeFetch(`${BASE}/app.bsky.feed.searchPosts?${params}`); +} + +// Compact a post for briefing output +function compactPost(post) { + const record = post?.record || post; + const author = post?.author; + return { + text: (record?.text || '').slice(0, 200), + author: author?.handle || author?.displayName || 'unknown', + date: record?.createdAt || null, + likes: post?.likeCount ?? 0, + }; +} + +// Categorize posts by topic bucket based on keyword matching +function categorize(posts, keywords) { + return posts.filter(p => + keywords.some(k => p.text?.toLowerCase().includes(k)) + ); +} + +// Briefing — search key geopolitical/market terms and categorize +export async function briefing() { + const searchQueries = [ + { label: 'conflict', q: 'Iran war OR missile strike OR sanctions' }, + { label: 'markets', q: 'market crash OR oil prices OR gold OR recession' }, + { label: 'health', q: 'pandemic OR outbreak OR epidemic' }, + ]; + + const allPosts = []; + const topicResults = {}; + + for (const { label, q } of searchQueries) { + const result = await searchPosts(q, { limit: 25 }); + const posts = (result?.posts || []).map(compactPost); + topicResults[label] = posts; + allPosts.push(...posts); + // Small delay between searches to be polite to the API + await delay(1500); + } + + return { + source: 'Bluesky', + timestamp: new Date().toISOString(), + topics: { + conflict: topicResults.conflict || [], + markets: topicResults.markets || [], + health: topicResults.health || [], + }, + }; +} + +// Run standalone +if (process.argv[1]?.endsWith('bluesky.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/comtrade.mjs b/apis/sources/comtrade.mjs new file mode 100644 index 0000000..5eb2959 --- /dev/null +++ b/apis/sources/comtrade.mjs @@ -0,0 +1,201 @@ +// UN Comtrade — Global Trade Data +// Public preview endpoint requires no key. Full API needs free registration. +// Tracks commodity trade flows between nations: crude oil, gas, gold, semiconductors, arms. +// Reporter codes: 842 (US), 156 (China), 276 (Germany), 392 (Japan), 826 (UK), 643 (Russia), 356 (India) + +import { safeFetch, daysAgo, today } from '../utils/fetch.mjs'; + +const BASE = 'https://comtradeapi.un.org/public/v1'; + +// Strategic commodity codes (HS classification) +const STRATEGIC_COMMODITIES = { + '2709': 'Crude Petroleum', + '2711': 'Natural Gas (LNG & Pipeline)', + '7108': 'Gold (unwrought/semi-manufactured)', + '8542': 'Semiconductors (Electronic Integrated Circuits)', + '93': 'Arms & Ammunition', + '2844': 'Radioactive Elements (Nuclear)', + '8471': 'Computers & Processing Units', + '2701': 'Coal', + '7601': 'Aluminium (unwrought)', + '2612': 'Uranium & Thorium Ores', +}; + +// Key reporter/partner country codes +const COUNTRIES = { + 842: 'United States', + 156: 'China', + 276: 'Germany', + 392: 'Japan', + 826: 'United Kingdom', + 643: 'Russia', + 356: 'India', + 410: 'South Korea', + 158: 'Taiwan', + 380: 'Italy', +}; + +// Get trade data for a specific reporter, commodity, and period +export async function getTradeData(opts = {}) { + const { + reporterCode = 842, // default: US + period = new Date().getFullYear(), + cmdCode = '2709', // default: crude oil + flowCode = 'M', // M = imports, X = exports + partnerCode = null, // null = all partners + } = opts; + + const params = new URLSearchParams({ + reporterCode: String(reporterCode), + period: String(period), + cmdCode, + flowCode, + }); + if (partnerCode) params.set('partnerCode', String(partnerCode)); + + return safeFetch(`${BASE}/preview/C/A/HS?${params}`, { timeout: 20000 }); +} + +// Get bilateral trade between two countries for a commodity +export async function getBilateralTrade(reporter, partner, cmdCode, period) { + return getTradeData({ + reporterCode: reporter, + partnerCode: partner, + cmdCode, + period: period || new Date().getFullYear(), + }); +} + +// Check multiple commodities for a given reporter +async function checkReporterCommodities(reporterCode, commodityCodes, period) { + const results = []; + for (const cmdCode of commodityCodes) { + const data = await getTradeData({ + reporterCode, + cmdCode, + period, + flowCode: 'M', // imports + }); + results.push({ + commodity: STRATEGIC_COMMODITIES[cmdCode] || cmdCode, + cmdCode, + data, + }); + } + return results; +} + +// Compact a trade record for briefing output +function compactRecord(rec) { + return { + reporter: rec.reporterDesc || rec.reporterCode, + partner: rec.partnerDesc || rec.partnerCode, + commodity: rec.cmdDesc || rec.cmdCode, + flow: rec.flowDesc || rec.flowCode, + value: rec.primaryValue || rec.cifvalue || rec.fobvalue || null, + quantity: rec.qty || rec.netWgt || null, + unit: rec.qtyUnitAbbr || rec.qtyUnitDesc || null, + period: rec.period, + }; +} + +// Detect anomalies in trade data (unusually large flows, new partners, etc.) +function detectAnomalies(tradeRecords) { + const signals = []; + if (!Array.isArray(tradeRecords) || tradeRecords.length === 0) return signals; + + const values = tradeRecords + .map(r => r.value) + .filter(v => typeof v === 'number' && v > 0); + + if (values.length > 2) { + const avg = values.reduce((a, b) => a + b, 0) / values.length; + const stdDev = Math.sqrt(values.reduce((a, v) => a + (v - avg) ** 2, 0) / values.length); + + tradeRecords.forEach(r => { + if (typeof r.value === 'number' && r.value > avg + 2 * stdDev) { + signals.push( + `OUTLIER: ${r.commodity} trade with ${r.partner} = $${(r.value / 1e9).toFixed(2)}B ` + + `(mean: $${(avg / 1e9).toFixed(2)}B)` + ); + } + }); + } + + return signals; +} + +// Briefing — check recent trade data for key commodities, detect anomalies +export async function briefing() { + const currentYear = new Date().getFullYear(); + const prevYear = currentYear - 1; + + // Key combinations to check: US imports of strategic commodities + const keyCommodities = ['2709', '2711', '7108', '8542', '93']; + const keyReporters = [842, 156]; // US, China + + const tradeFlows = []; + const signals = []; + + for (const reporter of keyReporters) { + for (const cmdCode of keyCommodities) { + // Try current year first, fall back to previous year + let data = await getTradeData({ + reporterCode: reporter, + cmdCode, + period: currentYear, + flowCode: 'M', + }); + + // Comtrade returns data in different structures; normalize + let records = data?.data || data?.dataset || []; + if (!Array.isArray(records)) records = []; + + // If no current year data, try previous year + if (records.length === 0) { + data = await getTradeData({ + reporterCode: reporter, + cmdCode, + period: prevYear, + flowCode: 'M', + }); + records = data?.data || data?.dataset || []; + if (!Array.isArray(records)) records = []; + } + + const compact = records.slice(0, 10).map(compactRecord); + if (compact.length > 0) { + tradeFlows.push({ + reporter: COUNTRIES[reporter] || reporter, + commodity: STRATEGIC_COMMODITIES[cmdCode] || cmdCode, + cmdCode, + topPartners: compact, + totalRecords: records.length, + }); + + // Run anomaly detection + const anomalies = detectAnomalies(compact); + signals.push(...anomalies); + } + } + } + + return { + source: 'UN Comtrade', + timestamp: new Date().toISOString(), + tradeFlows, + signals: signals.length > 0 + ? signals + : ['No significant trade anomalies detected in sampled commodities'], + status: tradeFlows.length > 0 ? 'ok' : 'no_data', + note: 'Comtrade data often lags 1-2 months. Recent periods may be incomplete.', + coveredCommodities: STRATEGIC_COMMODITIES, + coveredCountries: COUNTRIES, + }; +} + +// Run standalone +if (process.argv[1]?.endsWith('comtrade.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/eia.mjs b/apis/sources/eia.mjs new file mode 100644 index 0000000..2d0a760 --- /dev/null +++ b/apis/sources/eia.mjs @@ -0,0 +1,158 @@ +// EIA — US Energy Information Administration +// Oil prices, natural gas, crude inventories. Free API key required. +// Gracefully degrades without key. + +import { safeFetch } from '../utils/fetch.mjs'; +import '../utils/env.mjs'; + +const BASE = 'https://api.eia.gov/v2'; + +// Series definitions with their v2 API paths +const OIL_SERIES = { + wti: { + label: 'WTI Crude Oil ($/bbl)', + path: '/petroleum/pri/spt/data/', + params: { frequency: 'daily', 'data[0]': 'value', facets: { series: ['RWTC'] } }, + }, + brent: { + label: 'Brent Crude Oil ($/bbl)', + path: '/petroleum/pri/spt/data/', + params: { frequency: 'daily', 'data[0]': 'value', facets: { series: ['RBRTE'] } }, + }, +}; + +const GAS_SERIES = { + henryHub: { + label: 'Henry Hub Natural Gas ($/MMBtu)', + path: '/natural-gas/pri/fut/data/', + params: { frequency: 'daily', 'data[0]': 'value', facets: { series: ['RNGWHHD'] } }, + }, +}; + +const INVENTORY_SERIES = { + crudeStocks: { + label: 'US Crude Oil Inventories (thousand barrels)', + path: '/petroleum/stoc/wstk/data/', + params: { frequency: 'weekly', 'data[0]': 'value', facets: { series: ['WCESTUS1'] } }, + }, +}; + +// Build the URL for a v2 API query +function buildUrl(apiKey, path, params, length = 10) { + const url = new URL(`${BASE}${path}`); + url.searchParams.set('api_key', apiKey); + if (params.frequency) url.searchParams.set('frequency', params.frequency); + if (params['data[0]']) url.searchParams.set('data[0]', params['data[0]']); + url.searchParams.set('sort[0][column]', 'period'); + url.searchParams.set('sort[0][direction]', 'desc'); + url.searchParams.set('length', String(length)); + + // Add facets + if (params.facets) { + for (const [facetKey, facetValues] of Object.entries(params.facets)) { + facetValues.forEach((v, i) => { + url.searchParams.set(`facets[${facetKey}][]`, v); + }); + } + } + + return url.toString(); +} + +// Fetch a single EIA series +export async function fetchSeries(apiKey, seriesDef, length = 10) { + const url = buildUrl(apiKey, seriesDef.path, seriesDef.params, length); + return safeFetch(url); +} + +// Extract latest value from EIA response +function extractLatest(resp) { + const data = resp?.response?.data; + if (!data?.length) return null; + return { + value: parseFloat(data[0].value), + period: data[0].period, + unit: data[0]['unit-name'] || data[0].unit || null, + }; +} + +// Extract recent values for trend analysis +function extractRecent(resp, count = 5) { + const data = resp?.response?.data; + if (!data?.length) return []; + return data.slice(0, count).map(d => ({ + value: parseFloat(d.value), + period: d.period, + })); +} + +// Briefing — oil prices, gas prices, inventories +export async function briefing(apiKey) { + if (!apiKey) { + return { + source: 'EIA', + error: 'No EIA API key. Register free at https://www.eia.gov/opendata/register.php', + hint: 'Set EIA_API_KEY environment variable', + timestamp: new Date().toISOString(), + }; + } + + const [wtiResp, brentResp, gasResp, inventoryResp] = await Promise.all([ + fetchSeries(apiKey, OIL_SERIES.wti), + fetchSeries(apiKey, OIL_SERIES.brent), + fetchSeries(apiKey, GAS_SERIES.henryHub), + fetchSeries(apiKey, INVENTORY_SERIES.crudeStocks), + ]); + + const signals = []; + + // Oil prices + const wti = extractLatest(wtiResp); + const brent = extractLatest(brentResp); + const wtiRecent = extractRecent(wtiResp, 5); + const brentRecent = extractRecent(brentResp, 5); + + if (wti && wti.value > 100) signals.push(`WTI crude above $100 at $${wti.value}/bbl`); + if (wti && wti.value < 50) signals.push(`WTI crude below $50 at $${wti.value}/bbl — supply glut or demand destruction`); + if (brent && wti && (brent.value - wti.value) > 10) { + signals.push(`Brent-WTI spread wide at $${(brent.value - wti.value).toFixed(2)} — supply/logistics divergence`); + } + + // Gas prices + const gas = extractLatest(gasResp); + if (gas && gas.value > 6) signals.push(`Natural gas elevated at $${gas.value}/MMBtu`); + if (gas && gas.value > 9) signals.push(`Natural gas crisis-level at $${gas.value}/MMBtu`); + + // Inventories + const inv = extractLatest(inventoryResp); + const invRecent = extractRecent(inventoryResp, 5); + + // Check week-over-week inventory change + if (invRecent.length >= 2) { + const weekChange = invRecent[0].value - invRecent[1].value; + if (Math.abs(weekChange) > 5000) { + const direction = weekChange > 0 ? 'build' : 'draw'; + signals.push(`Large crude inventory ${direction}: ${weekChange > 0 ? '+' : ''}${(weekChange / 1000).toFixed(1)}M barrels`); + } + } + + return { + source: 'EIA', + timestamp: new Date().toISOString(), + oilPrices: { + wti: wti ? { ...wti, label: OIL_SERIES.wti.label, recent: wtiRecent } : null, + brent: brent ? { ...brent, label: OIL_SERIES.brent.label, recent: brentRecent } : null, + spread: wti && brent ? +(brent.value - wti.value).toFixed(2) : null, + }, + gasPrice: gas ? { ...gas, label: GAS_SERIES.henryHub.label } : null, + inventories: { + crudeStocks: inv ? { ...inv, label: INVENTORY_SERIES.crudeStocks.label, recent: invRecent } : null, + }, + signals, + }; +} + +if (process.argv[1]?.endsWith('eia.mjs')) { + const data = await briefing(process.env.EIA_API_KEY); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/epa.mjs b/apis/sources/epa.mjs new file mode 100644 index 0000000..9a9f0ce --- /dev/null +++ b/apis/sources/epa.mjs @@ -0,0 +1,206 @@ +// EPA RadNet — Radiation Monitoring Network +// No auth required. Government open data via Envirofacts REST API. +// Monitors ambient radiation levels across the US via fixed monitoring stations. +// Complements Safecast (citizen science) with official government readings. + +import { safeFetch } from '../utils/fetch.mjs'; + +const BASE = 'https://enviro.epa.gov/enviro/efservice'; + +// RadNet analytical results endpoint +const RADNET_ANALYTICAL = `${BASE}/RADNET_ANALYTICAL_RESULTS`; +// RadNet auxiliary data +const RADNET_AUX = `${BASE}/RADNET_AUX`; + +// Key US cities with RadNet monitoring stations +const MONITORING_STATIONS = { + washingtonDC: { label: 'Washington, DC', state: 'DC' }, + newYork: { label: 'New York, NY', state: 'NY' }, + losAngeles: { label: 'Los Angeles, CA', state: 'CA' }, + chicago: { label: 'Chicago, IL', state: 'IL' }, + seattle: { label: 'Seattle, WA', state: 'WA' }, + denver: { label: 'Denver, CO', state: 'CO' }, + honolulu: { label: 'Honolulu, HI', state: 'HI' }, + anchorage: { label: 'Anchorage, AK', state: 'AK' }, + miami: { label: 'Miami, FL', state: 'FL' }, + sanFrancisco: { label: 'San Francisco, CA', state: 'CA' }, +}; + +// Analyte types that indicate concerning radiation +const KEY_ANALYTES = [ + 'GROSS BETA', + 'GROSS ALPHA', + 'IODINE-131', + 'CESIUM-137', + 'CESIUM-134', + 'STRONTIUM-90', + 'TRITIUM', + 'URANIUM', + 'PLUTONIUM', +]; + +// Normal background radiation thresholds (pCi/L or pCi/m3 depending on medium) +const THRESHOLDS = { + 'GROSS BETA': { normal: 1.0, elevated: 5.0, unit: 'pCi/m3' }, + 'GROSS ALPHA': { normal: 0.05, elevated: 0.15, unit: 'pCi/m3' }, + 'IODINE-131': { normal: 0.01, elevated: 0.1, unit: 'pCi/m3' }, + 'CESIUM-137': { normal: 0.01, elevated: 0.1, unit: 'pCi/m3' }, + 'CESIUM-134': { normal: 0.001, elevated: 0.01, unit: 'pCi/m3' }, +}; + +// Get recent RadNet analytical results (JSON) +export async function getAnalyticalResults(opts = {}) { + const { rows = 50, startRow = 0 } = opts; + return safeFetch( + `${RADNET_ANALYTICAL}/ROWS/${startRow}:${startRow + rows}/JSON`, + { timeout: 25000 } + ); +} + +// Get results filtered by state +export async function getResultsByState(state, opts = {}) { + const { rows = 25, startRow = 0 } = opts; + return safeFetch( + `${RADNET_ANALYTICAL}/ANA_STATE/${state}/ROWS/${startRow}:${startRow + rows}/JSON`, + { timeout: 25000 } + ); +} + +// Get results filtered by analyte type +export async function getResultsByAnalyte(analyte, opts = {}) { + const { rows = 25, startRow = 0 } = opts; + const encoded = encodeURIComponent(analyte); + return safeFetch( + `${RADNET_ANALYTICAL}/ANA_TYPE/${encoded}/ROWS/${startRow}:${startRow + rows}/JSON`, + { timeout: 25000 } + ); +} + +// Compact a reading for briefing output +function compactReading(r) { + return { + location: r.ANA_CITY || r.LOCATION || 'Unknown', + state: r.ANA_STATE || r.STATE || null, + analyte: r.ANA_TYPE || r.ANALYTE_NAME || null, + result: r.ANA_RESULT != null ? parseFloat(r.ANA_RESULT) : null, + unit: r.RESULT_UNIT || r.ANA_UNIT || null, + collectDate: r.COLLECT_DATE || r.SAMPLE_DATE || null, + medium: r.SAMPLE_TYPE || r.MEDIUM || null, + }; +} + +// Check a reading against known thresholds +function checkReading(reading) { + if (reading.result === null || reading.result <= 0) return null; + const threshold = THRESHOLDS[reading.analyte?.toUpperCase()]; + if (!threshold) return null; + + if (reading.result > threshold.elevated) { + return { + level: 'ELEVATED', + reading, + threshold: threshold.elevated, + ratio: (reading.result / threshold.elevated).toFixed(1), + }; + } + if (reading.result > threshold.normal * 3) { + return { + level: 'ABOVE_NORMAL', + reading, + threshold: threshold.normal, + ratio: (reading.result / threshold.normal).toFixed(1), + }; + } + return null; +} + +// Briefing — get recent radiation readings from EPA network, flag anomalies +export async function briefing() { + const readings = []; + const signals = []; + + // Fetch recent analytical results (broad pull) + const recentData = await getAnalyticalResults({ rows: 100 }); + const recentRecords = Array.isArray(recentData) ? recentData : []; + + // Compact all readings + const allReadings = recentRecords.map(compactReading); + readings.push(...allReadings); + + // Also try to pull key analytes specifically + const analyteResults = await Promise.all( + ['GROSS BETA', 'IODINE-131', 'CESIUM-137'].map(async analyte => { + const data = await getResultsByAnalyte(analyte, { rows: 20 }); + const records = Array.isArray(data) ? data : []; + return { analyte, records: records.map(compactReading) }; + }) + ); + + for (const { analyte, records } of analyteResults) { + // Add any records not already in our list + for (const r of records) { + if (!readings.some(existing => + existing.location === r.location && + existing.collectDate === r.collectDate && + existing.analyte === r.analyte + )) { + readings.push(r); + } + } + } + + // Check all readings against thresholds + for (const reading of readings) { + const alert = checkReading(reading); + if (alert) { + if (alert.level === 'ELEVATED') { + signals.push( + `ELEVATED ${reading.analyte} at ${reading.location}, ${reading.state}: ` + + `${reading.result} ${reading.unit || ''} (${alert.ratio}x threshold) [${reading.collectDate}]` + ); + } else { + signals.push( + `ABOVE NORMAL ${reading.analyte} at ${reading.location}, ${reading.state}: ` + + `${reading.result} ${reading.unit || ''} (${alert.ratio}x normal) [${reading.collectDate}]` + ); + } + } + } + + // Summarize by state + const byState = {}; + for (const r of readings) { + const st = r.state || 'UNK'; + if (!byState[st]) byState[st] = { count: 0, analytes: new Set() }; + byState[st].count++; + if (r.analyte) byState[st].analytes.add(r.analyte); + } + + // Convert sets to arrays for JSON + const stateSummary = Object.fromEntries( + Object.entries(byState).map(([st, info]) => [ + st, + { count: info.count, analytes: [...info.analytes] }, + ]) + ); + + return { + source: 'EPA RadNet', + timestamp: new Date().toISOString(), + totalReadings: readings.length, + readings: readings.slice(0, 50), // cap for briefing size + stateSummary, + signals: signals.length > 0 + ? signals + : ['All EPA RadNet readings within normal background levels'], + monitoredAnalytes: KEY_ANALYTES, + thresholds: THRESHOLDS, + note: 'RadNet data may lag by hours to days. Near-real-time gamma data updates more frequently.', + }; +} + +// Run standalone +if (process.argv[1]?.endsWith('epa.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/firms.mjs b/apis/sources/firms.mjs new file mode 100644 index 0000000..efb4661 --- /dev/null +++ b/apis/sources/firms.mjs @@ -0,0 +1,150 @@ +// NASA FIRMS — Fire Information for Resource Management System +// Detects active fires/thermal anomalies globally within 3 hours of satellite pass. +// Detects military strikes, explosions, wildfires, industrial fires. + +import '../utils/env.mjs'; + +const FIRMS_BASE = 'https://firms.modaps.eosdis.nasa.gov/api/area/csv'; + +// Parse FIRMS CSV response into structured data +function parseCSV(rawText) { + if (!rawText || typeof rawText !== 'string') return []; + const lines = rawText.trim().split('\n'); + if (lines.length < 2) return []; + const headers = lines[0].split(','); + return lines.slice(1).map(line => { + const vals = line.split(','); + const obj = {}; + headers.forEach((h, i) => { obj[h.trim()] = vals[i]?.trim(); }); + return obj; + }); +} + +// Fetch fires in a bounding box +async function fetchFires(opts = {}) { + const { + west = -180, south = -90, east = 180, north = 90, + days = 1, + source = 'VIIRS_SNPP_NRT', + } = opts; + + const key = process.env.FIRMS_MAP_KEY; + if (!key) return { error: 'No FIRMS_MAP_KEY' }; + + const url = `${FIRMS_BASE}/${key}/${source}/${west},${south},${east},${north}/${days}`; + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 25000); + try { + const res = await fetch(url, { + signal: controller.signal, + headers: { 'User-Agent': 'Crucix/1.0' }, + }); + clearTimeout(timer); + if (!res.ok) return { error: `HTTP ${res.status}` }; + const text = await res.text(); + return parseCSV(text); + } catch (e) { + clearTimeout(timer); + return { error: e.message }; + } +} + +// Key conflict/hotspot zones +const HOTSPOTS = { + middleEast: { west: 30, south: 12, east: 65, north: 42, label: 'Middle East' }, + ukraine: { west: 22, south: 44, east: 41, north: 53, label: 'Ukraine' }, + iran: { west: 44, south: 25, east: 63, north: 40, label: 'Iran' }, + sudanHorn: { west: 21, south: 2, east: 52, north: 23, label: 'Sudan / Horn of Africa' }, + myanmar: { west: 92, south: 9, east: 102, north: 29, label: 'Myanmar' }, + southAsia: { west: 60, south: 5, east: 98, north: 37, label: 'South Asia' }, +}; + +// Analyze fire detections for potential military/strike activity +function analyzeFires(fires, regionLabel) { + if (!Array.isArray(fires) || fires.length === 0) { + return { region: regionLabel, totalDetections: 0, highConfidence: 0, highIntensity: [], summary: 'No detections' }; + } + + const highConf = fires.filter(f => f.confidence === 'h' || f.confidence === 'high'); + const nomConf = fires.filter(f => f.confidence === 'n' || f.confidence === 'nominal'); + + // High intensity fires (FRP > 10 MW) — potential strikes, industrial fires, large explosions + const highIntensity = fires + .filter(f => parseFloat(f.frp) > 10) + .map(f => ({ + lat: parseFloat(f.latitude), + lon: parseFloat(f.longitude), + brightness: parseFloat(f.bright_ti4), + frp: parseFloat(f.frp), + date: f.acq_date, + time: f.acq_time, + confidence: f.confidence, + daynight: f.daynight, + })) + .sort((a, b) => b.frp - a.frp) + .slice(0, 15); + + // Night detections are more significant (less likely agricultural burning) + const nightFires = fires.filter(f => f.daynight === 'N'); + + return { + region: regionLabel, + totalDetections: fires.length, + highConfidence: highConf.length, + nominalConfidence: nomConf.length, + nightDetections: nightFires.length, + highIntensity, + avgFRP: fires.reduce((sum, f) => sum + (parseFloat(f.frp) || 0), 0) / fires.length, + }; +} + +// Briefing +export async function briefing() { + const key = process.env.FIRMS_MAP_KEY; + if (!key) { + return { + source: 'NASA FIRMS', + timestamp: new Date().toISOString(), + status: 'no_key', + message: 'Set FIRMS_MAP_KEY for satellite fire/strike detection. Free at https://firms.modaps.eosdis.nasa.gov/api/area/', + }; + } + + // Fetch all hotspots in parallel + const entries = Object.entries(HOTSPOTS); + const rawResults = await Promise.all( + entries.map(async ([key, box]) => { + const fires = await fetchFires({ ...box, days: 2 }); + return { key, label: box.label, fires }; + }) + ); + + const hotspots = rawResults.map(r => { + if (r.fires?.error) return { region: r.label, error: r.fires.error }; + return analyzeFires(r.fires, r.label); + }); + + // Generate signals + const signals = []; + for (const h of hotspots) { + if (h.highIntensity?.length > 5) { + signals.push(`HIGH INTENSITY FIRES in ${h.region}: ${h.highIntensity.length} detections >10MW FRP`); + } + if (h.nightDetections > 20) { + signals.push(`ELEVATED NIGHT ACTIVITY in ${h.region}: ${h.nightDetections} night detections (potential strikes/combat)`); + } + } + + return { + source: 'NASA FIRMS', + timestamp: new Date().toISOString(), + status: 'active', + hotspots, + signals, + }; +} + +if (process.argv[1]?.endsWith('firms.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/fred.mjs b/apis/sources/fred.mjs new file mode 100644 index 0000000..0ca28fe --- /dev/null +++ b/apis/sources/fred.mjs @@ -0,0 +1,108 @@ +// FRED — Federal Reserve Economic Data +// 840,000+ time series. Free API key required. +// Key indicators: yield curve, CPI, unemployment, money supply, GDP, fed funds rate + +import { safeFetch, today, daysAgo } from '../utils/fetch.mjs'; + +const BASE = 'https://api.stlouisfed.org/fred'; + +// Key series IDs for macro intelligence +const KEY_SERIES = { + // Yield curve & rates + DFF: 'Fed Funds Rate', + DGS2: '2-Year Treasury Yield', + DGS10: '10-Year Treasury Yield', + DGS30: '30-Year Treasury Yield', + T10Y2Y: '10Y-2Y Spread (Yield Curve)', + T10Y3M: '10Y-3M Spread', + // Inflation + CPIAUCSL: 'CPI All Items', + CPILFESL: 'Core CPI (ex Food & Energy)', + PCEPI: 'PCE Price Index', + MICH: 'Michigan Inflation Expectations', + // Labor + UNRATE: 'Unemployment Rate', + PAYEMS: 'Nonfarm Payrolls', + ICSA: 'Initial Jobless Claims', + // Money & credit + M2SL: 'M2 Money Supply', + WALCL: 'Fed Balance Sheet Total Assets', + // Fear gauges + VIXCLS: 'VIX (Fear Index)', + BAMLH0A0HYM2: 'High Yield Spread (Credit Stress)', + // Commodities via FRED + DCOILWTICO: 'WTI Crude Oil', + GOLDAMGBD228NLBM: 'Gold Price (London Fix)', + // Housing + MORTGAGE30US: '30-Year Mortgage Rate', + // Global + DTWEXBGS: 'USD Trade Weighted Index', +}; + +// Get latest value for a series +async function getSeriesLatest(seriesId, apiKey) { + const params = new URLSearchParams({ + series_id: seriesId, + api_key: apiKey, + file_type: 'json', + sort_order: 'desc', + limit: '5', + observation_start: daysAgo(90), + }); + return safeFetch(`${BASE}/series/observations?${params}`); +} + +// Briefing — pull all key indicators +export async function briefing(apiKey) { + if (!apiKey) { + return { + source: 'FRED', + error: 'No FRED API key. Get one free at https://fred.stlouisfed.org/docs/api/api_key.html', + hint: 'Set FRED_API_KEY environment variable', + }; + } + + const entries = Object.entries(KEY_SERIES); + const results = await Promise.all( + entries.map(async ([id, label]) => { + const data = await getSeriesLatest(id, apiKey); + const obs = data?.observations; + if (!obs?.length) return { id, label, value: null, date: null, recent: [] }; + const latest = obs.find(o => o.value !== '.'); + const validObs = obs.filter(o => o.value !== '.'); + return { + id, + label, + value: latest ? parseFloat(latest.value) : null, + date: latest?.date || null, + recent: validObs.slice(0, 5).map(o => parseFloat(o.value)), + }; + }) + ); + + // Compute derived signals + const get = (id) => results.find(r => r.id === id)?.value; + const yieldCurve10y2y = get('T10Y2Y'); + const yieldCurve10y3m = get('T10Y3M'); + const vix = get('VIXCLS'); + const hySpread = get('BAMLH0A0HYM2'); + + const signals = []; + if (yieldCurve10y2y !== null && yieldCurve10y2y < 0) signals.push('YIELD CURVE INVERTED (10Y-2Y) — recession signal'); + if (yieldCurve10y3m !== null && yieldCurve10y3m < 0) signals.push('YIELD CURVE INVERTED (10Y-3M) — stronger recession signal'); + if (vix !== null && vix > 30) signals.push(`VIX ELEVATED at ${vix} — high fear/volatility`); + if (vix !== null && vix > 40) signals.push(`VIX EXTREME at ${vix} — crisis-level fear`); + if (hySpread !== null && hySpread > 5) signals.push(`HIGH YIELD SPREAD WIDE at ${hySpread}% — credit stress`); + + return { + source: 'FRED', + timestamp: new Date().toISOString(), + indicators: results.filter(r => r.value !== null), + signals, + }; +} + +if (process.argv[1]?.endsWith('fred.mjs')) { + const data = await briefing(process.env.FRED_API_KEY); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/gdelt.mjs b/apis/sources/gdelt.mjs new file mode 100644 index 0000000..73ad9ba --- /dev/null +++ b/apis/sources/gdelt.mjs @@ -0,0 +1,123 @@ +// GDELT — Global Database of Events, Language, and Tone +// No auth required. Updates every 15 minutes. Monitors news in 100+ languages. +// DOC 2.0 API: full-text search across last 3 months of global news +// GEO 2.0 API: geolocation mapping of events + +import { safeFetch } from '../utils/fetch.mjs'; + +const BASE = 'https://api.gdeltproject.org/api/v2'; + +// Search recent global events/articles by keyword +export async function searchEvents(query = '', opts = {}) { + const { + mode = 'ArtList', // ArtList, TimelineVol, TimelineVolInfo, TimelineTone, TimelineLang, TimelineSourceCountry + maxRecords = 75, + timespan = '24h', // e.g. "24h", "7d", "3m" + format = 'json', + sortBy = 'DateDesc', // DateDesc, DateAsc, ToneDesc, ToneAsc + } = opts; + + // If no query, use broad geopolitical terms + const q = query || 'conflict OR crisis OR military OR sanctions OR war OR economy'; + const params = new URLSearchParams({ + query: q, + mode, + maxrecords: String(maxRecords), + timespan, + format, + sort: sortBy, + }); + + return safeFetch(`${BASE}/doc/doc?${params}`); +} + +// Get tone/sentiment timeline for a topic +export async function toneTrend(query, timespan = '7d') { + const params = new URLSearchParams({ + query, + mode: 'TimelineTone', + timespan, + format: 'json', + }); + return safeFetch(`${BASE}/doc/doc?${params}`); +} + +// Get volume timeline for a topic (how much coverage) +export async function volumeTrend(query, timespan = '7d') { + const params = new URLSearchParams({ + query, + mode: 'TimelineVol', + timespan, + format: 'json', + }); + return safeFetch(`${BASE}/doc/doc?${params}`); +} + +// GEO API — geographic event mapping +export async function geoEvents(query = '', opts = {}) { + const { + mode = 'PointData', + timespan = '24h', + format = 'GeoJSON', + maxPoints = 500, + } = opts; + + const q = query || 'conflict OR military OR protest OR explosion'; + const params = new URLSearchParams({ + query: q, + mode, + timespan, + format, + maxpoints: String(maxPoints), + }); + + return safeFetch(`${BASE}/geo/geo?${params}`); +} + +// Compact article for briefing +function compactArticle(a) { + return { + title: a.title, + url: a.url, + date: a.seendate, + domain: a.domain, + language: a.language, + country: a.sourcecountry, + }; +} + +// GDELT rate limit: 1 request per 5 seconds +function delay(ms) { return new Promise(r => setTimeout(r, ms)); } + +// Briefing mode — get top global events summary (sequential due to rate limit) +export async function briefing() { + // Single broad query to stay within rate limits + const all = await searchEvents( + 'conflict OR military OR economy OR crisis OR war OR sanctions OR tariff OR strike OR outbreak', + { maxRecords: 50, timespan: '24h' } + ); + + const articles = (all?.articles || []).map(compactArticle); + + // Categorize by keyword matching in titles + const categorize = (keywords) => articles.filter(a => + keywords.some(k => a.title?.toLowerCase().includes(k)) + ); + + return { + source: 'GDELT', + timestamp: new Date().toISOString(), + totalArticles: articles.length, + allArticles: articles, + conflicts: categorize(['military', 'conflict', 'war', 'strike', 'missile', 'attack', 'bomb', 'troops']), + economy: categorize(['economy', 'recession', 'inflation', 'market', 'sanctions', 'tariff', 'trade', 'gdp']), + health: categorize(['pandemic', 'outbreak', 'epidemic', 'disease', 'virus', 'health']), + crisis: categorize(['crisis', 'disaster', 'emergency', 'refugee', 'famine']), + }; +} + +// Run standalone +if (process.argv[1]?.endsWith('gdelt.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/gscpi.mjs b/apis/sources/gscpi.mjs new file mode 100644 index 0000000..ff02534 --- /dev/null +++ b/apis/sources/gscpi.mjs @@ -0,0 +1,166 @@ +// GSCPI — NY Fed Global Supply Chain Pressure Index +// Measures global supply chain stress (standard deviations from historical average). +// Values above 0 = above average pressure. Above 1.0 = elevated. Below -1.0 = unusually loose. +// Data fetched directly from NY Fed — no API key required. + +const GSCPI_CSV_URL = 'https://www.newyorkfed.org/medialibrary/research/interactives/data/gscpi/gscpi_interactive_data.csv'; + +// Fetch and parse the GSCPI CSV from the NY Fed +// The CSV is wide-format: each column is a revision vintage, last column is latest estimate. +// Uses raw fetch instead of safeFetch because safeFetch truncates non-JSON to 500 chars. +export async function getGSCPI(months = 12) { + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 20000); + const res = await fetch(GSCPI_CSV_URL, { + signal: controller.signal, + headers: { 'User-Agent': 'Crucix/1.0' }, + }); + clearTimeout(timer); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const text = await res.text(); + return { data: parseCSV(text, months) }; + } catch (e) { + return { error: e.message || 'Failed to fetch GSCPI data', data: [] }; + } +} + +// Parse the wide-format CSV, extracting the latest vintage value for each date +function parseCSV(text, months) { + const lines = text.trim().split('\n').filter(l => l.trim() && !l.startsWith(',')); + if (lines.length < 2) return []; + + // Header row tells us column count; we want the last non-empty column for each row + const results = []; + + for (let i = 1; i < lines.length; i++) { + const cols = lines[i].split(','); + const dateStr = cols[0]?.trim(); + if (!dateStr) continue; + + // Find the last non-empty, non-#N/A value (latest vintage estimate) + let value = null; + for (let j = cols.length - 1; j >= 1; j--) { + const v = cols[j]?.trim(); + if (v && v !== '#N/A' && v !== '') { + const num = parseFloat(v); + if (!isNaN(num)) { + value = num; + break; + } + } + } + + if (value === null) continue; + + // Parse date from "31-Jan-2026" format to "2026-01" + const date = parseNYFedDate(dateStr); + if (date) { + results.push({ date, value }); + } + } + + // Sort newest first + results.sort((a, b) => b.date.localeCompare(a.date)); + + return results.slice(0, months); +} + +// Parse "31-Jan-2026" -> "2026-01" +function parseNYFedDate(str) { + const months = { + Jan: '01', Feb: '02', Mar: '03', Apr: '04', May: '05', Jun: '06', + Jul: '07', Aug: '08', Sep: '09', Oct: '10', Nov: '11', Dec: '12', + }; + const parts = str.split('-'); + if (parts.length !== 3) return null; + const mon = months[parts[1]]; + const year = parts[2]; + if (!mon || !year) return null; + return `${year}-${mon}`; +} + +// Detect trend from an array of {date, value} sorted newest-first +function detectTrend(history) { + if (history.length < 3) return 'insufficient data'; + + // Compare recent 3 months direction + const recent = history.slice(0, 3); + let rising = 0; + let falling = 0; + + for (let i = 0; i < recent.length - 1; i++) { + // history is newest-first, so recent[0] is latest + if (recent[i].value > recent[i + 1].value) rising++; + else if (recent[i].value < recent[i + 1].value) falling++; + } + + if (rising > falling) return 'rising'; + if (falling > rising) return 'falling'; + return 'stable'; +} + +// Briefing — latest GSCPI, trend, and signals +export async function briefing() { + const result = await getGSCPI(12); + + if (result.error) { + return { + source: 'NY Fed GSCPI', + error: result.error, + timestamp: new Date().toISOString(), + }; + } + + const history = result.data; + const trend = detectTrend(history); + const signals = []; + + const latest = history.length > 0 ? history[0] : null; + + if (latest) { + if (latest.value > 2.0) { + signals.push(`GSCPI extremely elevated at ${latest.value.toFixed(2)} — severe supply chain stress`); + } else if (latest.value > 1.0) { + signals.push(`GSCPI elevated at ${latest.value.toFixed(2)} — above-normal supply chain pressure`); + } else if (latest.value < -1.0) { + signals.push(`GSCPI at ${latest.value.toFixed(2)} — unusually loose supply chains`); + } + + if (trend === 'rising' && latest.value > 0) { + signals.push('Supply chain pressure trending higher'); + } + if (trend === 'falling' && latest.value > 1.0) { + signals.push('Supply chain pressure elevated but improving'); + } + } + + // Check month-over-month change + if (history.length >= 2) { + const mom = history[0].value - history[1].value; + if (Math.abs(mom) > 0.5) { + const dir = mom > 0 ? 'surged' : 'dropped'; + signals.push(`GSCPI ${dir} ${Math.abs(mom).toFixed(2)} points month-over-month`); + } + } + + return { + source: 'NY Fed GSCPI', + timestamp: new Date().toISOString(), + latest: latest ? { + value: latest.value, + date: latest.date, + interpretation: latest.value > 1.0 ? 'elevated' : + latest.value > 0 ? 'above average' : + latest.value > -1.0 ? 'below average' : 'unusually loose', + } : null, + trend, + history, + signals, + }; +} + +if (process.argv[1]?.endsWith('gscpi.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/kiwisdr.mjs b/apis/sources/kiwisdr.mjs new file mode 100644 index 0000000..a5232b1 --- /dev/null +++ b/apis/sources/kiwisdr.mjs @@ -0,0 +1,306 @@ +// KiwiSDR Network — Global software-defined radio receiver network +// No auth required. ~900 public HF receivers worldwide (0-30 MHz). +// Useful for SIGINT awareness: HF band activity, receiver distribution, +// detecting unusual radio configurations in conflict zones. +// Data source: receiverbook.de (embeds full receiver list as JS variable) + +import { safeFetch } from '../utils/fetch.mjs'; + +const RECEIVERBOOK_URL = 'https://www.receiverbook.de/map?type=kiwisdr'; + +// Fetch the full list of public KiwiSDR receivers from receiverbook.de +export async function getAllReceivers() { + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 20000); + const res = await fetch(RECEIVERBOOK_URL, { + headers: { 'User-Agent': 'Crucix/1.0' }, + signal: controller.signal, + }); + clearTimeout(timer); + if (!res.ok) return { error: `HTTP ${res.status}` }; + const html = await res.text(); + // Extract embedded JS: var receivers = [...]; + const match = html.match(/var\s+receivers\s*=\s*(\[[\s\S]*?\]);/); + if (!match) return { error: 'Could not parse receiver data from page' }; + const sites = JSON.parse(match[1]); + // Flatten: each site has a .receivers[] array of individual SDRs + const flat = []; + for (const site of sites) { + const [lon, lat] = site.location?.coordinates || [NaN, NaN]; + const country = site.label?.split(',').pop()?.trim() || ''; + for (const rx of (site.receivers || [site])) { + flat.push({ + name: rx.label || site.label || '', + location: site.label || '', + lat, lon, + country, + url: rx.url || site.url || '', + version: rx.version || '', + antenna: '', + users: 0, usersMax: 0, + offline: false, + snr: NaN, + tdoa: null, + bands: '', + }); + } + } + return flat; + } catch (e) { + return { error: e.message }; + } +} + +// Regions of intelligence interest with bounding boxes +const REGIONS_OF_INTEREST = { + middleEast: { lamin: 12, lomin: 30, lamax: 42, lomax: 65, label: 'Middle East' }, + ukraine: { lamin: 44, lomin: 22, lamax: 53, lomax: 41, label: 'Ukraine / Eastern Europe' }, + taiwan: { lamin: 20, lomin: 115, lamax: 28, lomax: 125, label: 'Taiwan Strait' }, + baltics: { lamin: 53, lomin: 19, lamax: 60, lomax: 29, label: 'Baltic Region' }, + southChinaSea: { lamin: 5, lomin: 105, lamax: 23, lomax: 122, label: 'South China Sea' }, + koreanPeninsula:{ lamin: 33, lomin: 124, lamax: 43, lomax: 132, label: 'Korean Peninsula' }, + iran: { lamin: 25, lomin: 44, lamax: 40, lomax: 63, label: 'Iran' }, + sahel: { lamin: 10, lomin: -17, lamax: 20, lomax: 25, label: 'Sahel / West Africa' }, +}; + +// HF band classifications for intelligence relevance +const HF_BANDS = { + vlf: { min: 0, max: 0.3, label: 'VLF (submarine/military comms)' }, + lf: { min: 0.3, max: 0.5, label: 'LF (navigation/time signals)' }, + mf: { min: 0.5, max: 1.8, label: 'MF (AM broadcast/maritime)' }, + hf160m: { min: 1.8, max: 2.0, label: '160m amateur' }, + hf80m: { min: 3.5, max: 4.0, label: '80m amateur' }, + hf60m: { min: 5.3, max: 5.4, label: '60m amateur/utility' }, + hf49m: { min: 5.9, max: 6.2, label: '49m shortwave broadcast' }, + hf40m: { min: 7.0, max: 7.3, label: '40m amateur' }, + hf31m: { min: 9.4, max: 9.9, label: '31m shortwave broadcast' }, + hf30m: { min: 10.1, max: 10.15,label: '30m amateur' }, + hf25m: { min: 11.6, max: 12.1, label: '25m shortwave broadcast' }, + hf20m: { min: 14.0, max: 14.35,label: '20m amateur' }, + hf17m: { min: 18.068,max: 18.168,label: '17m amateur' }, + hf15m: { min: 21.0, max: 21.45,label: '15m amateur' }, + hf11m: { min: 25.67, max: 26.1, label: '11m broadcast/CB' }, + hfMilitary:{ min: 2.0, max: 30.0, label: 'HF military/utility (general)' }, +}; + +// Check if a receiver falls within a bounding box +function inBounds(rx, box) { + if (isNaN(rx.lat) || isNaN(rx.lon)) return false; + return rx.lat >= box.lamin && rx.lat <= box.lamax && rx.lon >= box.lomin && rx.lon <= box.lomax; +} + +// Map a receiver to a continent based on coordinates +function getContinent(lat, lon) { + if (isNaN(lat) || isNaN(lon)) return 'Unknown'; + if (lat >= 15 && lat <= 72 && lon >= -170 && lon <= -50) return 'North America'; + if (lat >= -60 && lat < 15 && lon >= -90 && lon <= -30) return 'South America'; + if (lat >= 35 && lat <= 72 && lon >= -25 && lon <= 45) return 'Europe'; + if (lat >= -35 && lat <= 37 && lon >= -25 && lon <= 55) return 'Africa'; + if (lat >= 0 && lat <= 72 && lon >= 45 && lon <= 180) return 'Asia'; + if (lat >= -50 && lat <= 0 && lon >= 95 && lon <= 180) return 'Oceania'; + if (lat >= 35 && lat < 45 && lon >= 25 && lon <= 45) return 'Middle East'; + return 'Other'; +} + +// Classify the frequency range of a receiver +function classifyFrequency(rx) { + // KiwiSDR receivers typically cover 0-30 MHz + // Some entries have frequency info in various fields + const maxFreq = parseFloat(rx.max_freq ?? rx.sdr_hu?.max_freq ?? 30); + const minFreq = parseFloat(rx.min_freq ?? rx.sdr_hu?.min_freq ?? 0); + return { minFreq, maxFreq }; +} + +// Normalize receiver data (already flat from getAllReceivers) +function normalizeReceiver(rx, idx) { + return { + name: (rx.name || `Receiver-${idx}`).slice(0, 100), + location: (rx.location || '').slice(0, 80), + lat: parseFloat(rx.lat) || NaN, + lon: parseFloat(rx.lon) || NaN, + users: parseInt(rx.users ?? 0, 10), + usersMax: parseInt(rx.usersMax ?? 0, 10), + antenna: (rx.antenna || '').slice(0, 80), + bands: (rx.bands || '').slice(0, 60), + offline: rx.offline === true, + snr: parseFloat(rx.snr ?? NaN), + tdoa: rx.tdoa ?? null, + country: rx.country || '', + }; +} + +// Briefing — analyze the global KiwiSDR network +export async function briefing() { + const raw = await getAllReceivers(); + + // Handle errors + if (raw?.error) { + return { + source: 'KiwiSDR', + timestamp: new Date().toISOString(), + status: 'error', + message: raw.error, + }; + } + + // The API may return an array directly or an object with a receivers list + let rxList; + if (Array.isArray(raw)) { + rxList = raw; + } else if (raw && typeof raw === 'object') { + // Try common keys + rxList = raw.receivers || raw.rx || raw.sdrs || raw.data || Object.values(raw); + // If the object values are receiver objects, flatten + if (!Array.isArray(rxList)) { + rxList = Object.values(raw).filter(v => v && typeof v === 'object' && !Array.isArray(v)); + } + } else { + return { + source: 'KiwiSDR', + timestamp: new Date().toISOString(), + status: 'error', + message: 'Unexpected data format from KiwiSDR API', + }; + } + + // Normalize all receivers + const allRx = rxList.map((rx, i) => normalizeReceiver(rx, i)); + const onlineRx = allRx.filter(r => !r.offline); + const offlineRx = allRx.filter(r => r.offline); + + // --- Geographic distribution by country --- + const byCountry = {}; + for (const rx of onlineRx) { + const c = rx.country || 'Unknown'; + byCountry[c] = (byCountry[c] || 0) + 1; + } + // Sort by count descending, take top 20 + const topCountries = Object.entries(byCountry) + .sort((a, b) => b[1] - a[1]) + .slice(0, 20) + .map(([country, count]) => ({ country, count })); + + // --- Continental distribution --- + const byContinent = {}; + for (const rx of onlineRx) { + const continent = getContinent(rx.lat, rx.lon); + byContinent[continent] = (byContinent[continent] || 0) + 1; + } + + // --- Receivers in regions of interest --- + const conflictZoneReceivers = {}; + for (const [key, box] of Object.entries(REGIONS_OF_INTEREST)) { + const rxInRegion = onlineRx.filter(rx => inBounds(rx, box)); + conflictZoneReceivers[key] = { + region: box.label, + count: rxInRegion.length, + receivers: rxInRegion.slice(0, 10).map(rx => ({ + name: rx.name, + location: rx.location, + lat: rx.lat, + lon: rx.lon, + users: rx.users, + antenna: rx.antenna, + country: rx.country, + })), + }; + } + + // --- Activity analysis (users connected) --- + const activeRx = onlineRx + .filter(r => r.users > 0) + .sort((a, b) => b.users - a.users); + + const totalUsers = onlineRx.reduce((sum, r) => sum + r.users, 0); + const totalCapacity = onlineRx.reduce((sum, r) => sum + r.usersMax, 0); + + const topActive = activeRx.slice(0, 15).map(rx => ({ + name: rx.name, + location: rx.location, + country: rx.country, + users: rx.users, + usersMax: rx.usersMax, + lat: rx.lat, + lon: rx.lon, + antenna: rx.antenna, + })); + + // --- TDOA-capable receivers (direction finding / geolocation) --- + const tdoaCapable = onlineRx.filter(r => r.tdoa !== null && r.tdoa > 0); + + // --- Antenna analysis (identify unusual/specialized setups) --- + const antennaTypes = {}; + for (const rx of onlineRx) { + if (rx.antenna) { + const key = rx.antenna.toLowerCase().trim(); + antennaTypes[key] = (antennaTypes[key] || 0) + 1; + } + } + + // --- Utilization metrics --- + const utilizationPct = totalCapacity > 0 + ? ((totalUsers / totalCapacity) * 100).toFixed(1) + : '0.0'; + + const highUtilization = onlineRx + .filter(r => r.usersMax > 0 && (r.users / r.usersMax) >= 0.8) + .map(rx => ({ + name: rx.name, + location: rx.location, + country: rx.country, + users: rx.users, + usersMax: rx.usersMax, + })); + + // --- Generate signals --- + const signals = []; + + // High user count (unusual listening activity) + if (totalUsers > onlineRx.length * 0.5) { + signals.push(`HIGH LISTENER ACTIVITY: ${totalUsers} total users across ${onlineRx.length} receivers (${utilizationPct}% utilization)`); + } + + // Conflict zone coverage + for (const [key, info] of Object.entries(conflictZoneReceivers)) { + if (info.count > 0) { + const activeInZone = info.receivers.filter(r => r.users > 0); + if (activeInZone.length > 0) { + signals.push(`ACTIVE LISTENING in ${info.region}: ${activeInZone.length}/${info.count} receivers have users connected`); + } + } + } + + // High utilization receivers + if (highUtilization.length > 5) { + signals.push(`${highUtilization.length} receivers at >80% capacity — elevated HF monitoring demand`); + } + + return { + source: 'KiwiSDR', + timestamp: new Date().toISOString(), + status: 'active', + network: { + totalReceivers: allRx.length, + online: onlineRx.length, + offline: offlineRx.length, + totalUsers, + totalCapacity, + utilizationPct: parseFloat(utilizationPct), + tdoaCapable: tdoaCapable.length, + }, + geographic: { + byContinent, + topCountries, + }, + conflictZones: conflictZoneReceivers, + topActive, + highUtilization: highUtilization.slice(0, 10), + signals, + }; +} + +if (process.argv[1]?.endsWith('kiwisdr.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/noaa.mjs b/apis/sources/noaa.mjs new file mode 100644 index 0000000..8762ad1 --- /dev/null +++ b/apis/sources/noaa.mjs @@ -0,0 +1,75 @@ +// NOAA / National Weather Service — Severe weather alerts & climate events +// No auth required. Real-time alerts. + +import { safeFetch } from '../utils/fetch.mjs'; + +const NWS_BASE = 'https://api.weather.gov'; + +// Get all active weather alerts (US) +export async function getActiveAlerts(opts = {}) { + const { + severity = null, // Extreme, Severe, Moderate, Minor + urgency = null, // Immediate, Expected, Future + event = null, // e.g. "Tornado Warning", "Hurricane Warning" + limit = 50, + } = opts; + + const params = new URLSearchParams({ limit: String(limit), status: 'actual' }); + if (severity) params.set('severity', severity); + if (urgency) params.set('urgency', urgency); + if (event) params.set('event', event); + + return safeFetch(`${NWS_BASE}/alerts/active?${params}`, { + headers: { 'Accept': 'application/geo+json' }, + }); +} + +// Get severe alerts only +export async function getSevereAlerts() { + return getActiveAlerts({ severity: 'Extreme,Severe' }); +} + +// Briefing — severe weather events that could impact markets/supply chains +export async function briefing() { + const alerts = await getSevereAlerts(); + const features = alerts?.features || []; + + // Categorize by impact type + const hurricanes = features.filter(f => /hurricane|typhoon|tropical/i.test(f.properties?.event)); + const tornadoes = features.filter(f => /tornado/i.test(f.properties?.event)); + const floods = features.filter(f => /flood/i.test(f.properties?.event)); + const winter = features.filter(f => /blizzard|ice storm|winter/i.test(f.properties?.event)); + const fire = features.filter(f => /fire/i.test(f.properties?.event)); + const other = features.filter(f => { + const e = f.properties?.event || ''; + return !/hurricane|typhoon|tropical|tornado|flood|blizzard|ice storm|winter|fire/i.test(e); + }); + + return { + source: 'NOAA/NWS', + timestamp: new Date().toISOString(), + totalSevereAlerts: features.length, + summary: { + hurricanes: hurricanes.length, + tornadoes: tornadoes.length, + floods: floods.length, + winterStorms: winter.length, + wildfires: fire.length, + other: other.length, + }, + topAlerts: features.slice(0, 15).map(f => ({ + event: f.properties?.event, + severity: f.properties?.severity, + urgency: f.properties?.urgency, + headline: f.properties?.headline, + areas: f.properties?.areaDesc, + onset: f.properties?.onset, + expires: f.properties?.expires, + })), + }; +} + +if (process.argv[1]?.endsWith('noaa.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/ofac.mjs b/apis/sources/ofac.mjs new file mode 100644 index 0000000..38e3742 --- /dev/null +++ b/apis/sources/ofac.mjs @@ -0,0 +1,143 @@ +// OFAC — US Treasury Office of Foreign Assets Control Sanctions +// No auth required. Monitors the Specially Designated Nationals (SDN) list +// and consolidated sanctions list for changes. + +import { safeFetch } from '../utils/fetch.mjs'; + +const EXPORTS_BASE = 'https://sanctionslistservice.ofac.treas.gov/api/PublicationPreview/exports'; + +// SDN list endpoints +const SDN_XML_URL = `${EXPORTS_BASE}/SDN.XML`; +const SDN_ADVANCED_URL = `${EXPORTS_BASE}/SDN_ADVANCED.XML`; +const CONS_ADVANCED_URL = `${EXPORTS_BASE}/CONS_ADVANCED.XML`; + +// Parse basic info from SDN XML (publish date, entry count) +function parseSDNMetadata(xml) { + if (!xml || xml.error) return { error: xml?.error || 'No data returned' }; + + const raw = xml.rawText || ''; + + // Extract publish date + const publishDate = raw.match(/(.*?)<\/Publish_Date>/)?.[1] + || raw.match(/(.*?)<\/publish_date>/i)?.[1] + || null; + + // Count SDN entries + const entryMatches = raw.match(//gi); + const entryCount = entryMatches ? entryMatches.length : null; + + // Extract record count if present + const recordCount = raw.match(/(.*?)<\/Record_Count>/)?.[1] + || raw.match(/(.*?)<\/records_count>/i)?.[1] + || null; + + return { + publishDate, + entryCount, + recordCount: recordCount ? parseInt(recordCount, 10) : null, + hasData: raw.length > 0, + dataSize: raw.length, + }; +} + +// Fetch SDN list metadata (smaller initial chunk via timeout) +export async function getSDNMetadata() { + // The full SDN XML is large; safeFetch will get the first 500 chars + // which should include the header/publish date + const data = await safeFetch(SDN_XML_URL, { timeout: 20000 }); + return parseSDNMetadata(data); +} + +// Fetch advanced SDN data (includes more structured info) +export async function getSDNAdvanced() { + const data = await safeFetch(SDN_ADVANCED_URL, { timeout: 20000 }); + return parseSDNMetadata(data); +} + +// Fetch consolidated list metadata +export async function getConsolidatedMetadata() { + const data = await safeFetch(CONS_ADVANCED_URL, { timeout: 20000 }); + return parseSDNMetadata(data); +} + +// Parse recent SDN entries from XML snippet +function parseRecentEntries(xml) { + if (!xml || xml.error) return []; + + const raw = xml.rawText || ''; + const entries = []; + const entryRegex = /([\s\S]*?)<\/sdnEntry>/gi; + let match; + let count = 0; + + while ((match = entryRegex.exec(raw)) !== null && count < 20) { + const content = match[1]; + const uid = content.match(/(.*?)<\/uid>/i)?.[1]; + const lastName = content.match(/(.*?)<\/lastName>/i)?.[1]; + const firstName = content.match(/(.*?)<\/firstName>/i)?.[1]; + const sdnType = content.match(/(.*?)<\/sdnType>/i)?.[1]; + + // Extract programs + const programs = []; + const progRegex = /(.*?)<\/program>/gi; + let progMatch; + while ((progMatch = progRegex.exec(content)) !== null) { + programs.push(progMatch[1]); + } + + if (uid || lastName) { + entries.push({ + uid, + name: [firstName, lastName].filter(Boolean).join(' '), + type: sdnType, + programs, + }); + count++; + } + } + + return entries; +} + +// Briefing — report on sanctions list status and metadata +export async function briefing() { + const [sdnMeta, advancedMeta] = await Promise.all([ + getSDNMetadata(), + getSDNAdvanced(), + ]); + + // Try to extract any entries visible in the advanced data + const sampleEntries = parseRecentEntries( + await safeFetch(SDN_ADVANCED_URL, { timeout: 25000 }) + ); + + return { + source: 'OFAC Sanctions', + timestamp: new Date().toISOString(), + lastUpdated: sdnMeta.publishDate || advancedMeta.publishDate || 'unknown', + sdnList: { + publishDate: sdnMeta.publishDate, + entryCount: sdnMeta.entryCount, + recordCount: sdnMeta.recordCount, + dataAvailable: sdnMeta.hasData, + }, + advancedList: { + publishDate: advancedMeta.publishDate, + entryCount: advancedMeta.entryCount, + recordCount: advancedMeta.recordCount, + dataAvailable: advancedMeta.hasData, + }, + sampleEntries: sampleEntries.slice(0, 10), + endpoints: { + sdnXml: SDN_XML_URL, + sdnAdvanced: SDN_ADVANCED_URL, + consolidatedAdvanced: CONS_ADVANCED_URL, + }, + }; +} + +// Run standalone +if (process.argv[1]?.endsWith('ofac.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/opensanctions.mjs b/apis/sources/opensanctions.mjs new file mode 100644 index 0000000..035463b --- /dev/null +++ b/apis/sources/opensanctions.mjs @@ -0,0 +1,112 @@ +// OpenSanctions — Global Sanctions & PEP Aggregator +// No auth required for basic queries. Aggregates sanctions data from +// OFAC, EU, UN, and 30+ other sources into a unified searchable dataset. + +import { safeFetch } from '../utils/fetch.mjs'; + +const BASE = 'https://api.opensanctions.org'; + +// Search sanctioned entities by name/keyword +export async function searchEntities(query, opts = {}) { + const { limit = 20, schema, topics } = opts; + + const params = new URLSearchParams({ + q: query, + limit: String(limit), + }); + if (schema) params.set('schema', schema); // e.g. "Person", "Company", "Organization" + if (topics) params.set('topics', topics); // e.g. "sanction", "crime", "poi" + + return safeFetch(`${BASE}/search/default?${params}`, { timeout: 15000 }); +} + +// Get available datasets/collections +export async function getCollections() { + return safeFetch(`${BASE}/collections`, { timeout: 15000 }); +} + +// Get details about a specific dataset +export async function getDataset(name) { + return safeFetch(`${BASE}/datasets/${name}`, { timeout: 15000 }); +} + +// Get a specific entity by ID +export async function getEntity(entityId) { + return safeFetch(`${BASE}/entities/${entityId}`, { timeout: 15000 }); +} + +// Compact entity for briefing output +function compactEntity(e) { + return { + id: e.id, + name: e.caption || e.name, + schema: e.schema, + datasets: e.datasets, + topics: e.topics, + countries: e.properties?.country || [], + lastSeen: e.last_seen, + firstSeen: e.first_seen, + }; +} + +// Compact search results +function compactSearchResult(result, query) { + const entities = (result?.results || []).map(compactEntity); + return { + query, + totalResults: result?.total || 0, + entities: entities.slice(0, 10), + }; +} + +// Key entities/subjects to monitor for sanctions intelligence +const BRIEFING_QUERIES = [ + 'Iran', + 'Russia', + 'North Korea', + 'Syria', + 'Venezuela', + 'Wagner', +]; + +// Briefing — search for notable sanctioned entities across key targets +export async function briefing() { + // Run searches in parallel + const results = await Promise.all( + BRIEFING_QUERIES.map(async (query) => { + const data = await searchEntities(query, { limit: 10, topics: 'sanction' }); + return compactSearchResult(data, query); + }) + ); + + // Also fetch dataset metadata for context + const collections = await getCollections(); + const datasetSummary = Array.isArray(collections) + ? collections.slice(0, 10).map(c => ({ + name: c.name, + title: c.title, + entityCount: c.entity_count, + lastUpdated: c.updated_at, + })) + : []; + + // Aggregate totals + const totalSanctionedEntities = results.reduce( + (sum, r) => sum + (r.totalResults || 0), 0 + ); + + return { + source: 'OpenSanctions', + timestamp: new Date().toISOString(), + recentSearches: results, + totalSanctionedEntities, + datasets: datasetSummary, + monitoringTargets: BRIEFING_QUERIES, + }; +} + +// Run standalone +if (process.argv[1]?.endsWith('opensanctions.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/opensky.mjs b/apis/sources/opensky.mjs new file mode 100644 index 0000000..5ec0712 --- /dev/null +++ b/apis/sources/opensky.mjs @@ -0,0 +1,96 @@ +// OpenSky Network — Real-time flight tracking +// Free for research. 4,000 API credits/day (no auth), 8,000 with account. +// Tracks all aircraft with ADS-B transponders including many military. + +import { safeFetch } from '../utils/fetch.mjs'; + +const BASE = 'https://opensky-network.org/api'; + +// Get all current flights (global state vector) +export async function getAllFlights() { + return safeFetch(`${BASE}/states/all`, { timeout: 30000 }); +} + +// Get flights in a bounding box (lat/lon) +export async function getFlightsInArea(lamin, lomin, lamax, lomax) { + const params = new URLSearchParams({ + lamin: String(lamin), + lomin: String(lomin), + lamax: String(lamax), + lomax: String(lomax), + }); + return safeFetch(`${BASE}/states/all?${params}`, { timeout: 20000 }); +} + +// Get flights by specific aircraft (ICAO24 hex codes) +export async function getFlightsByIcao(icao24List) { + const icao = Array.isArray(icao24List) ? icao24List : [icao24List]; + const params = icao.map(i => `icao24=${i}`).join('&'); + return safeFetch(`${BASE}/states/all?${params}`, { timeout: 20000 }); +} + +// Get departures from an airport in a time range +export async function getDepartures(airportIcao, begin, end) { + const params = new URLSearchParams({ + airport: airportIcao, + begin: String(Math.floor(begin / 1000)), + end: String(Math.floor(end / 1000)), + }); + return safeFetch(`${BASE}/flights/departure?${params}`); +} + +// Get arrivals at an airport +export async function getArrivals(airportIcao, begin, end) { + const params = new URLSearchParams({ + airport: airportIcao, + begin: String(Math.floor(begin / 1000)), + end: String(Math.floor(end / 1000)), + }); + return safeFetch(`${BASE}/flights/arrival?${params}`); +} + +// Key hotspot regions for monitoring +const HOTSPOTS = { + middleEast: { lamin: 12, lomin: 30, lamax: 42, lomax: 65, label: 'Middle East' }, + taiwan: { lamin: 20, lomin: 115, lamax: 28, lomax: 125, label: 'Taiwan Strait' }, + ukraine: { lamin: 44, lomin: 22, lamax: 53, lomax: 41, label: 'Ukraine Region' }, + baltics: { lamin: 53, lomin: 19, lamax: 60, lomax: 29, label: 'Baltic Region' }, + southChinaSea: { lamin: 5, lomin: 105, lamax: 23, lomax: 122, label: 'South China Sea' }, + koreanPeninsula: { lamin: 33, lomin: 124, lamax: 43, lomax: 132, label: 'Korean Peninsula' }, +}; + +// Briefing — check hotspot regions for flight activity +export async function briefing() { + const hotspotEntries = Object.entries(HOTSPOTS); + const results = await Promise.all( + hotspotEntries.map(async ([key, box]) => { + const data = await getFlightsInArea(box.lamin, box.lomin, box.lamax, box.lomax); + const states = data?.states || []; + return { + region: box.label, + key, + totalAircraft: states.length, + // states format: [icao24, callsign, origin_country, ...] + byCountry: states.reduce((acc, s) => { + const country = s[2] || 'Unknown'; + acc[country] = (acc[country] || 0) + 1; + return acc; + }, {}), + // Flag potentially interesting (military often have no callsign or specific patterns) + noCallsign: states.filter(s => !s[1]?.trim()).length, + highAltitude: states.filter(s => s[7] && s[7] > 12000).length, // >12km altitude + }; + }) + ); + + return { + source: 'OpenSky', + timestamp: new Date().toISOString(), + hotspots: results, + }; +} + +if (process.argv[1]?.endsWith('opensky.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/patents.mjs b/apis/sources/patents.mjs new file mode 100644 index 0000000..565eebf --- /dev/null +++ b/apis/sources/patents.mjs @@ -0,0 +1,205 @@ +// USPTO PatentsView — Patent Intelligence +// No auth required. Tracks patent filings in strategic technology areas. +// API v1: https://search.patentsview.org/api/v1/patent/ +// Useful for detecting R&D trends, tech competition, state-backed innovation. + +import { safeFetch, daysAgo } from '../utils/fetch.mjs'; + +const BASE = 'https://search.patentsview.org/api/v1'; + +// Strategic technology domains and their search terms +const STRATEGIC_DOMAINS = { + ai: { + label: 'Artificial Intelligence', + terms: ['artificial intelligence', 'machine learning', 'deep learning', 'neural network', 'large language model'], + }, + quantum: { + label: 'Quantum Computing', + terms: ['quantum computing', 'quantum processor', 'qubit', 'quantum entanglement', 'quantum cryptography'], + }, + nuclear: { + label: 'Nuclear Technology', + terms: ['nuclear fusion', 'nuclear reactor', 'nuclear fuel', 'uranium enrichment', 'small modular reactor'], + }, + hypersonic: { + label: 'Hypersonic & Advanced Propulsion', + terms: ['hypersonic', 'scramjet', 'directed energy weapon', 'railgun', 'advanced propulsion'], + }, + semiconductor: { + label: 'Semiconductor & Chip Technology', + terms: ['semiconductor', 'integrated circuit', 'lithography', 'chip fabrication', 'transistor'], + }, + biotech: { + label: 'Biotechnology & Synthetic Biology', + terms: ['synthetic biology', 'gene editing', 'CRISPR', 'mRNA', 'bioweapon'], + }, + space: { + label: 'Space & Satellite Technology', + terms: ['satellite', 'space launch', 'orbital', 'space debris', 'anti-satellite'], + }, +}; + +// Search patents by keyword query +export async function searchPatents(query, opts = {}) { + const { + since = daysAgo(90), + limit = 10, + sort = 'patent_date', + sortDir = 'desc', + } = opts; + + // PatentsView v1 API uses query params with JSON values + const q = JSON.stringify({ + _and: [ + { _gte: { patent_date: since } }, + { _text_any: { patent_abstract: query } }, + ], + }); + + const f = JSON.stringify([ + 'patent_id', + 'patent_title', + 'patent_date', + 'patent_abstract', + 'assignee_organization', + 'patent_type', + ]); + + const o = JSON.stringify({ [sort]: sortDir }); + + const params = new URLSearchParams({ + q, + f, + o, + s: String(limit), + }); + + return safeFetch(`${BASE}/patent/?${params}`, { timeout: 20000 }); +} + +// Search by assignee organization +export async function searchByAssignee(orgName, opts = {}) { + const { since = daysAgo(180), limit = 10 } = opts; + + const q = JSON.stringify({ + _and: [ + { _gte: { patent_date: since } }, + { _contains: { assignee_organization: orgName } }, + ], + }); + + const f = JSON.stringify([ + 'patent_id', + 'patent_title', + 'patent_date', + 'patent_abstract', + 'assignee_organization', + ]); + + const o = JSON.stringify({ patent_date: 'desc' }); + + const params = new URLSearchParams({ + q, + f, + o, + s: String(limit), + }); + + return safeFetch(`${BASE}/patent/?${params}`, { timeout: 20000 }); +} + +// Compact patent record for briefing output +function compactPatent(p) { + return { + id: p.patent_id, + title: p.patent_title, + date: p.patent_date, + assignee: p.assignee_organization || 'Unknown', + type: p.patent_type, + }; +} + +// Search a single domain, combining its keyword terms +async function searchDomain(domain, since) { + const terms = domain.terms.join(' '); + const data = await searchPatents(terms, { since, limit: 10 }); + + // PatentsView v1 returns { patents: [...] } or similar + const patents = data?.patents || data?.results || []; + if (!Array.isArray(patents)) return []; + return patents.map(compactPatent); +} + +// Briefing — search recent patents in key strategic tech areas +export async function briefing() { + const since = daysAgo(90); + const domainEntries = Object.entries(STRATEGIC_DOMAINS); + const recentPatents = {}; + const signals = []; + + // Run all domain searches in parallel + const results = await Promise.all( + domainEntries.map(async ([key, domain]) => { + const patents = await searchDomain(domain, since); + return { key, label: domain.label, patents }; + }) + ); + + let totalFound = 0; + for (const { key, label, patents } of results) { + recentPatents[key] = patents; + totalFound += patents.length; + + if (patents.length > 0) { + // Identify dominant assignees (potential state-backed programs) + const assigneeCounts = {}; + patents.forEach(p => { + if (p.assignee && p.assignee !== 'Unknown') { + assigneeCounts[p.assignee] = (assigneeCounts[p.assignee] || 0) + 1; + } + }); + + // Flag organizations with high patent density in strategic areas + Object.entries(assigneeCounts).forEach(([org, count]) => { + if (count >= 3) { + signals.push(`HIGH ACTIVITY: ${org} filed ${count} ${label} patents in last 90 days`); + } + }); + } + } + + // Track key defense/intelligence organizations specifically + const watchOrgs = [ + 'Raytheon', 'Lockheed Martin', 'Northrop Grumman', 'BAE Systems', + 'China Academy', 'Huawei', 'SMIC', 'Samsung', 'TSMC', + 'US Department', 'Navy', 'Air Force', 'Army', 'DARPA', + ]; + + for (const { patents } of results) { + for (const p of patents) { + if (watchOrgs.some(org => p.assignee?.toLowerCase().includes(org.toLowerCase()))) { + signals.push(`WATCH ORG: "${p.title}" by ${p.assignee} (${p.date})`); + } + } + } + + return { + source: 'USPTO Patents', + timestamp: new Date().toISOString(), + searchWindow: `${since} to ${new Date().toISOString().split('T')[0]}`, + totalFound, + recentPatents, + signals: signals.length > 0 + ? signals + : ['No unusual patent filing patterns detected in strategic domains'], + domains: Object.fromEntries( + domainEntries.map(([key, domain]) => [key, domain.label]) + ), + }; +} + +// Run standalone +if (process.argv[1]?.endsWith('patents.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/reddit.mjs b/apis/sources/reddit.mjs new file mode 100644 index 0000000..29606cf --- /dev/null +++ b/apis/sources/reddit.mjs @@ -0,0 +1,107 @@ +// Reddit — social sentiment intelligence +// Reddit now requires OAuth for API access (public JSON API returns 403). +// Gracefully degrades when not authenticated. +// To enable: register an app at https://www.reddit.com/prefs/apps/ and set +// REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET in .env + +import { safeFetch } from '../utils/fetch.mjs'; +import '../utils/env.mjs'; + +function delay(ms) { return new Promise(r => setTimeout(r, ms)); } + +const SUBREDDITS = [ + 'worldnews', + 'geopolitics', + 'economics', + 'wallstreetbets', + 'commodities', +]; + +// Get OAuth token using client credentials flow (application-only) +async function getToken() { + const clientId = process.env.REDDIT_CLIENT_ID; + const clientSecret = process.env.REDDIT_CLIENT_SECRET; + if (!clientId || !clientSecret) return null; + + try { + const auth = Buffer.from(`${clientId}:${clientSecret}`).toString('base64'); + const res = await fetch('https://www.reddit.com/api/v1/access_token', { + method: 'POST', + headers: { + 'Authorization': `Basic ${auth}`, + 'Content-Type': 'application/x-www-form-urlencoded', + 'User-Agent': 'Crucix/1.0 intelligence-engine', + }, + body: 'grant_type=client_credentials', + }); + if (!res.ok) return null; + const data = await res.json(); + return data.access_token || null; + } catch { + return null; + } +} + +// Fetch hot posts — tries OAuth first, then falls back to public endpoint +export async function getHot(subreddit, opts = {}) { + const { limit = 10, token = null } = opts; + + if (token) { + // Use OAuth endpoint + return safeFetch(`https://oauth.reddit.com/r/${subreddit}/hot?limit=${limit}&raw_json=1`, { + headers: { + 'Authorization': `Bearer ${token}`, + 'User-Agent': 'Crucix/1.0 intelligence-engine', + }, + }); + } + + // Try public endpoint (may 403) + return safeFetch(`https://www.reddit.com/r/${subreddit}/hot.json?limit=${limit}&raw_json=1`, { + headers: { 'User-Agent': 'Crucix/1.0 intelligence-engine' }, + }); +} + +function compactPost(child) { + const d = child?.data; + if (!d) return null; + return { + title: d.title, + score: d.score ?? 0, + comments: d.num_comments ?? 0, + url: d.url, + created: d.created_utc ? new Date(d.created_utc * 1000).toISOString() : null, + }; +} + +export async function briefing() { + const token = await getToken(); + + if (!token && !process.env.REDDIT_CLIENT_ID) { + return { + source: 'Reddit', + timestamp: new Date().toISOString(), + status: 'no_key', + message: 'Reddit requires OAuth. Register at https://www.reddit.com/prefs/apps/ (script type), set REDDIT_CLIENT_ID and REDDIT_CLIENT_SECRET in .env', + }; + } + + const subredditResults = {}; + for (const sub of SUBREDDITS) { + const result = await getHot(sub, { limit: 10, token }); + const children = result?.data?.children || []; + subredditResults[sub] = children.map(compactPost).filter(Boolean); + await delay(token ? 1000 : 2000); + } + + return { + source: 'Reddit', + timestamp: new Date().toISOString(), + subreddits: subredditResults, + }; +} + +if (process.argv[1]?.endsWith('reddit.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/reliefweb.mjs b/apis/sources/reliefweb.mjs new file mode 100644 index 0000000..c86172d --- /dev/null +++ b/apis/sources/reliefweb.mjs @@ -0,0 +1,152 @@ +// ReliefWeb — UN OCHA humanitarian crisis tracking +// Requires approved appname since Nov 2025. Register at https://apidoc.reliefweb.int/parameters#appname +// Falls back to HDX (Humanitarian Data Exchange) if ReliefWeb API returns 403. + +import { safeFetch } from '../utils/fetch.mjs'; + +const BASE = 'https://api.reliefweb.int/v1'; +// Register your own appname at https://apidoc.reliefweb.int/parameters#appname +// and replace this value. Without an approved appname the API returns 403. +const APPNAME = process.env.RELIEFWEB_APPNAME || 'crucix'; + +const HDX_BASE = 'https://data.humdata.org/api/3/action'; + +// POST-based search for reports (ReliefWeb API v1 POST format) +async function rwPost(endpoint, body) { + const url = `${BASE}/${endpoint}?appname=${APPNAME}`; + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 15000); + const res = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'User-Agent': 'Crucix/1.0', + }, + body: JSON.stringify(body), + signal: controller.signal, + }); + clearTimeout(timer); + if (!res.ok) { + const errBody = await res.text().catch(() => ''); + throw new Error(`HTTP ${res.status}: ${errBody.slice(0, 200)}`); + } + return await res.json(); + } catch (e) { + return { error: e.message, source: url }; + } +} + +// Search recent reports via ReliefWeb API (POST method) +export async function searchReports(opts = {}) { + const { query = '', limit = 15 } = opts; + const body = { + limit, + fields: { + include: [ + 'title', + 'date.created', + 'country.name', + 'disaster_type.name', + 'url_alias', + 'source.name', + ], + }, + sort: ['date.created:desc'], + }; + if (query) { + body.query = { value: query }; + } + return rwPost('reports', body); +} + +// Get active disasters via ReliefWeb API (POST method) +export async function getDisasters(opts = {}) { + const { limit = 15 } = opts; + const body = { + limit, + fields: { + include: ['name', 'date.created', 'country.name', 'type.name', 'status'], + }, + filter: { + field: 'status', + value: 'ongoing', + }, + sort: ['date.created:desc'], + }; + return rwPost('disasters', body); +} + +// Fallback: search HDX (Humanitarian Data Exchange) for crisis datasets +async function hdxFallback(limit = 15) { + const data = await safeFetch( + `${HDX_BASE}/package_search?q=crisis+OR+disaster+OR+emergency&rows=${limit}&sort=metadata_modified+desc` + ); + if (data?.result?.results) { + return data.result.results.map(pkg => ({ + title: pkg.title, + date: pkg.metadata_modified, + source: pkg.dataset_source || pkg.organization?.title, + countries: pkg.groups?.map(g => g.display_name), + url: `https://data.humdata.org/dataset/${pkg.name}`, + })); + } + return []; +} + +// Briefing — get latest humanitarian crises +export async function briefing() { + const [reports, disasters] = await Promise.all([ + searchReports({ limit: 15 }), + getDisasters({ limit: 15 }), + ]); + + const rwFailed = !!reports?.error || !!disasters?.error; + + let latestReports = []; + let activeDisasters = []; + let hdxDatasets = []; + + if (!rwFailed) { + latestReports = (reports?.data || []).map(r => ({ + title: r.fields?.title, + date: r.fields?.date?.created, + countries: r.fields?.country?.map(c => c.name), + disasterType: r.fields?.disaster_type?.map(d => d.name), + source: r.fields?.source?.map(s => s.name), + url: r.fields?.url_alias + ? `https://reliefweb.int${r.fields.url_alias}` + : null, + })); + activeDisasters = (disasters?.data || []).map(d => ({ + name: d.fields?.name, + date: d.fields?.date?.created, + countries: d.fields?.country?.map(c => c.name), + type: d.fields?.type?.map(t => t.name), + status: d.fields?.status, + })); + } else { + // Fallback to HDX when ReliefWeb returns 403 (unapproved appname) + hdxDatasets = await hdxFallback(15); + } + + return { + source: rwFailed ? 'HDX (Humanitarian Data Exchange) — ReliefWeb fallback' : 'ReliefWeb (UN OCHA)', + timestamp: new Date().toISOString(), + ...(rwFailed + ? { + rwError: reports?.error || disasters?.error, + rwNote: 'ReliefWeb API requires an approved appname since Nov 2025. Set RELIEFWEB_APPNAME env var after registering at https://apidoc.reliefweb.int/parameters#appname', + hdxDatasets, + } + : { + latestReports, + activeDisasters, + }), + }; +} + +if (process.argv[1]?.endsWith('reliefweb.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/safecast.mjs b/apis/sources/safecast.mjs new file mode 100644 index 0000000..f925fd7 --- /dev/null +++ b/apis/sources/safecast.mjs @@ -0,0 +1,82 @@ +// Safecast — Global radiation monitoring (150M+ readings) +// No auth required. CC0 public domain. Citizen-science network. + +import { safeFetch } from '../utils/fetch.mjs'; + +const BASE = 'https://api.safecast.org'; + +// Get recent measurements in an area +export async function getMeasurements(opts = {}) { + const { + latitude = null, + longitude = null, + distance = 100, // km + limit = 50, + since = null, + } = opts; + + const params = new URLSearchParams({ limit: String(limit) }); + if (latitude && longitude) { + params.set('latitude', String(latitude)); + params.set('longitude', String(longitude)); + params.set('distance', String(distance * 1000)); // meters + } + if (since) params.set('since', since); + + return safeFetch(`${BASE}/measurements.json?${params}`); +} + +// Key nuclear sites to monitor +const NUCLEAR_SITES = { + zaporizhzhia: { lat: 47.51, lon: 34.58, label: 'Zaporizhzhia NPP (Ukraine)', radius: 100 }, + chernobyl: { lat: 51.39, lon: 30.1, label: 'Chernobyl Exclusion Zone', radius: 50 }, + bushehr: { lat: 28.83, lon: 50.89, label: 'Bushehr NPP (Iran)', radius: 100 }, + yongbyon: { lat: 39.8, lon: 125.75, label: 'Yongbyon (North Korea)', radius: 100 }, + fukushima: { lat: 37.42, lon: 141.03, label: 'Fukushima Daiichi', radius: 50 }, + dimona: { lat: 31.0, lon: 35.15, label: 'Dimona (Israel)', radius: 100 }, +}; + +// Briefing — check radiation levels near key nuclear sites +export async function briefing() { + const results = await Promise.all( + Object.entries(NUCLEAR_SITES).map(async ([key, site]) => { + const data = await getMeasurements({ + latitude: site.lat, + longitude: site.lon, + distance: site.radius, + limit: 10, + }); + + const measurements = Array.isArray(data) ? data : []; + const values = measurements.map(m => m.value).filter(v => typeof v === 'number'); + const avgCPM = values.length > 0 ? values.reduce((a, b) => a + b, 0) / values.length : null; + + return { + site: site.label, + key, + recentReadings: values.length, + avgCPM, + maxCPM: values.length > 0 ? Math.max(...values) : null, + // Normal background: 10-80 CPM. >100 CPM warrants attention. + anomaly: avgCPM !== null && avgCPM > 100, + lastReading: measurements[0]?.captured_at || null, + }; + }) + ); + + const anomalies = results.filter(r => r.anomaly); + + return { + source: 'Safecast', + timestamp: new Date().toISOString(), + sites: results, + signals: anomalies.length > 0 + ? anomalies.map(a => `ELEVATED RADIATION at ${a.site}: ${a.avgCPM?.toFixed(1)} CPM (normal: 10-80)`) + : ['All monitored nuclear sites within normal radiation levels'], + }; +} + +if (process.argv[1]?.endsWith('safecast.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/ships.mjs b/apis/sources/ships.mjs new file mode 100644 index 0000000..97ac2ba --- /dev/null +++ b/apis/sources/ships.mjs @@ -0,0 +1,66 @@ +// Ship/Vessel Tracking — aisstream.io (free real-time global AIS) +// Also includes fallback to public vessel tracking data +// Detects: dark ships, sanctions evasion, naval deployments, port congestion + +import { safeFetch } from '../utils/fetch.mjs'; + +// aisstream.io requires a WebSocket connection for real-time data +// For briefing mode, we'll use snapshot-based approaches + +// MarineTraffic-style density estimation via public endpoints +// The real power comes from running a persistent WebSocket listener + +// Key maritime chokepoints to monitor +const CHOKEPOINTS = { + straitOfHormuz: { label: 'Strait of Hormuz', lat: 26.5, lon: 56.5, note: '20% of world oil' }, + suezCanal: { label: 'Suez Canal', lat: 30.5, lon: 32.3, note: '12% of world trade' }, + straitOfMalacca: { label: 'Strait of Malacca', lat: 2.5, lon: 101.5, note: '25% of world trade' }, + babElMandeb: { label: 'Bab el-Mandeb', lat: 12.6, lon: 43.3, note: 'Red Sea gateway' }, + taiwanStrait: { label: 'Taiwan Strait', lat: 24.0, lon: 119.0, note: '88% of largest container ships' }, + bosporusStrait: { label: 'Bosphorus', lat: 41.1, lon: 29.1, note: 'Black Sea access' }, + panamaCanal: { label: 'Panama Canal', lat: 9.1, lon: -79.7, note: '5% of world trade' }, + capeOfGoodHope: { label: 'Cape of Good Hope', lat: -34.4, lon: 18.5, note: 'Suez alternative' }, +}; + +// For non-realtime briefing, use web-searchable vessel data +export async function briefing() { + const hasKey = !!process.env.AISSTREAM_API_KEY; + + return { + source: 'Maritime/AIS', + timestamp: new Date().toISOString(), + status: hasKey ? 'ready' : 'limited', + message: hasKey + ? 'AIS stream connected — use WebSocket listener for real-time data' + : 'Set AISSTREAM_API_KEY for real-time global vessel tracking (free at aisstream.io)', + chokepoints: CHOKEPOINTS, + monitoringCapabilities: [ + 'Dark ship detection (AIS transponder shutoffs)', + 'Sanctions evasion (ship-to-ship transfers)', + 'Naval deployment tracking', + 'Port congestion (vessel dwell time)', + 'Chokepoint traffic anomalies', + 'Oil tanker route changes', + ], + hint: 'For now, I can use web search to check maritime news and shipping disruptions', + }; +} + +// WebSocket listener setup (for persistent monitoring) +export function getWebSocketConfig(apiKey) { + return { + url: 'wss://stream.aisstream.io/v0/stream', + message: JSON.stringify({ + APIKey: apiKey, + BoundingBoxes: Object.values(CHOKEPOINTS).map(cp => [ + [cp.lat - 2, cp.lon - 2], + [cp.lat + 2, cp.lon + 2], + ]), + }), + }; +} + +if (process.argv[1]?.endsWith('ships.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/telegram.mjs b/apis/sources/telegram.mjs new file mode 100644 index 0000000..2f0f849 --- /dev/null +++ b/apis/sources/telegram.mjs @@ -0,0 +1,336 @@ +// Telegram — public channel intelligence from conflict zones and OSINT analysts +// Primary mode: Bot API with TELEGRAM_BOT_TOKEN (getUpdates, getChat) +// Fallback mode: Scrape public channel web previews at https://t.me/s/{channel} +// Monitors conflict zones (Ukraine, Middle East), geopolitics, and OSINT channels. + +import { safeFetch } from '../utils/fetch.mjs'; +import '../utils/env.mjs'; + +function delay(ms) { return new Promise(r => setTimeout(r, ms)); } + +// Curated list of well-known public OSINT / conflict / geopolitics channels +// All verified to have public web previews enabled at https://t.me/s/{id} +const CHANNELS = [ + { id: 'intelslava', label: 'Intel Slava Z', topic: 'conflict', note: 'Conflict updates, pro-Russian perspective' }, + { id: 'legitimniy', label: 'Legitimniy', topic: 'conflict', note: 'Ukrainian politics & conflict analysis' }, + { id: 'wartranslated', label: 'War Translated', topic: 'conflict', note: 'Conflict translations & OSINT' }, + { id: 'ukraine_frontline', label: 'Ukraine Frontline', topic: 'conflict', note: 'Frontline situation updates' }, + { id: 'middleeastosint', label: 'Middle East OSINT', topic: 'osint', note: 'Middle East open source intel' }, + { id: 'mod_russia', label: 'Russian MoD', topic: 'conflict', note: 'Russian Ministry of Defense official' }, + { id: 'CIG_telegram', label: 'Conflict Intel Team', topic: 'osint', note: 'Conflict Intelligence Team analysis' }, + { id: 'RVvoenkor', label: 'Voenkor RV', topic: 'conflict', note: 'Russian military correspondent' }, + { id: 'readovkanews', label: 'Readovka', topic: 'conflict', note: 'Russian conflict news aggregator' }, + { id: 'DeepStateUA', label: 'DeepState Ukraine', topic: 'conflict', note: 'Ukrainian frontline maps & analysis' }, + { id: 'operativnoZSU', label: 'ZSU Operative', topic: 'conflict', note: 'Ukrainian armed forces updates' }, + { id: 'GeneralStaffZSU', label: 'General Staff ZSU', topic: 'conflict', note: 'Ukrainian General Staff official' }, +]; + +// Urgent keywords that flag high-priority posts +const URGENT_KEYWORDS = [ + 'breaking', 'urgent', 'alert', 'missile', 'strike', 'explosion', + 'nuclear', 'chemical', 'ceasefire', 'escalation', 'invasion', + 'offensive', 'airstrike', 'casualties', 'retreat', 'advance', + 'nato', 'mobilization', 'coup', 'assassination', 'drone', +]; + +// ─── Bot API mode ─────────────────────────────────────────────────────────── + +const botBase = () => `https://api.telegram.org/bot${process.env.TELEGRAM_BOT_TOKEN}`; + +// Get recent updates the bot has received +export async function getUpdates(opts = {}) { + const { limit = 100, offset = 0 } = opts; + const params = new URLSearchParams({ limit: String(limit), offset: String(offset) }); + return safeFetch(`${botBase()}/getUpdates?${params}`); +} + +// Get info about a chat/channel by username +export async function getChat(chatId) { + const params = new URLSearchParams({ chat_id: chatId.startsWith('@') ? chatId : `@${chatId}` }); + return safeFetch(`${botBase()}/getChat?${params}`); +} + +// Compact a Bot API message for briefing output +function compactBotMessage(msg) { + return { + text: (msg.text || msg.caption || '').slice(0, 300), + date: msg.date ? new Date(msg.date * 1000).toISOString() : null, + chat: msg.chat?.title || msg.chat?.username || 'unknown', + views: msg.views || 0, + hasMedia: !!(msg.photo || msg.video || msg.document), + }; +} + +// Fetch updates via Bot API and organize by channel +async function fetchBotUpdates() { + const result = await getUpdates({ limit: 100 }); + if (!result?.ok || !Array.isArray(result.result)) { + return { error: result?.description || 'Bot API request failed' }; + } + + const messages = result.result + .map(u => u.message || u.channel_post || u.edited_channel_post) + .filter(Boolean) + .map(compactBotMessage); + + return { messages, count: messages.length }; +} + +// ─── Web preview scraping fallback ────────────────────────────────────────── + +// Fetch raw HTML from a URL (safeFetch truncates non-JSON to 500 chars, too short) +async function fetchHTML(url, timeoutMs = 15000) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const res = await fetch(url, { + signal: controller.signal, + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept-Language': 'en-US,en;q=0.9', + }, + }); + clearTimeout(timer); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + return await res.text(); + } catch (e) { + clearTimeout(timer); + return null; + } +} + +// Parse messages from Telegram web preview HTML (https://t.me/s/channel) +// The HTML contains
blocks with message content. +function parseWebPreview(html, channelId) { + if (!html) return []; + + const messages = []; + + // Each message sits inside a tgme_widget_message_wrap div + // We extract using the data-post attribute which has the format "channel/msgId" + const msgBlockRegex = /class="tgme_widget_message_wrap[^"]*"[\s\S]*?data-post="([^"]*)"([\s\S]*?)(?=class="tgme_widget_message_wrap|$)/gi; + // Simpler: split on message boundaries using data-post + const postRegex = /data-post="([^"]+)"([\s\S]*?)(?=data-post="|$)/gi; + + let match; + while ((match = postRegex.exec(html)) !== null && messages.length < 20) { + const postId = match[1]; // e.g. "intelslava/12345" + const block = match[2]; + + // Extract message text from tgme_widget_message_text + const textMatch = block.match(/class="tgme_widget_message_text[^"]*"[^>]*>([\s\S]*?)<\/div>/i); + let text = ''; + if (textMatch) { + text = textMatch[1] + .replace(//gi, '\n') // preserve line breaks + .replace(/<[^>]+>/g, '') // strip HTML tags + .replace(/&/g, '&') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/ /g, ' ') + .trim() + .slice(0, 300); + } + + // Extract view count + const viewsMatch = block.match(/class="tgme_widget_message_views"[^>]*>([\s\S]*?)<\/span>/i); + let views = 0; + if (viewsMatch) { + const raw = viewsMatch[1].trim(); + if (raw.endsWith('K')) views = parseFloat(raw) * 1000; + else if (raw.endsWith('M')) views = parseFloat(raw) * 1000000; + else views = parseInt(raw, 10) || 0; + } + + // Extract datetime + const timeMatch = block.match(/datetime="([^"]+)"/i); + const date = timeMatch ? timeMatch[1] : null; + + // Check for media (photos, videos) + const hasMedia = /tgme_widget_message_photo|tgme_widget_message_video/i.test(block); + + if (text || hasMedia) { + messages.push({ + postId, + text, + date, + views, + hasMedia, + channel: channelId, + }); + } + } + + return messages; +} + +// Scrape a single channel's web preview +async function scrapeChannel(channelId) { + const url = `https://t.me/s/${channelId}`; + const html = await fetchHTML(url); + if (!html) return { channel: channelId, error: 'Failed to fetch', posts: [] }; + + // Extract channel title from page + const titleMatch = html.match(/class="tgme_channel_info_header_title[^"]*"[^>]*>([\s\S]*?)<\/span>/i) + || html.match(/(.*?)<\/title>/i); + const title = titleMatch + ? titleMatch[1].replace(/<[^>]+>/g, '').trim() + : channelId; + + const posts = parseWebPreview(html, channelId); + + return { channel: channelId, title, posts, postCount: posts.length }; +} + +// ─── Analysis helpers ─────────────────────────────────────────────────────── + +// Flag urgent/high-priority posts +function flagUrgent(post) { + const lower = (post.text || '').toLowerCase(); + const matched = URGENT_KEYWORDS.filter(k => lower.includes(k)); + return matched.length > 0 ? matched : null; +} + +// Score a post's significance (views + urgency + length) +function significanceScore(post) { + let score = 0; + score += Math.min(post.views / 1000, 50); // views weight (capped) + const urgentFlags = flagUrgent(post); + if (urgentFlags) score += urgentFlags.length * 10; // urgency weight + if (post.text?.length > 100) score += 5; // substantive text bonus + if (post.hasMedia) score += 3; // media bonus + return score; +} + +// Group posts by topic based on the channel config +function groupByTopic(allPosts, channelMeta) { + const groups = {}; + for (const post of allPosts) { + const meta = channelMeta.find(c => c.id === post.channel); + const topic = meta?.topic || 'other'; + if (!groups[topic]) groups[topic] = []; + groups[topic].push(post); + } + return groups; +} + +// ─── Briefing ─────────────────────────────────────────────────────────────── + +export async function briefing() { + const token = process.env.TELEGRAM_BOT_TOKEN; + + // Try Bot API first if token is available + if (token) { + try { + const botData = await fetchBotUpdates(); + if (!botData.error && botData.count > 0) { + const enriched = botData.messages.map(m => ({ + ...m, + urgentFlags: flagUrgent(m), + score: significanceScore(m), + })); + + const urgent = enriched.filter(m => m.urgentFlags).sort((a, b) => b.score - a.score); + const top = enriched.sort((a, b) => b.score - a.score).slice(0, 15); + + return { + source: 'Telegram', + timestamp: new Date().toISOString(), + status: 'bot_api', + totalMessages: botData.count, + urgentPosts: urgent.slice(0, 10), + topPosts: top, + note: 'Data from Bot API getUpdates. Bot must be added to channels to receive posts.', + }; + } + // If bot returned no messages, fall through to web scraping + } catch { /* fall through to scraping */ } + } + + // Fallback: scrape public channel web previews (no auth needed) + const results = []; + const errors = []; + + // Fetch channels in batches of 3 to avoid rate limiting + for (let i = 0; i < CHANNELS.length; i += 3) { + const batch = CHANNELS.slice(i, i + 3); + const batchResults = await Promise.all( + batch.map(ch => scrapeChannel(ch.id)) + ); + results.push(...batchResults); + + // Delay between batches to be respectful + if (i + 3 < CHANNELS.length) await delay(1500); + } + + // Collect all posts and separate errors + const allPosts = []; + const channelSummaries = []; + + for (const r of results) { + const meta = CHANNELS.find(c => c.id === r.channel); + if (r.error) { + errors.push({ channel: r.channel, error: r.error }); + } + // Enrich posts with urgency flags and scores + const enriched = (r.posts || []).map(p => ({ + ...p, + urgentFlags: flagUrgent(p), + score: significanceScore(p), + })); + allPosts.push(...enriched); + + channelSummaries.push({ + channel: r.channel, + title: r.title || meta?.label || r.channel, + topic: meta?.topic || 'other', + postCount: r.postCount || 0, + reachable: !r.error, + }); + } + + // Sort all posts by significance + allPosts.sort((a, b) => b.score - a.score); + + // Separate urgent posts + const urgentPosts = allPosts.filter(p => p.urgentFlags).slice(0, 15); + + // Group by topic + const byTopic = groupByTopic(allPosts, CHANNELS); + const topicSummary = {}; + for (const [topic, posts] of Object.entries(byTopic)) { + topicSummary[topic] = { + totalPosts: posts.length, + urgentCount: posts.filter(p => p.urgentFlags).length, + topPosts: posts.sort((a, b) => b.score - a.score).slice(0, 5), + }; + } + + return { + source: 'Telegram', + timestamp: new Date().toISOString(), + status: token ? 'bot_api_empty_fallback_scrape' : 'web_scrape', + method: 'Public channel web preview scraping (no auth required)', + channelsMonitored: channelSummaries.length, + channelsReachable: channelSummaries.filter(c => c.reachable).length, + totalPosts: allPosts.length, + urgentPosts, + byTopic: topicSummary, + channels: channelSummaries, + errors: errors.length > 0 ? errors : undefined, + topPosts: allPosts.slice(0, 15), + hint: token + ? undefined + : 'Set TELEGRAM_BOT_TOKEN in .env for Bot API access. Create a bot via @BotFather on Telegram.', + }; +} + +// ─── CLI runner ───────────────────────────────────────────────────────────── + +if (process.argv[1]?.endsWith('telegram.mjs')) { + console.log('Telegram OSINT — fetching public channel intelligence...\n'); + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/treasury.mjs b/apis/sources/treasury.mjs new file mode 100644 index 0000000..89d27f7 --- /dev/null +++ b/apis/sources/treasury.mjs @@ -0,0 +1,80 @@ +// US Treasury Fiscal Data — Government debt, spending, yields +// No auth required. Daily updates. + +import { safeFetch, today, daysAgo } from '../utils/fetch.mjs'; + +const BASE = 'https://api.fiscaldata.treasury.gov/services/api/fiscal_service'; + +// Debt to the Penny (daily national debt) +export async function getDebtToThePenny(days = 30) { + const params = new URLSearchParams({ + 'fields': 'record_date,tot_pub_debt_out_amt,intragov_hold_amt,debt_held_public_amt', + 'sort': '-record_date', + 'page[size]': '30', + 'filter': `record_date:gte:${daysAgo(days)}`, + }); + return safeFetch(`${BASE}/v2/accounting/od/debt_to_penny?${params}`); +} + +// Daily Treasury Statement (government cash flow) +export async function getDailyStatement(days = 7) { + const params = new URLSearchParams({ + 'fields': 'record_date,account_type,close_today_bal', + 'sort': '-record_date', + 'page[size]': '20', + 'filter': `record_date:gte:${daysAgo(days)}`, + }); + return safeFetch(`${BASE}/v1/accounting/dts/deposits_withdrawals_operating_cash?${params}`); +} + +// Treasury yield curves (average interest rates on debt) +export async function getAvgInterestRates() { + const params = new URLSearchParams({ + 'fields': 'record_date,security_desc,avg_interest_rate_amt', + 'sort': '-record_date', + 'page[size]': '50', + 'filter': `record_date:gte:${daysAgo(30)}`, + }); + return safeFetch(`${BASE}/v2/accounting/od/avg_interest_rates?${params}`); +} + +// Briefing — key treasury data +export async function briefing() { + const [debt, rates] = await Promise.all([ + getDebtToThePenny(14), + getAvgInterestRates(), + ]); + + const debtData = debt?.data || []; + const latestDebt = debtData[0]; + const signals = []; + + if (latestDebt) { + const totalDebt = parseFloat(latestDebt.tot_pub_debt_out_amt); + if (totalDebt > 36_000_000_000_000) { + signals.push(`National debt at $${(totalDebt / 1e12).toFixed(2)}T`); + } + } + + return { + source: 'US Treasury', + timestamp: new Date().toISOString(), + debt: debtData.slice(0, 5).map(d => ({ + date: d.record_date, + totalDebt: d.tot_pub_debt_out_amt, + publicDebt: d.debt_held_public_amt, + intragovDebt: d.intragov_hold_amt, + })), + interestRates: (rates?.data || []).slice(0, 20).map(r => ({ + date: r.record_date, + security: r.security_desc, + rate: r.avg_interest_rate_amt, + })), + signals, + }; +} + +if (process.argv[1]?.endsWith('treasury.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/usaspending.mjs b/apis/sources/usaspending.mjs new file mode 100644 index 0000000..a2d980d --- /dev/null +++ b/apis/sources/usaspending.mjs @@ -0,0 +1,119 @@ +// USAspending — Federal spending, defense contracts, procurement signals +// No auth required. Updated daily. + +import { safeFetch, daysAgo } from '../utils/fetch.mjs'; + +const BASE = 'https://api.usaspending.gov/api/v2'; + +// Award type codes — required by the spending_by_award endpoint +// Contracts: A=BPA Call, B=Purchase Order, C=Delivery Order, D=Definitive Contract +// Grants: 02=Block Grant, 03=Formula Grant, 04=Project Grant, 05=Cooperative Agreement +// Direct payments: 06=Direct Payment (unrestricted), 07=Direct Payment (specified use) +// Loans: 08=Direct Loan, 09=Guaranteed/Insured Loan +// IDVs: IDV_A=GWAC, IDV_B=IDC, IDV_B_A=IDC / IDV, IDV_B_B=IDC / Multiple Award, +// IDV_B_C=IDC / FSS, IDV_C=FSS, IDV_D=BOA, IDV_E=BPA +const CONTRACT_CODES = ['A', 'B', 'C', 'D']; +const ALL_AWARD_CODES = ['A', 'B', 'C', 'D', '02', '03', '04', '05', '06', '07', '08', '09']; + +// Search recent awards/contracts +export async function searchAwards(opts = {}) { + const { + keywords = ['defense', 'military'], + limit = 20, + sortField = 'Award Amount', + order = 'desc', + awardTypeCodes = CONTRACT_CODES, + days = 30, + } = opts; + + const body = { + filters: { + keywords, + time_period: [{ start_date: daysAgo(days), end_date: daysAgo(0) }], + award_type_codes: awardTypeCodes, + }, + fields: [ + 'Award ID', + 'Recipient Name', + 'Award Amount', + 'Description', + 'Awarding Agency', + 'Start Date', + 'Award Type', + ], + limit, + page: 1, + sort: sortField, + order, + }; + + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 15000); + const res = await fetch(`${BASE}/search/spending_by_award/`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + signal: controller.signal, + }); + clearTimeout(timer); + if (!res.ok) { + const errBody = await res.text().catch(() => ''); + return { error: `HTTP ${res.status}: ${errBody.slice(0, 300)}`, results: [] }; + } + return res.json(); + } catch (e) { + return { error: e.message, results: [] }; + } +} + +// Get top agencies by spending +export async function getAgencySpending() { + return safeFetch(`${BASE}/references/toptier_agencies/`); +} + +// Search for defense-specific spending +export async function getDefenseSpending(days = 30) { + return searchAwards({ + keywords: ['defense', 'military', 'missile', 'ammunition', 'aircraft', 'naval'], + limit: 20, + sortField: 'Award Amount', + order: 'desc', + awardTypeCodes: CONTRACT_CODES, + days, + }); +} + +// Briefing +export async function briefing() { + const [defense, agencies] = await Promise.all([ + getDefenseSpending(14), + getAgencySpending(), + ]); + + return { + source: 'USAspending', + timestamp: new Date().toISOString(), + recentDefenseContracts: (defense?.results || []).slice(0, 10).map(r => ({ + awardId: r['Award ID'], + recipient: r['Recipient Name'], + amount: r['Award Amount'], + description: r['Description'], + agency: r['Awarding Agency'], + date: r['Start Date'], + type: r['Award Type'], + })), + topAgencies: (agencies?.results || []).slice(0, 10).map(a => ({ + name: a.agency_name, + budget: a.budget_authority_amount, + obligations: a.obligated_amount, + outlays: a.outlay_amount, + })), + ...(defense?.error ? { defenseError: defense.error } : {}), + }; +} + +if (process.argv[1]?.endsWith('usaspending.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/who.mjs b/apis/sources/who.mjs new file mode 100644 index 0000000..f3adde1 --- /dev/null +++ b/apis/sources/who.mjs @@ -0,0 +1,89 @@ +// WHO — World Health Organization Global Health Observatory +// No auth required. Disease outbreak monitoring. + +import { safeFetch } from '../utils/fetch.mjs'; + +const GHO_BASE = 'https://ghoapi.azureedge.net/api'; +const DON_API = 'https://www.who.int/api/news/diseaseoutbreaknews'; + +// Get GHO indicator data +export async function getIndicator(code, opts = {}) { + const { filter = '', top = 20 } = opts; + let url = `${GHO_BASE}/${code}?$top=${top}&$orderby=TimeDim desc`; + if (filter) url += `&$filter=${filter}`; + return safeFetch(url); +} + +// Key health indicators +const INDICATORS = { + MDG_0000000020: 'TB incidence (per 100k)', + MALARIA_EST_CASES: 'Malaria estimated cases', + WHOSIS_000001: 'Life expectancy at birth', + UHC_INDEX_REPORTED: 'UHC Service Coverage Index', +}; + +// Get Disease Outbreak News via WHO JSON API +// The old RSS feed at /feeds/entity/don/en/rss.xml returns 404. +// This JSON endpoint returns ~50 items; OData $orderby is ignored by +// the server, so we sort client-side by PublicationDate descending. +export async function getOutbreakNews() { + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), 15000); + const res = await fetch(DON_API, { + signal: controller.signal, + headers: { 'User-Agent': 'Crucix/1.0' }, + }); + clearTimeout(timer); + + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new Error(`HTTP ${res.status}: ${body.slice(0, 200)}`); + } + + const data = await res.json(); + const items = data?.value || []; + + // Sort by PublicationDate descending (server ignores $orderby) + items.sort((a, b) => { + const da = new Date(a.PublicationDate || 0); + const db = new Date(b.PublicationDate || 0); + return db - da; + }); + + return items.map(item => ({ + title: item.Title, + date: item.PublicationDate, + donId: item.DonId || null, + url: item.ItemDefaultUrl + ? `https://www.who.int/emergencies/disease-outbreak-news${item.ItemDefaultUrl}` + : null, + summary: (item.Summary || item.Overview || '').replace(/<[^>]*>/g, '').slice(0, 300) || null, + })); + } catch (e) { + return { error: e.message }; + } +} + +// Briefing +export async function briefing() { + const outbreaks = await getOutbreakNews(); + + return { + source: 'WHO', + timestamp: new Date().toISOString(), + diseaseOutbreakNews: Array.isArray(outbreaks) ? outbreaks.slice(0, 15) : [], + outbreakError: Array.isArray(outbreaks) ? null : outbreaks.error, + monitoringCapabilities: [ + 'Disease Outbreak News (DONs)', + 'Global health indicators (GHO)', + 'Pandemic early warning signals', + 'Cross-reference with GDELT health event mentions', + ], + }; +} + +if (process.argv[1]?.endsWith('who.mjs')) { + const data = await briefing(); + console.log(JSON.stringify(data, null, 2)); +} diff --git a/apis/sources/yfinance.mjs b/apis/sources/yfinance.mjs new file mode 100644 index 0000000..1699ffa --- /dev/null +++ b/apis/sources/yfinance.mjs @@ -0,0 +1,130 @@ +// Yahoo Finance — Live market quotes (no API key required) +// Provides real-time prices for stocks, ETFs, crypto, commodities +// Replaces the need for Alpaca or any paid market data provider + +import { safeFetch } from '../utils/fetch.mjs'; + +const BASE = 'https://query1.finance.yahoo.com/v8/finance/chart'; + +// Symbols to track — covers broad market, rates, commodities, crypto, volatility +const SYMBOLS = { + // Indexes / ETFs + SPY: 'S&P 500', + QQQ: 'Nasdaq 100', + DIA: 'Dow Jones', + IWM: 'Russell 2000', + // Rates / Credit + TLT: '20Y+ Treasury', + HYG: 'High Yield Corp', + LQD: 'IG Corporate', + // Commodities + 'GC=F': 'Gold', + 'SI=F': 'Silver', + 'CL=F': 'WTI Crude', + 'BZ=F': 'Brent Crude', + 'NG=F': 'Natural Gas', + // Crypto + 'BTC-USD': 'Bitcoin', + 'ETH-USD': 'Ethereum', + // Volatility + '^VIX': 'VIX', +}; + +async function fetchQuote(symbol) { + try { + const url = `${BASE}/${encodeURIComponent(symbol)}?range=5d&interval=1d&includePrePost=false`; + const data = await safeFetch(url, { + timeout: 8000, + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', + }, + }); + + const result = data?.chart?.result?.[0]; + if (!result) return null; + + const meta = result.meta || {}; + const quotes = result.indicators?.quote?.[0] || {}; + const closes = quotes.close || []; + const timestamps = result.timestamp || []; + + // Get current price and previous close + const price = meta.regularMarketPrice ?? closes[closes.length - 1]; + const prevClose = meta.chartPreviousClose ?? meta.previousClose ?? closes[closes.length - 2]; + const change = price && prevClose ? price - prevClose : 0; + const changePct = prevClose ? (change / prevClose) * 100 : 0; + + // Build 5-day history + const history = []; + for (let i = 0; i < timestamps.length; i++) { + if (closes[i] != null) { + history.push({ + date: new Date(timestamps[i] * 1000).toISOString().split('T')[0], + close: Math.round(closes[i] * 100) / 100, + }); + } + } + + return { + symbol, + name: SYMBOLS[symbol] || meta.shortName || symbol, + price: Math.round(price * 100) / 100, + prevClose: Math.round((prevClose || 0) * 100) / 100, + change: Math.round(change * 100) / 100, + changePct: Math.round(changePct * 100) / 100, + currency: meta.currency || 'USD', + exchange: meta.exchangeName || '', + marketState: meta.marketState || 'UNKNOWN', + history, + }; + } catch (e) { + return { symbol, name: SYMBOLS[symbol] || symbol, error: e.message }; + } +} + +export async function briefing() { + return collect(); +} + +export async function collect() { + const symbols = Object.keys(SYMBOLS); + const results = await Promise.allSettled( + symbols.map(s => fetchQuote(s)) + ); + + const quotes = {}; + let ok = 0; + let failed = 0; + + for (const r of results) { + const q = r.status === 'fulfilled' ? r.value : null; + if (q && !q.error) { + quotes[q.symbol] = q; + ok++; + } else { + failed++; + const sym = q?.symbol || 'unknown'; + quotes[sym] = q || { symbol: sym, error: 'fetch failed' }; + } + } + + // Categorize for easy dashboard consumption + return { + quotes, + summary: { + totalSymbols: symbols.length, + ok, + failed, + timestamp: new Date().toISOString(), + }, + indexes: pickGroup(quotes, ['SPY', 'QQQ', 'DIA', 'IWM']), + rates: pickGroup(quotes, ['TLT', 'HYG', 'LQD']), + commodities: pickGroup(quotes, ['GC=F', 'SI=F', 'CL=F', 'BZ=F', 'NG=F']), + crypto: pickGroup(quotes, ['BTC-USD', 'ETH-USD']), + volatility: pickGroup(quotes, ['^VIX']), + }; +} + +function pickGroup(quotes, symbols) { + return symbols.map(s => quotes[s]).filter(Boolean); +} diff --git a/apis/utils/env.mjs b/apis/utils/env.mjs new file mode 100644 index 0000000..0b60ee8 --- /dev/null +++ b/apis/utils/env.mjs @@ -0,0 +1,32 @@ +// Load .env file for API keys +// Searches: project root .env first, then apis/.env as fallback +import { readFileSync } from 'fs'; +import { resolve, dirname } from 'path'; +import { fileURLToPath } from 'url'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const paths = [ + resolve(__dirname, '..', '..', '.env'), // project root + resolve(__dirname, '..', '.env'), // apis/.env (legacy) +]; + +function loadEnv(filePath) { + try { + const content = readFileSync(filePath, 'utf-8'); + let loaded = 0; + for (const line of content.split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + const eq = trimmed.indexOf('='); + if (eq === -1) continue; + const key = trimmed.slice(0, eq).trim(); + const val = trimmed.slice(eq + 1).trim(); + if (!process.env[key]) { process.env[key] = val; loaded++; } + } + return loaded; + } catch { return -1; } +} + +for (const p of paths) { + if (loadEnv(p) >= 0) break; +} diff --git a/apis/utils/fetch.mjs b/apis/utils/fetch.mjs new file mode 100644 index 0000000..312859d --- /dev/null +++ b/apis/utils/fetch.mjs @@ -0,0 +1,42 @@ +// Shared fetch utility with timeout, retries, and error handling + +export async function safeFetch(url, opts = {}) { + const { timeout = 15000, retries = 1, headers = {} } = opts; + let lastError; + for (let i = 0; i <= retries; i++) { + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeout); + const res = await fetch(url, { + signal: controller.signal, + headers: { 'User-Agent': 'Crucix/1.0', ...headers }, + }); + clearTimeout(timer); + if (!res.ok) { + const body = await res.text().catch(() => ''); + throw new Error(`HTTP ${res.status}: ${body.slice(0, 200)}`); + } + const text = await res.text(); + try { return JSON.parse(text); } catch { return { rawText: text.slice(0, 500) }; } + } catch (e) { + lastError = e; + // GDELT needs 5s between requests, others are fine with shorter delays + if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1))); + } + } + return { error: lastError?.message || 'Unknown error', source: url }; +} + +export function ago(hours) { + return new Date(Date.now() - hours * 3600000).toISOString(); +} + +export function today() { + return new Date().toISOString().split('T')[0]; +} + +export function daysAgo(n) { + const d = new Date(); + d.setDate(d.getDate() - n); + return d.toISOString().split('T')[0]; +} diff --git a/crucix.config.mjs b/crucix.config.mjs new file mode 100644 index 0000000..3992772 --- /dev/null +++ b/crucix.config.mjs @@ -0,0 +1,19 @@ +// Crucix Configuration — all settings with env var overrides + +import './apis/utils/env.mjs'; // Load .env first + +export default { + port: parseInt(process.env.PORT) || 3117, + refreshIntervalMinutes: parseInt(process.env.REFRESH_INTERVAL_MINUTES) || 15, + + llm: { + provider: process.env.LLM_PROVIDER || null, // anthropic | openai | gemini | codex + apiKey: process.env.LLM_API_KEY || null, + model: process.env.LLM_MODEL || null, + }, + + telegram: { + botToken: process.env.TELEGRAM_BOT_TOKEN || null, + chatId: process.env.TELEGRAM_CHAT_ID || null, + }, +}; diff --git a/dashboard/inject.mjs b/dashboard/inject.mjs new file mode 100644 index 0000000..57497ed --- /dev/null +++ b/dashboard/inject.mjs @@ -0,0 +1,483 @@ +#!/usr/bin/env node +// Crucix Dashboard Data Synthesizer +// Reads runs/latest.json, fetches RSS news, generates signal-based ideas, +// and injects everything into dashboard/public/jarvis.html +// +// Exports synthesize(), generateIdeas(), fetchAllNews() for use by server.mjs + +import { readFileSync, writeFileSync } from 'fs'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +import { exec } from 'child_process'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = join(__dirname, '..'); + +// === Helpers === +const cyrillic = /[\u0400-\u04FF]/; +function isEnglish(text) { + if (!text) return false; + return !cyrillic.test(text.substring(0, 80)); +} + +// === Geo-tagging keyword map === +const geoKeywords = { + 'Ukraine':[49,32],'Russia':[56,38],'Moscow':[55.7,37.6],'Kyiv':[50.4,30.5], + 'China':[35,105],'Beijing':[39.9,116.4],'Iran':[32,53],'Tehran':[35.7,51.4], + 'Israel':[31.5,35],'Gaza':[31.4,34.4],'Palestine':[31.9,35.2], + 'Syria':[35,38],'Iraq':[33,44],'Saudi':[24,45],'Yemen':[15,48],'Lebanon':[34,36], + 'India':[20,78],'Japan':[36,138],'Korea':[37,127],'Pyongyang':[39,125.7], + 'Taiwan':[23.5,121],'Philippines':[13,122],'Myanmar':[20,96], + 'Canada':[56,-96],'Mexico':[23,-102],'Brazil':[-14,-51],'Argentina':[-38,-63], + 'Colombia':[4,-74],'Venezuela':[7,-66],'Cuba':[22,-80],'Chile':[-35,-71], + 'Germany':[51,10],'France':[46,2],'UK':[54,-2],'Britain':[54,-2],'London':[51.5,-0.1], + 'Spain':[40,-4],'Italy':[42,12],'Poland':[52,20],'NATO':[50,4],'EU':[50,4], + 'Turkey':[39,35],'Greece':[39,22],'Romania':[46,25],'Finland':[64,26],'Sweden':[62,15], + 'Africa':[0,20],'Nigeria':[10,8],'South Africa':[-30,25],'Kenya':[-1,38], + 'Egypt':[27,30],'Libya':[27,17],'Sudan':[13,30],'Ethiopia':[9,38], + 'Somalia':[5,46],'Congo':[-4,22],'Uganda':[1,32],'Morocco':[32,-6], + 'Pakistan':[30,70],'Afghanistan':[33,65],'Bangladesh':[24,90], + 'Australia':[-25,134],'Indonesia':[-2,118],'Thailand':[15,100], + 'US':[39,-98],'America':[39,-98],'Washington':[38.9,-77],'Pentagon':[38.9,-77], + 'Trump':[38.9,-77],'White House':[38.9,-77], + 'Wall Street':[40.7,-74],'New York':[40.7,-74],'California':[37,-120], + 'Nepal':[28,84],'Cambodia':[12.5,105],'Malawi':[-13.5,34],'Burundi':[-3.4,29.9], + 'Oman':[21,57],'Netherlands':[52.1,5.3],'Gabon':[-0.8,11.6], + 'Peru':[-10,-76],'Ecuador':[-2,-78],'Bolivia':[-17,-65], + 'Singapore':[1.35,103.8],'Malaysia':[4.2,101.9],'Vietnam':[16,108], + 'Algeria':[28,3],'Tunisia':[34,9],'Zimbabwe':[-20,30],'Mozambique':[-18,35], +}; + +function geoTagText(text) { + if (!text) return null; + for (const [keyword, [lat, lon]] of Object.entries(geoKeywords)) { + if (text.includes(keyword)) { + return { lat, lon, region: keyword }; + } + } + return null; +} + +// === RSS Fetching === +async function fetchRSS(url, source) { + try { + const res = await fetch(url, { signal: AbortSignal.timeout(8000) }); + const xml = await res.text(); + const items = []; + const itemRegex = /<item>([\s\S]*?)<\/item>/g; + let match; + while ((match = itemRegex.exec(xml)) !== null) { + const block = match[1]; + const title = (block.match(/<title>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?<\/title>/)?.[1] || '').trim(); + const pubDate = block.match(/<pubDate>(.*?)<\/pubDate>/)?.[1] || ''; + if (title && title !== source) items.push({ title, date: pubDate, source }); + } + return items; + } catch (e) { + console.log(`RSS fetch failed (${source}):`, e.message); + return []; + } +} + +export async function fetchAllNews() { + const feeds = [ + ['http://feeds.bbci.co.uk/news/world/rss.xml', 'BBC'], + ['https://rss.nytimes.com/services/xml/rss/nyt/World.xml', 'NYT'], + ['https://feeds.aljazeera.com/xml/rss/all.xml', 'Al Jazeera'], + ]; + + const results = await Promise.allSettled( + feeds.map(([url, source]) => fetchRSS(url, source)) + ); + + const allNews = results + .filter(r => r.status === 'fulfilled') + .flatMap(r => r.value); + + // De-duplicate and geo-tag + const seen = new Set(); + const geoNews = []; + for (const item of allNews) { + const key = item.title.substring(0, 40).toLowerCase(); + if (seen.has(key)) continue; + seen.add(key); + const geo = geoTagText(item.title); + if (geo) { + geoNews.push({ + title: item.title.substring(0, 100), + source: item.source, + date: item.date, + lat: geo.lat + (Math.random() - 0.5) * 2, + lon: geo.lon + (Math.random() - 0.5) * 2, + region: geo.region + }); + } + } + + geoNews.sort((a, b) => new Date(b.date || 0) - new Date(a.date || 0)); + return geoNews.slice(0, 30); +} + +// === Leverageable Ideas from Signals === +export function generateIdeas(V2) { + const ideas = []; + const vix = V2.fred.find(f => f.id === 'VIXCLS'); + const hy = V2.fred.find(f => f.id === 'BAMLH0A0HYM2'); + const spread = V2.fred.find(f => f.id === 'T10Y2Y'); + + if (V2.tg.urgent.length > 3 && V2.energy.wti > 68) { + ideas.push({ + title: 'Conflict-Energy Nexus Active', + text: `${V2.tg.urgent.length} urgent conflict signals with WTI at $${V2.energy.wti}. Geopolitical risk premium may expand. Consider energy exposure.`, + type: 'long', confidence: 'Medium', horizon: 'swing' + }); + } + if (vix && vix.value > 20) { + ideas.push({ + title: 'Elevated Volatility Regime', + text: `VIX at ${vix.value} — fear premium elevated. Portfolio hedges justified. Short-term equity upside is capped.`, + type: 'hedge', confidence: vix.value > 25 ? 'High' : 'Medium', horizon: 'tactical' + }); + } + if (vix && vix.value > 20 && hy && hy.value > 3) { + ideas.push({ + title: 'Safe Haven Demand Rising', + text: `VIX ${vix.value} + HY spread ${hy.value}% = risk-off building. Gold, treasuries, quality dividends may outperform.`, + type: 'hedge', confidence: 'Medium', horizon: 'tactical' + }); + } + if (V2.energy.wtiRecent.length > 1) { + const latest = V2.energy.wtiRecent[0]; + const oldest = V2.energy.wtiRecent[V2.energy.wtiRecent.length - 1]; + const pct = ((latest - oldest) / oldest * 100).toFixed(1); + if (Math.abs(pct) > 3) { + ideas.push({ + title: pct > 0 ? 'Oil Momentum Building' : 'Oil Under Pressure', + text: `WTI moved ${pct > 0 ? '+' : ''}${pct}% recently to $${V2.energy.wti}/bbl. ${pct > 0 ? 'Energy and commodity names benefit.' : 'Demand concerns may be emerging.'}`, + type: pct > 0 ? 'long' : 'watch', confidence: 'Medium', horizon: 'swing' + }); + } + } + if (spread) { + ideas.push({ + title: spread.value > 0 ? 'Yield Curve Normalizing' : 'Yield Curve Inverted', + text: `10Y-2Y spread at ${spread.value.toFixed(2)}. ${spread.value > 0 ? 'Recession signal fading — cyclical rotation possible.' : 'Inversion persists — defensive positioning warranted.'}`, + type: 'watch', confidence: 'Medium', horizon: 'strategic' + }); + } + const debt = parseFloat(V2.treasury.totalDebt); + if (debt > 35e12) { + ideas.push({ + title: 'Fiscal Trajectory Supports Hard Assets', + text: `National debt at $${(debt / 1e12).toFixed(1)}T. Long-term gold, bitcoin, and real asset appreciation thesis intact.`, + type: 'long', confidence: 'High', horizon: 'strategic' + }); + } + const totalThermal = V2.thermal.reduce((s, t) => s + t.det, 0); + if (totalThermal > 30000 && V2.tg.urgent.length > 2) { + ideas.push({ + title: 'Satellite Confirms Conflict Intensity', + text: `${totalThermal.toLocaleString()} thermal detections + ${V2.tg.urgent.length} urgent OSINT flags. Defense sector procurement may accelerate.`, + type: 'watch', confidence: 'Medium', horizon: 'swing' + }); + } + + // Yield Curve + Labor Interaction + const unemployment = V2.bls.find(b => b.id === 'LNS14000000' || b.id === 'UNRATE'); + const payrolls = V2.bls.find(b => b.id === 'CES0000000001' || b.id === 'PAYEMS'); + if (spread && unemployment && payrolls) { + const weakLabor = (unemployment.value > 4.3) || (payrolls.momChange && payrolls.momChange < -50); + if (spread.value > 0.3 && weakLabor) { + ideas.push({ + title: 'Steepening Curve Meets Weak Labor', + text: `10Y-2Y at ${spread.value.toFixed(2)} + UE ${unemployment.value}%. Curve steepening with deteriorating employment = recession positioning warranted.`, + type: 'hedge', confidence: 'High', horizon: 'tactical' + }); + } + } + + // ACLED Conflict + Energy Momentum + const conflictEvents = V2.acled?.totalEvents || 0; + if (conflictEvents > 50 && V2.energy.wtiRecent.length > 1) { + const wtiMove = V2.energy.wtiRecent[0] - V2.energy.wtiRecent[V2.energy.wtiRecent.length - 1]; + if (wtiMove > 2) { + ideas.push({ + title: 'Conflict Fueling Energy Momentum', + text: `${conflictEvents} ACLED events this week + WTI up $${wtiMove.toFixed(1)}. Conflict-energy transmission channel active.`, + type: 'long', confidence: 'Medium', horizon: 'swing' + }); + } + } + + // Defense + Conflict Intensity + const totalFatalities = V2.acled?.totalFatalities || 0; + const totalThermalAll = V2.thermal.reduce((s, t) => s + t.det, 0); + if (totalFatalities > 500 && totalThermalAll > 20000) { + ideas.push({ + title: 'Defense Procurement Acceleration Signal', + text: `${totalFatalities.toLocaleString()} conflict fatalities + ${totalThermalAll.toLocaleString()} thermal detections. Defense contractors may see accelerated procurement.`, + type: 'long', confidence: 'Medium', horizon: 'swing' + }); + } + + // HY Spread + VIX Divergence + if (hy && vix) { + const hyWide = hy.value > 3.5; + const vixLow = vix.value < 18; + const hyTight = hy.value < 2.5; + const vixHigh = vix.value > 25; + if (hyWide && vixLow) { + ideas.push({ + title: 'Credit Stress Ignored by Equity Vol', + text: `HY spread ${hy.value.toFixed(1)}% (wide) but VIX only ${vix.value.toFixed(0)} (complacent). Equity may be underpricing credit deterioration.`, + type: 'watch', confidence: 'Medium', horizon: 'tactical' + }); + } else if (hyTight && vixHigh) { + ideas.push({ + title: 'Equity Fear Exceeds Credit Stress', + text: `VIX at ${vix.value.toFixed(0)} but HY spread only ${hy.value.toFixed(1)}%. Equity vol may be overshooting — credit markets aren't confirming.`, + type: 'watch', confidence: 'Medium', horizon: 'tactical' + }); + } + } + + // Supply Chain + Inflation Pipeline + const ppi = V2.bls.find(b => b.id === 'WPUFD49104' || b.id === 'PCU--PCU--'); + const cpi = V2.bls.find(b => b.id === 'CUUR0000SA0' || b.id === 'CPIAUCSL'); + if (ppi && cpi && V2.gscpi) { + const supplyPressure = V2.gscpi.value > 0.5; + const ppiRising = ppi.momChangePct > 0.3; + if (supplyPressure && ppiRising) { + ideas.push({ + title: 'Inflation Pipeline Building Pressure', + text: `GSCPI at ${V2.gscpi.value.toFixed(2)} (${V2.gscpi.interpretation}) + PPI momentum +${ppi.momChangePct?.toFixed(1)}%. Input costs flowing through — CPI may follow.`, + type: 'long', confidence: 'Medium', horizon: 'strategic' + }); + } + } + + return ideas.slice(0, 8); +} + +// === Synthesize raw sweep data into dashboard format === +export async function synthesize(data) { + const air = (data.sources.OpenSky?.hotspots || []).map(h => ({ + region: h.region, total: h.totalAircraft || 0, noCallsign: h.noCallsign || 0, + highAlt: h.highAltitude || 0, + top: Object.entries(h.byCountry || {}).sort((a, b) => b[1] - a[1]).slice(0, 5) + })); + const thermal = (data.sources.FIRMS?.hotspots || []).map(h => ({ + region: h.region, det: h.totalDetections || 0, night: h.nightDetections || 0, + hc: h.highConfidence || 0, + fires: (h.highIntensity || []).slice(0, 8).map(f => ({ lat: f.lat, lon: f.lon, frp: f.frp || 0 })) + })); + const tSignals = data.sources.FIRMS?.signals || []; + const chokepoints = Object.values(data.sources.Maritime?.chokepoints || {}).map(c => ({ + label: c.label || c.name, note: c.note || '', lat: c.lat || 0, lon: c.lon || 0 + })); + const nuke = (data.sources.Safecast?.sites || []).map(s => ({ + site: s.site, anom: s.anomaly || false, cpm: s.avgCPM, n: s.recentReadings || 0 + })); + const nukeSignals = (data.sources.Safecast?.signals || []).filter(s => s); + const sdrData = data.sources.KiwiSDR || {}; + const sdrNet = sdrData.network || {}; + const sdrConflict = sdrData.conflictZones || {}; + const sdrZones = Object.values(sdrConflict).map(z => ({ + region: z.region, count: z.count || 0, + receivers: (z.receivers || []).slice(0, 5).map(r => ({ name: r.name || '', lat: r.lat || 0, lon: r.lon || 0 })) + })); + const tgData = data.sources.Telegram || {}; + const tgUrgent = (tgData.urgentPosts || []).filter(p => isEnglish(p.text)).map(p => ({ + channel: p.channel, text: p.text?.substring(0, 200), views: p.views, date: p.date, urgentFlags: p.urgentFlags || [] + })); + const tgTop = (tgData.topPosts || []).filter(p => isEnglish(p.text)).map(p => ({ + channel: p.channel, text: p.text?.substring(0, 200), views: p.views, date: p.date, urgentFlags: [] + })); + const who = (data.sources.WHO?.diseaseOutbreakNews || []).slice(0, 10).map(w => ({ + title: w.title?.substring(0, 120), date: w.date, summary: w.summary?.substring(0, 150) + })); + const fred = (data.sources.FRED?.indicators || []).map(f => ({ + id: f.id, label: f.label, value: f.value, date: f.date, + recent: f.recent || [], + momChange: f.momChange, momChangePct: f.momChangePct + })); + const energyData = data.sources.EIA || {}; + const oilPrices = energyData.oilPrices || {}; + const wtiRecent = (oilPrices.wti?.recent || []).map(d => d.value); + const energy = { + wti: oilPrices.wti?.value, brent: oilPrices.brent?.value, + natgas: energyData.gasPrice?.value, crudeStocks: energyData.inventories?.crudeStocks?.value, + wtiRecent, signals: energyData.signals || [] + }; + const bls = data.sources.BLS?.indicators || []; + const treasuryData = data.sources.Treasury || {}; + const debtArr = treasuryData.debt || []; + const treasury = { totalDebt: debtArr[0]?.totalDebt || '0', signals: treasuryData.signals || [] }; + const gscpi = data.sources.GSCPI?.latest || null; + const defense = (data.sources.USAspending?.recentDefenseContracts || []).slice(0, 5).map(c => ({ + recipient: c.recipient?.substring(0, 40), amount: c.amount, desc: c.description?.substring(0, 80) + })); + const noaa = { totalAlerts: data.sources.NOAA?.totalSevereAlerts || 0 }; + + // ACLED conflict events + const acledData = data.sources.ACLED || {}; + const acled = acledData.error ? { totalEvents: 0, totalFatalities: 0, byRegion: {}, byType: {}, deadliestEvents: [] } : { + totalEvents: acledData.totalEvents || 0, + totalFatalities: acledData.totalFatalities || 0, + byRegion: acledData.byRegion || {}, + byType: acledData.byType || {}, + deadliestEvents: (acledData.deadliestEvents || []).slice(0, 15).map(e => ({ + date: e.date, type: e.type, country: e.country, location: e.location, + fatalities: e.fatalities || 0, lat: e.lat || null, lon: e.lon || null + })) + }; + + // GDELT news articles + const gdeltData = data.sources.GDELT || {}; + const gdelt = { + totalArticles: gdeltData.totalArticles || 0, + conflicts: (gdeltData.conflicts || []).length, + economy: (gdeltData.economy || []).length, + health: (gdeltData.health || []).length, + crisis: (gdeltData.crisis || []).length, + topTitles: (gdeltData.allArticles || []).slice(0, 5).map(a => a.title?.substring(0, 80)) + }; + + const health = Object.entries(data.sources).map(([name, src]) => ({ + n: name, err: Boolean(src.error), stale: Boolean(src.stale) + })); + + // === Yahoo Finance live market data === + const yfData = data.sources.YFinance || {}; + const yfQuotes = yfData.quotes || {}; + const markets = { + indexes: (yfData.indexes || []).map(q => ({ + symbol: q.symbol, name: q.name, price: q.price, + change: q.change, changePct: q.changePct, history: q.history || [] + })), + rates: (yfData.rates || []).map(q => ({ + symbol: q.symbol, name: q.name, price: q.price, + change: q.change, changePct: q.changePct + })), + commodities: (yfData.commodities || []).map(q => ({ + symbol: q.symbol, name: q.name, price: q.price, + change: q.change, changePct: q.changePct, history: q.history || [] + })), + crypto: (yfData.crypto || []).map(q => ({ + symbol: q.symbol, name: q.name, price: q.price, + change: q.change, changePct: q.changePct + })), + vix: yfQuotes['^VIX'] ? { + value: yfQuotes['^VIX'].price, + change: yfQuotes['^VIX'].change, + changePct: yfQuotes['^VIX'].changePct, + } : null, + timestamp: yfData.summary?.timestamp || null, + }; + + // Override stale EIA prices with live Yahoo Finance data if available + const yfWti = yfQuotes['CL=F']; + const yfBrent = yfQuotes['BZ=F']; + const yfNatgas = yfQuotes['NG=F']; + if (yfWti?.price) energy.wti = yfWti.price; + if (yfBrent?.price) energy.brent = yfBrent.price; + if (yfNatgas?.price) energy.natgas = yfNatgas.price; + if (yfWti?.history?.length) energy.wtiRecent = yfWti.history.map(h => h.close); + + // Fetch RSS + const news = await fetchAllNews(); + + const V2 = { + meta: data.crucix, air, thermal, tSignals, chokepoints, nuke, nukeSignals, + sdr: { total: sdrNet.totalReceivers || 0, online: sdrNet.online || 0, zones: sdrZones }, + tg: { posts: tgData.totalPosts || 0, urgent: tgUrgent, topPosts: tgTop }, + who, fred, energy, bls, treasury, gscpi, defense, noaa, acled, gdelt, health, news, + markets, // Live Yahoo Finance market data + ideas: [], ideasSource: 'disabled', + // newsFeed for ticker (merged RSS + GDELT + Telegram) + newsFeed: buildNewsFeed(news, gdeltData, tgUrgent, tgTop), + }; + + return V2; +} + +// === Unified News Feed for Ticker === +function buildNewsFeed(rssNews, gdeltData, tgUrgent, tgTop) { + const feed = []; + + // RSS news + for (const n of rssNews) { + feed.push({ + headline: n.title, source: n.source, type: 'rss', + timestamp: n.date, region: n.region, urgent: false + }); + } + + // GDELT top articles + for (const title of (gdeltData.allArticles || []).slice(0, 10).map(a => a.title)) { + if (title) { + const geo = geoTagText(title); + feed.push({ + headline: title.substring(0, 100), source: 'GDELT', type: 'gdelt', + timestamp: new Date().toISOString(), region: geo?.region || 'Global', urgent: false + }); + } + } + + // Telegram urgent + for (const p of tgUrgent.slice(0, 10)) { + const text = (p.text || '').replace(/[\u{1F1E0}-\u{1F1FF}]/gu, '').trim(); + feed.push({ + headline: text.substring(0, 100), source: p.channel?.toUpperCase() || 'TELEGRAM', + type: 'telegram', timestamp: p.date, region: 'OSINT', urgent: true + }); + } + + // Telegram top (non-urgent) + for (const p of tgTop.slice(0, 5)) { + const text = (p.text || '').replace(/[\u{1F1E0}-\u{1F1FF}]/gu, '').trim(); + feed.push({ + headline: text.substring(0, 100), source: p.channel?.toUpperCase() || 'TELEGRAM', + type: 'telegram', timestamp: p.date, region: 'OSINT', urgent: false + }); + } + + // Sort by timestamp descending, limit to 50 + feed.sort((a, b) => new Date(b.timestamp || 0) - new Date(a.timestamp || 0)); + return feed.slice(0, 50); +} + +// === CLI Mode: inject into HTML file === +async function cliInject() { + const data = JSON.parse(readFileSync(join(ROOT, 'runs/latest.json'), 'utf8')); + + console.log('Fetching RSS news feeds...'); + const V2 = await synthesize(data); + console.log(`Generated ${V2.ideas.length} leverageable ideas`); + + const json = JSON.stringify(V2); + console.log('\n--- Synthesis ---'); + console.log('Size:', json.length, 'bytes | Air:', V2.air.length, '| Thermal:', V2.thermal.length, + '| News:', V2.news.length, '| Ideas:', V2.ideas.length, '| Sources:', V2.health.length); + + const htmlPath = join(ROOT, 'dashboard/public/jarvis.html'); + let html = readFileSync(htmlPath, 'utf8'); + html = html.replace(/^(let|const) D = .*;\s*$/m, 'let D = ' + json + ';'); + writeFileSync(htmlPath, html); + console.log('Data injected into jarvis.html!'); + + // Auto-open dashboard in default browser + const openCmd = process.platform === 'win32' ? 'start ""' : + process.platform === 'darwin' ? 'open' : 'xdg-open'; + const dashUrl = htmlPath.replace(/\\/g, '/'); + exec(`${openCmd} "${dashUrl}"`, (err) => { + if (err) console.log('Could not auto-open browser:', err.message); + else console.log('Dashboard opened in browser!'); + }); +} + +// Run CLI if invoked directly +const isMain = process.argv[1] && fileURLToPath(import.meta.url).includes(process.argv[1].replace(/\\/g, '/')); +if (isMain) { + cliInject(); +} diff --git a/dashboard/public/jarvis.html b/dashboard/public/jarvis.html new file mode 100644 index 0000000..2dbc177 --- /dev/null +++ b/dashboard/public/jarvis.html @@ -0,0 +1,1033 @@ +<!DOCTYPE html> +<html lang="en"> +<head> +<meta charset="UTF-8"> +<meta name="viewport" content="width=device-width, initial-scale=1.0"> +<title>CRUCIX — Intelligence Terminal + + + + + + + + +
+
CRUCIX
+
+
TERMINAL ACTIVE
+
+
+
+
+
+
+
+
+
+
+ +
+
SCROLL TO ZOOM · DRAG TO PAN
+
+ + + +
+ +
+
+
+
+
+
+
+ + + diff --git a/lib/alerts/telegram.mjs b/lib/alerts/telegram.mjs new file mode 100644 index 0000000..66ecdbd --- /dev/null +++ b/lib/alerts/telegram.mjs @@ -0,0 +1,162 @@ +// Telegram Alerter — sends breaking news alerts via Telegram Bot API (LLM-gated) + +const TELEGRAM_API = 'https://api.telegram.org'; + +export class TelegramAlerter { + constructor({ botToken, chatId }) { + this.botToken = botToken; + this.chatId = chatId; + } + + get isConfigured() { + return !!(this.botToken && this.chatId); + } + + /** + * Send a message via Telegram Bot API. + * @param {string} message - markdown-formatted message + * @returns {Promise} - true if sent successfully + */ + async sendAlert(message) { + if (!this.isConfigured) return false; + + try { + const res = await fetch(`${TELEGRAM_API}/bot${this.botToken}/sendMessage`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + chat_id: this.chatId, + text: message, + parse_mode: 'Markdown', + disable_web_page_preview: true, + }), + signal: AbortSignal.timeout(15000), + }); + + if (!res.ok) { + const err = await res.text().catch(() => ''); + console.error(`[Telegram] Send failed (${res.status}): ${err.substring(0, 100)}`); + return false; + } + + return true; + } catch (err) { + console.error('[Telegram] Send error:', err.message); + return false; + } + } + + /** + * Evaluate delta signals with LLM and send alert if warranted. + * @param {LLMProvider} llmProvider - configured LLM provider + * @param {object} delta - delta from current sweep + * @param {MemoryManager} memory - memory manager for dedup + * @returns {Promise} - true if alert was sent + */ + async evaluateAndAlert(llmProvider, delta, memory) { + if (!this.isConfigured || !llmProvider?.isConfigured) return false; + if (!delta?.summary?.criticalChanges) return false; + + // Filter out already-alerted signals + const alerted = memory.getAlertedSignals(); + const newSignals = [ + ...(delta.signals?.new || []), + ...(delta.signals?.escalated || []), + ].filter(s => { + const key = s.key || s.label || s.text?.substring(0, 40); + return !alerted[key]; + }); + + if (newSignals.length === 0) return false; + + // Ask LLM if these signals warrant an immediate alert + const systemPrompt = `You are an intelligence alert evaluator. You receive new/escalated signals from an OSINT monitoring system. Your job is to determine if any warrant an IMMEDIATE alert to the user. + +Alert criteria (ALL must be true): +1. Material market impact likely (>1% move in major index, or >5% move in sector/commodity) +2. Time-sensitive — acting in the next few hours matters +3. Not routine data (scheduled economic releases don't count unless they're a major surprise) + +Respond with ONLY valid JSON: +{ + "shouldAlert": true/false, + "reason": "1-2 sentence explanation", + "headline": "Alert headline if shouldAlert is true", + "signals": ["key signals that triggered alert"] +}`; + + const userMessage = `New/escalated signals since last sweep:\n${newSignals.map(s => { + if (s.changePct !== undefined) return `- ${s.label}: ${s.previous} → ${s.current} (${s.changePct > 0 ? '+' : ''}${s.changePct.toFixed(1)}%)`; + if (s.text) return `- NEW OSINT: ${s.text.substring(0, 120)}`; + return `- ${s.label || JSON.stringify(s)}`; + }).join('\n')} + +Delta summary: direction=${delta.summary.direction}, total changes=${delta.summary.totalChanges}, critical=${delta.summary.criticalChanges}`; + + try { + const result = await llmProvider.complete(systemPrompt, userMessage, { maxTokens: 512, timeout: 30000 }); + const evaluation = parseEvaluation(result.text); + + if (!evaluation?.shouldAlert) { + console.log('[Telegram] LLM says no alert needed:', evaluation?.reason || 'unknown'); + return false; + } + + // Build and send alert message + const message = formatAlertMessage(evaluation, delta); + const sent = await this.sendAlert(message); + + if (sent) { + // Mark signals as alerted + for (const s of newSignals) { + const key = s.key || s.label || s.text?.substring(0, 40); + memory.markAsAlerted(key, new Date().toISOString()); + } + console.log('[Telegram] Alert sent:', evaluation.headline); + } + + return sent; + } catch (err) { + console.error('[Telegram] LLM evaluation failed:', err.message); + return false; + } + } +} + +function parseEvaluation(text) { + if (!text) return null; + let cleaned = text.trim(); + if (cleaned.startsWith('```')) { + cleaned = cleaned.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, ''); + } + try { + return JSON.parse(cleaned); + } catch { + const match = cleaned.match(/\{[\s\S]*\}/); + if (match) { + try { return JSON.parse(match[0]); } catch { /* give up */ } + } + return null; + } +} + +function formatAlertMessage(evaluation, delta) { + const lines = [ + `🚨 *CRUCIX ALERT*`, + ``, + `*${evaluation.headline}*`, + ``, + evaluation.reason, + ``, + `Direction: ${delta.summary.direction.toUpperCase()}`, + `Critical changes: ${delta.summary.criticalChanges}`, + ]; + + if (evaluation.signals?.length) { + lines.push('', `Key signals: ${evaluation.signals.join(', ')}`); + } + + lines.push('', `_${new Date().toLocaleTimeString()} UTC_`); + + return lines.join('\n'); +} diff --git a/lib/delta/engine.mjs b/lib/delta/engine.mjs new file mode 100644 index 0000000..721e25a --- /dev/null +++ b/lib/delta/engine.mjs @@ -0,0 +1,117 @@ +// Delta Engine — compares two synthesized sweep results and produces structured changes + +// Metrics we track for delta computation +const NUMERIC_METRICS = [ + { key: 'vix', extract: d => d.fred?.find(f => f.id === 'VIXCLS')?.value, label: 'VIX', threshold: 5 }, + { key: 'hy_spread', extract: d => d.fred?.find(f => f.id === 'BAMLH0A0HYM2')?.value, label: 'HY Spread', threshold: 5 }, + { key: '10y2y', extract: d => d.fred?.find(f => f.id === 'T10Y2Y')?.value, label: '10Y-2Y Spread', threshold: 10 }, + { key: 'wti', extract: d => d.energy?.wti, label: 'WTI Crude', threshold: 3 }, + { key: 'brent', extract: d => d.energy?.brent, label: 'Brent Crude', threshold: 3 }, + { key: 'natgas', extract: d => d.energy?.natgas, label: 'Natural Gas', threshold: 5 }, + { key: 'unemployment', extract: d => d.bls?.find(b => b.id === 'LNS14000000' || b.id === 'UNRATE')?.value, label: 'Unemployment', threshold: 2 }, + { key: 'fed_funds', extract: d => d.fred?.find(f => f.id === 'DFF')?.value, label: 'Fed Funds Rate', threshold: 1 }, + { key: '10y_yield', extract: d => d.fred?.find(f => f.id === 'DGS10')?.value, label: '10Y Yield', threshold: 3 }, + { key: 'usd_index', extract: d => d.fred?.find(f => f.id === 'DTWEXBGS')?.value, label: 'USD Index', threshold: 1 }, + { key: 'mortgage', extract: d => d.fred?.find(f => f.id === 'MORTGAGE30US')?.value, label: '30Y Mortgage', threshold: 2 }, +]; + +const COUNT_METRICS = [ + { key: 'urgent_posts', extract: d => d.tg?.urgent?.length || 0, label: 'Urgent OSINT Posts' }, + { key: 'thermal_total', extract: d => d.thermal?.reduce((s, t) => s + t.det, 0) || 0, label: 'Thermal Detections' }, + { key: 'air_total', extract: d => d.air?.reduce((s, a) => s + a.total, 0) || 0, label: 'Air Activity' }, + { key: 'who_alerts', extract: d => d.who?.length || 0, label: 'WHO Alerts' }, + { key: 'conflict_events', extract: d => d.acled?.totalEvents || 0, label: 'Conflict Events' }, + { key: 'conflict_fatalities', extract: d => d.acled?.totalFatalities || 0, label: 'Conflict Fatalities' }, + { key: 'sdr_online', extract: d => d.sdr?.online || 0, label: 'SDR Receivers' }, + { key: 'news_count', extract: d => d.news?.length || 0, label: 'News Items' }, + { key: 'sources_ok', extract: d => d.meta?.sourcesOk || 0, label: 'Sources OK' }, +]; + +export function computeDelta(current, previous) { + if (!previous) return null; + + const signals = { new: [], escalated: [], deescalated: [], unchanged: [] }; + let criticalChanges = 0; + + // Numeric metrics: track % change + for (const m of NUMERIC_METRICS) { + const curr = m.extract(current); + const prev = m.extract(previous); + if (curr == null || prev == null) continue; + + const pctChange = prev !== 0 ? ((curr - prev) / Math.abs(prev)) * 100 : 0; + + if (Math.abs(pctChange) > m.threshold) { + const entry = { + key: m.key, label: m.label, from: prev, to: curr, + pctChange: parseFloat(pctChange.toFixed(2)), + direction: pctChange > 0 ? 'up' : 'down', + }; + if (pctChange > 0) signals.escalated.push(entry); + else signals.deescalated.push(entry); + if (Math.abs(pctChange) > 10) criticalChanges++; + } else { + signals.unchanged.push(m.key); + } + } + + // Count metrics: track absolute change + for (const m of COUNT_METRICS) { + const curr = m.extract(current); + const prev = m.extract(previous); + const diff = curr - prev; + + if (Math.abs(diff) > 0) { + const entry = { + key: m.key, label: m.label, from: prev, to: curr, + change: diff, direction: diff > 0 ? 'up' : 'down', + }; + if (diff > 0) signals.escalated.push(entry); + else signals.deescalated.push(entry); + } else { + signals.unchanged.push(m.key); + } + } + + // New urgent posts (check by text content) + const prevUrgentTexts = new Set((previous.tg?.urgent || []).map(p => p.text?.substring(0, 60))); + for (const post of (current.tg?.urgent || [])) { + const key = post.text?.substring(0, 60); + if (key && !prevUrgentTexts.has(key)) { + signals.new.push({ key: 'tg_urgent', item: post, reason: 'New urgent OSINT post' }); + criticalChanges++; + } + } + + // Nuclear anomaly change + const currAnom = current.nuke?.some(n => n.anom) || false; + const prevAnom = previous.nuke?.some(n => n.anom) || false; + if (currAnom && !prevAnom) { + signals.new.push({ key: 'nuke_anomaly', reason: 'Nuclear anomaly detected' }); + criticalChanges += 5; // Critical + } else if (!currAnom && prevAnom) { + signals.deescalated.push({ key: 'nuke_anomaly', label: 'Nuclear Anomaly', direction: 'resolved' }); + } + + // Determine overall direction + let direction = 'mixed'; + const riskUp = signals.escalated.filter(s => + ['vix', 'hy_spread', 'urgent_posts', 'conflict_events', 'thermal_total'].includes(s.key) + ).length; + const riskDown = signals.deescalated.filter(s => + ['vix', 'hy_spread', 'urgent_posts', 'conflict_events', 'thermal_total'].includes(s.key) + ).length; + if (riskUp > riskDown + 1) direction = 'risk-off'; + else if (riskDown > riskUp + 1) direction = 'risk-on'; + + return { + timestamp: current.meta?.timestamp || new Date().toISOString(), + previous: previous.meta?.timestamp || null, + signals, + summary: { + totalChanges: signals.new.length + signals.escalated.length + signals.deescalated.length, + criticalChanges, + direction, + }, + }; +} diff --git a/lib/delta/index.mjs b/lib/delta/index.mjs new file mode 100644 index 0000000..b7f3a1d --- /dev/null +++ b/lib/delta/index.mjs @@ -0,0 +1,2 @@ +export { computeDelta } from './engine.mjs'; +export { MemoryManager } from './memory.mjs'; diff --git a/lib/delta/memory.mjs b/lib/delta/memory.mjs new file mode 100644 index 0000000..9dcae5a --- /dev/null +++ b/lib/delta/memory.mjs @@ -0,0 +1,139 @@ +// Memory Manager — hot/cold storage for sweep history and alert tracking + +import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs'; +import { join } from 'path'; +import { computeDelta } from './engine.mjs'; + +const MAX_HOT_RUNS = 3; + +export class MemoryManager { + constructor(runsDir) { + this.runsDir = runsDir; + this.memoryDir = join(runsDir, 'memory'); + this.hotPath = join(this.memoryDir, 'hot.json'); + this.coldDir = join(this.memoryDir, 'cold'); + + // Ensure dirs exist + for (const dir of [this.memoryDir, this.coldDir]) { + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + } + + // Load hot memory from disk + this.hot = this._loadHot(); + } + + _loadHot() { + try { + return JSON.parse(readFileSync(this.hotPath, 'utf8')); + } catch { + return { runs: [], alertedSignals: {} }; + } + } + + _saveHot() { + try { + writeFileSync(this.hotPath, JSON.stringify(this.hot, null, 2)); + } catch (err) { + console.error('[Memory] Failed to save hot memory:', err.message); + } + } + + // Add a new run to hot memory + addRun(synthesizedData) { + const previous = this.getLastRun(); + const delta = computeDelta(synthesizedData, previous); + + // Compact the data for storage (strip large arrays) + const compact = this._compactForStorage(synthesizedData); + + this.hot.runs.unshift({ + timestamp: synthesizedData.meta?.timestamp || new Date().toISOString(), + data: compact, + delta, + }); + + // Keep only MAX_HOT_RUNS + if (this.hot.runs.length > MAX_HOT_RUNS) { + const archived = this.hot.runs.splice(MAX_HOT_RUNS); + this._archiveToCold(archived); + } + + this._saveHot(); + return delta; + } + + // Get last run's synthesized data + getLastRun() { + if (this.hot.runs.length === 0) return null; + return this.hot.runs[0].data; + } + + // Get last N runs + getRunHistory(n = 3) { + return this.hot.runs.slice(0, n); + } + + // Get the delta from the most recent run + getLastDelta() { + if (this.hot.runs.length === 0) return null; + return this.hot.runs[0].delta; + } + + // Track what signals have been alerted on + getAlertedSignals() { + return this.hot.alertedSignals || {}; + } + + markAsAlerted(signalKey, timestamp) { + this.hot.alertedSignals[signalKey] = timestamp || new Date().toISOString(); + this._saveHot(); + } + + // Clean up old alerted signals (older than 24h) + pruneAlertedSignals() { + const cutoff = Date.now() - 24 * 60 * 60 * 1000; + for (const [key, ts] of Object.entries(this.hot.alertedSignals)) { + if (new Date(ts).getTime() < cutoff) { + delete this.hot.alertedSignals[key]; + } + } + this._saveHot(); + } + + // Compact data for storage — strip heavy arrays + _compactForStorage(data) { + return { + meta: data.meta, + fred: data.fred, + energy: data.energy, + bls: data.bls, + treasury: data.treasury, + gscpi: data.gscpi, + tg: { posts: data.tg?.posts, urgent: (data.tg?.urgent || []).map(p => ({ text: p.text?.substring(0, 80), date: p.date })) }, + thermal: (data.thermal || []).map(t => ({ region: t.region, det: t.det, night: t.night, hc: t.hc })), + air: (data.air || []).map(a => ({ region: a.region, total: a.total })), + nuke: (data.nuke || []).map(n => ({ site: n.site, anom: n.anom, cpm: n.cpm })), + who: (data.who || []).map(w => ({ title: w.title })), + acled: { totalEvents: data.acled?.totalEvents, totalFatalities: data.acled?.totalFatalities }, + sdr: { total: data.sdr?.total, online: data.sdr?.online }, + ideas: (data.ideas || []).map(i => ({ title: i.title, type: i.type, confidence: i.confidence })), + }; + } + + // Archive old runs to cold storage + _archiveToCold(runs) { + if (runs.length === 0) return; + const dateKey = new Date().toISOString().split('T')[0]; // YYYY-MM-DD + const coldPath = join(this.coldDir, `${dateKey}.json`); + + let existing = []; + try { existing = JSON.parse(readFileSync(coldPath, 'utf8')); } catch { } + + existing.push(...runs); + try { + writeFileSync(coldPath, JSON.stringify(existing, null, 2)); + } catch (err) { + console.error('[Memory] Failed to archive to cold storage:', err.message); + } + } +} diff --git a/lib/llm/anthropic.mjs b/lib/llm/anthropic.mjs new file mode 100644 index 0000000..075bb5c --- /dev/null +++ b/lib/llm/anthropic.mjs @@ -0,0 +1,49 @@ +// Anthropic Claude Provider — raw fetch, no SDK + +import { LLMProvider } from './provider.mjs'; + +export class AnthropicProvider extends LLMProvider { + constructor(config) { + super(config); + this.name = 'anthropic'; + this.apiKey = config.apiKey; + this.model = config.model || 'claude-sonnet-4-20250514'; + } + + get isConfigured() { return !!this.apiKey; } + + async complete(systemPrompt, userMessage, opts = {}) { + const res = await fetch('https://api.anthropic.com/v1/messages', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-api-key': this.apiKey, + 'anthropic-version': '2023-06-01', + }, + body: JSON.stringify({ + model: this.model, + max_tokens: opts.maxTokens || 4096, + system: systemPrompt, + messages: [{ role: 'user', content: userMessage }], + }), + signal: AbortSignal.timeout(opts.timeout || 60000), + }); + + if (!res.ok) { + const err = await res.text().catch(() => ''); + throw new Error(`Anthropic API ${res.status}: ${err.substring(0, 200)}`); + } + + const data = await res.json(); + const text = data.content?.[0]?.text || ''; + + return { + text, + usage: { + inputTokens: data.usage?.input_tokens || 0, + outputTokens: data.usage?.output_tokens || 0, + }, + model: data.model || this.model, + }; + } +} diff --git a/lib/llm/codex.mjs b/lib/llm/codex.mjs new file mode 100644 index 0000000..98b3b3d --- /dev/null +++ b/lib/llm/codex.mjs @@ -0,0 +1,147 @@ +// OpenAI Codex Provider — uses ChatGPT subscription via chatgpt.com/backend-api/codex/responses +// Auth: reads ~/.codex/auth.json (created by `npx @openai/codex login`) +// SSE streaming, codex-specific models only (gpt-5.2-codex, gpt-5.3-codex) + +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { homedir } from 'os'; +import { LLMProvider } from './provider.mjs'; + +const CODEX_ENDPOINT = 'https://chatgpt.com/backend-api/codex/responses'; +const AUTH_PATH = join(homedir(), '.codex', 'auth.json'); + +export class CodexProvider extends LLMProvider { + constructor(config) { + super(config); + this.name = 'codex'; + this.model = config.model || 'gpt-5.2-codex'; + this._creds = null; + } + + get isConfigured() { + return !!this._getCredentials(); + } + + _getCredentials() { + if (this._creds) return this._creds; + + // Try env vars first + const token = process.env.CODEX_ACCESS_TOKEN || process.env.OPENAI_OAUTH_TOKEN; + const accountId = process.env.CODEX_ACCOUNT_ID; + if (token && accountId) { + this._creds = { accessToken: token, accountId }; + return this._creds; + } + + // Try ~/.codex/auth.json + try { + const auth = JSON.parse(readFileSync(AUTH_PATH, 'utf8')); + // Tokens may be nested under auth.tokens (newer format) or top-level + const tokens = auth.tokens || auth; + const accessToken = tokens.access_token || tokens.token || auth.access_token || auth.token; + if (accessToken) { + this._creds = { + accessToken, + accountId: tokens.account_id || auth.account_id || accountId || '', + }; + return this._creds; + } + } catch { /* no auth file */ } + + return null; + } + + _clearCredentials() { + this._creds = null; + } + + async complete(systemPrompt, userMessage, opts = {}) { + const creds = this._getCredentials(); + if (!creds) throw new Error('Codex: No credentials found. Run `npx @openai/codex login`'); + + const headers = { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${creds.accessToken}`, + }; + if (creds.accountId) headers['ChatGPT-Account-Id'] = creds.accountId; + + const body = { + model: this.model, + instructions: systemPrompt || '', + input: [{ type: 'message', role: 'user', content: userMessage }], + stream: true, + store: false, + }; + + const res = await fetch(CODEX_ENDPOINT, { + method: 'POST', + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(opts.timeout || 90000), + }); + + if (res.status === 401 || res.status === 403) { + this._clearCredentials(); + throw new Error(`Codex auth failed (${res.status}). Run \`npx @openai/codex login\` to refresh.`); + } + + if (!res.ok) { + const err = await res.text().catch(() => ''); + throw new Error(`Codex API ${res.status}: ${err.substring(0, 200)}`); + } + + // Parse SSE stream + const text = await this._parseSSE(res); + + return { + text, + usage: { inputTokens: 0, outputTokens: 0 }, // Codex doesn't always return usage + model: this.model, + }; + } + + async _parseSSE(res) { + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let text = ''; + let buffer = ''; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + + for (const line of lines) { + if (!line.startsWith('data: ')) continue; + const payload = line.slice(6).trim(); + if (payload === '[DONE]') return text; + + try { + const event = JSON.parse(payload); + // Handle text deltas + if (event.type === 'response.output_text.delta') { + text += event.delta || ''; + } + // Handle completed response + if (event.type === 'response.completed') { + const output = event.response?.output; + if (output && Array.isArray(output)) { + for (const item of output) { + if (item.type === 'message' && item.content) { + for (const part of item.content) { + if (part.type === 'output_text') text = part.text || text; + } + } + } + } + } + } catch { /* skip malformed events */ } + } + } + + return text; + } +} diff --git a/lib/llm/gemini.mjs b/lib/llm/gemini.mjs new file mode 100644 index 0000000..9d4ed6e --- /dev/null +++ b/lib/llm/gemini.mjs @@ -0,0 +1,48 @@ +// Google Gemini Provider — raw fetch, no SDK + +import { LLMProvider } from './provider.mjs'; + +export class GeminiProvider extends LLMProvider { + constructor(config) { + super(config); + this.name = 'gemini'; + this.apiKey = config.apiKey; + this.model = config.model || 'gemini-2.0-flash'; + } + + get isConfigured() { return !!this.apiKey; } + + async complete(systemPrompt, userMessage, opts = {}) { + const url = `https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent?key=${this.apiKey}`; + + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + systemInstruction: { parts: [{ text: systemPrompt }] }, + contents: [{ parts: [{ text: userMessage }] }], + generationConfig: { + maxOutputTokens: opts.maxTokens || 4096, + }, + }), + signal: AbortSignal.timeout(opts.timeout || 60000), + }); + + if (!res.ok) { + const err = await res.text().catch(() => ''); + throw new Error(`Gemini API ${res.status}: ${err.substring(0, 200)}`); + } + + const data = await res.json(); + const text = data.candidates?.[0]?.content?.parts?.[0]?.text || ''; + + return { + text, + usage: { + inputTokens: data.usageMetadata?.promptTokenCount || 0, + outputTokens: data.usageMetadata?.candidatesTokenCount || 0, + }, + model: this.model, + }; + } +} diff --git a/lib/llm/ideas.mjs b/lib/llm/ideas.mjs new file mode 100644 index 0000000..08e6603 --- /dev/null +++ b/lib/llm/ideas.mjs @@ -0,0 +1,189 @@ +// LLM-Powered Trade Ideas — generates actionable ideas from sweep data + delta context + +/** + * Generate LLM-enhanced trade ideas from sweep data. + * @param {LLMProvider} provider - configured LLM provider + * @param {object} sweepData - synthesized dashboard data + * @param {object|null} delta - delta from last sweep + * @param {Array} previousIdeas - ideas from previous runs (for dedup) + * @returns {Promise} - array of idea objects + */ +export async function generateLLMIdeas(provider, sweepData, delta, previousIdeas = []) { + if (!provider?.isConfigured) return null; + + let context; + try { + context = compactSweepForLLM(sweepData, delta, previousIdeas); + } catch (err) { + console.error('[LLM Ideas] Failed to compact sweep data:', err.message); + return null; + } + + const systemPrompt = `You are a quantitative analyst at a macro intelligence firm. You receive structured OSINT + economic data from 25 sources and produce 5-8 actionable trade ideas. + +Rules: +- Each idea must cite specific data points from the input +- Include entry rationale, risk factors, and time horizon +- Blend geopolitical, economic, and market signals — cross-correlate across domains +- Be specific: name instruments (tickers, futures, ETFs), not vague sectors +- If delta shows significant changes, lead with those +- Do NOT repeat ideas from the "previous ideas" list unless conditions have materially changed +- Rate confidence: HIGH (multiple confirming signals), MEDIUM (thesis supported), LOW (speculative) + +Output ONLY valid JSON array. Each object: +{ + "title": "Short title (max 10 words)", + "type": "LONG|SHORT|HEDGE|WATCH|AVOID", + "ticker": "Primary instrument", + "confidence": "HIGH|MEDIUM|LOW", + "rationale": "2-3 sentence explanation citing specific data", + "risk": "Key risk factor", + "horizon": "Intraday|Days|Weeks|Months", + "signals": ["signal1", "signal2"] +}`; + + try { + const result = await provider.complete(systemPrompt, context, { maxTokens: 4096, timeout: 90000 }); + const ideas = parseIdeasResponse(result.text); + if (ideas && ideas.length > 0) { + return ideas; + } + console.warn('[LLM Ideas] No valid ideas parsed from response'); + return null; + } catch (err) { + console.error('[LLM Ideas] Generation failed:', err.message); + return null; + } +} + +/** + * Compact sweep data to ~8KB for token efficiency. + */ +function compactSweepForLLM(data, delta, previousIdeas) { + const sections = []; + + // Economic indicators + if (data.fred?.length) { + const key = data.fred.filter(f => ['VIXCLS', 'DFF', 'DGS10', 'DGS2', 'T10Y2Y', 'BAMLH0A0HYM2', 'DTWEXBGS', 'MORTGAGE30US'].includes(f.id)); + sections.push(`ECONOMIC: ${key.map(f => `${f.id}=${f.value}${f.momChange ? ` (${f.momChange > 0 ? '+' : ''}${f.momChange})` : ''}`).join(', ')}`); + } + + // Energy + if (data.energy) { + sections.push(`ENERGY: WTI=$${data.energy.wti}, Brent=$${data.energy.brent}, NatGas=$${data.energy.natgas}, CrudeStocks=${data.energy.crudeStocks}bbl`); + } + + // BLS + if (data.bls?.length) { + sections.push(`LABOR: ${data.bls.map(b => `${b.id}=${b.value}`).join(', ')}`); + } + + // Treasury + if (data.treasury) { + sections.push(`TREASURY: totalDebt=$${data.treasury}T`); + } + + // Supply chain + if (data.gscpi) { + sections.push(`SUPPLY_CHAIN: GSCPI=${data.gscpi.value} (${data.gscpi.interpretation})`); + } + + // Geopolitical signals + const urgentPosts = (data.tg?.urgent || []).slice(0, 5); + if (urgentPosts.length) { + sections.push(`URGENT_OSINT:\n${urgentPosts.map(p => `- ${(p.text || '').substring(0, 120)}`).join('\n')}`); + } + + // Thermal / fire detections + if (data.thermal?.length) { + const hotRegions = data.thermal.filter(t => t.det > 10).map(t => `${t.region}: ${t.det} detections (${t.hc} high-conf)`); + if (hotRegions.length) sections.push(`THERMAL: ${hotRegions.join(', ')}`); + } + + // Air activity + if (data.air?.length) { + const airSum = data.air.map(a => `${a.region}: ${a.total} aircraft`); + sections.push(`AIR_ACTIVITY: ${airSum.join(', ')}`); + } + + // Nuclear + if (data.nuke?.length) { + const anomalies = data.nuke.filter(n => n.anom); + if (anomalies.length) sections.push(`NUCLEAR_ANOMALY: ${anomalies.map(n => `${n.site}: ${n.cpm}cpm`).join(', ')}`); + } + + // WHO alerts + if (data.who?.length) { + sections.push(`WHO_ALERTS: ${data.who.slice(0, 3).map(w => w.title).join('; ')}`); + } + + // Defense spending + if (data.defense?.length) { + const topContracts = data.defense.slice(0, 3).map(d => `$${((d.amount || 0) / 1e6).toFixed(0)}M to ${d.recipient}`); + sections.push(`DEFENSE_CONTRACTS: ${topContracts.join(', ')}`); + } + + // Delta context + if (delta?.summary) { + sections.push(`\nDELTA_SINCE_LAST_SWEEP: direction=${delta.summary.direction}, changes=${delta.summary.totalChanges}, critical=${delta.summary.criticalChanges}`); + if (delta.signals?.escalated?.length) { + sections.push(`ESCALATED: ${delta.signals.escalated.map(s => `${s.label}: ${s.previous}→${s.current} (${(s.changePct||0) > 0 ? '+' : ''}${(s.changePct||0).toFixed(1)}%)`).join(', ')}`); + } + if (delta.signals?.new?.length) { + sections.push(`NEW_SIGNALS: ${delta.signals.new.map(s => s.label || s.text?.substring(0, 60)).join('; ')}`); + } + } + + // Previous ideas (for dedup) + if (previousIdeas.length) { + sections.push(`\nPREVIOUS_IDEAS (avoid repeating):\n${previousIdeas.map(i => `- ${i.title} [${i.type}]`).join('\n')}`); + } + + return sections.join('\n'); +} + +/** + * Parse LLM response into ideas array. Handles markdown code blocks. + */ +function parseIdeasResponse(text) { + if (!text) return null; + + // Strip markdown code block wrappers + let cleaned = text.trim(); + if (cleaned.startsWith('```')) { + cleaned = cleaned.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, ''); + } + + try { + const parsed = JSON.parse(cleaned); + if (!Array.isArray(parsed)) return null; + + // Validate each idea has required fields + return parsed.filter(idea => + idea.title && idea.type && idea.confidence + ).map(idea => ({ + title: idea.title, + type: idea.type, + ticker: idea.ticker || '', + confidence: idea.confidence, + rationale: idea.rationale || '', + risk: idea.risk || '', + horizon: idea.horizon || '', + signals: idea.signals || [], + source: 'llm', + })); + } catch { + // Try to extract JSON array from mixed text + const match = cleaned.match(/\[[\s\S]*\]/); + if (match) { + try { + const arr = JSON.parse(match[0]); + return arr.filter(i => i.title && i.type).map(idea => ({ + ...idea, + source: 'llm', + })); + } catch { /* give up */ } + } + return null; + } +} diff --git a/lib/llm/index.mjs b/lib/llm/index.mjs new file mode 100644 index 0000000..5711350 --- /dev/null +++ b/lib/llm/index.mjs @@ -0,0 +1,37 @@ +// LLM Factory — creates the configured provider or returns null + +import { AnthropicProvider } from './anthropic.mjs'; +import { OpenAIProvider } from './openai.mjs'; +import { GeminiProvider } from './gemini.mjs'; +import { CodexProvider } from './codex.mjs'; + +export { LLMProvider } from './provider.mjs'; +export { AnthropicProvider } from './anthropic.mjs'; +export { OpenAIProvider } from './openai.mjs'; +export { GeminiProvider } from './gemini.mjs'; +export { CodexProvider } from './codex.mjs'; + +/** + * Create an LLM provider based on config. + * @param {{ provider: string|null, apiKey: string|null, model: string|null }} llmConfig + * @returns {LLMProvider|null} + */ +export function createLLMProvider(llmConfig) { + if (!llmConfig?.provider) return null; + + const { provider, apiKey, model } = llmConfig; + + switch (provider.toLowerCase()) { + case 'anthropic': + return new AnthropicProvider({ apiKey, model }); + case 'openai': + return new OpenAIProvider({ apiKey, model }); + case 'gemini': + return new GeminiProvider({ apiKey, model }); + case 'codex': + return new CodexProvider({ model }); + default: + console.warn(`[LLM] Unknown provider "${provider}". LLM features disabled.`); + return null; + } +} diff --git a/lib/llm/openai.mjs b/lib/llm/openai.mjs new file mode 100644 index 0000000..7161215 --- /dev/null +++ b/lib/llm/openai.mjs @@ -0,0 +1,50 @@ +// OpenAI Provider — raw fetch, no SDK + +import { LLMProvider } from './provider.mjs'; + +export class OpenAIProvider extends LLMProvider { + constructor(config) { + super(config); + this.name = 'openai'; + this.apiKey = config.apiKey; + this.model = config.model || 'gpt-4o'; + } + + get isConfigured() { return !!this.apiKey; } + + async complete(systemPrompt, userMessage, opts = {}) { + const res = await fetch('https://api.openai.com/v1/chat/completions', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${this.apiKey}`, + }, + body: JSON.stringify({ + model: this.model, + max_tokens: opts.maxTokens || 4096, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: userMessage }, + ], + }), + signal: AbortSignal.timeout(opts.timeout || 60000), + }); + + if (!res.ok) { + const err = await res.text().catch(() => ''); + throw new Error(`OpenAI API ${res.status}: ${err.substring(0, 200)}`); + } + + const data = await res.json(); + const text = data.choices?.[0]?.message?.content || ''; + + return { + text, + usage: { + inputTokens: data.usage?.prompt_tokens || 0, + outputTokens: data.usage?.completion_tokens || 0, + }, + model: data.model || this.model, + }; + } +} diff --git a/lib/llm/provider.mjs b/lib/llm/provider.mjs new file mode 100644 index 0000000..ce36932 --- /dev/null +++ b/lib/llm/provider.mjs @@ -0,0 +1,18 @@ +// Base LLM Provider — all providers implement this interface + +export class LLMProvider { + constructor(config) { + this.config = config; + this.name = 'base'; + } + + /** + * Complete a prompt with system + user messages + * @returns {{ text: string, usage: { inputTokens: number, outputTokens: number }, model: string }} + */ + async complete(systemPrompt, userMessage, opts = {}) { + throw new Error(`${this.name}: complete() not implemented`); + } + + get isConfigured() { return false; } +} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..42c3545 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,830 @@ +{ + "name": "crucix", + "version": "2.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "crucix", + "version": "2.0.0", + "license": "ISC", + "dependencies": { + "express": "^5.1.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/accepts": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", + "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", + "license": "MIT", + "dependencies": { + "mime-types": "^3.0.0", + "negotiator": "^1.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/body-parser": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz", + "integrity": "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==", + "license": "MIT", + "dependencies": { + "bytes": "^3.1.2", + "content-type": "^1.0.5", + "debug": "^4.4.3", + "http-errors": "^2.0.0", + "iconv-lite": "^0.7.0", + "on-finished": "^2.4.1", + "qs": "^6.14.1", + "raw-body": "^3.0.1", + "type-is": "^2.0.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.0.1.tgz", + "integrity": "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.2.2.tgz", + "integrity": "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==", + "license": "MIT", + "engines": { + "node": ">=6.6.0" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", + "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", + "license": "MIT", + "dependencies": { + "accepts": "^2.0.0", + "body-parser": "^2.2.1", + "content-disposition": "^1.0.0", + "content-type": "^1.0.5", + "cookie": "^0.7.1", + "cookie-signature": "^1.2.1", + "debug": "^4.4.0", + "depd": "^2.0.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "finalhandler": "^2.1.0", + "fresh": "^2.0.0", + "http-errors": "^2.0.0", + "merge-descriptors": "^2.0.0", + "mime-types": "^3.0.0", + "on-finished": "^2.4.1", + "once": "^1.4.0", + "parseurl": "^1.3.3", + "proxy-addr": "^2.0.7", + "qs": "^6.14.0", + "range-parser": "^1.2.1", + "router": "^2.2.0", + "send": "^1.1.0", + "serve-static": "^2.2.0", + "statuses": "^2.0.1", + "type-is": "^2.0.1", + "vary": "^1.1.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", + "integrity": "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "on-finished": "^2.4.1", + "parseurl": "^1.3.3", + "statuses": "^2.0.1" + }, + "engines": { + "node": ">= 18.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz", + "integrity": "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "license": "MIT" + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", + "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/merge-descriptors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-2.0.0.tgz", + "integrity": "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz", + "integrity": "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", + "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.0.tgz", + "integrity": "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", + "integrity": "sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.7.0", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/router": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/router/-/router-2.2.0.tgz", + "integrity": "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.0", + "depd": "^2.0.0", + "is-promise": "^4.0.0", + "parseurl": "^1.3.3", + "path-to-regexp": "^8.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz", + "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", + "license": "MIT", + "dependencies": { + "content-type": "^1.0.5", + "media-typer": "^1.1.0", + "mime-types": "^3.0.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..87d96f5 --- /dev/null +++ b/package.json @@ -0,0 +1,22 @@ +{ + "name": "crucix", + "version": "2.0.0", + "description": "Local intelligence engine — 26 OSINT sources, live dashboard, auto-refresh, optional LLM layer.", + "type": "module", + "scripts": { + "dev": "node server.mjs", + "sweep": "node apis/briefing.mjs", + "inject": "node dashboard/inject.mjs", + "brief": "node apis/briefing.mjs", + "brief:save": "node apis/save-briefing.mjs" + }, + "keywords": ["osint", "intelligence", "dashboard", "geopolitical"], + "author": "Crucix", + "license": "ISC", + "engines": { + "node": ">=22" + }, + "dependencies": { + "express": "^5.1.0" + } +} diff --git a/server.mjs b/server.mjs new file mode 100644 index 0000000..049aae0 --- /dev/null +++ b/server.mjs @@ -0,0 +1,234 @@ +#!/usr/bin/env node +// Crucix Intelligence Engine — Dev Server +// Serves the Jarvis dashboard, runs sweep cycle, pushes live updates via SSE + +import express from 'express'; +import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs'; +import { dirname, join } from 'path'; +import { fileURLToPath } from 'url'; +import { exec } from 'child_process'; +import config from './crucix.config.mjs'; +import { fullBriefing } from './apis/briefing.mjs'; +import { synthesize, generateIdeas } from './dashboard/inject.mjs'; +import { MemoryManager } from './lib/delta/index.mjs'; +import { createLLMProvider } from './lib/llm/index.mjs'; +import { generateLLMIdeas } from './lib/llm/ideas.mjs'; +import { TelegramAlerter } from './lib/alerts/telegram.mjs'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const ROOT = __dirname; +const RUNS_DIR = join(ROOT, 'runs'); +const MEMORY_DIR = join(RUNS_DIR, 'memory'); + +// Ensure directories exist +for (const dir of [RUNS_DIR, MEMORY_DIR, join(MEMORY_DIR, 'cold')]) { + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); +} + +// === State === +let currentData = null; // Current synthesized dashboard data +let lastSweepTime = null; // Timestamp of last sweep +let sweepInProgress = false; +const startTime = Date.now(); +const sseClients = new Set(); + +// === Delta/Memory === +const memory = new MemoryManager(RUNS_DIR); + +// === LLM + Telegram === +const llmProvider = createLLMProvider(config.llm); +const telegramAlerter = new TelegramAlerter(config.telegram); + +if (llmProvider) console.log(`[Crucix] LLM enabled: ${llmProvider.name} (${llmProvider.model})`); +if (telegramAlerter.isConfigured) console.log('[Crucix] Telegram alerts enabled'); + +// === Express Server === +const app = express(); +app.use(express.static(join(ROOT, 'dashboard/public'))); + +// Serve jarvis.html as the root page +app.get('/', (req, res) => { + res.sendFile(join(ROOT, 'dashboard/public/jarvis.html')); +}); + +// API: current data +app.get('/api/data', (req, res) => { + if (!currentData) return res.status(503).json({ error: 'No data yet — first sweep in progress' }); + res.json(currentData); +}); + +// API: health check +app.get('/api/health', (req, res) => { + res.json({ + status: 'ok', + uptime: Math.floor((Date.now() - startTime) / 1000), + lastSweep: lastSweepTime, + nextSweep: lastSweepTime + ? new Date(new Date(lastSweepTime).getTime() + config.refreshIntervalMinutes * 60000).toISOString() + : null, + sweepInProgress, + sourcesOk: currentData?.meta?.sourcesOk || 0, + sourcesFailed: currentData?.meta?.sourcesFailed || 0, + llmEnabled: !!config.llm.provider, + llmProvider: config.llm.provider, + telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId), + refreshIntervalMinutes: config.refreshIntervalMinutes, + }); +}); + +// SSE: live updates +app.get('/events', (req, res) => { + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + 'Connection': 'keep-alive', + 'Access-Control-Allow-Origin': '*', + }); + res.write('data: {"type":"connected"}\n\n'); + sseClients.add(res); + req.on('close', () => sseClients.delete(res)); +}); + +function broadcast(data) { + const msg = `data: ${JSON.stringify(data)}\n\n`; + for (const client of sseClients) { + try { client.write(msg); } catch { sseClients.delete(client); } + } +} + +// === Sweep Cycle === +async function runSweepCycle() { + if (sweepInProgress) { + console.log('[Crucix] Sweep already in progress, skipping'); + return; + } + + sweepInProgress = true; + broadcast({ type: 'sweep_start', timestamp: new Date().toISOString() }); + console.log(`\n${'='.repeat(60)}`); + console.log(`[Crucix] Starting sweep at ${new Date().toLocaleTimeString()}`); + console.log(`${'='.repeat(60)}`); + + try { + // 1. Run the full briefing sweep + const rawData = await fullBriefing(); + + // 2. Save to runs/latest.json + writeFileSync(join(RUNS_DIR, 'latest.json'), JSON.stringify(rawData, null, 2)); + lastSweepTime = new Date().toISOString(); + + // 3. Synthesize into dashboard format + console.log('[Crucix] Synthesizing dashboard data...'); + const synthesized = await synthesize(rawData); + + // 4. Delta computation + memory + const delta = memory.addRun(synthesized); + synthesized.delta = delta; + + // 5. LLM-powered trade ideas (LLM-only feature) — isolated so failures don't kill sweep + if (llmProvider?.isConfigured) { + try { + console.log('[Crucix] Generating LLM trade ideas...'); + const previousIdeas = memory.getLastRun()?.ideas || []; + const llmIdeas = await generateLLMIdeas(llmProvider, synthesized, delta, previousIdeas); + if (llmIdeas) { + synthesized.ideas = llmIdeas; + synthesized.ideasSource = 'llm'; + console.log(`[Crucix] LLM generated ${llmIdeas.length} ideas`); + } else { + synthesized.ideas = []; + synthesized.ideasSource = 'llm-failed'; + } + } catch (llmErr) { + console.error('[Crucix] LLM ideas failed (non-fatal):', llmErr.message); + synthesized.ideas = []; + synthesized.ideasSource = 'llm-failed'; + } + } else { + synthesized.ideas = []; + synthesized.ideasSource = 'disabled'; + } + + // 6. Telegram alert evaluation (LLM-gated) + if (telegramAlerter.isConfigured && llmProvider?.isConfigured && delta?.summary?.criticalChanges > 0) { + telegramAlerter.evaluateAndAlert(llmProvider, delta, memory).catch(err => { + console.error('[Crucix] Telegram alert error:', err.message); + }); + } + + // Prune old alerted signals + memory.pruneAlertedSignals(); + + currentData = synthesized; + + // 6. Push to all connected browsers + broadcast({ type: 'update', data: currentData }); + + console.log(`[Crucix] Sweep complete — ${currentData.meta.sourcesOk}/${currentData.meta.sourcesQueried} sources OK`); + console.log(`[Crucix] ${currentData.ideas.length} ideas (${synthesized.ideasSource}) | ${currentData.news.length} news | ${currentData.newsFeed.length} feed items`); + if (delta?.summary) console.log(`[Crucix] Delta: ${delta.summary.totalChanges} changes, ${delta.summary.criticalChanges} critical, direction: ${delta.summary.direction}`); + console.log(`[Crucix] Next sweep at ${new Date(Date.now() + config.refreshIntervalMinutes * 60000).toLocaleTimeString()}`); + + } catch (err) { + console.error('[Crucix] Sweep failed:', err.message); + broadcast({ type: 'sweep_error', error: err.message }); + } finally { + sweepInProgress = false; + } +} + +// === Startup === +async function start() { + const port = config.port; + + console.log(` + ╔══════════════════════════════════════════════╗ + ║ CRUCIX INTELLIGENCE ENGINE ║ + ║ Local Palantir · 26 Sources ║ + ╠══════════════════════════════════════════════╣ + ║ Dashboard: http://localhost:${port}${' '.repeat(14 - String(port).length)}║ + ║ Health: http://localhost:${port}/api/health${' '.repeat(4 - String(port).length)}║ + ║ Refresh: Every ${config.refreshIntervalMinutes} min${' '.repeat(20 - String(config.refreshIntervalMinutes).length)}║ + ║ LLM: ${(config.llm.provider || 'disabled').padEnd(31)}║ + ║ Alerts: ${config.telegram.botToken ? 'Telegram enabled' : 'disabled'}${' '.repeat(config.telegram.botToken ? 14 : 23)}║ + ╚══════════════════════════════════════════════╝ + `); + + app.listen(port, () => { + console.log(`[Crucix] Server running on http://localhost:${port}`); + + // Auto-open browser + const openCmd = process.platform === 'win32' ? 'start ""' : + process.platform === 'darwin' ? 'open' : 'xdg-open'; + exec(`${openCmd} "http://localhost:${port}"`, (err) => { + if (err) console.log('[Crucix] Could not auto-open browser:', err.message); + }); + + // Try to load existing data first for instant display + try { + const existing = JSON.parse(readFileSync(join(RUNS_DIR, 'latest.json'), 'utf8')); + synthesize(existing).then(data => { + currentData = data; + console.log('[Crucix] Loaded existing data from runs/latest.json'); + broadcast({ type: 'update', data: currentData }); + }).catch(() => {}); + } catch { /* no existing data */ } + + // Run first sweep + console.log('[Crucix] Running initial sweep...'); + runSweepCycle(); + + // Schedule recurring sweeps + setInterval(runSweepCycle, config.refreshIntervalMinutes * 60 * 1000); + }); +} + +// Graceful error handling +process.on('unhandledRejection', (err) => { + console.error('[Crucix] Unhandled rejection:', err.message || err); +}); +process.on('uncaughtException', (err) => { + console.error('[Crucix] Uncaught exception:', err.message || err); +}); + +start();