From 85f97bb2a655017f46399a84e38a4ce8ac79e897 Mon Sep 17 00:00:00 2001 From: MrSphay Date: Sat, 16 May 2026 21:18:34 +0200 Subject: [PATCH] feat: harden intelligence runtime and llm providers --- Dockerfile | 11 +- apis/briefing.mjs | 19 ++- apis/sources/yfinance.mjs | 6 +- apis/utils/fetch.mjs | 77 ++++++++++- crucix.config.mjs | 33 ++++- dashboard/inject.mjs | 29 ++++- docker-compose.yml | 22 +++- lib/alerts/discord.mjs | 12 +- lib/alerts/telegram.mjs | 12 +- lib/intelligence-store.mjs | 112 ++++++++++++++++ lib/llm/index.mjs | 36 +++++- lib/llm/ollama.mjs | 57 ++------ lib/llm/openai-compatible.mjs | 77 +++++++++++ lib/llm/openai.mjs | 53 ++------ lib/llm/openrouter.mjs | 54 ++------ package-lock.json | 12 +- package.json | 9 +- server.mjs | 193 ++++++++++++++++++++++++---- test/fetch-utils.test.mjs | 36 ++++++ test/llm-ollama.test.mjs | 10 +- test/llm-openai-compatible.test.mjs | 61 +++++++++ test/llm-openrouter.test.mjs | 2 +- 22 files changed, 732 insertions(+), 201 deletions(-) create mode 100644 lib/intelligence-store.mjs create mode 100644 lib/llm/openai-compatible.mjs create mode 100644 test/fetch-utils.test.mjs create mode 100644 test/llm-openai-compatible.test.mjs diff --git a/Dockerfile b/Dockerfile index af812de..f92c1d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,16 +4,21 @@ WORKDIR /app # Copy package files first for better layer caching COPY package*.json ./ -RUN npm install --production +RUN npm ci --omit=dev # Copy source COPY . . +RUN mkdir -p /app/runs /app/runs/memory /app/runs/memory/cold && chown -R node:node /app # Default port (override with -e PORT=xxxx) EXPOSE 3117 +ENV PORT=3117 \ + AUTO_OPEN_BROWSER=false \ + NODE_ENV=production # Health check -HEALTHCHECK --interval=60s --timeout=10s --retries=3 \ - CMD wget -qO- http://localhost:3117/api/health || exit 1 +HEALTHCHECK --interval=60s --timeout=10s --start-period=45s --retries=3 \ + CMD node -e "fetch('http://127.0.0.1:'+(process.env.PORT||3117)+'/api/health').then(r=>{if(![200,503].includes(r.status))process.exit(1);return r.json()}).then(j=>{if(['error'].includes(j.status))process.exit(1)}).catch(()=>process.exit(1))" +USER node CMD ["node", "server.mjs"] diff --git a/apis/briefing.mjs b/apis/briefing.mjs index 1ae2cff..2cb5f2e 100644 --- a/apis/briefing.mjs +++ b/apis/briefing.mjs @@ -58,7 +58,15 @@ export async function runSource(name, fn, ...args) { timer = setTimeout(() => reject(new Error(`Source ${name} timed out after ${SOURCE_TIMEOUT_MS / 1000}s`)), SOURCE_TIMEOUT_MS); }); const data = await Promise.race([dataPromise, timeoutPromise]); - return { name, status: 'ok', durationMs: Date.now() - start, data }; + const hasError = Boolean(data?.error); + const isDegraded = hasError || ['no_credentials', 'degraded', 'failed'].includes(data?.status); + return { + name, + status: isDegraded ? 'degraded' : 'ok', + durationMs: Date.now() - start, + data, + error: hasError ? data.error : null, + }; } catch (e) { return { name, status: 'error', durationMs: Date.now() - start, error: e.message }; } finally { @@ -127,14 +135,15 @@ export async function fullBriefing() { totalDurationMs: totalMs, sourcesQueried: sources.length, sourcesOk: sources.filter(s => s.status === 'ok').length, - sourcesFailed: sources.filter(s => s.status !== 'ok').length, + sourcesDegraded: sources.filter(s => s.status === 'degraded').length, + sourcesFailed: sources.filter(s => s.status === 'error' || s.status === 'failed').length, }, sources: Object.fromEntries( - sources.filter(s => s.status === 'ok').map(s => [s.name, s.data]) + sources.filter(s => s.status === 'ok' || s.status === 'degraded').map(s => [s.name, s.data]) ), - errors: sources.filter(s => s.status !== 'ok').map(s => ({ name: s.name, error: s.error })), + errors: sources.filter(s => s.status !== 'ok').map(s => ({ name: s.name, status: s.status, error: s.error || s.data?.message || 'degraded' })), timing: Object.fromEntries( - sources.map(s => [s.name, { status: s.status, ms: s.durationMs }]) + sources.map(s => [s.name, { status: s.status, ms: s.durationMs, error: s.error || null }]) ), }; diff --git a/apis/sources/yfinance.mjs b/apis/sources/yfinance.mjs index aa31345..bbb5a18 100644 --- a/apis/sources/yfinance.mjs +++ b/apis/sources/yfinance.mjs @@ -35,13 +35,17 @@ async function fetchQuote(symbol) { const url = `${BASE}/${encodeURIComponent(symbol)}?range=5d&interval=1d&includePrePost=false`; const data = await safeFetch(url, { timeout: 8000, + retries: 2, + source: `YFinance:${symbol}`, headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', + 'Accept': 'application/json,text/plain,*/*', }, }); + if (data?.error) throw new Error(data.error); const result = data?.chart?.result?.[0]; - if (!result) return null; + if (!result) throw new Error(data?.chart?.error?.description || 'Yahoo response missing chart result'); const meta = result.meta || {}; const quotes = result.indicators?.quote?.[0] || {}; diff --git a/apis/utils/fetch.mjs b/apis/utils/fetch.mjs index 312859d..47b17d3 100644 --- a/apis/utils/fetch.mjs +++ b/apis/utils/fetch.mjs @@ -1,9 +1,47 @@ -// Shared fetch utility with timeout, retries, and error handling +// Shared fetch utility with timeout, retries, metrics, and error handling + +const fetchMetrics = { + requests: 0, + ok: 0, + failed: 0, + bytes: 0, + byHost: {}, + bySource: {}, + recent: [], +}; + +function metricBucket(map, key) { + if (!map[key]) map[key] = { requests: 0, ok: 0, failed: 0, bytes: 0, lastStatus: null, lastError: null, lastMs: 0 }; + return map[key]; +} + +function recordFetchMetric({ url, source = 'unknown', ok, status, bytes, durationMs, error }) { + let host = 'unknown'; + try { host = new URL(url).host; } catch { } + fetchMetrics.requests++; + fetchMetrics.bytes += bytes || 0; + if (ok) fetchMetrics.ok++; else fetchMetrics.failed++; + for (const bucket of [metricBucket(fetchMetrics.byHost, host), metricBucket(fetchMetrics.bySource, source)]) { + bucket.requests++; + bucket.bytes += bytes || 0; + bucket.lastStatus = status || null; + bucket.lastMs = durationMs || 0; + bucket.lastError = error || null; + if (ok) bucket.ok++; else bucket.failed++; + } + fetchMetrics.recent.unshift({ at: new Date().toISOString(), source, host, ok, status, bytes: bytes || 0, durationMs, error: error || null }); + fetchMetrics.recent = fetchMetrics.recent.slice(0, 100); +} + +export function getFetchMetrics() { + return JSON.parse(JSON.stringify(fetchMetrics)); +} export async function safeFetch(url, opts = {}) { - const { timeout = 15000, retries = 1, headers = {} } = opts; + const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts; let lastError; for (let i = 0; i <= retries; i++) { + const started = Date.now(); try { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeout); @@ -12,14 +50,23 @@ export async function safeFetch(url, opts = {}) { headers: { 'User-Agent': 'Crucix/1.0', ...headers }, }); clearTimeout(timer); + const status = res.status; if (!res.ok) { const body = await res.text().catch(() => ''); + recordFetchMetric({ url, source, ok: false, status, bytes: body.length, durationMs: Date.now() - started, error: `HTTP ${res.status}` }); throw new Error(`HTTP ${res.status}: ${body.slice(0, 200)}`); } const text = await res.text(); + recordFetchMetric({ url, source, ok: true, status, bytes: text.length, durationMs: Date.now() - started }); + const trimmed = text.trim(); + const contentType = res.headers.get('content-type') || ''; + if (contentType.includes('text/html') || trimmed.startsWith(' setTimeout(r, 2000 * (i + 1))); } @@ -27,6 +74,32 @@ export async function safeFetch(url, opts = {}) { return { error: lastError?.message || 'Unknown error', source: url }; } +export async function safeFetchText(url, opts = {}) { + const { timeout = 15000, retries = 1, headers = {}, source = undefined } = opts; + let lastError; + for (let i = 0; i <= retries; i++) { + const started = Date.now(); + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeout); + const res = await fetch(url, { + signal: controller.signal, + headers: { 'User-Agent': 'Crucix/1.0', ...headers }, + }); + clearTimeout(timer); + const text = await res.text(); + recordFetchMetric({ url, source, ok: res.ok, status: res.status, bytes: text.length, durationMs: Date.now() - started, error: res.ok ? null : `HTTP ${res.status}` }); + if (!res.ok) throw new Error(`HTTP ${res.status}: ${text.slice(0, 200)}`); + return { text, status: res.status, bytes: text.length }; + } catch (e) { + lastError = e; + recordFetchMetric({ url, source, ok: false, status: null, bytes: 0, durationMs: Date.now() - started, error: e.message }); + if (i < retries) await new Promise(r => setTimeout(r, 2000 * (i + 1))); + } + } + return { error: lastError?.message || 'Unknown error' }; +} + export function ago(hours) { return new Date(Date.now() - hours * 3600000).toISOString(); } diff --git a/crucix.config.mjs b/crucix.config.mjs index 887d760..8792df3 100644 --- a/crucix.config.mjs +++ b/crucix.config.mjs @@ -2,22 +2,47 @@ import "./apis/utils/env.mjs"; // Load .env first +function intEnv(name, fallback) { + const value = parseInt(process.env[name], 10); + return Number.isFinite(value) ? value : fallback; +} + +function floatEnv(name, fallback) { + const value = parseFloat(process.env[name]); + return Number.isFinite(value) ? value : fallback; +} + +function boolEnv(name, fallback = false) { + const value = process.env[name]; + if (value == null || value === '') return fallback; + return ['1', 'true', 'yes', 'on'].includes(String(value).toLowerCase()); +} + export default { - port: parseInt(process.env.PORT) || 3117, - refreshIntervalMinutes: parseInt(process.env.REFRESH_INTERVAL_MINUTES) || 15, + port: intEnv('PORT', 3117), + refreshIntervalMinutes: intEnv('REFRESH_INTERVAL_MINUTES', 15), + autoOpenBrowser: boolEnv('AUTO_OPEN_BROWSER', false), + staleDataMaxAgeMinutes: intEnv('STALE_DATA_MAX_AGE_MINUTES', 60), + sweepToken: process.env.SWEEP_TOKEN || null, llm: { provider: process.env.LLM_PROVIDER || null, // anthropic | openai | gemini | codex | openrouter | minimax | mistral | ollama | grok apiKey: process.env.LLM_API_KEY || null, model: process.env.LLM_MODEL || null, - baseUrl: process.env.OLLAMA_BASE_URL || null, + baseUrl: process.env.LLM_BASE_URL || process.env.OPENAI_BASE_URL || process.env.OLLAMA_BASE_URL || null, + temperature: floatEnv('LLM_TEMPERATURE', 0.2), + maxTokens: intEnv('LLM_MAX_TOKENS', 2000), + timeoutMs: intEnv('LLM_TIMEOUT_MS', 90000), + openRouterSiteUrl: process.env.OPENROUTER_SITE_URL || 'https://git.wilkensxl.de/MrSphay/intelligence-terminal', + openRouterAppName: process.env.OPENROUTER_APP_NAME || 'Intelligence Terminal', }, telegram: { botToken: process.env.TELEGRAM_BOT_TOKEN || null, chatId: process.env.TELEGRAM_CHAT_ID || null, - botPollingInterval: parseInt(process.env.TELEGRAM_POLL_INTERVAL) || 5000, + botPollingInterval: intEnv('TELEGRAM_POLL_INTERVAL', 5000), channels: process.env.TELEGRAM_CHANNELS || null, // Comma-separated extra channel IDs + briefVerbosity: process.env.BRIEF_VERBOSITY || 'standard', }, discord: { diff --git a/dashboard/inject.mjs b/dashboard/inject.mjs index 1b067a7..1c78935 100644 --- a/dashboard/inject.mjs +++ b/dashboard/inject.mjs @@ -12,6 +12,7 @@ import { exec } from 'child_process'; import config from '../crucix.config.mjs'; import { createLLMProvider } from '../lib/llm/index.mjs'; import { generateLLMIdeas } from '../lib/llm/ideas.mjs'; +import { safeFetchText } from '../apis/utils/fetch.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const ROOT = join(__dirname, '..'); @@ -146,10 +147,14 @@ function loadOpenSkyFallback(currentTimestamp) { } // === RSS Fetching === +const rssHealth = {}; + async function fetchRSS(url, source) { + const started = Date.now(); try { - const res = await fetch(url, { signal: AbortSignal.timeout(8000) }); - const xml = await res.text(); + const fetched = await safeFetchText(url, { signal: AbortSignal.timeout(8000), source: `RSS:${source}`, timeout: 12000, retries: 1 }); + if (fetched.error) throw new Error(fetched.error); + const xml = fetched.text; const items = []; const itemRegex = /([\s\S]*?)<\/item>/g; let match; @@ -160,9 +165,11 @@ async function fetchRSS(url, source) { const pubDate = block.match(/(.*?)<\/pubDate>/)?.[1] || ''; if (title && title !== source) items.push({ title, date: pubDate, source, url: link || undefined }); } + rssHealth[source] = { status: items.length ? 'ok' : 'degraded', items: items.length, durationMs: Date.now() - started, url }; return items; } catch (e) { console.log(`RSS fetch failed (${source}):`, e.message); + rssHealth[source] = { status: 'failed', items: 0, durationMs: Date.now() - started, url, error: e.message }; return []; } } @@ -176,6 +183,7 @@ const RSS_SOURCE_FALLBACKS = { const REGIONAL_NEWS_SOURCES = ['MercoPress', 'Indian Express', 'The Hindu', 'SBS Australia']; export async function fetchAllNews() { + for (const key of Object.keys(rssHealth)) delete rssHealth[key]; const feeds = [ // Global ['http://feeds.bbci.co.uk/news/world/rss.xml', 'BBC'], @@ -256,6 +264,10 @@ export async function fetchAllNews() { return selected.slice(0, 50); } +export function getRSSHealth() { + return { ...rssHealth }; +} + // === Leverageable Ideas from Signals === export function generateIdeas(V2) { const ideas = []; @@ -540,7 +552,12 @@ export async function synthesize(data) { }; const health = Object.entries(data.sources).map(([name, src]) => ({ - n: name, err: Boolean(src.error), stale: Boolean(src.stale) + n: name, + status: data.timing?.[name]?.status || (src.error ? 'degraded' : 'ok'), + err: Boolean(src.error || data.timing?.[name]?.status === 'error'), + stale: Boolean(src.stale), + message: src.error || src.message || data.timing?.[name]?.error || null, + ms: data.timing?.[name]?.ms || 0, })); // === Yahoo Finance live market data === @@ -595,6 +612,7 @@ export async function synthesize(data) { // Fetch RSS const news = await fetchAllNews(); + const newsHealth = getRSSHealth(); const V2 = { meta: data.crucix, air, thermal, tSignals, chokepoints, nuke, nukeSignals, @@ -610,6 +628,11 @@ export async function synthesize(data) { tg: { posts: tgData.totalPosts || 0, urgent: tgUrgent, topPosts: tgTop }, who, fred, energy, metals, bls, treasury, gscpi, defense, noaa, epa, acled, gdelt, space, health, news, markets, // Live Yahoo Finance market data + newsMeta: { + rssHealth: newsHealth, + rssOk: Object.values(newsHealth).filter(s => s.status === 'ok').length, + rssFailed: Object.values(newsHealth).filter(s => s.status === 'failed').length, + }, ideas: [], ideasSource: 'disabled', // newsFeed for ticker (merged RSS + GDELT + Telegram) newsFeed: buildNewsFeed(news, gdeltData, tgUrgent, tgTop), diff --git a/docker-compose.yml b/docker-compose.yml index cdad7b3..f152ece 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,24 @@ services: - crucix: - build: . + intelligence-terminal: + image: git.wilkensxl.de/mrsphay/intelligence-terminal:latest + build: + context: . + dockerfile: Dockerfile + container_name: intelligence-terminal + env_file: + - path: .env + required: false + environment: + PORT: ${PORT:-3117} + AUTO_OPEN_BROWSER: "false" ports: - "${PORT:-3117}:${PORT:-3117}" - env_file: - - .env volumes: - ./runs:/app/runs restart: unless-stopped + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://127.0.0.1:'+(process.env.PORT||3117)+'/api/health').then(r=>{if(![200,503].includes(r.status))process.exit(1);return r.json()}).then(j=>{if(j.status==='error')process.exit(1)}).catch(()=>process.exit(1))"] + interval: 60s + timeout: 10s + start_period: 45s + retries: 3 diff --git a/lib/alerts/discord.mjs b/lib/alerts/discord.mjs index ba0ce3f..0d73e05 100644 --- a/lib/alerts/discord.mjs +++ b/lib/alerts/discord.mjs @@ -59,9 +59,6 @@ export class DiscordAlerter { intents: [GatewayIntentBits.Guilds], }); - // Register slash commands - await this._registerCommands(REST, Routes, SlashCommandBuilder); - // Handle slash command interactions this._client.on('interactionCreate', async (interaction) => { if (!interaction.isChatInputCommand()) return; @@ -71,9 +68,10 @@ export class DiscordAlerter { // Connect await this._client.login(this.botToken); - this._client.once('ready', () => { + this._client.once('clientReady', async () => { this._ready = true; console.log(`[Discord] Bot online as ${this._client.user.tag}`); + await this._registerCommands(REST, Routes, SlashCommandBuilder); }); } catch (err) { @@ -123,11 +121,13 @@ export class DiscordAlerter { try { if (this.guildId) { // Guild commands (instant, for development) - await rest.put(Routes.applicationGuildCommands(this._client?.user?.id || 'me', this.guildId), { body: commands }); + const appId = this._client?.application?.id || this._client?.user?.id; + if (!appId) throw new Error('Discord application id unavailable after login'); + await rest.put(Routes.applicationGuildCommands(appId, this.guildId), { body: commands }); console.log(`[Discord] Registered ${commands.length} guild slash commands`); } else { // Global commands (can take up to 1h to propagate) - const appId = this._client?.application?.id; + const appId = this._client?.application?.id || this._client?.user?.id; if (appId) { await rest.put(Routes.applicationCommands(appId), { body: commands }); console.log(`[Discord] Registered ${commands.length} global slash commands`); diff --git a/lib/alerts/telegram.mjs b/lib/alerts/telegram.mjs index f7bba56..5a1c343 100644 --- a/lib/alerts/telegram.mjs +++ b/lib/alerts/telegram.mjs @@ -41,6 +41,8 @@ export class TelegramAlerter { this._commandHandlers = {}; // Registered command callbacks this._pollingInterval = null; this._botUsername = null; + this._pollFailureCount = 0; + this._lastPollErrorLogAt = 0; } get isConfigured() { @@ -353,6 +355,7 @@ export class TelegramAlerter { const data = await res.json(); if (!data.ok || !Array.isArray(data.result)) return; + this._pollFailureCount = 0; for (const update of data.result) { this._lastUpdateId = Math.max(this._lastUpdateId, update.update_id); @@ -366,9 +369,14 @@ export class TelegramAlerter { await this._handleMessage(msg); } } catch (err) { - // Silent — polling failures are non-fatal + this._pollFailureCount++; if (!err.message?.includes('aborted')) { - console.error('[Telegram] Poll error:', err.message); + const now = Date.now(); + const quietMs = Math.min(300000, 30000 * this._pollFailureCount); + if (now - this._lastPollErrorLogAt > quietMs) { + this._lastPollErrorLogAt = now; + console.error(`[Telegram] Poll degraded (${this._pollFailureCount} consecutive failures):`, err.message); + } } } } diff --git a/lib/intelligence-store.mjs b/lib/intelligence-store.mjs new file mode 100644 index 0000000..10daf40 --- /dev/null +++ b/lib/intelligence-store.mjs @@ -0,0 +1,112 @@ +// Phase-1 intelligence memory. Uses node:sqlite when available and degrades to no-op. + +import { existsSync, mkdirSync, writeFileSync } from 'fs'; +import { dirname, join } from 'path'; + +export class IntelligenceStore { + constructor(dbPath) { + this.dbPath = dbPath; + this.db = null; + this.available = false; + this.reason = null; + } + + async init() { + const dir = dirname(this.dbPath); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + try { + const sqlite = await import('node:sqlite'); + const DatabaseSync = sqlite.DatabaseSync; + this.db = new DatabaseSync(this.dbPath); + this.db.exec(` + CREATE TABLE IF NOT EXISTS runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + sources_ok INTEGER DEFAULT 0, + sources_degraded INTEGER DEFAULT 0, + sources_failed INTEGER DEFAULT 0, + direction TEXT, + summary_json TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS predictions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + created_at TEXT NOT NULL, + title TEXT NOT NULL, + type TEXT, + confidence TEXT, + source TEXT, + payload_json TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS entities ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL, + name TEXT NOT NULL, + kind TEXT NOT NULL, + count INTEGER DEFAULT 1, + UNIQUE(name, kind) + ); + `); + this.available = true; + } catch (err) { + this.available = false; + this.reason = err.message; + if (!existsSync(this.dbPath)) { + writeFileSync(this.dbPath, ''); + } + } + return this; + } + + recordRun(data, delta) { + if (!this.available || !this.db) return; + const meta = data.meta || {}; + const timestamp = meta.timestamp || new Date().toISOString(); + this.db.prepare(`INSERT INTO runs (timestamp, sources_ok, sources_degraded, sources_failed, direction, summary_json) + VALUES (?, ?, ?, ?, ?, ?)`).run( + timestamp, + meta.sourcesOk || 0, + meta.sourcesDegraded || 0, + meta.sourcesFailed || 0, + delta?.summary?.direction || null, + JSON.stringify({ meta, delta: delta?.summary || null }), + ); + for (const idea of data.ideas || []) { + this.db.prepare(`INSERT INTO predictions (created_at, title, type, confidence, source, payload_json) + VALUES (?, ?, ?, ?, ?, ?)`).run( + timestamp, + idea.title || 'Untitled idea', + idea.type || null, + idea.confidence || null, + idea.source || data.ideasSource || null, + JSON.stringify(idea), + ); + } + this._recordEntities(data, timestamp); + } + + status() { + return { available: this.available, path: this.dbPath, reason: this.reason }; + } + + _recordEntities(data, timestamp) { + const names = []; + for (const item of data.acled?.deadliestEvents || []) { + if (item.country) names.push([item.country, 'country']); + if (item.location) names.push([item.location, 'location']); + } + for (const item of data.news || []) { + if (item.region) names.push([item.region, 'region']); + } + for (const [name, kind] of names.slice(0, 200)) { + this.db.prepare(`INSERT INTO entities (first_seen, last_seen, name, kind, count) + VALUES (?, ?, ?, ?, 1) + ON CONFLICT(name, kind) DO UPDATE SET last_seen=excluded.last_seen, count=count+1`).run( + timestamp, + timestamp, + String(name).slice(0, 160), + kind, + ); + } + } +} diff --git a/lib/llm/index.mjs b/lib/llm/index.mjs index 21fd64a..241b7e6 100644 --- a/lib/llm/index.mjs +++ b/lib/llm/index.mjs @@ -9,6 +9,7 @@ import { MiniMaxProvider } from "./minimax.mjs"; import { MistralProvider } from "./mistral.mjs"; import { OllamaProvider } from "./ollama.mjs"; import { GrokProvider } from "./grok.mjs"; +import { OpenAICompatibleProvider } from "./openai-compatible.mjs"; export { LLMProvider } from "./provider.mjs"; export { AnthropicProvider } from "./anthropic.mjs"; @@ -20,6 +21,7 @@ export { MiniMaxProvider } from "./minimax.mjs"; export { MistralProvider } from "./mistral.mjs"; export { OllamaProvider } from "./ollama.mjs"; export { GrokProvider } from "./grok.mjs"; +export { OpenAICompatibleProvider } from "./openai-compatible.mjs"; /** * Create an LLM provider based on config. @@ -30,14 +32,42 @@ export function createLLMProvider(llmConfig) { if (!llmConfig?.provider) return null; const { provider, apiKey, model } = llmConfig; + const common = { + apiKey, + model, + baseUrl: llmConfig.baseUrl, + temperature: llmConfig.temperature, + maxTokens: llmConfig.maxTokens, + timeoutMs: llmConfig.timeoutMs, + openRouterSiteUrl: llmConfig.openRouterSiteUrl, + openRouterAppName: llmConfig.openRouterAppName, + }; switch (provider.toLowerCase()) { case "anthropic": return new AnthropicProvider({ apiKey, model }); case "openai": - return new OpenAIProvider({ apiKey, model }); + return new OpenAIProvider(common); + case "openai-compatible": + case "compatible": + case "local-openai": + return new OpenAICompatibleProvider({ + ...common, + name: provider.toLowerCase(), + model: model || 'local-model', + requiresApiKey: Boolean(apiKey), + }); + case "lmstudio": + case "lm-studio": + return new OpenAICompatibleProvider({ + ...common, + name: 'lmstudio', + baseUrl: llmConfig.baseUrl || 'http://localhost:1234/v1', + model: model || 'local-model', + requiresApiKey: false, + }); case "openrouter": - return new OpenRouterProvider({ apiKey, model }); + return new OpenRouterProvider(common); case "gemini": return new GeminiProvider({ apiKey, model }); case "codex": @@ -47,7 +77,7 @@ export function createLLMProvider(llmConfig) { case "mistral": return new MistralProvider({ apiKey, model }); case "ollama": - return new OllamaProvider({ model, baseUrl: llmConfig.baseUrl }); + return new OllamaProvider(common); case 'grok': return new GrokProvider({ apiKey, model }); default: diff --git a/lib/llm/ollama.mjs b/lib/llm/ollama.mjs index 5bb509a..2ef4830 100644 --- a/lib/llm/ollama.mjs +++ b/lib/llm/ollama.mjs @@ -1,49 +1,20 @@ -// Ollama Provider — raw fetch, no SDK -// Uses Ollama's OpenAI-compatible Chat Completions API -// No API key required — fully local inference +// Ollama Provider — OpenAI-compatible Chat Completions API -import { LLMProvider } from './provider.mjs'; +import { OpenAICompatibleProvider } from './openai-compatible.mjs'; -export class OllamaProvider extends LLMProvider { +export class OllamaProvider extends OpenAICompatibleProvider { constructor(config) { - super(config); - this.name = 'ollama'; - this.baseUrl = (config.baseUrl || 'http://localhost:11434').replace(/\/+$/, ''); - this.model = config.model || 'llama3.1:8b'; - } - - get isConfigured() { return !!this.model; } - - async complete(systemPrompt, userMessage, opts = {}) { - const res = await fetch(`${this.baseUrl}/v1/chat/completions`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - model: this.model, - max_tokens: opts.maxTokens || 4096, - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: userMessage }, - ], - }), - signal: AbortSignal.timeout(opts.timeout || 120000), + const rawBaseUrl = config.baseUrl || 'http://localhost:11434'; + const baseUrl = rawBaseUrl.replace(/\/+$/, '').endsWith('/v1') + ? rawBaseUrl + : `${rawBaseUrl.replace(/\/+$/, '')}/v1`; + super({ + ...config, + name: 'ollama', + baseUrl, + model: config.model || 'llama3.1:8b', + requiresApiKey: false, + timeoutMs: config.timeoutMs || 120000, }); - - if (!res.ok) { - const err = await res.text().catch(() => ''); - throw new Error(`Ollama API ${res.status}: ${err.substring(0, 200)}`); - } - - const data = await res.json(); - const text = data.choices?.[0]?.message?.content || ''; - - return { - text, - usage: { - inputTokens: data.usage?.prompt_tokens || 0, - outputTokens: data.usage?.completion_tokens || 0, - }, - model: data.model || this.model, - }; } } diff --git a/lib/llm/openai-compatible.mjs b/lib/llm/openai-compatible.mjs new file mode 100644 index 0000000..99e39e3 --- /dev/null +++ b/lib/llm/openai-compatible.mjs @@ -0,0 +1,77 @@ +// OpenAI-compatible provider for OpenRouter, LM Studio, Ollama, Groq, xAI, and local endpoints. + +import { LLMProvider } from './provider.mjs'; + +const DEFAULT_BASE_URL = 'https://api.openai.com/v1'; + +export class OpenAICompatibleProvider extends LLMProvider { + constructor(config = {}) { + super(config); + this.name = config.name || 'openai-compatible'; + this.apiKey = config.apiKey || null; + this.baseUrl = (config.baseUrl || DEFAULT_BASE_URL).replace(/\/+$/, ''); + this.model = config.model || 'gpt-4o-mini'; + this.temperature = config.temperature ?? 0.2; + this.maxTokens = config.maxTokens || 2000; + this.timeoutMs = config.timeoutMs || 90000; + this.extraHeaders = config.extraHeaders || {}; + this.requiresApiKey = config.requiresApiKey ?? true; + this.useMaxCompletionTokens = Boolean(config.useMaxCompletionTokens); + } + + get isConfigured() { + return Boolean(this.model && (!this.requiresApiKey || this.apiKey)); + } + + get status() { + if (!this.model) return { state: 'misconfigured', reason: 'LLM_MODEL is required' }; + if (this.requiresApiKey && !this.apiKey) return { state: 'misconfigured', reason: 'LLM_API_KEY is required' }; + return { state: 'configured', provider: this.name, model: this.model, baseUrl: this.baseUrl }; + } + + async complete(systemPrompt, userMessage, opts = {}) { + const maxTokens = opts.maxTokens || this.maxTokens; + const timeout = opts.timeout || this.timeoutMs; + const body = { + model: this.model, + temperature: opts.temperature ?? this.temperature, + messages: [ + { role: 'system', content: systemPrompt }, + { role: 'user', content: userMessage }, + ], + }; + body[this.useMaxCompletionTokens ? 'max_completion_tokens' : 'max_tokens'] = maxTokens; + + const headers = { + 'Content-Type': 'application/json', + ...this.extraHeaders, + }; + if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`; + + const res = await fetch(`${this.baseUrl}/chat/completions`, { + method: 'POST', + headers, + body: JSON.stringify(body), + signal: AbortSignal.timeout(timeout), + }); + + if (!res.ok) { + const err = await res.text().catch(() => ''); + const detail = err.substring(0, 300); + if (res.status === 401 || res.status === 403) throw new Error(`${this.name} auth failed (${res.status}): ${detail}`); + if (res.status === 408 || res.status === 429) throw new Error(`${this.name} rate limited or timed out (${res.status}): ${detail}`); + throw new Error(`${this.name} API ${res.status}: ${detail}`); + } + + const data = await res.json(); + const text = data.choices?.[0]?.message?.content || data.choices?.[0]?.text || ''; + return { + text, + usage: { + inputTokens: data.usage?.prompt_tokens || data.usage?.input_tokens || 0, + outputTokens: data.usage?.completion_tokens || data.usage?.output_tokens || 0, + }, + model: data.model || this.model, + }; + } +} diff --git a/lib/llm/openai.mjs b/lib/llm/openai.mjs index 8db5987..2cfa397 100644 --- a/lib/llm/openai.mjs +++ b/lib/llm/openai.mjs @@ -1,50 +1,15 @@ -// OpenAI Provider — raw fetch, no SDK +// OpenAI Provider — OpenAI-compatible chat completions -import { LLMProvider } from './provider.mjs'; +import { OpenAICompatibleProvider } from './openai-compatible.mjs'; -export class OpenAIProvider extends LLMProvider { +export class OpenAIProvider extends OpenAICompatibleProvider { constructor(config) { - super(config); - this.name = 'openai'; - this.apiKey = config.apiKey; - this.model = config.model || 'gpt-5.4'; - } - - get isConfigured() { return !!this.apiKey; } - - async complete(systemPrompt, userMessage, opts = {}) { - const res = await fetch('https://api.openai.com/v1/chat/completions', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${this.apiKey}`, - }, - body: JSON.stringify({ - model: this.model, - max_completion_tokens: opts.maxTokens || 4096, - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: userMessage }, - ], - }), - signal: AbortSignal.timeout(opts.timeout || 60000), + super({ + ...config, + name: 'openai', + baseUrl: config.baseUrl || 'https://api.openai.com/v1', + model: config.model || 'gpt-4o-mini', + useMaxCompletionTokens: true, }); - - if (!res.ok) { - const err = await res.text().catch(() => ''); - throw new Error(`OpenAI API ${res.status}: ${err.substring(0, 200)}`); - } - - const data = await res.json(); - const text = data.choices?.[0]?.message?.content || ''; - - return { - text, - usage: { - inputTokens: data.usage?.prompt_tokens || 0, - outputTokens: data.usage?.completion_tokens || 0, - }, - model: data.model || this.model, - }; } } diff --git a/lib/llm/openrouter.mjs b/lib/llm/openrouter.mjs index 4805de8..569bd68 100644 --- a/lib/llm/openrouter.mjs +++ b/lib/llm/openrouter.mjs @@ -1,52 +1,20 @@ -// OpenRouter Provider — raw fetch, no SDK +// OpenRouter Provider — OpenAI-compatible chat completions -import { LLMProvider } from './provider.mjs'; +import { OpenAICompatibleProvider } from './openai-compatible.mjs'; -export class OpenRouterProvider extends LLMProvider { +export class OpenRouterProvider extends OpenAICompatibleProvider { constructor(config) { - super(config); - this.name = 'openrouter'; - this.apiKey = config.apiKey; - this.model = config.model || 'openrouter/auto'; - } - - get isConfigured() { return !!this.apiKey; } - - async complete(systemPrompt, userMessage, opts = {}) { - const res = await fetch('https://openrouter.ai/api/v1/chat/completions', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${this.apiKey}`, + super({ + ...config, + name: 'openrouter', + baseUrl: config.baseUrl || 'https://openrouter.ai/api/v1', + model: config.model || 'openrouter/free', + extraHeaders: { 'HTTP-Referer': 'https://github.com/calesthio/Crucix', 'X-Title': 'Crucix', + ...(config.openRouterSiteUrl ? { 'HTTP-Referer': config.openRouterSiteUrl } : {}), + ...(config.openRouterAppName ? { 'X-Title': config.openRouterAppName } : {}), }, - body: JSON.stringify({ - model: this.model, - max_tokens: opts.maxTokens || 4096, - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: userMessage }, - ], - }), - signal: AbortSignal.timeout(opts.timeout || 60000), }); - - if (!res.ok) { - const err = await res.text().catch(() => ''); - throw new Error(`OpenRouter API ${res.status}: ${err.substring(0, 200)}`); - } - - const data = await res.json(); - const text = data.choices?.[0]?.message?.content || ''; - - return { - text, - usage: { - inputTokens: data.usage?.prompt_tokens || 0, - outputTokens: data.usage?.completion_tokens || 0, - }, - model: data.model || this.model, - }; } } diff --git a/package-lock.json b/package-lock.json index 2dba6a6..b803cdd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -706,9 +706,9 @@ "license": "MIT" }, "node_modules/lodash": { - "version": "4.17.23", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", - "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", "license": "MIT", "optional": true }, @@ -839,9 +839,9 @@ } }, "node_modules/path-to-regexp": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", - "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.2.tgz", + "integrity": "sha512-qRcuIdP69NPm4qbACK+aDogI5CBDMi1jKe0ry5rSQJz8JVLsC7jV8XpiJjGRLLol3N+R5ihGYcrPLTno6pAdBA==", "license": "MIT", "funding": { "type": "opencollective", diff --git a/package.json b/package.json index 5b90bf2..4f3e7cc 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,9 @@ "brief": "node apis/briefing.mjs", "brief:save": "node apis/save-briefing.mjs", "diag": "node diag.mjs", + "test": "npm run test:unit", + "test:unit": "node --test test/llm-openrouter.test.mjs test/llm-ollama.test.mjs test/llm-openai-compatible.test.mjs test/fetch-utils.test.mjs", + "compose:config": "docker compose config", "clean": "node scripts/clean.mjs", "fresh-start": "npm run clean && npm start" }, @@ -30,7 +33,9 @@ "express": "^5.1.0" }, "optionalDependencies": { - "discord.js": "^14.25.1" }, + "discord.js": "^14.25.1" + }, "overrides": { - "undici": "^7.24.4" } + "undici": "^7.24.4" + } } diff --git a/server.mjs b/server.mjs index 3c094b4..8f46df8 100644 --- a/server.mjs +++ b/server.mjs @@ -16,6 +16,8 @@ import { createLLMProvider } from './lib/llm/index.mjs'; import { generateLLMIdeas } from './lib/llm/ideas.mjs'; import { TelegramAlerter } from './lib/alerts/telegram.mjs'; import { DiscordAlerter } from './lib/alerts/discord.mjs'; +import { getFetchMetrics } from './apis/utils/fetch.mjs'; +import { IntelligenceStore } from './lib/intelligence-store.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); const ROOT = __dirname; @@ -30,6 +32,9 @@ for (const dir of [RUNS_DIR, MEMORY_DIR, join(MEMORY_DIR, 'cold')]) { // === State === let currentData = null; // Current synthesized dashboard data let lastSweepTime = null; // Timestamp of last sweep +let lastSuccessfulSweepTime = null; +let lastSweepError = null; +let bootstrapDataLoaded = false; let sweepStartedAt = null; // Timestamp when current/last sweep started let sweepInProgress = false; const startTime = Date.now(); @@ -37,6 +42,8 @@ const sseClients = new Set(); // === Delta/Memory === const memory = new MemoryManager(RUNS_DIR); +const intelligenceStore = new IntelligenceStore(join(RUNS_DIR, 'intelligence.db')); +await intelligenceStore.init(); // === LLM + Telegram + Discord === const llmProvider = createLLMProvider(config.llm); @@ -44,6 +51,7 @@ const telegramAlerter = new TelegramAlerter(config.telegram); const discordAlerter = new DiscordAlerter(config.discord || {}); if (llmProvider) console.log(`[Crucix] LLM enabled: ${llmProvider.name} (${llmProvider.model})`); +else if (config.llm.provider) console.warn(`[Crucix] LLM provider "${config.llm.provider}" is not configured; LLM features disabled`); if (telegramAlerter.isConfigured) { console.log('[Crucix] Telegram alerts enabled'); @@ -84,6 +92,7 @@ if (telegramAlerter.isConfigured) { telegramAlerter.onCommand('/brief', async () => { if (!currentData) return '⏳ No data yet — waiting for first sweep to complete.'; + return buildBrief(currentData); const tg = currentData.tg || {}; const energy = currentData.energy || {}; @@ -181,6 +190,7 @@ if (discordAlerter.isConfigured) { discordAlerter.onCommand('brief', async () => { if (!currentData) return '⏳ No data yet — waiting for first sweep to complete.'; + return buildBrief(currentData); const tg = currentData.tg || {}; const energy = currentData.energy || {}; @@ -261,25 +271,34 @@ app.get('/api/data', (req, res) => { // API: health check app.get('/api/health', (req, res) => { + const health = buildHealth(); + const httpStatus = health.status === 'error' ? 500 : health.status === 'starting' ? 503 : 200; + res.status(httpStatus).json(health); +}); + +// API: network/source/LLM metrics +app.get('/api/metrics', (req, res) => { res.json({ - status: 'ok', - uptime: Math.floor((Date.now() - startTime) / 1000), - lastSweep: lastSweepTime, - nextSweep: lastSweepTime - ? new Date(new Date(lastSweepTime).getTime() + config.refreshIntervalMinutes * 60000).toISOString() - : null, - sweepInProgress, - sweepStartedAt, - sourcesOk: currentData?.meta?.sourcesOk || 0, - sourcesFailed: currentData?.meta?.sourcesFailed || 0, - llmEnabled: !!config.llm.provider, - llmProvider: config.llm.provider, - telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId), - refreshIntervalMinutes: config.refreshIntervalMinutes, - language: currentLanguage, + generatedAt: new Date().toISOString(), + fetch: getFetchMetrics(), + sources: currentData?.sourceHealth || currentData?.health || [], + news: currentData?.newsMeta || {}, + llm: getLLMStatus(), + memory: intelligenceStore.status(), }); }); +app.post('/api/sweep', express.json(), (req, res) => { + const remote = req.ip || ''; + const local = remote.includes('127.0.0.1') || remote === '::1' || remote === '::ffff:127.0.0.1'; + const token = req.get('x-crucix-token') || req.query.token || req.body?.token; + if (config.sweepToken && token !== config.sweepToken) return res.status(401).json({ error: 'Invalid sweep token' }); + if (!config.sweepToken && !local) return res.status(403).json({ error: 'Manual sweep is local-only unless SWEEP_TOKEN is set' }); + if (sweepInProgress) return res.status(409).json({ status: 'already_running', sweepStartedAt }); + runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message)); + res.status(202).json({ status: 'accepted' }); +}); + // API: available locales app.get('/api/locales', (req, res) => { res.json({ @@ -308,6 +327,101 @@ function broadcast(data) { } } +function dataAgeMs() { + const ts = currentData?.meta?.timestamp || lastSuccessfulSweepTime || lastSweepTime; + const ms = ts ? Date.now() - new Date(ts).getTime() : null; + return Number.isFinite(ms) ? ms : null; +} + +function getLLMStatus() { + if (!config.llm.provider) return { state: 'disabled' }; + if (!llmProvider) return { state: 'misconfigured', provider: config.llm.provider }; + return typeof llmProvider.status === 'object' + ? llmProvider.status + : { state: llmProvider.isConfigured ? 'configured' : 'misconfigured', provider: llmProvider.name, model: llmProvider.model }; +} + +function buildHealth() { + const ageMs = dataAgeMs(); + const stale = ageMs != null && ageMs > config.staleDataMaxAgeMinutes * 60 * 1000; + const sourcesFailed = currentData?.meta?.sourcesFailed || 0; + const sourcesDegraded = currentData?.meta?.sourcesDegraded || 0; + const status = lastSweepError + ? 'error' + : !currentData + ? 'starting' + : stale + ? 'stale' + : (sourcesFailed > 0 || sourcesDegraded > 0) + ? 'degraded' + : 'healthy'; + return { + status, + uptime: Math.floor((Date.now() - startTime) / 1000), + lastSweep: lastSweepTime, + lastSuccessfulSweep: lastSuccessfulSweepTime, + nextSweep: lastSweepTime + ? new Date(new Date(lastSweepTime).getTime() + config.refreshIntervalMinutes * 60000).toISOString() + : null, + dataAgeSeconds: ageMs == null ? null : Math.floor(ageMs / 1000), + stale, + bootstrapDataLoaded, + sweepInProgress, + sweepStartedAt, + lastSweepError, + sourcesOk: currentData?.meta?.sourcesOk || 0, + sourcesDegraded, + sourcesFailed, + sourceHealth: currentData?.sourceHealth || currentData?.health || [], + llm: getLLMStatus(), + telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId), + discordEnabled: !!(config.discord?.botToken || config.discord?.webhookUrl), + refreshIntervalMinutes: config.refreshIntervalMinutes, + language: currentLanguage, + memory: intelligenceStore.status(), + }; +} + +function buildBrief(data) { + const verbosity = config.telegram.briefVerbosity || 'standard'; + const delta = memory.getLastDelta(); + const sourceRows = (data.sourceHealth || data.health || []).slice(0, verbosity === 'audit' ? 12 : 6); + const degraded = sourceRows.filter(s => s.status && s.status !== 'ok'); + const evidence = [ + ...(data.newsFeed || []).filter(n => n.url).slice(0, 4), + ...(data.news || []).filter(n => n.url).slice(0, 4), + ].slice(0, verbosity === 'compact' ? 3 : 6); + const ideas = (data.ideas || []).slice(0, verbosity === 'compact' ? 2 : 4); + const vix = data.fred?.find(f => f.id === 'VIXCLS'); + const id = `evt-${Buffer.from(`${data.meta?.timestamp || Date.now()}-${delta?.summary?.direction || 'mixed'}`).toString('base64url').slice(0, 10)}`; + + const lines = [ + '*CRUCIX BRIEF*', + `_${new Date().toISOString().replace('T', ' ').substring(0, 19)} UTC_`, + `Event ID: \`${id}\``, + '', + `Direction: *${(delta?.summary?.direction || 'mixed').toUpperCase()}* | Changes: ${delta?.summary?.totalChanges || 0} | Critical: ${delta?.summary?.criticalChanges || 0}`, + `VIX: ${vix?.value || '--'} | WTI: $${data.energy?.wti || '--'} | Gold: $${data.metals?.gold || '--'} | Sources: ${data.meta?.sourcesOk || 0}/${data.meta?.sourcesQueried || 0} OK`, + '', + '*Source Integrity*', + degraded.length + ? degraded.map(s => `- ${s.name || s.n}: ${s.status || 'degraded'}${s.error || s.message ? ` (${String(s.error || s.message).slice(0, 80)})` : ''}`).join('\n') + : '- Strong: no degraded source in the sampled health set', + '', + '*Top Evidence*', + evidence.length + ? evidence.map((n, idx) => `${idx + 1}. ${n.source || 'source'}: ${n.headline || n.title || 'link'}\n${n.url}`).join('\n') + : '- No direct links available in the current sweep', + ]; + + if (ideas.length) { + lines.push('', '*Why This Matters*'); + for (const idea of ideas) lines.push(`- ${idea.title}: ${(idea.rationale || idea.text || '').slice(0, 140)}`); + } + lines.push('', '*What To Do Next*', '- Open the dashboard, verify the evidence links, and compare source health before acting.'); + return lines.join('\n'); +} + // === Sweep Cycle === async function runSweepCycle() { if (sweepInProgress) { @@ -323,6 +437,7 @@ async function runSweepCycle() { console.log(`${'='.repeat(60)}`); try { + lastSweepError = null; // 1. Run the full briefing sweep const rawData = await fullBriefing(); @@ -333,6 +448,18 @@ async function runSweepCycle() { // 3. Synthesize into dashboard format console.log('[Crucix] Synthesizing dashboard data...'); const synthesized = await synthesize(rawData); + synthesized.meta = { + ...synthesized.meta, + generatedAt: new Date().toISOString(), + stale: false, + bootstrap: false, + }; + synthesized.sourceHealth = Object.entries(rawData.timing || {}).map(([name, info]) => ({ + name, + status: info.status || 'unknown', + ms: info.ms || 0, + error: info.error || rawData.errors?.find(e => e.name === name)?.error || null, + })); // 4. Delta computation + memory const delta = memory.addRun(synthesized); @@ -380,6 +507,8 @@ async function runSweepCycle() { memory.pruneAlertedSignals(); currentData = synthesized; + lastSuccessfulSweepTime = lastSweepTime; + intelligenceStore.recordRun(currentData, delta); // 6. Push to all connected browsers broadcast({ type: 'update', data: currentData }); @@ -391,6 +520,7 @@ async function runSweepCycle() { } catch (err) { console.error('[Crucix] Sweep failed:', err.message); + lastSweepError = err.message; broadcast({ type: 'sweep_error', error: err.message }); } finally { sweepInProgress = false; @@ -433,21 +563,36 @@ async function start() { server.on('listening', async () => { console.log(`[Crucix] Server running on http://localhost:${port}`); - // Auto-open browser - // NOTE: On Windows, `start` in PowerShell is an alias for Start-Service, not cmd's start. - // We must use `cmd /c start ""` to ensure it works in both cmd.exe and PowerShell. - const openCmd = process.platform === 'win32' ? 'cmd /c start ""' : - process.platform === 'darwin' ? 'open' : 'xdg-open'; - exec(`${openCmd} "http://localhost:${port}"`, (err) => { - if (err) console.log('[Crucix] Could not auto-open browser:', err.message); - }); + if (config.autoOpenBrowser) { + const openCmd = process.platform === 'win32' ? 'cmd /c start ""' : + process.platform === 'darwin' ? 'open' : 'xdg-open'; + exec(`${openCmd} "http://localhost:${port}"`, (err) => { + if (err) console.log('[Crucix] Could not auto-open browser:', err.message); + }); + } else { + console.log('[Crucix] Auto-open browser disabled (AUTO_OPEN_BROWSER=false)'); + } // Try to load existing data first for instant display (await so dashboard shows immediately) try { const existing = JSON.parse(readFileSync(join(RUNS_DIR, 'latest.json'), 'utf8')); const data = await synthesize(existing); + data.meta = { + ...data.meta, + generatedAt: new Date().toISOString(), + stale: true, + bootstrap: true, + bootstrapReason: 'Loaded from runs/latest.json while initial sweep starts', + }; + data.sourceHealth = Object.entries(existing.timing || {}).map(([name, info]) => ({ + name, + status: info.status || 'bootstrap', + ms: info.ms || 0, + error: info.error || null, + })); currentData = data; - console.log('[Crucix] Loaded existing data from runs/latest.json — dashboard ready instantly'); + bootstrapDataLoaded = true; + console.log('[Crucix] Loaded existing data from runs/latest.json as stale bootstrap; initial sweep will refresh it'); broadcast({ type: 'update', data: currentData }); } catch { console.log('[Crucix] No existing data found — first sweep required'); diff --git a/test/fetch-utils.test.mjs b/test/fetch-utils.test.mjs new file mode 100644 index 0000000..2dcee45 --- /dev/null +++ b/test/fetch-utils.test.mjs @@ -0,0 +1,36 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { safeFetch, safeFetchText, getFetchMetrics } from '../apis/utils/fetch.mjs'; + +test('safeFetch reports HTML as degraded JSON response', async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => ({ + ok: true, + status: 200, + headers: { get: () => 'text/html' }, + text: async () => 'not json', + }); + try { + const data = await safeFetch('https://example.test/json', { retries: 0, source: 'unit' }); + assert.match(data.error, /Expected JSON/); + assert.ok(getFetchMetrics().bySource.unit.requests >= 1); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test('safeFetchText returns text and byte count', async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => ({ + ok: true, + status: 200, + text: async () => 'hello', + }); + try { + const data = await safeFetchText('https://example.test/rss', { retries: 0, source: 'rss-unit' }); + assert.equal(data.text, 'hello'); + assert.equal(data.bytes, 5); + } finally { + globalThis.fetch = originalFetch; + } +}); diff --git a/test/llm-ollama.test.mjs b/test/llm-ollama.test.mjs index f1b31e1..2190cd2 100644 --- a/test/llm-ollama.test.mjs +++ b/test/llm-ollama.test.mjs @@ -13,19 +13,19 @@ describe('OllamaProvider', () => { const provider = new OllamaProvider({}); assert.equal(provider.name, 'ollama'); assert.equal(provider.model, 'llama3.1:8b'); - assert.equal(provider.baseUrl, 'http://localhost:11434'); + assert.equal(provider.baseUrl, 'http://localhost:11434/v1'); assert.equal(provider.isConfigured, true); }); it('should accept custom model and base URL', () => { const provider = new OllamaProvider({ model: 'qwen2.5:14b', baseUrl: 'http://192.168.1.10:11434' }); assert.equal(provider.model, 'qwen2.5:14b'); - assert.equal(provider.baseUrl, 'http://192.168.1.10:11434'); + assert.equal(provider.baseUrl, 'http://192.168.1.10:11434/v1'); }); it('should strip trailing slashes from base URL', () => { const provider = new OllamaProvider({ baseUrl: 'http://localhost:11434/' }); - assert.equal(provider.baseUrl, 'http://localhost:11434'); + assert.equal(provider.baseUrl, 'http://localhost:11434/v1'); }); it('should throw on API error', async () => { @@ -38,7 +38,7 @@ describe('OllamaProvider', () => { await assert.rejects( () => provider.complete('system', 'user'), (err) => { - assert.match(err.message, /Ollama API 404/); + assert.match(err.message, /ollama API 404/); return true; } ); @@ -164,7 +164,7 @@ describe('createLLMProvider — ollama', () => { it('should pass baseUrl from config', () => { const provider = createLLMProvider({ provider: 'ollama', apiKey: null, model: 'mistral:7b', baseUrl: 'http://gpu-box:11434' }); assert.ok(provider instanceof OllamaProvider); - assert.equal(provider.baseUrl, 'http://gpu-box:11434'); + assert.equal(provider.baseUrl, 'http://gpu-box:11434/v1'); assert.equal(provider.model, 'mistral:7b'); }); }); diff --git a/test/llm-openai-compatible.test.mjs b/test/llm-openai-compatible.test.mjs new file mode 100644 index 0000000..cedbe5d --- /dev/null +++ b/test/llm-openai-compatible.test.mjs @@ -0,0 +1,61 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { OpenAICompatibleProvider } from '../lib/llm/openai-compatible.mjs'; +import { createLLMProvider } from '../lib/llm/index.mjs'; + +test('OpenAI-compatible provider sends configurable payload', async () => { + const provider = new OpenAICompatibleProvider({ + name: 'local-openai', + baseUrl: 'http://localhost:1234/v1/', + apiKey: 'local-key', + model: 'local-model', + temperature: 0.1, + maxTokens: 123, + timeoutMs: 5000, + }); + + let capturedUrl; + let capturedBody; + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url, opts) => { + capturedUrl = url; + capturedBody = JSON.parse(opts.body); + assert.equal(opts.headers.Authorization, 'Bearer local-key'); + return { + ok: true, + json: async () => ({ + choices: [{ message: { content: 'ok' } }], + usage: { prompt_tokens: 2, completion_tokens: 3 }, + model: 'local-model', + }), + }; + }; + + try { + const result = await provider.complete('system', 'user'); + assert.equal(capturedUrl, 'http://localhost:1234/v1/chat/completions'); + assert.equal(capturedBody.model, 'local-model'); + assert.equal(capturedBody.temperature, 0.1); + assert.equal(capturedBody.max_tokens, 123); + assert.equal(result.text, 'ok'); + assert.deepEqual(result.usage, { inputTokens: 2, outputTokens: 3 }); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test('factory supports lmstudio and openai-compatible aliases', () => { + const lmstudio = createLLMProvider({ provider: 'lmstudio', model: 'local-model' }); + assert.ok(lmstudio instanceof OpenAICompatibleProvider); + assert.equal(lmstudio.baseUrl, 'http://localhost:1234/v1'); + assert.equal(lmstudio.isConfigured, true); + + const compatible = createLLMProvider({ + provider: 'openai-compatible', + baseUrl: 'http://llm:8000/v1', + apiKey: 'token', + model: 'qwen', + }); + assert.ok(compatible instanceof OpenAICompatibleProvider); + assert.equal(compatible.baseUrl, 'http://llm:8000/v1'); +}); diff --git a/test/llm-openrouter.test.mjs b/test/llm-openrouter.test.mjs index bc13ac6..7a40250 100644 --- a/test/llm-openrouter.test.mjs +++ b/test/llm-openrouter.test.mjs @@ -80,7 +80,7 @@ test('OpenRouterProvider Unit Tests', async (t) => { provider.complete('system', 'user'), { name: 'Error', - message: 'OpenRouter API 401: Unauthorized access' + message: 'openrouter auth failed (401): Unauthorized access' } ); } finally {