Compare commits
7 Commits
codex/issu
...
bb139799d7
| Author | SHA1 | Date | |
|---|---|---|---|
| bb139799d7 | |||
| 8605d0baab | |||
| 53470cc701 | |||
| 4262c7e939 | |||
| e933586b22 | |||
| 8e096b2697 | |||
| b309bd690e |
@@ -6,6 +6,7 @@ PORT=3117
|
||||
REFRESH_INTERVAL_MINUTES=15
|
||||
AUTO_OPEN_BROWSER=false
|
||||
STALE_DATA_MAX_AGE_MINUTES=60
|
||||
TERMINAL_ACTIONS_ENABLED=true
|
||||
SWEEP_TOKEN=
|
||||
BRIEF_VERBOSITY=standard
|
||||
|
||||
|
||||
@@ -38,7 +38,12 @@ jobs:
|
||||
run: docker compose config
|
||||
|
||||
- name: Build Docker image
|
||||
run: docker build -t "${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${IMAGE_NAME}:${GITHUB_SHA}" .
|
||||
shell: bash
|
||||
run: |
|
||||
image="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${IMAGE_NAME}"
|
||||
build_tag="build-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_NUMBER:-0}"
|
||||
echo "BUILD_IMAGE=${image}:${build_tag}" >> "$GITHUB_ENV"
|
||||
docker build -t "${image}:${build_tag}" .
|
||||
|
||||
- name: Publish Docker image
|
||||
if: ${{ env.REGISTRY_TOKEN != '' }}
|
||||
@@ -47,8 +52,9 @@ jobs:
|
||||
image="${REGISTRY_HOST}/${REGISTRY_NAMESPACE}/${IMAGE_NAME}"
|
||||
date_tag="$(date -u +%Y%m%d)"
|
||||
echo "${REGISTRY_TOKEN}" | docker login "${REGISTRY_HOST}" -u "${REGISTRY_USERNAME}" --password-stdin
|
||||
docker tag "${image}:${GITHUB_SHA}" "${image}:latest"
|
||||
docker tag "${image}:${GITHUB_SHA}" "${image}:${date_tag}"
|
||||
docker tag "${BUILD_IMAGE}" "${image}:${GITHUB_SHA}"
|
||||
docker tag "${BUILD_IMAGE}" "${image}:latest"
|
||||
docker tag "${BUILD_IMAGE}" "${image}:${date_tag}"
|
||||
docker push "${image}:${GITHUB_SHA}"
|
||||
docker push "${image}:latest"
|
||||
docker push "${image}:${date_tag}"
|
||||
|
||||
@@ -135,6 +135,7 @@ PORT=3117
|
||||
REFRESH_INTERVAL_MINUTES=15
|
||||
AUTO_OPEN_BROWSER=false
|
||||
STALE_DATA_MAX_AGE_MINUTES=60
|
||||
TERMINAL_ACTIONS_ENABLED=true
|
||||
SWEEP_TOKEN=
|
||||
BRIEF_VERBOSITY=standard
|
||||
|
||||
@@ -187,6 +188,8 @@ LLM_MODEL=your-model
|
||||
|
||||
For Pangolin or another reverse proxy, forward HTTP traffic to `intelligence-terminal:3117` (or the `PORT` you set). Missing API keys do not crash sweeps; affected sources are reported as degraded in `/api/health`.
|
||||
|
||||
The dashboard Terminal Actions panel can trigger `status`, `sweep`, and `brief` through `/api/action`. Leave `TERMINAL_ACTIONS_ENABLED=true` for a private home-server deployment. For an internet-exposed deployment, set `SWEEP_TOKEN` and pass it through trusted automation, or set `TERMINAL_ACTIONS_ENABLED=false` to disable browser-triggered actions. If you protect actions with `SWEEP_TOKEN`, the browser can send it from `localStorage.crucix_sweep_token`.
|
||||
|
||||
#### Build And Publish Your Gitea Image
|
||||
|
||||
```bash
|
||||
@@ -324,7 +327,7 @@ These three unlock the most valuable economic and satellite data. Each takes abo
|
||||
|
||||
| Key | Source | How to Get |
|
||||
|-----|--------|------------|
|
||||
| `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2 |
|
||||
| `ACLED_EMAIL` + `ACLED_PASSWORD` | Armed conflict event data | [acleddata.com/register](https://acleddata.com/register/) — free, OAuth2. `ACLED_USER` / `ACLED_USERNAME` are accepted as email aliases |
|
||||
| `AISSTREAM_API_KEY` | Maritime AIS vessel tracking | [aisstream.io](https://aisstream.io/) — free |
|
||||
| `ADSB_API_KEY` | Unfiltered flight tracking | [RapidAPI](https://rapidapi.com/adsbexchange/api/adsbexchange-com1) — ~$10/mo |
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
// ACLED — Armed Conflict Location & Event Data
|
||||
// ACLED - Armed Conflict Location & Event Data.
|
||||
// Auth strategy (tries in order):
|
||||
// 1. Cookie-based session: POST /user/login?_format=json → session cookie
|
||||
// 2. OAuth Bearer token: POST /oauth/token → Authorization header
|
||||
// Set ACLED_EMAIL and ACLED_PASSWORD in .env (your myACLED login credentials).
|
||||
// Data endpoint: GET https://acleddata.com/api/acled/read
|
||||
// 1. OAuth Bearer token: POST /oauth/token -> Authorization header
|
||||
// 2. Cookie-based session: POST /user/login?_format=json -> session cookie
|
||||
// Set ACLED_EMAIL and ACLED_PASSWORD in .env. ACLED_USER or ACLED_USERNAME are
|
||||
// accepted as aliases for ACLED_EMAIL.
|
||||
|
||||
import { daysAgo } from '../utils/fetch.mjs';
|
||||
import '../utils/env.mjs';
|
||||
@@ -12,155 +12,135 @@ const LOGIN_URL = 'https://acleddata.com/user/login?_format=json';
|
||||
const TOKEN_URL = 'https://acleddata.com/oauth/token';
|
||||
const API_BASE = 'https://acleddata.com/api/acled/read';
|
||||
|
||||
// Session cache
|
||||
let sessionCache = { cookies: null, token: null, method: null, expires: 0 };
|
||||
|
||||
function acledCredentials() {
|
||||
return {
|
||||
email: process.env.ACLED_EMAIL || process.env.ACLED_USER || process.env.ACLED_USERNAME,
|
||||
password: process.env.ACLED_PASSWORD,
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeAuthHeaders(headers) {
|
||||
const safe = { ...headers };
|
||||
if (safe.Authorization) safe.Authorization = 'Bearer [redacted]';
|
||||
if (safe.Cookie) safe.Cookie = '[redacted]';
|
||||
return safe;
|
||||
}
|
||||
|
||||
function classifyAuthFailure(status, body = '') {
|
||||
if (status === 401 || status === 403) return 'auth_denied';
|
||||
if (status >= 500) return 'auth_endpoint_failed';
|
||||
if (/invalid|denied|unauthorized|forbidden/i.test(body)) return 'auth_denied';
|
||||
return 'auth_endpoint_failed';
|
||||
}
|
||||
|
||||
export function resetAcledSessionForTests() {
|
||||
export function resetAcledSessionCache() {
|
||||
sessionCache = { cookies: null, token: null, method: null, expires: 0 };
|
||||
}
|
||||
|
||||
// Strategy 1: Cookie-based session login (mirrors browser login)
|
||||
async function loginCookie(email, password) {
|
||||
export function getAcledConfig(env = process.env) {
|
||||
const email = env.ACLED_EMAIL || env.ACLED_USER || env.ACLED_USERNAME || '';
|
||||
const password = env.ACLED_PASSWORD || '';
|
||||
const missing = [];
|
||||
if (!email) missing.push('ACLED_EMAIL');
|
||||
if (!password) missing.push('ACLED_PASSWORD');
|
||||
return { email, password, configured: missing.length === 0, missing };
|
||||
}
|
||||
|
||||
function acledError(status, error, message, extra = {}) {
|
||||
return { status, error, message, ...extra };
|
||||
}
|
||||
|
||||
function safeText(value, max = 200) {
|
||||
return String(value || '').replace(/Bearer\s+[A-Za-z0-9._-]+/gi, 'Bearer [redacted]').slice(0, max);
|
||||
}
|
||||
|
||||
async function fetchWithTimeout(fetchImpl, url, init, timeoutMs) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 15000);
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const res = await fetch(LOGIN_URL, {
|
||||
return await fetchImpl(url, { ...init, signal: controller.signal });
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
export async function loginCookie(email, password, opts = {}) {
|
||||
const fetchImpl = opts.fetchImpl || globalThis.fetch;
|
||||
try {
|
||||
const res = await fetchWithTimeout(fetchImpl, LOGIN_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ name: email, pass: password }),
|
||||
redirect: 'manual',
|
||||
signal: controller.signal,
|
||||
});
|
||||
clearTimeout(timer);
|
||||
}, 15000);
|
||||
|
||||
// Collect Set-Cookie headers
|
||||
const setCookies = res.headers.getSetCookie?.() || [];
|
||||
const cookieStr = setCookies.map(c => c.split(';')[0]).join('; ');
|
||||
|
||||
if (res.ok && cookieStr) {
|
||||
return { cookies: cookieStr };
|
||||
}
|
||||
|
||||
// Some Drupal sites return 303 redirect on successful login — cookies still set
|
||||
if (res.status >= 300 && res.status < 400 && cookieStr) {
|
||||
return { cookies: cookieStr };
|
||||
if ((res.ok || (res.status >= 300 && res.status < 400)) && cookieStr) {
|
||||
return { ok: true, cookies: cookieStr };
|
||||
}
|
||||
|
||||
const errText = await res.text().catch(() => '');
|
||||
return {
|
||||
error: `Cookie login failed (HTTP ${res.status}): ${errText.slice(0, 200)}`,
|
||||
code: classifyAuthFailure(res.status, errText),
|
||||
};
|
||||
return acledError('auth_failed', `acled_cookie_http_${res.status}`, `ACLED cookie login failed with HTTP ${res.status}`, {
|
||||
detail: safeText(errText),
|
||||
});
|
||||
} catch (e) {
|
||||
clearTimeout(timer);
|
||||
const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : '';
|
||||
return { error: `Cookie login error: ${e.message}${cause}`, code: 'auth_endpoint_failed' };
|
||||
return acledError('auth_failed', 'acled_cookie_request_failed', `ACLED cookie login error: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Strategy 2: OAuth2 password grant
|
||||
async function loginOAuth(email, password) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 15000);
|
||||
export async function loginOAuth(email, password, opts = {}) {
|
||||
const fetchImpl = opts.fetchImpl || globalThis.fetch;
|
||||
try {
|
||||
const body = new URLSearchParams({
|
||||
username: email,
|
||||
password: password,
|
||||
password,
|
||||
grant_type: 'password',
|
||||
client_id: 'acled',
|
||||
});
|
||||
|
||||
const res = await fetch(TOKEN_URL, {
|
||||
const res = await fetchWithTimeout(fetchImpl, TOKEN_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||
body: body.toString(),
|
||||
signal: controller.signal,
|
||||
});
|
||||
clearTimeout(timer);
|
||||
}, 15000);
|
||||
|
||||
if (!res.ok) {
|
||||
const errText = await res.text().catch(() => '');
|
||||
return {
|
||||
error: `OAuth failed (HTTP ${res.status}): ${errText.slice(0, 200)}`,
|
||||
code: classifyAuthFailure(res.status, errText),
|
||||
};
|
||||
return acledError('auth_failed', `acled_oauth_http_${res.status}`, `ACLED OAuth failed with HTTP ${res.status}`, {
|
||||
detail: safeText(errText),
|
||||
});
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
if (!data.access_token) {
|
||||
return { error: `OAuth response missing access_token: ${JSON.stringify(data).slice(0, 200)}`, code: 'auth_endpoint_failed' };
|
||||
return acledError('auth_failed', 'acled_oauth_missing_access_token', 'ACLED OAuth response did not include access_token');
|
||||
}
|
||||
|
||||
return { token: data.access_token };
|
||||
return { ok: true, token: data.access_token };
|
||||
} catch (e) {
|
||||
clearTimeout(timer);
|
||||
const cause = e.cause ? ` → ${e.cause.message || e.cause.code || e.cause}` : '';
|
||||
return { error: `OAuth error: ${e.message}${cause}`, code: 'auth_endpoint_failed' };
|
||||
return acledError('auth_failed', 'acled_oauth_request_failed', `ACLED OAuth error: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Try both auth strategies
|
||||
async function authenticate() {
|
||||
const { email, password } = acledCredentials();
|
||||
if (!email || !password) {
|
||||
return { error: 'No ACLED credentials. Set ACLED_EMAIL or ACLED_USER and ACLED_PASSWORD in .env.', code: 'no_credentials' };
|
||||
export async function authenticate(opts = {}) {
|
||||
const env = opts.env || process.env;
|
||||
const fetchImpl = opts.fetchImpl || globalThis.fetch;
|
||||
const config = getAcledConfig(env);
|
||||
if (!config.configured) {
|
||||
return acledError('no_credentials', 'missing_acled_credentials', 'No ACLED credentials. Set ACLED_EMAIL and ACLED_PASSWORD in .env.', {
|
||||
missing: config.missing,
|
||||
});
|
||||
}
|
||||
|
||||
// Return cached session if still valid
|
||||
if (sessionCache.method && Date.now() < sessionCache.expires) {
|
||||
return sessionCache;
|
||||
}
|
||||
|
||||
const errors = [];
|
||||
const debug = process.argv.includes('--debug');
|
||||
|
||||
// Try OAuth first (official programmatic method per ACLED docs)
|
||||
const oauthResult = await loginOAuth(email, password);
|
||||
if (oauthResult.token) {
|
||||
if (debug) console.error('[ACLED DEBUG] OAuth OK');
|
||||
const diagnostics = [];
|
||||
const oauthResult = await loginOAuth(config.email, config.password, { fetchImpl });
|
||||
if (oauthResult.ok) {
|
||||
sessionCache = { cookies: null, token: oauthResult.token, method: 'oauth', expires: Date.now() + 23 * 60 * 60 * 1000 };
|
||||
return sessionCache;
|
||||
}
|
||||
errors.push(`OAuth: ${oauthResult.error}`);
|
||||
if (debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
|
||||
diagnostics.push({ method: 'oauth', status: oauthResult.status, error: oauthResult.error, message: oauthResult.message });
|
||||
if (opts.debug) console.error(`[ACLED DEBUG] OAuth failed: ${oauthResult.error}`);
|
||||
|
||||
// Fall back to cookie-based session
|
||||
const cookieResult = await loginCookie(email, password);
|
||||
if (cookieResult.cookies) {
|
||||
if (debug) console.error('[ACLED DEBUG] Cookie OK');
|
||||
const cookieResult = await loginCookie(config.email, config.password, { fetchImpl });
|
||||
if (cookieResult.ok) {
|
||||
sessionCache = { cookies: cookieResult.cookies, token: null, method: 'cookie', expires: Date.now() + 12 * 60 * 60 * 1000 };
|
||||
return sessionCache;
|
||||
}
|
||||
errors.push(`Cookie: ${cookieResult.error}`);
|
||||
diagnostics.push({ method: 'cookie', status: cookieResult.status, error: cookieResult.error, message: cookieResult.message });
|
||||
if (opts.debug) console.error(`[ACLED DEBUG] Cookie login failed: ${cookieResult.error}`);
|
||||
|
||||
const code = [oauthResult.code, cookieResult.code].includes('auth_denied') ? 'auth_denied' : 'auth_endpoint_failed';
|
||||
return { error: `All ACLED auth methods failed.\n${errors.join('\n')}`, code };
|
||||
return acledError('auth_failed', 'acled_auth_failed', 'All ACLED auth methods failed.', { diagnostics });
|
||||
}
|
||||
|
||||
// Build headers based on auth method
|
||||
function authHeaders(session) {
|
||||
const headers = { 'User-Agent': 'Crucix/1.0', 'Content-Type': 'application/json' };
|
||||
const headers = { 'User-Agent': 'Crucix/2.0', 'Content-Type': 'application/json' };
|
||||
if (session.method === 'cookie' && session.cookies) {
|
||||
headers['Cookie'] = session.cookies;
|
||||
} else if (session.method === 'oauth' && session.token) {
|
||||
@@ -169,7 +149,6 @@ function authHeaders(session) {
|
||||
return headers;
|
||||
}
|
||||
|
||||
// Event type constants
|
||||
export const EVENT_TYPES = [
|
||||
'Battles',
|
||||
'Explosions/Remote violence',
|
||||
@@ -179,7 +158,6 @@ export const EVENT_TYPES = [
|
||||
'Strategic developments',
|
||||
];
|
||||
|
||||
// Query conflict events with flexible filters
|
||||
export async function getEvents(opts = {}) {
|
||||
const {
|
||||
limit = 500,
|
||||
@@ -188,10 +166,13 @@ export async function getEvents(opts = {}) {
|
||||
eventType,
|
||||
country,
|
||||
region,
|
||||
env = process.env,
|
||||
fetchImpl = globalThis.fetch,
|
||||
debug = process.argv.includes('--debug'),
|
||||
} = opts;
|
||||
|
||||
const session = await authenticate();
|
||||
if (session.error) return { error: session.error, code: session.code || 'auth_failed' };
|
||||
const session = await authenticate({ env, fetchImpl, debug });
|
||||
if (session.error) return session;
|
||||
|
||||
const params = new URLSearchParams({ _format: 'json', limit: String(limit) });
|
||||
if (eventDateStart && eventDateEnd) {
|
||||
@@ -202,62 +183,43 @@ export async function getEvents(opts = {}) {
|
||||
if (country) params.set('country', country);
|
||||
if (region) params.set('region', String(region));
|
||||
|
||||
const debug = process.argv.includes('--debug');
|
||||
try {
|
||||
const url = `${API_BASE}?${params}`;
|
||||
const hdrs = authHeaders(session);
|
||||
if (debug) {
|
||||
console.error(`[ACLED DEBUG] Data request: GET ${url}`);
|
||||
console.error(`[ACLED DEBUG] Headers: ${JSON.stringify(sanitizeAuthHeaders(hdrs))}`);
|
||||
}
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), 25000);
|
||||
const res = await fetch(url, {
|
||||
headers: hdrs,
|
||||
signal: controller.signal,
|
||||
});
|
||||
clearTimeout(timer);
|
||||
if (debug) console.error(`[ACLED DEBUG] Data request: GET ${url}`);
|
||||
const res = await fetchWithTimeout(fetchImpl, url, { headers: authHeaders(session) }, 25000);
|
||||
if (debug) console.error(`[ACLED DEBUG] Data response: HTTP ${res.status}`);
|
||||
|
||||
if (!res.ok) {
|
||||
const errText = await res.text().catch(() => '');
|
||||
if (debug) console.error(`[ACLED DEBUG] Error body: ${errText.slice(0, 500)}`);
|
||||
if (res.status === 401 || res.status === 403) {
|
||||
// Clear cache and report
|
||||
sessionCache = { cookies: null, token: null, method: null, expires: 0 };
|
||||
const hint = res.status === 403
|
||||
? '\n→ Fix: Log in at https://acleddata.com/user/login, then:\n'
|
||||
+ ' 1. Accept Terms of Use (profile → Terms of Use button → check the box)\n'
|
||||
+ ' 2. Complete all required profile fields\n'
|
||||
+ ' 3. Ensure your account has the "API" access group\n'
|
||||
+ ' Contact access@acleddata.com if issues persist.'
|
||||
: '';
|
||||
return {
|
||||
error: `ACLED data access denied (HTTP ${res.status}, auth method: ${session.method}). Response: ${errText.slice(0, 300)}${hint}`,
|
||||
code: 'auth_denied',
|
||||
};
|
||||
return acledError('access_denied', `acled_data_http_${res.status}`, `ACLED data access denied with HTTP ${res.status}`, {
|
||||
authMethod: session.method,
|
||||
detail: safeText(errText, 300),
|
||||
hint: 'Accept ACLED terms, complete profile fields, and confirm API access for the account.',
|
||||
});
|
||||
}
|
||||
return { error: `HTTP ${res.status}: ${errText.slice(0, 200)}`, code: 'api_failed' };
|
||||
return acledError('api_failed', `acled_data_http_${res.status}`, `ACLED data request failed with HTTP ${res.status}`, {
|
||||
detail: safeText(errText),
|
||||
});
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
// ACLED may return a 200 with an error status in the body
|
||||
if (data?.status && data.status !== 200) {
|
||||
return { error: `ACLED API error: status ${data.status} - ${data.message || 'Unknown error'}`, code: 'api_failed' };
|
||||
return acledError('api_failed', `acled_api_status_${data.status}`, `ACLED API returned status ${data.status}`, {
|
||||
detail: safeText(data.message),
|
||||
});
|
||||
}
|
||||
|
||||
return data;
|
||||
} catch (e) {
|
||||
if (e.name === 'AbortError') {
|
||||
return { error: 'ACLED data request timed out (25s)', code: 'api_failed' };
|
||||
return acledError('api_failed', 'acled_data_timeout', 'ACLED data request timed out after 25s');
|
||||
}
|
||||
const rootCause = e.cause ? `${e.cause.message || e.cause.code || e.cause}` : '';
|
||||
return { error: `ACLED data error: ${e.message}${rootCause ? ' - ' + rootCause : ''}`, code: 'api_failed' };
|
||||
return acledError('api_failed', 'acled_data_request_failed', `ACLED data error: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Summarize events by a given field
|
||||
function groupBy(events, field) {
|
||||
const map = {};
|
||||
for (const e of events) {
|
||||
@@ -269,15 +231,18 @@ function groupBy(events, field) {
|
||||
return map;
|
||||
}
|
||||
|
||||
// Briefing — last 7 days of global conflict events
|
||||
export async function briefing() {
|
||||
const { email, password } = acledCredentials();
|
||||
if (!email || !password) {
|
||||
export async function briefing(opts = {}) {
|
||||
const env = opts.env || process.env;
|
||||
const fetchImpl = opts.fetchImpl || globalThis.fetch;
|
||||
const config = getAcledConfig(env);
|
||||
if (!config.configured) {
|
||||
return {
|
||||
source: 'ACLED',
|
||||
timestamp: new Date().toISOString(),
|
||||
status: 'no_credentials',
|
||||
message: 'Set ACLED_EMAIL or ACLED_USER and ACLED_PASSWORD in .env. Register at https://acleddata.com/user/register',
|
||||
error: 'missing_acled_credentials',
|
||||
missing: config.missing,
|
||||
message: 'Set ACLED_EMAIL and ACLED_PASSWORD in .env. Register at https://acleddata.com/user/register',
|
||||
};
|
||||
}
|
||||
|
||||
@@ -288,16 +253,25 @@ export async function briefing() {
|
||||
eventDateStart: start,
|
||||
eventDateEnd: end,
|
||||
limit: 2000,
|
||||
env,
|
||||
fetchImpl,
|
||||
debug: opts.debug,
|
||||
});
|
||||
|
||||
if (data?.error) {
|
||||
const status = data.code === 'auth_denied' ? 'auth_failed' : 'api_failed';
|
||||
return { source: 'ACLED', timestamp: new Date().toISOString(), status, error: data.error };
|
||||
return {
|
||||
source: 'ACLED',
|
||||
timestamp: new Date().toISOString(),
|
||||
status: data.status || 'api_failed',
|
||||
error: data.error,
|
||||
message: data.message,
|
||||
detail: data.detail,
|
||||
hint: data.hint,
|
||||
diagnostics: data.diagnostics,
|
||||
};
|
||||
}
|
||||
|
||||
let events = data?.data || [];
|
||||
|
||||
// Enrich all events with numeric lat/lon
|
||||
events = events.map(e => ({
|
||||
...e,
|
||||
lat: parseFloat(e.latitude) || null,
|
||||
@@ -311,7 +285,6 @@ export async function briefing() {
|
||||
const byRegion = groupBy(events, 'region');
|
||||
const byType = groupBy(events, 'event_type');
|
||||
const byCountry = groupBy(events, 'country');
|
||||
|
||||
const topCountries = Object.entries(byCountry)
|
||||
.sort((a, b) => b[1].count - a[1].count)
|
||||
.slice(0, 10)
|
||||
@@ -336,7 +309,7 @@ export async function briefing() {
|
||||
return {
|
||||
source: 'ACLED',
|
||||
timestamp: new Date().toISOString(),
|
||||
status: 'live',
|
||||
status: 'ok',
|
||||
period: { start, end },
|
||||
totalEvents: events.length,
|
||||
totalFatalities,
|
||||
@@ -344,7 +317,6 @@ export async function briefing() {
|
||||
byType,
|
||||
topCountries,
|
||||
deadliestEvents,
|
||||
message: events.length === 0 ? 'ACLED returned a valid empty event set for the requested period.' : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ export default {
|
||||
autoOpenBrowser: boolEnv('AUTO_OPEN_BROWSER', false),
|
||||
staleDataMaxAgeMinutes: intEnv('STALE_DATA_MAX_AGE_MINUTES', 60),
|
||||
sweepToken: process.env.SWEEP_TOKEN || null,
|
||||
terminalActionsEnabled: boolEnv('TERMINAL_ACTIONS_ENABLED', true),
|
||||
|
||||
llm: {
|
||||
provider: process.env.LLM_PROVIDER || null, // anthropic | openai | gemini | codex | openrouter | minimax | mistral | ollama | grok
|
||||
|
||||
@@ -83,6 +83,13 @@ html,body{height:100%;background:var(--bg);color:var(--text);font-family:var(--s
|
||||
.sensor-actions{display:flex;gap:6px;align-items:center}
|
||||
.mini-btn{border:1px solid rgba(100,240,200,0.18);background:rgba(100,240,200,0.04);color:var(--dim);font-family:var(--mono);font-size:9px;padding:3px 6px;cursor:pointer}
|
||||
.mini-btn:hover{color:var(--accent);border-color:rgba(100,240,200,0.4)}
|
||||
.action-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:6px;margin-bottom:10px}
|
||||
.action-btn{border:1px solid rgba(68,204,255,0.24);background:rgba(68,204,255,0.06);color:var(--text);font-family:var(--mono);font-size:9px;padding:7px 6px;cursor:pointer;text-transform:uppercase;letter-spacing:.08em}
|
||||
.action-btn:hover{border-color:rgba(68,204,255,0.55);color:var(--accent2);background:rgba(68,204,255,0.12)}
|
||||
.action-btn[disabled]{opacity:.45;cursor:wait}
|
||||
.terminal-output{min-height:58px;max-height:180px;overflow:auto;border:1px solid rgba(255,255,255,0.05);background:rgba(0,0,0,0.22);padding:8px;font-family:var(--mono);font-size:10px;line-height:1.45;color:var(--dim);white-space:pre-wrap}
|
||||
.terminal-output strong{color:var(--accent)}
|
||||
.terminal-output .err{color:var(--danger)}
|
||||
.layer-left{display:flex;align-items:center;gap:8px}
|
||||
.ldot{width:10px;height:10px;border-radius:50%;flex-shrink:0}
|
||||
.ldot.air{background:var(--accent);box-shadow:0 0 6px rgba(100,240,200,0.4)}
|
||||
@@ -404,6 +411,8 @@ let lowPerfMode = localStorage.getItem('crucix_low_perf') === 'true';
|
||||
let isFlat = shouldStartFlat();
|
||||
let layerModes = JSON.parse(localStorage.getItem('crucix_layer_modes') || '{}');
|
||||
let spaceDisplayMode = localStorage.getItem('crucix_space_display') || 'icons';
|
||||
let terminalOutput = 'Ready. Live data is loaded from /api/data in server mode.';
|
||||
let terminalBusy = false;
|
||||
let currentRegion = 'world';
|
||||
let flatSvg, flatProjection, flatPath, flatG, flatZoom, flatW, flatH;
|
||||
|
||||
@@ -1564,6 +1573,46 @@ function renderLower(){
|
||||
document.getElementById('lowerGrid').innerHTML=`${tickerPanel}${osintPanel}${macroPanel}${ideasPanel}`;
|
||||
}
|
||||
|
||||
async function runTerminalAction(action){
|
||||
if(terminalBusy) return;
|
||||
terminalBusy = true;
|
||||
terminalOutput = `> ${action}\nRunning...`;
|
||||
renderRight();
|
||||
try{
|
||||
const res = await fetch('/api/action', {
|
||||
method:'POST',
|
||||
headers:{
|
||||
'Content-Type':'application/json',
|
||||
...(localStorage.getItem('crucix_sweep_token') ? {'x-crucix-token': localStorage.getItem('crucix_sweep_token')} : {})
|
||||
},
|
||||
body:JSON.stringify({action})
|
||||
});
|
||||
const payload = await res.json().catch(()=>({error:'Invalid server response'}));
|
||||
if(!res.ok) throw new Error(payload.error || `HTTP ${res.status}`);
|
||||
if(action === 'status'){
|
||||
const h = payload.health || {};
|
||||
terminalOutput = [
|
||||
'> status',
|
||||
`State: ${h.status || '--'}`,
|
||||
`Last sweep: ${h.lastSuccessfulSweep || h.lastSweep || '--'}`,
|
||||
`Data age: ${h.dataAgeSeconds != null ? h.dataAgeSeconds + 's' : '--'}`,
|
||||
`Sources: ${h.sourcesOk || 0} ok / ${h.sourcesDegraded || 0} degraded / ${h.sourcesFailed || 0} failed`,
|
||||
`LLM: ${h.llm?.state || '--'}`,
|
||||
`Sweep active: ${h.sweepInProgress ? 'yes' : 'no'}`
|
||||
].join('\n');
|
||||
}else if(action === 'brief'){
|
||||
terminalOutput = `> brief\n${payload.text || 'No briefing text returned.'}`;
|
||||
}else if(action === 'sweep'){
|
||||
terminalOutput = `> sweep\n${payload.status === 'already_running' ? 'Sweep already running.' : 'Sweep accepted. The dashboard will update when the sweep finishes.'}`;
|
||||
}
|
||||
}catch(err){
|
||||
terminalOutput = `> ${action}\nERROR: ${err.message}`;
|
||||
}finally{
|
||||
terminalBusy = false;
|
||||
renderRight();
|
||||
}
|
||||
}
|
||||
|
||||
// === RIGHT RAIL ===
|
||||
function renderRight(){
|
||||
const mobile = isMobileLayout();
|
||||
@@ -1605,6 +1654,15 @@ function renderRight(){
|
||||
const deltaHtml = hasDelta ? deltaRows.join('') : `<div style="padding:12px;text-align:center;color:var(--dim);font-family:var(--mono);font-size:10px">${t('delta.noChanges','No changes since last sweep')}</div>`;
|
||||
|
||||
document.getElementById('rightRail').innerHTML=`
|
||||
<div class="g-panel right-actions">
|
||||
<div class="sec-head"><h3>Terminal Actions</h3><span class="badge">${terminalBusy?'RUNNING':'READY'}</span></div>
|
||||
<div class="action-grid">
|
||||
<button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('status')">Status</button>
|
||||
<button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('sweep')">Sweep</button>
|
||||
<button class="action-btn" ${terminalBusy?'disabled':''} onclick="runTerminalAction('brief')">Brief</button>
|
||||
</div>
|
||||
<div class="terminal-output">${terminalOutput.replace(/[&<>]/g,c=>({'&':'&','<':'<','>':'>'}[c])).replace(/\n/g,'<br>')}</div>
|
||||
</div>
|
||||
<div class="g-panel right-signals">
|
||||
<div class="sec-head"><h3>${t('panels.crossSourceSignals','Cross-Source Signals')}</h3><span class="badge">${t('badges.worldview','WORLDVIEW')}</span></div>
|
||||
${signals}
|
||||
@@ -1839,10 +1897,10 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
const hasInlineData = !!(D && D.meta);
|
||||
const canProbeApi = location.protocol !== 'file:';
|
||||
|
||||
if (canProbeApi && !hasInlineData) {
|
||||
if (canProbeApi) {
|
||||
// Server mode: always fetch live data from API (ignore any stale inline D)
|
||||
fetch('/api/data')
|
||||
.then(r => r.json())
|
||||
.then(r => { if(!r.ok) throw new Error(`HTTP ${r.status}`); return r.json(); })
|
||||
.then(data => { D = data; init(); connectSSE(); })
|
||||
.catch(() => {
|
||||
// Should not reach here — server routes to loading.html when no data
|
||||
|
||||
@@ -1,18 +1,489 @@
|
||||
# Agent Handoff
|
||||
|
||||
## Current Release Goal
|
||||
Last updated: 2026-05-17
|
||||
|
||||
Source branch: `codex/production-intelligence-terminal`
|
||||
## Repository State
|
||||
|
||||
Registry image:
|
||||
Project: Crucix fork / Intelligence Terminal
|
||||
|
||||
Local workspace:
|
||||
|
||||
```text
|
||||
C:\Users\MrSphay\Documents\Codex\Crucix\intelligence-terminal
|
||||
```
|
||||
|
||||
Remotes:
|
||||
|
||||
```text
|
||||
origin https://git.wilkensxl.de/MrSphay/intelligence-terminal.git
|
||||
upstream https://github.com/calesthio/Crucix.git
|
||||
```
|
||||
|
||||
Current branch tip:
|
||||
|
||||
```text
|
||||
Run `git rev-parse HEAD` after clone/pull. This handoff was updated by the `docs: sync issue tracker and handoff` commit after the implementation commit below.
|
||||
```
|
||||
|
||||
Latest implementation commit before issue-sync documentation:
|
||||
|
||||
```text
|
||||
53470cc701ec322080a89d220aef449b25850590
|
||||
```
|
||||
|
||||
Both pushed branches currently point to this commit:
|
||||
|
||||
```text
|
||||
origin/codex/production-intelligence-terminal
|
||||
origin/main
|
||||
```
|
||||
|
||||
Gitea repository:
|
||||
|
||||
```text
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal
|
||||
```
|
||||
|
||||
Default branch observed through the Gitea API:
|
||||
|
||||
```text
|
||||
codex/production-intelligence-terminal
|
||||
```
|
||||
|
||||
## Agent Kit Requirements Applied
|
||||
|
||||
The mandatory kit was cloned and reviewed first:
|
||||
|
||||
```text
|
||||
C:\Users\MrSphay\Documents\Codex\Crucix\agent-kit
|
||||
```
|
||||
|
||||
Rules applied from the kit:
|
||||
|
||||
- Keep agent context in source control: `AGENTS.md`, `.codex/project.md`, and this handoff file.
|
||||
- Use Gitea Ubuntu runners for heavy verification and package publishing.
|
||||
- Keep Docker/Dockge operation first-class.
|
||||
- Do not commit secrets, `.env`, private logs, tokens, or generated `runs/` data.
|
||||
- Add report-only maintenance workflows for security, dependency checks, repo cleanup, release dry runs, and template compliance.
|
||||
- Poll pushed Gitea Actions until terminal state when a token is available.
|
||||
|
||||
## What Was Implemented
|
||||
|
||||
### Docker And Runtime
|
||||
|
||||
- Docker image is Docker-first and Dockge/Pangolin suitable.
|
||||
- Browser auto-open is disabled by default through `AUTO_OPEN_BROWSER=false`.
|
||||
- Runtime health checks now work in the container without `wget` or host browser tools.
|
||||
- `runs` is persisted through a volume.
|
||||
- A later fix added `docker-entrypoint.sh` to prepare `/app/runs` before dropping privileges, so mounted volumes work with the non-root Node runtime.
|
||||
- `docker-compose.yml` uses the Gitea Registry image by default:
|
||||
|
||||
```text
|
||||
git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
## Notes
|
||||
### API And Health
|
||||
|
||||
- The repository is Docker-first and should stay suitable for Dockge/Pangolin.
|
||||
- Use `.env.example` as the operator-facing source of truth for configuration.
|
||||
- Source health and network metrics are available through `/api/health` and `/api/metrics`.
|
||||
- If Gitea Registry authentication is unavailable locally, build and push with the commands documented in `README.md`.
|
||||
Added or hardened:
|
||||
|
||||
- `GET /api/health`
|
||||
- `GET /api/data`
|
||||
- `GET /api/metrics`
|
||||
- `POST /api/sweep`
|
||||
- `POST /api/action`
|
||||
|
||||
Health now reports:
|
||||
|
||||
- `starting`
|
||||
- `healthy`
|
||||
- `degraded`
|
||||
- `stale`
|
||||
- `error`
|
||||
|
||||
It also reports:
|
||||
|
||||
- last sweep timestamps
|
||||
- stale/bootstrap state
|
||||
- data age
|
||||
- source health
|
||||
- source errors
|
||||
- LLM configuration state
|
||||
- Telegram/Discord enabled state
|
||||
- memory store state
|
||||
|
||||
### Live Data And Source Degradation
|
||||
|
||||
- Existing `runs/latest.json` is only treated as bootstrap/stale data until a real sweep completes.
|
||||
- Sweeps update `sourceHealth`, SSE/API data, and memory state.
|
||||
- RSS/news feed failures no longer silently look like fresh valid data.
|
||||
- `safeFetch` now tracks request counts, failures, bytes, source labels, hosts, and recent fetch events.
|
||||
- `safeFetch` has better timeout/retry/backoff/error behavior and reports HTML-as-API-error cases.
|
||||
- Yahoo Finance fetches are more explicit about source errors and HTML/API failures.
|
||||
- ACLED missing credentials now degrade transparently.
|
||||
- Telegram polling has quieter network-error backoff logs.
|
||||
|
||||
### LLM Integration
|
||||
|
||||
Added unified OpenAI-compatible provider layer:
|
||||
|
||||
```text
|
||||
lib/llm/openai-compatible.mjs
|
||||
```
|
||||
|
||||
Supported provider paths include:
|
||||
|
||||
- `openrouter`
|
||||
- `openai`
|
||||
- `openai-compatible`
|
||||
- `local-openai`
|
||||
- `lmstudio`
|
||||
- `lm-studio`
|
||||
- `ollama`
|
||||
|
||||
Relevant environment keys:
|
||||
|
||||
```text
|
||||
LLM_PROVIDER
|
||||
LLM_BASE_URL
|
||||
LLM_API_KEY
|
||||
LLM_MODEL
|
||||
LLM_TEMPERATURE
|
||||
LLM_MAX_TOKENS
|
||||
LLM_TIMEOUT_MS
|
||||
OPENROUTER_SITE_URL
|
||||
OPENROUTER_APP_NAME
|
||||
```
|
||||
|
||||
OpenRouter Free and local OpenAI-compatible endpoints are documented in `README.md` and `.env.example`.
|
||||
|
||||
### Memory
|
||||
|
||||
Added Phase-1 SQLite memory:
|
||||
|
||||
```text
|
||||
lib/intelligence-store.mjs
|
||||
runs/intelligence.db
|
||||
```
|
||||
|
||||
It uses `node:sqlite` when available and gracefully falls back when unavailable.
|
||||
|
||||
### Dashboard
|
||||
|
||||
Implemented:
|
||||
|
||||
- interactive Sensor Grid layer modes
|
||||
- focus/hide/normal states persisted in `localStorage`
|
||||
- Space Watch icon/orbit toggle
|
||||
- map/globe filtering consistency
|
||||
- flat map label redraw handling
|
||||
- live server-mode data loading from `/api/data` even when `jarvis.html` still contains an offline inline snapshot
|
||||
- Terminal Actions panel with `Status`, `Sweep`, and `Brief` buttons
|
||||
|
||||
Important UI markers in the final code:
|
||||
|
||||
```text
|
||||
layerModes
|
||||
spaceDisplayMode
|
||||
toggleSpaceDisplay()
|
||||
shouldShowType()
|
||||
runTerminalAction()
|
||||
```
|
||||
|
||||
### Briefings
|
||||
|
||||
Brief output now includes:
|
||||
|
||||
- Source Integrity
|
||||
- evidence links
|
||||
- event IDs
|
||||
- configurable verbosity through `BRIEF_VERBOSITY`
|
||||
|
||||
### Documentation
|
||||
|
||||
Updated:
|
||||
|
||||
- `README.md`
|
||||
- `.env.example`
|
||||
- `docs/sources/README.md`
|
||||
- `docs/sources/opensky.md`
|
||||
- `docs/sources/acled.md`
|
||||
- `docs/sources/telegram.md`
|
||||
- `docs/sources/firms.md`
|
||||
- `docs/sources/maritime.md`
|
||||
- `docs/security-review.md`
|
||||
- `docs/release-checklist.md`
|
||||
|
||||
README includes:
|
||||
|
||||
- Gitea Registry pull example
|
||||
- Dockge-compatible compose example
|
||||
- full `.env` examples
|
||||
- OpenRouter Free setup
|
||||
- LM Studio setup
|
||||
- Ollama setup
|
||||
- local OpenAI-compatible setup
|
||||
- Pangolin/reverse proxy notes
|
||||
|
||||
## Registry And Images
|
||||
|
||||
Registry image:
|
||||
|
||||
```text
|
||||
git.wilkensxl.de/mrsphay/intelligence-terminal
|
||||
```
|
||||
|
||||
Verified package tags through Gitea API:
|
||||
|
||||
```text
|
||||
latest
|
||||
20260517
|
||||
e933586b220656a2858d2215b934b22d1f08a908
|
||||
53470cc701ec322080a89d220aef449b25850590
|
||||
```
|
||||
|
||||
Successful pull test:
|
||||
|
||||
```bash
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
Observed digest:
|
||||
|
||||
```text
|
||||
sha256:780a41413921bd9a676461eca1cd1372591f523be4b7c9513d9bc085cbe7922d
|
||||
```
|
||||
|
||||
## Gitea Actions
|
||||
|
||||
Workflows present:
|
||||
|
||||
```text
|
||||
.gitea/workflows/build.yml
|
||||
.gitea/workflows/security-scan.yml
|
||||
.gitea/workflows/repo-cleanup.yml
|
||||
.gitea/workflows/dependency-check.yml
|
||||
.gitea/workflows/release-dry-run.yml
|
||||
.gitea/workflows/template-compliance.yml
|
||||
```
|
||||
|
||||
Final runs for commit `53470cc701ec322080a89d220aef449b25850590` were polled through the Gitea API and succeeded:
|
||||
|
||||
```text
|
||||
build.yml on main: success
|
||||
build.yml on codex/production-intelligence-terminal: success
|
||||
release-dry-run.yml on main: success
|
||||
release-dry-run.yml on codex/production-intelligence-terminal: success
|
||||
template-compliance.yml on main: success
|
||||
template-compliance.yml on codex/production-intelligence-terminal: success
|
||||
```
|
||||
|
||||
Relevant run URLs:
|
||||
|
||||
```text
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/23
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/24
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/25
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/26
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/27
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/actions/runs/28
|
||||
```
|
||||
|
||||
Repository secret expected by the registry publish workflow:
|
||||
|
||||
```text
|
||||
REGISTRY_TOKEN
|
||||
```
|
||||
|
||||
Local token note:
|
||||
|
||||
- `GITEA_TOKEN` was visible in the final Codex process.
|
||||
- It was used only for Gitea API checks and not printed.
|
||||
|
||||
## Issue Sync
|
||||
|
||||
Open upstream GitHub issues were reviewed on 2026-05-17 from:
|
||||
|
||||
```text
|
||||
https://github.com/calesthio/Crucix/issues
|
||||
```
|
||||
|
||||
The upstream list contained 24 open issues. Issues already handled by this fork were not copied as open work, including the Docker stale-dashboard incident (#105), map label redraw (#70), Sensor Grid controls (#72), space display toggle (#51), source docs (#52), Dockge/CasaOS docs (#78), LLM timeout (#87), inject/static helper confusion (#100), network metrics (#101), Telegram polling backoff (#104), and briefing/evidence context (#75).
|
||||
|
||||
Issues not relevant to this fork were also not copied, including the Wallpaper Engine redesign (#41), the fork-inflation discussion (#107), empty/unclear placeholders (#79/#80), and the general use-case discussion (#93).
|
||||
|
||||
The following Gitea issues were created for real remaining work:
|
||||
|
||||
```text
|
||||
#1 Reddit source must stop unauthenticated .json scraping
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/1
|
||||
|
||||
#2 Send operator alerts when dashboard data remains stale
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/2
|
||||
|
||||
#3 ACLED credentialed integration needs regression test and diagnostics
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/3
|
||||
|
||||
#4 Complete memory and prediction loop beyond Phase-1 SQLite
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/4
|
||||
|
||||
#5 Remove old inline dashboard snapshot from production builds
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/5
|
||||
|
||||
#6 Harden Terminal Actions for public reverse-proxy deployments
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/6
|
||||
|
||||
#7 Replace ADS-B stub with real disabled/degraded source handling
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/7
|
||||
|
||||
#8 Clean inherited public-demo and upstream marketing references
|
||||
https://git.wilkensxl.de/MrSphay/intelligence-terminal/issues/8
|
||||
```
|
||||
|
||||
## Verification Already Performed
|
||||
|
||||
Local lightweight checks:
|
||||
|
||||
```bash
|
||||
npm run test:unit
|
||||
npm audit --omit=dev --audit-level=high
|
||||
docker compose --env-file .env.example config
|
||||
node --check server.mjs
|
||||
node --check dashboard/inject.mjs
|
||||
node --check lib/llm/openai-compatible.mjs
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Unit test result:
|
||||
|
||||
```text
|
||||
21 tests passing
|
||||
0 failing
|
||||
```
|
||||
|
||||
Audit result:
|
||||
|
||||
```text
|
||||
0 high vulnerabilities
|
||||
```
|
||||
|
||||
Docker build and smoke test were performed locally earlier:
|
||||
|
||||
```bash
|
||||
docker build -t git.wilkensxl.de/mrsphay/intelligence-terminal:latest .
|
||||
docker run --rm -d --name intelligence-terminal-smoke -p 127.0.0.1::3117 -e AUTO_OPEN_BROWSER=false git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
Smoke test observations:
|
||||
|
||||
- Server booted.
|
||||
- No `xdg-open` error.
|
||||
- Initial sweep completed.
|
||||
- `/api/health` moved from `starting` to `degraded` with transparent source errors.
|
||||
- Degraded state was expected without all optional API keys.
|
||||
|
||||
Additional checks after fixing the dashboard live-data bug and Terminal Actions:
|
||||
|
||||
```bash
|
||||
node --check server.mjs
|
||||
npm run test:unit
|
||||
docker compose --env-file .env.example config
|
||||
git diff --check
|
||||
```
|
||||
|
||||
The dashboard script was also syntax-checked after extracting script blocks from `dashboard/public/jarvis.html`.
|
||||
|
||||
## Important Commits
|
||||
|
||||
```text
|
||||
7e85a54 chore: apply agent kit project structure
|
||||
85f97bb feat: harden intelligence runtime and llm providers
|
||||
42b7fc2 docs: add registry dockge and dashboard operations
|
||||
d072390 ci: align gitea workflows with agent kit
|
||||
0559481 ci: fix gitea registry publish login
|
||||
f3c9331 ci: fix agent kit compliance checks
|
||||
c2d572e fix: prepare runs volume before dropping privileges
|
||||
8e096b2 ci: harden gitea workflow reruns
|
||||
e933586 merge: reconcile main with production branch
|
||||
4262c7e docs: expand agent handoff
|
||||
53470cc fix: load live dashboard data and add terminal actions
|
||||
```
|
||||
|
||||
The large implementation commit `85f97bb` and the dashboard/action fix `53470cc` are contained in both:
|
||||
|
||||
```text
|
||||
origin/codex/production-intelligence-terminal
|
||||
origin/main
|
||||
```
|
||||
|
||||
## How To Continue In A Fresh Codex Environment
|
||||
|
||||
1. Clone the Gitea repository:
|
||||
|
||||
```bash
|
||||
git clone https://git.wilkensxl.de/MrSphay/intelligence-terminal.git
|
||||
cd intelligence-terminal
|
||||
git checkout codex/production-intelligence-terminal
|
||||
```
|
||||
|
||||
2. Confirm the expected commit:
|
||||
|
||||
```bash
|
||||
git rev-parse HEAD
|
||||
```
|
||||
|
||||
Expected:
|
||||
|
||||
```text
|
||||
The branch tip should include commit 53470cc701ec322080a89d220aef449b25850590 and the later `docs: sync issue tracker and handoff` commit.
|
||||
```
|
||||
|
||||
3. Read these files first:
|
||||
|
||||
```text
|
||||
AGENTS.md
|
||||
.codex/project.md
|
||||
docs/agent-handoff.md
|
||||
README.md
|
||||
.env.example
|
||||
```
|
||||
|
||||
4. If checking Actions, use `GITEA_TOKEN` from the environment. Do not print it.
|
||||
|
||||
PowerShell check:
|
||||
|
||||
```powershell
|
||||
if ($env:GITEA_TOKEN) { "GITEA_TOKEN=set" } else { "GITEA_TOKEN=missing" }
|
||||
```
|
||||
|
||||
5. Useful commands:
|
||||
|
||||
```bash
|
||||
npm run test:unit
|
||||
docker compose --env-file .env.example config
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
6. Start with Dockge/Pangolin using the README compose example and a `.env` based on `.env.example`.
|
||||
|
||||
## Remaining Risks And Follow-Ups
|
||||
|
||||
- Some sources will report `degraded` until optional keys are set, especially ACLED, FRED, EIA, and Cloudflare Radar.
|
||||
- OpenSky can rate-limit with HTTP 429; this is now visible in health instead of hidden.
|
||||
- GDELT/OFAC can time out under runner/network conditions; health reports this explicitly.
|
||||
- Browser-level visual verification of the full dashboard should be repeated after any future UI change.
|
||||
- The project still inherits the original Crucix broad source surface. Future work should prefer focused source-by-source tests over broad refactors.
|
||||
- If a new Codex environment sees non-fast-forward branch pushes, fetch first and preserve remote commits. Do not force-push without explicit approval.
|
||||
|
||||
## Operator Pull Command
|
||||
|
||||
For deployment:
|
||||
|
||||
```bash
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:latest
|
||||
```
|
||||
|
||||
For a pinned deployment:
|
||||
|
||||
```bash
|
||||
docker pull git.wilkensxl.de/mrsphay/intelligence-terminal:20260517
|
||||
```
|
||||
|
||||
@@ -5,12 +5,21 @@
|
||||
- Shell execution: browser auto-open is gated by `AUTO_OPEN_BROWSER` and defaults to false.
|
||||
- Secrets: `.env` remains ignored; `.env.example` contains no real keys.
|
||||
- External network calls: source fetches use timeout/retry diagnostics and expose degraded state.
|
||||
- Manual actions: `/api/sweep` is local-only unless `SWEEP_TOKEN` is configured.
|
||||
- Manual actions: `/api/sweep` and `/api/action` are gated by `TERMINAL_ACTIONS_ENABLED` and local-only or `SWEEP_TOKEN` authorization.
|
||||
- File writes: runtime writes are limited to `runs/`.
|
||||
- HTML injection: dashboard data is JSON-injected only by the CLI path; server mode serves data through API/SSE.
|
||||
|
||||
## Terminal Actions
|
||||
|
||||
- `TERMINAL_ACTIONS_ENABLED=true` enables dashboard-triggered `status`, `sweep`, and `brief` actions through `POST /api/action`.
|
||||
- If `SWEEP_TOKEN` is set, callers must send the token through `x-sweep-token`, `Authorization: Bearer ...`, or the `token` request body field.
|
||||
- If `SWEEP_TOKEN` is empty, actions are accepted only from local loopback addresses.
|
||||
- For private Dockge/LAN deployments, this is intended to make the terminal operable from the browser.
|
||||
- For Pangolin or other internet-exposed deployments, set `SWEEP_TOKEN` or `TERMINAL_ACTIONS_ENABLED=false` until the public reverse-proxy hardening issue is completed.
|
||||
|
||||
## Residual Risk
|
||||
|
||||
- External feeds can return malformed, stale, or adversarial content. UI rendering should continue to sanitize titles and URLs.
|
||||
- LLM outputs are advisory only and must not be treated as financial advice.
|
||||
- `node:sqlite` availability depends on the Node 22 build; when unavailable the memory database degrades to a no-op placeholder.
|
||||
- Browser-stored sweep tokens are acceptable for a trusted home-server UI, but should not be treated as a strong auth boundary on a public endpoint.
|
||||
|
||||
@@ -2,9 +2,11 @@
|
||||
|
||||
Provides conflict events, fatalities, event types, and locations.
|
||||
|
||||
- Auth: `ACLED_EMAIL` or `ACLED_USER`, plus `ACLED_PASSWORD`.
|
||||
- Auth: `ACLED_EMAIL` and `ACLED_PASSWORD`. `ACLED_USER` or `ACLED_USERNAME` may be used as aliases for `ACLED_EMAIL`.
|
||||
- Flow: OAuth password grant is tried first, then cookie session fallback.
|
||||
- Failure modes: missing credentials (`no_credentials`), rejected credentials or access denied (`auth_failed`), token/API endpoint failure (`api_failed`), and valid empty event sets (`totalEvents: 0`).
|
||||
- Behavior: missing or rejected credentials produce degraded source health with the ACLED error text.
|
||||
- Debug logs redact bearer tokens and cookies.
|
||||
- Test: set credentials, run `node apis/sources/acled.mjs`, then check `/api/health`.
|
||||
- Failure modes are classified as `no_credentials`, `auth_failed`, `access_denied`, or `api_failed`.
|
||||
- Behavior: missing, rejected, or unauthorized credentials produce degraded source health with a concise operator message.
|
||||
- Secret handling: debug output never prints bearer tokens, cookies, or the configured password.
|
||||
- Test: run `node --test test/acled-source.test.mjs`; with real credentials, run `node apis/sources/acled.mjs`, then check `/api/health`.
|
||||
|
||||
`access_denied` normally means the login worked but the account cannot read API data. Check that ACLED terms are accepted, required profile fields are complete, and API access is enabled for the account.
|
||||
|
||||
45
server.mjs
45
server.mjs
@@ -289,14 +289,28 @@ app.get('/api/metrics', (req, res) => {
|
||||
});
|
||||
|
||||
app.post('/api/sweep', express.json(), (req, res) => {
|
||||
const remote = req.ip || '';
|
||||
const local = remote.includes('127.0.0.1') || remote === '::1' || remote === '::ffff:127.0.0.1';
|
||||
const token = req.get('x-crucix-token') || req.query.token || req.body?.token;
|
||||
if (config.sweepToken && token !== config.sweepToken) return res.status(401).json({ error: 'Invalid sweep token' });
|
||||
if (!config.sweepToken && !local) return res.status(403).json({ error: 'Manual sweep is local-only unless SWEEP_TOKEN is set' });
|
||||
if (sweepInProgress) return res.status(409).json({ status: 'already_running', sweepStartedAt });
|
||||
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
|
||||
res.status(202).json({ status: 'accepted' });
|
||||
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
|
||||
triggerSweep(res);
|
||||
});
|
||||
|
||||
app.post('/api/action', express.json(), async (req, res) => {
|
||||
if (!canRunTerminalAction(req)) return res.status(403).json({ error: 'Terminal actions disabled or unauthorized' });
|
||||
const action = String(req.body?.action || req.query.action || '').toLowerCase();
|
||||
|
||||
if (action === 'status') {
|
||||
return res.json({ ok: true, action, health: buildHealth() });
|
||||
}
|
||||
|
||||
if (action === 'brief') {
|
||||
if (!currentData) return res.status(503).json({ ok: false, action, error: 'No data yet — first sweep in progress' });
|
||||
return res.json({ ok: true, action, text: buildBrief(currentData) });
|
||||
}
|
||||
|
||||
if (action === 'sweep') {
|
||||
return triggerSweep(res);
|
||||
}
|
||||
|
||||
res.status(400).json({ ok: false, error: 'Unknown action', actions: ['status', 'brief', 'sweep'] });
|
||||
});
|
||||
|
||||
// API: available locales
|
||||
@@ -333,6 +347,20 @@ function dataAgeMs() {
|
||||
return Number.isFinite(ms) ? ms : null;
|
||||
}
|
||||
|
||||
function canRunTerminalAction(req) {
|
||||
const remote = req.ip || '';
|
||||
const local = remote.includes('127.0.0.1') || remote === '::1' || remote === '::ffff:127.0.0.1';
|
||||
const token = req.get('x-crucix-token') || req.query.token || req.body?.token;
|
||||
if (config.sweepToken) return token === config.sweepToken;
|
||||
return Boolean(config.terminalActionsEnabled || local);
|
||||
}
|
||||
|
||||
function triggerSweep(res) {
|
||||
if (sweepInProgress) return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
|
||||
runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
|
||||
return res.status(202).json({ ok: true, status: 'accepted' });
|
||||
}
|
||||
|
||||
function getLLMStatus() {
|
||||
if (!config.llm.provider) return { state: 'disabled' };
|
||||
if (!llmProvider) return { state: 'misconfigured', provider: config.llm.provider };
|
||||
@@ -376,6 +404,7 @@ function buildHealth() {
|
||||
llm: getLLMStatus(),
|
||||
telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId),
|
||||
discordEnabled: !!(config.discord?.botToken || config.discord?.webhookUrl),
|
||||
terminalActionsEnabled: Boolean(config.terminalActionsEnabled || config.sweepToken),
|
||||
refreshIntervalMinutes: config.refreshIntervalMinutes,
|
||||
language: currentLanguage,
|
||||
memory: intelligenceStore.status(),
|
||||
|
||||
95
test/acled-source.test.mjs
Normal file
95
test/acled-source.test.mjs
Normal file
@@ -0,0 +1,95 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { authenticate, briefing, resetAcledSessionCache } from '../apis/sources/acled.mjs';
|
||||
|
||||
function jsonResponse(status, body, ok = status >= 200 && status < 300) {
|
||||
return {
|
||||
ok,
|
||||
status,
|
||||
headers: { getSetCookie: () => [] },
|
||||
json: async () => body,
|
||||
text: async () => JSON.stringify(body),
|
||||
};
|
||||
}
|
||||
|
||||
test('ACLED reports missing credentials without network access', async () => {
|
||||
resetAcledSessionCache();
|
||||
let calls = 0;
|
||||
const data = await briefing({
|
||||
env: {},
|
||||
fetchImpl: async () => {
|
||||
calls++;
|
||||
throw new Error('unexpected network access');
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(calls, 0);
|
||||
assert.equal(data.status, 'no_credentials');
|
||||
assert.equal(data.error, 'missing_acled_credentials');
|
||||
assert.deepEqual(data.missing, ['ACLED_EMAIL', 'ACLED_PASSWORD']);
|
||||
});
|
||||
|
||||
test('ACLED accepts ACLED_USER as email alias and returns empty valid result', async () => {
|
||||
resetAcledSessionCache();
|
||||
const urls = [];
|
||||
const data = await briefing({
|
||||
env: { ACLED_USER: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
|
||||
fetchImpl: async url => {
|
||||
urls.push(String(url));
|
||||
if (String(url).includes('/oauth/token')) {
|
||||
return jsonResponse(200, { access_token: 'token' });
|
||||
}
|
||||
return jsonResponse(200, { status: 200, data: [] });
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(data.status, 'ok');
|
||||
assert.equal(data.totalEvents, 0);
|
||||
assert.ok(urls.some(url => url.includes('/oauth/token')));
|
||||
assert.ok(urls.some(url => url.includes('/api/acled/read')));
|
||||
});
|
||||
|
||||
test('ACLED classifies auth failure without exposing credentials', async () => {
|
||||
resetAcledSessionCache();
|
||||
const result = await authenticate({
|
||||
env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'super-secret' },
|
||||
fetchImpl: async url => {
|
||||
if (String(url).includes('/oauth/token')) {
|
||||
return jsonResponse(401, { error: 'invalid_grant' }, false);
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
status: 403,
|
||||
headers: { getSetCookie: () => [] },
|
||||
text: async () => 'forbidden',
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.status, 'auth_failed');
|
||||
assert.equal(result.error, 'acled_auth_failed');
|
||||
assert.equal(result.diagnostics.length, 2);
|
||||
assert.doesNotMatch(JSON.stringify(result), /super-secret/);
|
||||
});
|
||||
|
||||
test('ACLED classifies data access denied distinctly from auth failure', async () => {
|
||||
resetAcledSessionCache();
|
||||
const data = await briefing({
|
||||
env: { ACLED_EMAIL: 'analyst@example.test', ACLED_PASSWORD: 'secret' },
|
||||
fetchImpl: async url => {
|
||||
if (String(url).includes('/oauth/token')) {
|
||||
return jsonResponse(200, { access_token: 'token' });
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
status: 403,
|
||||
headers: { getSetCookie: () => [] },
|
||||
text: async () => 'terms not accepted',
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(data.status, 'access_denied');
|
||||
assert.equal(data.error, 'acled_data_http_403');
|
||||
assert.match(data.hint, /Accept ACLED terms/);
|
||||
});
|
||||
@@ -34,129 +34,3 @@ test('safeFetchText returns text and byte count', async () => {
|
||||
globalThis.fetch = originalFetch;
|
||||
}
|
||||
});
|
||||
|
||||
function jsonResponse(payload, ok = true, status = 200) {
|
||||
return {
|
||||
ok,
|
||||
status,
|
||||
headers: { getSetCookie: () => [], get: () => 'application/json' },
|
||||
text: async () => JSON.stringify(payload),
|
||||
json: async () => payload,
|
||||
};
|
||||
}
|
||||
|
||||
function textResponse(text, ok = false, status = 500) {
|
||||
return {
|
||||
ok,
|
||||
status,
|
||||
headers: { getSetCookie: () => [], get: () => 'text/plain' },
|
||||
text: async () => text,
|
||||
json: async () => JSON.parse(text),
|
||||
};
|
||||
}
|
||||
|
||||
async function withAcledEnv(mockFetch, fn) {
|
||||
const originalFetch = globalThis.fetch;
|
||||
const saved = {
|
||||
ACLED_EMAIL: process.env.ACLED_EMAIL,
|
||||
ACLED_USER: process.env.ACLED_USER,
|
||||
ACLED_USERNAME: process.env.ACLED_USERNAME,
|
||||
ACLED_PASSWORD: process.env.ACLED_PASSWORD,
|
||||
};
|
||||
globalThis.fetch = mockFetch;
|
||||
delete process.env.ACLED_EMAIL;
|
||||
delete process.env.ACLED_USER;
|
||||
delete process.env.ACLED_USERNAME;
|
||||
delete process.env.ACLED_PASSWORD;
|
||||
const acled = await import('../apis/sources/acled.mjs');
|
||||
acled.resetAcledSessionForTests();
|
||||
try {
|
||||
return await fn(acled);
|
||||
} finally {
|
||||
globalThis.fetch = originalFetch;
|
||||
for (const [key, value] of Object.entries(saved)) {
|
||||
if (value === undefined) delete process.env[key];
|
||||
else process.env[key] = value;
|
||||
}
|
||||
acled.resetAcledSessionForTests();
|
||||
}
|
||||
}
|
||||
|
||||
test('ACLED credentialed OAuth success returns live events and supports ACLED_USER', async () => {
|
||||
const responses = [
|
||||
jsonResponse({ access_token: 'secret-token' }),
|
||||
jsonResponse({
|
||||
status: 200,
|
||||
data: [{
|
||||
event_date: '2026-05-17',
|
||||
event_type: 'Protests',
|
||||
sub_event_type: 'Peaceful protest',
|
||||
country: 'Example',
|
||||
region: 'Example Region',
|
||||
location: 'Example City',
|
||||
fatalities: '0',
|
||||
latitude: '1.23',
|
||||
longitude: '4.56',
|
||||
}],
|
||||
}),
|
||||
];
|
||||
|
||||
await withAcledEnv(async () => responses.shift(), async ({ briefing }) => {
|
||||
process.env.ACLED_USER = 'operator@example.test';
|
||||
process.env.ACLED_PASSWORD = 'password';
|
||||
const data = await briefing();
|
||||
|
||||
assert.equal(data.status, 'live');
|
||||
assert.equal(data.totalEvents, 1);
|
||||
assert.equal(data.topCountries.Example.count, 1);
|
||||
});
|
||||
});
|
||||
|
||||
test('ACLED rejected credentials return auth_failed diagnostics', async () => {
|
||||
const responses = [
|
||||
textResponse('invalid credentials', false, 401),
|
||||
textResponse('forbidden', false, 403),
|
||||
];
|
||||
|
||||
await withAcledEnv(async () => responses.shift(), async ({ briefing }) => {
|
||||
process.env.ACLED_EMAIL = 'operator@example.test';
|
||||
process.env.ACLED_PASSWORD = 'wrong-password';
|
||||
const data = await briefing();
|
||||
|
||||
assert.equal(data.status, 'auth_failed');
|
||||
assert.match(data.error, /All ACLED auth methods failed/);
|
||||
});
|
||||
});
|
||||
|
||||
test('ACLED token endpoint failure returns api_failed diagnostics', async () => {
|
||||
const responses = [
|
||||
textResponse('temporary outage', false, 503),
|
||||
textResponse('temporary outage', false, 503),
|
||||
];
|
||||
|
||||
await withAcledEnv(async () => responses.shift(), async ({ briefing }) => {
|
||||
process.env.ACLED_EMAIL = 'operator@example.test';
|
||||
process.env.ACLED_PASSWORD = 'password';
|
||||
const data = await briefing();
|
||||
|
||||
assert.equal(data.status, 'api_failed');
|
||||
assert.match(data.error, /All ACLED auth methods failed/);
|
||||
});
|
||||
});
|
||||
|
||||
test('ACLED valid empty response is live with zero events', async () => {
|
||||
const responses = [
|
||||
jsonResponse({ access_token: 'secret-token' }),
|
||||
jsonResponse({ status: 200, data: [] }),
|
||||
];
|
||||
|
||||
await withAcledEnv(async () => responses.shift(), async ({ briefing }) => {
|
||||
process.env.ACLED_EMAIL = 'operator@example.test';
|
||||
process.env.ACLED_PASSWORD = 'password';
|
||||
const data = await briefing();
|
||||
|
||||
assert.equal(data.status, 'live');
|
||||
assert.equal(data.totalEvents, 0);
|
||||
assert.match(data.message, /valid empty/);
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user