feat: alert operators on stale data
All checks were successful
Codex Template Compliance / template-compliance (pull_request) Successful in 5s
Build / test-and-image (pull_request) Successful in 53s

This commit is contained in:
2026-05-17 13:58:32 +02:00
parent 8605d0baab
commit e574ad1c3d
6 changed files with 151 additions and 0 deletions

View File

@@ -18,6 +18,7 @@ import { TelegramAlerter } from './lib/alerts/telegram.mjs';
import { DiscordAlerter } from './lib/alerts/discord.mjs';
import { getFetchMetrics } from './apis/utils/fetch.mjs';
import { IntelligenceStore } from './lib/intelligence-store.mjs';
import { formatStaleAlert, shouldSendStaleAlert } from './lib/stale-alerts.mjs';
const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT = __dirname;
@@ -39,6 +40,7 @@ let sweepStartedAt = null; // Timestamp when current/last sweep started
let sweepInProgress = false;
const startTime = Date.now();
const sseClients = new Set();
const staleAlertState = {};
// === Delta/Memory ===
const memory = new MemoryManager(RUNS_DIR);
@@ -411,6 +413,31 @@ function buildHealth() {
};
}
async function notifyIfDataStale(context = 'scheduled sweep') {
const health = buildHealth();
const decision = shouldSendStaleAlert(health, staleAlertState, {
cooldownMs: config.staleAlertCooldownMinutes * 60 * 1000,
});
if (!decision.send) return false;
const dashboardUrl = config.dashboardUrl || `http://localhost:${config.port}`;
const message = formatStaleAlert(health, { dashboardUrl, context });
const sends = [];
if (telegramAlerter.isConfigured) sends.push(telegramAlerter.sendMessage(message));
if (discordAlerter.isConfigured) sends.push(discordAlerter.sendAlert(message));
if (sends.length === 0) {
console.warn('[Crucix] Data is stale but no operator alert channel is configured');
return false;
}
const results = await Promise.allSettled(sends);
const sent = results.some(r => r.status === 'fulfilled' && (r.value === true || r.value?.ok === true));
if (sent) console.warn('[Crucix] Operator stale-data alert sent');
else console.warn('[Crucix] Operator stale-data alert attempted but no channel accepted it');
return sent;
}
function buildBrief(data) {
const verbosity = config.telegram.briefVerbosity || 'standard';
const delta = memory.getLastDelta();
@@ -553,6 +580,9 @@ async function runSweepCycle() {
broadcast({ type: 'sweep_error', error: err.message });
} finally {
sweepInProgress = false;
await notifyIfDataStale(lastSweepError ? 'failed sweep' : 'completed sweep').catch(err => {
console.error('[Crucix] Stale-data operator alert failed:', err.message);
});
}
}