feat: alert operators on stale data
This commit is contained in:
30
server.mjs
30
server.mjs
@@ -18,6 +18,7 @@ import { TelegramAlerter } from './lib/alerts/telegram.mjs';
|
||||
import { DiscordAlerter } from './lib/alerts/discord.mjs';
|
||||
import { getFetchMetrics } from './apis/utils/fetch.mjs';
|
||||
import { IntelligenceStore } from './lib/intelligence-store.mjs';
|
||||
import { formatStaleAlert, shouldSendStaleAlert } from './lib/stale-alerts.mjs';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = __dirname;
|
||||
@@ -39,6 +40,7 @@ let sweepStartedAt = null; // Timestamp when current/last sweep started
|
||||
let sweepInProgress = false;
|
||||
const startTime = Date.now();
|
||||
const sseClients = new Set();
|
||||
const staleAlertState = {};
|
||||
|
||||
// === Delta/Memory ===
|
||||
const memory = new MemoryManager(RUNS_DIR);
|
||||
@@ -411,6 +413,31 @@ function buildHealth() {
|
||||
};
|
||||
}
|
||||
|
||||
async function notifyIfDataStale(context = 'scheduled sweep') {
|
||||
const health = buildHealth();
|
||||
const decision = shouldSendStaleAlert(health, staleAlertState, {
|
||||
cooldownMs: config.staleAlertCooldownMinutes * 60 * 1000,
|
||||
});
|
||||
if (!decision.send) return false;
|
||||
|
||||
const dashboardUrl = config.dashboardUrl || `http://localhost:${config.port}`;
|
||||
const message = formatStaleAlert(health, { dashboardUrl, context });
|
||||
const sends = [];
|
||||
if (telegramAlerter.isConfigured) sends.push(telegramAlerter.sendMessage(message));
|
||||
if (discordAlerter.isConfigured) sends.push(discordAlerter.sendAlert(message));
|
||||
|
||||
if (sends.length === 0) {
|
||||
console.warn('[Crucix] Data is stale but no operator alert channel is configured');
|
||||
return false;
|
||||
}
|
||||
|
||||
const results = await Promise.allSettled(sends);
|
||||
const sent = results.some(r => r.status === 'fulfilled' && (r.value === true || r.value?.ok === true));
|
||||
if (sent) console.warn('[Crucix] Operator stale-data alert sent');
|
||||
else console.warn('[Crucix] Operator stale-data alert attempted but no channel accepted it');
|
||||
return sent;
|
||||
}
|
||||
|
||||
function buildBrief(data) {
|
||||
const verbosity = config.telegram.briefVerbosity || 'standard';
|
||||
const delta = memory.getLastDelta();
|
||||
@@ -553,6 +580,9 @@ async function runSweepCycle() {
|
||||
broadcast({ type: 'sweep_error', error: err.message });
|
||||
} finally {
|
||||
sweepInProgress = false;
|
||||
await notifyIfDataStale(lastSweepError ? 'failed sweep' : 'completed sweep').catch(err => {
|
||||
console.error('[Crucix] Stale-data operator alert failed:', err.message);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user