// ============================================
// COOKBOOK RUNNING SUB-MODULE
// Running tasks tab: task cards, status monitoring,
// stop/restart, diagnosis, auto-fix, background monitor
// ============================================
import uiModule from './ui.js';
import { _diagnose, _showDiagnosis, _clearDiagnosis } from './cookbook-diagnosis.js';
import { registerMenuDismiss } from './escMenuStack.js';
import { computeProgressSignal } from './cookbookProgressSignal.js';
// Human-friendly badge label for a task's internal status. Avoids surfacing
// the word "error" in the sidebar — a server the user stopped or one that
// quit cleanly reads as "stopped", not "error".
function _statusLabel(status, type) {
if (status === 'running' && type === 'download') return 'downloading';
if (status === 'done' && type === 'download') return 'finished';
if (status === 'error') return 'stopped';
return status || '';
}
// Single source of truth for what a task's status badge shows + its style class.
// Crucially, a serve task that's still coming up shows its live phase
// ("loading 45%", "warming up", …) rather than the generic "running" — they're
// the same state, so the badge shouldn't flip between two different labels on
// every re-render. Returns { text, cls } where cls is appended after
// "cookbook-task-status" ('' = the neutral loading style).
function _taskBadge(task) {
if (task._unreachable && task.status === 'running') return { text: 'unreachable', cls: 'cookbook-task-error' };
if (task.type === 'serve' && task.status === 'running' && task.progress) {
// Same green "running" pill — just with dynamic phase text, so it doesn't
// read as a different status while the server is coming up.
return { text: task.progress, cls: 'cookbook-task-running' };
}
return { text: _statusLabel(task.status, task.type), cls: 'cookbook-task-' + task.status };
}
// A download task whose tmux output still shows an active per-shard line
// (e.g. "model-00012-of-00082.safetensors: 56%|") is NOT actually finished —
// the cookbook just lost track. The clear pill becomes a "reconnect" affordance
// in that case (click → revive the row + reattach the poll loop).
function _downloadOutputLooksActive(task) {
if (!task || task.type !== 'download') return false;
const out = task.output || '';
if (!out) return false;
if (out.includes('DOWNLOAD_OK') || out.includes('DOWNLOAD_FAILED')) return false;
// An active shard line: filename + a colon + a percentage that isn't 100%.
// We catch any in-flight shard or "Downloading 'X' to ..." line (no %).
return /model-\d+-of-\d+\.[a-z]+:\s+(?!100%)\d+%/i.test(out)
|| /Downloading\s+'[^']+'\s+to\s+'[^']*\.incomplete'/i.test(out);
}
function _canClearTask(task) {
if (!task || task.status === 'running') return false;
if (task.type === 'serve' && (task.status === 'ready' || task._serveReady)) return false;
// If the tmux output still shows an in-flight download, the task isn't
// actually finished — hide the clear/check pill so it doesn't show on a
// task that's still doing work. (The next render will reflect this and
// ideally the self-heal flips status back to running.)
if (_downloadOutputLooksActive(task)) return false;
return ['done', 'stopped', 'error', 'crashed', 'failed'].includes(task.status);
}
function _clearPillLabel(task) {
if (_downloadOutputLooksActive(task)) return 'reconnect';
return 'clear';
}
// A pip dependency/driver install (payload._dep) reports success with the
// runner's "=== Process exited with code 0 ===" sentinel and pip's
// "Successfully installed" line — never the HuggingFace download markers
// (DONE / 100% / /snapshots/ / DOWNLOAD_OK) that the download heuristics look
// for. Without this, a clean install whose tmux pane has already gone away is
// misread as crashed/stopped even though pip exited 0. Prefer the authoritative
// exit-code sentinel; fall back to pip's success line when no sentinel was
// captured (and there's no install error in the same output).
function _depInstallSucceeded(output) {
const text = String(output || '');
if (!text) return false;
const exitMatch = text.match(/=== Process exited with code (-?\d+) ===/);
if (exitMatch) return Number(exitMatch[1]) === 0;
return /\b(?:Successfully installed|Requirement already satisfied)\b/.test(text)
&& !/\bERROR\b|No matching distribution|Could not find a version|Traceback \(most recent call last\)/.test(text);
}
function _shouldOfferCrashReport(task) {
if (!task) return false;
if (task._unreachable && task.type === 'serve') return true;
return ['error', 'crashed', 'failed'].includes(task.status);
}
function _serveTaskLooksAwqOnLocalBackend(task, outputText = '') {
const repo = `${task?.payload?.repo_id || ''} ${task?.name || ''}`.toLowerCase();
const cmd = `${task?.payload?._cmd || ''} ${outputText || ''}`.toLowerCase();
return /\b(awq|gptq|fp8)\b/.test(repo) && /(llama-server|llama_cpp\.server|ollama|ggml_cuda_enable_unified_memory)/.test(cmd);
}
function _serveTaskLooksAwqWithoutUsableAccelerator(task, outputText = '') {
const repo = `${task?.payload?.repo_id || ''} ${task?.name || ''}`.toLowerCase();
const out = String(outputText || '').toLowerCase();
return /\b(awq|gptq|fp8)\b/.test(repo)
&& /(no accelerator|no cuda runtime|failed to infer device type|triton is not supported|0 active driver)/i.test(out);
}
async function _openDownloadForGgufTask(task) {
const raw = task?.payload?.repo_id || task?.name || '';
const modelName = String(raw)
.split('/').pop()
.replace(/[-_](?:AWQ|GPTQ|FP8|4bit|8bit|Int4|Int8).*$/i, '')
.replace(/[-_]+$/g, '')
|| String(raw).split('/').pop()
|| raw;
const cookbook = window.cookbookModule;
if (cookbook && typeof cookbook.open === 'function') {
cookbook.open({ tab: 'Search' });
} else {
document.getElementById('tool-cookbook-btn')?.click();
}
setTimeout(async () => {
const modal = document.getElementById('cookbook-modal');
const tab = modal?.querySelector('.cookbook-tab[data-backend="Search"]');
if (tab && !tab.classList.contains('active')) tab.click();
const search = document.getElementById('hwfit-search');
if (search) {
search.value = modelName;
search.dispatchEvent(new Event('input', { bubbles: true }));
search.focus();
}
const quant = document.getElementById('hwfit-quant');
if (quant) {
quant.value = 'Q4_K_M';
quant.dispatchEvent(new Event('change', { bubbles: true }));
}
try {
const hwfit = await import('./cookbook-hwfit.js');
if (typeof hwfit._hwfitFetch === 'function') hwfit._hwfitFetch(true);
} catch {}
}, 80);
}
function _terminalServeDiagnosis(task, outputText) {
const out = String(outputText || task?.output || '');
if (!task || task.type !== 'serve' || !['stopped', 'error', 'crashed', 'failed'].includes(task.status) || !out.trim()) return null;
// Pip tasks (Reinstall vLLM, Upgrade torch, etc.) ride on the serve task
// type so they get a tmux session + show up in Running tab — but they are
// NOT serve invocations. Their output is pip's own; the generic
// "Serve stopped before the model became reachable" message + Edit-serve
// fix make no sense. Bail so the panel just shows pip's output.
const _isPipTask = ((task.payload?.repo_id || '').startsWith('pip-'))
|| /python3? -m pip\b/.test(task.payload?._cmd || '');
if (_isPipTask) return null;
if (_serveTaskLooksAwqOnLocalBackend(task, out)) {
return {
message: 'AWQ/GPTQ/FP8 cannot be served through llama.cpp/Ollama unified-memory mode.',
suggestion: 'Suggested action: use vLLM/SGLang on a compatible CUDA/ROCm GPU server, or download a GGUF version for llama.cpp/Ollama/unified-memory serving.',
fixes: [
{ label: 'Find GGUF download', action: () => _openDownloadForGgufTask(task) },
{ label: 'Edit serve', action: (panel) => _openServeEditForTask(task) },
],
};
}
if (_serveTaskLooksAwqWithoutUsableAccelerator(task, out)) {
return {
message: 'AWQ/GPTQ/FP8 needs a working vLLM/SGLang accelerator path; this server did not expose one.',
suggestion: 'Suggested action: choose a CUDA/ROCm server where vLLM/SGLang can see the GPU, or download a GGUF version and serve it with llama.cpp/Ollama.',
fixes: [
{ label: 'Find GGUF download', action: () => _openDownloadForGgufTask(task) },
{ label: 'Edit serve', action: (panel) => _openServeEditForTask(task) },
],
};
}
return _diagnose(out) || {
message: /Native llama-server not found|building llama-server|llama\.cpp/i.test(out)
? 'llama.cpp build stopped before the server became reachable.'
: 'Serve stopped before the model became reachable.',
suggestion: /Native llama-server not found|building llama-server|llama\.cpp/i.test(out)
? 'Suggested action: copy the troubleshooting bundle, then edit serve settings. For the quickest local/CPU path, use Ollama or a prebuilt llama-server; source builds can take several minutes and fail if build dependencies are incomplete.'
: 'Suggested action: copy the troubleshooting bundle, then edit serve settings or relaunch with a CPU/backend fallback.',
fixes: [{ label: 'Edit serve', action: (panel) => _openServeEditForTask(task) }],
};
}
function _redactCrashReportText(text) {
if (!text) return '';
return String(text)
.replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]{12,}/gi, '$1[redacted]')
.replace(/\b(hf_[A-Za-z0-9]{16,})\b/g, '[redacted-hf-token]')
.replace(/\b(sk-[A-Za-z0-9_-]{16,})\b/g, '[redacted-api-key]')
.replace(/\b(xox[baprs]-[A-Za-z0-9-]{16,})\b/g, '[redacted-slack-token]')
.replace(/\b(AIza[0-9A-Za-z_-]{20,})\b/g, '[redacted-google-key]')
.replace(/\b((?:HF_TOKEN|HUGGING_FACE_HUB_TOKEN|OPENAI_API_KEY|ANTHROPIC_API_KEY|BRAVE_API_KEY|TAVILY_API_KEY|SERPER_API_KEY|GOOGLE_API_KEY|API_KEY|TOKEN|PASSWORD)\s*=\s*)(['"]?)[^\s'"\\]+/gi, '$1$2[redacted]')
.replace(/\b(--(?:api-key|token|hf-token|password)\s+)([^\s]+)/gi, '$1[redacted]');
}
function _lastLines(text, count = 160) {
const clean = _redactCrashReportText(text || '').trimEnd();
if (!clean) return '(no captured output)';
return clean.split('\n').slice(-count).join('\n');
}
function _codeFence(text) {
return String(text || '').replace(/```/g, '` ` `');
}
function _taskHostLabel(task) {
if (!task?.remoteHost) return 'local';
return task.remoteHost + (task.sshPort ? `:${task.sshPort}` : '');
}
function _taskPort(task) {
const cmd = task?.payload?._cmd || '';
const match = cmd.match(/--port\s+(\d+)/);
return match ? match[1] : '';
}
function _buildCrashReport(task, outputText) {
const capturedOutput = outputText || task?.output || '';
const cmd = _redactCrashReportText(task?.payload?._cmd || '');
const diag = _diagnose(capturedOutput);
const started = task?.ts ? new Date(task.ts).toISOString() : '';
const report = [
'## Odysseus Cookbook crash report',
'',
'Please review this report for secrets before posting it publicly.',
'',
'### Task',
`- ID: \`${task?.sessionId || task?.id || 'unknown'}\``,
`- Type: \`${task?.type || 'unknown'}\``,
`- Status: \`${task?._unreachable ? 'unreachable' : (task?.status || 'unknown')}\``,
`- Model/repo: \`${task?.payload?.repo_id || task?.name || 'unknown'}\``,
`- Host: \`${_taskHostLabel(task)}\``,
];
if (task?.platform) report.push(`- Platform: \`${task.platform}\``);
if (started) report.push(`- Started: \`${started}\``);
const port = _taskPort(task);
if (port) report.push(`- Port: \`${port}\``);
if (diag?.message) report.push(`- Diagnosis: ${diag.message}`);
if (cmd) {
report.push('', '### Command', '```bash', _codeFence(cmd), '```');
}
report.push('', '### Last captured output', '```text', _codeFence(_lastLines(capturedOutput)), '```');
return report.join('\n');
}
// Shared state/functions injected by init()
let _envState;
let _sshCmd;
let _getPort;
let _sshPrefix;
let _getPlatform;
let _isWindows;
let _buildEnvPrefix;
let _loadPresets;
let _savePresets;
let _copyText;
let _persistEnvState;
let _refreshDependencies;
let modelLogo;
let esc;
let _detectBackend;
let _detectToolParser;
let _detectModelOptimizations;
let _buildServeCmd;
// When a new action is started (download / dependency / serve), this holds the
// new task's id so the next render collapses every other card and leaves only
// the new one open. Consumed (cleared) by _renderRunningTab.
let _soloExpandTaskId = null;
// Storage keys
const TASKS_KEY = 'cookbook-tasks';
const STORAGE_KEY = 'cookbook-presets';
const SERVE_STATE_KEY = 'cookbook-serve-state';
// Polling / timeout intervals
const TASK_POLL_INTERVAL_MS = 3000; // delay between reconnect-loop iterations
const BG_MONITOR_INTERVAL_MS = 5000; // background task status poll
const STALE_PROGRESS_MS = 5 * 60 * 1000; // download with no progress this long = stale
const STARTUP_STALE_PROGRESS_MS = 45 * 1000; // 0%-forever startup stall: retry much sooner
// ── Phase detection (mirrors Python _parse_serve_phase in cookbook_routes.py) ──
// Single source of truth for serve task status. KEEP IN SYNC with the Python version.
export function _parseServePhase(snapshot) {
if (!snapshot) return {};
// Strip newlines so tmux line-wrapping doesn't break regex matching
const flat = snapshot.replace(/\s+/g, ' ');
const loadMatches = [...flat.matchAll(/Loading safetensors.*?(\d+)%/g)];
// "Downloading (incomplete total...)" tracks real aggregate bytes; prefer it
// over "Fetching N files" which only counts fully-closed files and lags badly
// with hf_transfer's parallel-chunk strategy (often sits at 0/N for most of the run).
const downloadingMatches = [...flat.matchAll(/Downloading.*?(\d+)%/g)];
const fetchingMatches = [...flat.matchAll(/Fetching.*?(\d+)%/g)];
const dlMatches = downloadingMatches.length ? downloadingMatches : fetchingMatches;
// "Avg generation throughput: X tokens/s, Running: N reqs"
const tpsMatches = [...flat.matchAll(/(?:Avg )?generation throughput:\s*([\d.]+)\s*tokens\/s.*?Running:\s*(\d+)\s*reqs/g)];
// Throughput FIRST — its log line contains "GPU KV cache usage" which would
// otherwise false-match the warmup check
if (tpsMatches.length) {
const m = tpsMatches[tpsMatches.length - 1];
const tps = parseFloat(m[1]);
const reqs = parseInt(m[2]);
return {
phase: reqs > 0 ? `${m[1]} tok/s` : 'idle',
status: 'ready',
tps,
reqs,
};
}
if (flat.includes('Application startup complete')) {
return { phase: 'ready', status: 'ready' };
}
if (/Ollama API ready on port\s+\d+/i.test(flat)) {
return { phase: 'ready', status: 'ready' };
}
const llamaBuildMatches = [...flat.matchAll(/\[\s*(\d{1,3})%\]\s*(?:Building|Linking)/gi)];
if (llamaBuildMatches.length) {
const pct = Math.min(100, parseInt(llamaBuildMatches[llamaBuildMatches.length - 1][1], 10));
return { phase: `building llama.cpp ${pct}%`, status: 'running', pct };
}
if (/Native llama-server not found|building from source/i.test(flat)) {
if (/Cloning into ['"]?llama\.cpp/i.test(flat) && !/Receiving objects:\s*100%/i.test(flat)) {
return { phase: 'cloning llama.cpp', status: 'running' };
}
if (/Configuring incomplete|CMake Error/i.test(flat)) {
return {};
}
if (/CMAKE_BUILD_TYPE|Detecting CXX|Found Threads|Including CPU backend|CUDA nvcc found|building llama-server/i.test(flat)) {
return { phase: 'configuring llama.cpp', status: 'running' };
}
return { phase: 'building llama.cpp', status: 'running' };
}
// HTTP access logs (e.g. GET /v1/models 200 OK) mean the server is up
if (/(?:GET|POST)\s+\/[^\s]*\s+HTTP\/[\d.]+"\s*\d{3}/.test(flat)) {
return { phase: 'idle', status: 'ready' };
}
if (flat.includes('Loading weights took')) {
return { phase: 'initializing', status: 'running' };
}
// "GPU KV cache" alone (during allocation) — not "GPU KV cache usage" (runtime log)
if (flat.includes('GPU KV cache') && !flat.includes('GPU KV cache usage')) {
return { phase: 'warming up', status: 'running' };
}
if (loadMatches.length) {
const pct = parseInt(loadMatches[loadMatches.length - 1][1]);
return { phase: `loading ${pct}%`, status: 'running', pct };
}
if (dlMatches.length) {
const pct = parseInt(dlMatches[dlMatches.length - 1][1]);
return { phase: `downloading ${pct}%`, status: 'running', pct };
}
return {};
}
// ── Port auto-increment ──
function _nextAvailablePort() {
const tasks = _loadTasks();
const presets = _loadPresets();
const usedPorts = new Set();
tasks.forEach(t => {
if (t.type === 'serve' && (t.status === 'running' || t.status === 'queued')) {
const m = t.payload?._cmd?.match(/--port\s+(\d+)/);
if (m) usedPorts.add(parseInt(m[1]));
}
});
presets.forEach(p => {
if (p.port) usedPorts.add(parseInt(p.port));
});
let port = 8000;
while (usedPorts.has(port)) port++;
return String(port);
}
// ── Endpoint cleanup ──
async function _removeEndpointByUrl(baseUrl) {
try {
const res = await fetch('/api/model-endpoints', { credentials: 'same-origin' });
if (!res.ok) return;
const endpoints = await res.json();
const hostPort = baseUrl.replace(/^https?:\/\//, '').replace(/\/.*$/, '');
const ep = endpoints.find(e => e.base_url === baseUrl)
|| endpoints.find(e => e.base_url.includes(hostPort));
if (ep) {
await fetch(`/api/model-endpoints/${ep.id}`, { method: 'DELETE', credentials: 'same-origin' });
_refreshModelsAfterEndpointChange();
}
} catch {}
}
function _refreshModelsAfterEndpointChange() {
const pickerLabel = document.getElementById('model-picker-label');
if (pickerLabel) {
pickerLabel.dataset.prevHtml = pickerLabel.innerHTML;
pickerLabel.innerHTML = 'refreshing…';
}
if (window.modelsModule && window.modelsModule.refreshModels) {
window.modelsModule.refreshModels(true);
}
setTimeout(() => {
if (!window.sessionModule) return;
const currentModel = window.sessionModule.getCurrentModel ? window.sessionModule.getCurrentModel() : null;
if (currentModel) {
const items = (window.modelsModule && window.modelsModule.getCachedItems) ? window.modelsModule.getCachedItems() : [];
const allModels = [];
items.forEach(item => {
if (item.offline) return;
(item.models || []).concat(item.models_extra || []).forEach(m => allModels.push({ mid: m, url: item.url, endpointId: item.endpoint_id }));
});
const stillExists = allModels.some(m => m.mid === currentModel);
if (!stillExists && allModels.length > 0) {
const fallback = allModels[0];
if (window.sessionModule.createDirectChat) {
window.sessionModule.createDirectChat(fallback.url, fallback.mid, fallback.endpointId);
}
}
}
if (window.sessionModule.updateModelPicker) {
window.sessionModule.updateModelPicker();
}
}, 1500);
}
function _appendCookbookEndpointScope(fd, remoteHost) {
const host = String(remoteHost || '').trim();
if (!host || host === 'local' || host === 'localhost' || host === '127.0.0.1') {
fd.append('container_local', 'true');
}
}
function _connectHostFromRemote(remoteHost, fallback = 'localhost') {
const host = String(remoteHost || '').trim();
if (!host || host === 'local') return fallback;
return host.includes('@') ? host.split('@').pop() : host;
}
function _isAnyBindHost(host) {
const h = String(host || '').trim().toLowerCase();
return h === '0.0.0.0' || h === '::' || h === '[::]';
}
function _endpointFromAdvertisedUrl(rawUrl, currentHost, fallbackPort = '11434') {
try {
const u = new URL(rawUrl);
const host = _isAnyBindHost(u.hostname) ? currentHost : (u.hostname || currentHost);
const port = u.port || fallbackPort;
const bracketedHost = host.includes(':') && !host.startsWith('[') ? `[${host}]` : host;
return { host, port, baseUrl: `${u.protocol}//${bracketedHost}${port ? `:${port}` : ''}/v1` };
} catch {
return null;
}
}
// ── Download queue — runs one at a time per server ──
function _processQueue() {
const tasks = _loadPrunedTasks();
const running = tasks.filter(t => t.type === 'download' && t.status === 'running');
const queued = tasks.filter(t => t.type === 'download' && t.status === 'queued');
if (!queued.length) return;
const busyHosts = new Set(running.map(t => t.remoteHost || 'local'));
for (const task of queued) {
const host = task.remoteHost || 'local';
if (busyHosts.has(host)) continue;
busyHosts.add(host);
_startQueuedDownload(task);
}
}
async function _startQueuedDownload(task) {
if (!task.payload) {
_updateTask(task.sessionId, { status: 'error', output: 'No payload' });
_renderRunningTab();
return;
}
// Flip to 'running' SYNCHRONOUSLY (before the async POST) so a concurrent
// _processQueue — or a second "Start now" — can't see it as still 'queued' and
// launch the same download a second time. Without this, finishing another
// download mid-POST re-queued this one into a duplicate task.
{
const _pre = _loadTasks();
const _pt = _pre.find(t => t.sessionId === task.sessionId);
if (_pt) {
if (_pt.status === 'running' && _pt._startLaunched) return; // already being started
_pt.status = 'running';
_pt._startLaunched = true;
_saveTasks(_pre);
}
}
try {
const res = await fetch('/api/model/download', {
method: 'POST', credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(task.payload),
});
if (!res.ok) {
const errText = await res.text().catch(() => '');
_updateTask(task.sessionId, { status: 'error', output: `HTTP ${res.status}: ${errText.slice(0, 200)}` });
_renderRunningTab();
return;
}
const data = await res.json();
if (!data.ok) {
_updateTask(task.sessionId, { status: 'error', output: data.error || 'Unknown error' });
_renderRunningTab();
return;
}
const oldId = task.sessionId;
const launchedTask = { ...task, sessionId: data.session_id, id: data.session_id, status: 'running' };
const key = _downloadDedupeKey(launchedTask);
let found = false;
const tasks = _loadTasks().filter(t => {
if (t.sessionId === oldId) {
found = true;
t.sessionId = data.session_id;
t.id = data.session_id;
t.status = 'running';
t._startLaunched = true;
return true;
}
if (t.sessionId === data.session_id) return false;
return !(key && t.type === 'download' && t.status === 'queued' && _downloadDedupeKey(t) === key);
});
if (!found) tasks.push(_stripTaskSecrets(launchedTask));
_saveTasks(tasks);
_renderRunningTab();
_startBackgroundMonitor();
await new Promise(r => setTimeout(r, 2000));
_renderRunningTab();
} catch (e) {
_updateTask(task.sessionId, { status: 'error', output: e.message || 'Network error' });
_renderRunningTab();
}
}
// ── Task CRUD ──
function _serveOutputLooksReady(task) {
const out = String(task?.output || '');
return !!task?._serveReady
|| /Application startup complete/i.test(out)
|| /Ollama API ready on port\s+\d+/i.test(out)
|| /(?:GET|POST)\s+\/[^\s]*\s+HTTP\/[\d.]+"\s*2\d\d/i.test(out);
}
function _normalizeTaskForDisplay(task) {
if (!task || typeof task !== 'object') return task;
// Pip tasks (Reinstall vLLM / Upgrade torch / etc.) ride on the serve task
// type so they get tmux + the Running tab. They are NOT serves — their
// "ready" markers are pip's `Successfully installed` / `Requirement already
// satisfied`, not "Application startup complete".
const _isPipTask = ((task.payload?.repo_id || '').startsWith('pip-'))
|| /python3? -m pip\b/.test(task.payload?._cmd || '');
if (_isPipTask) {
// Override stale status: any pip task whose output carries pip's own
// success markers gets displayed as `done` regardless of what's in
// localStorage. Old pre-fix runs landed in error/stopped state and
// stuck there even after we taught the rest of the flow about pip
// tasks — this is the catch-all that flips them to Finished on render.
const out = String(task.output || '');
const ranOk = /Successfully installed|Requirement already (?:satisfied|up-to-date)/i.test(out)
&& !/error:|ERROR:/.test(out.slice(-1024));
if (ranOk && task.status !== 'done' && task.status !== 'running') {
return { ...task, status: 'done' };
}
return task;
}
if (task.type === 'serve' && task.status === 'done' && !_serveOutputLooksReady(task)) {
return { ...task, status: 'error' };
}
return task;
}
export function _loadTasks() {
try { return (JSON.parse(localStorage.getItem(TASKS_KEY)) || []).map(_normalizeTaskForDisplay); }
catch { return []; }
}
function _downloadRepoKey(task) {
return String(task?.payload?.repo_id || task?.repo_id || task?.repo || task?.name || '').trim();
}
function _downloadHostKey(task) {
return String(task?.remoteHost || task?.payload?.remote_host || 'local').trim() || 'local';
}
function _downloadDedupeKey(task) {
if (!task || task.type !== 'download') return '';
const repo = _downloadRepoKey(task);
if (!repo) return '';
return `${_downloadHostKey(task)}\n${repo}`;
}
function _pruneQueuedDownloadDuplicates(tasks) {
if (!Array.isArray(tasks) || !tasks.length) return tasks || [];
const launched = new Set();
for (const task of tasks) {
if (task?.type !== 'download' || task.status === 'queued') continue;
const key = _downloadDedupeKey(task);
if (key) launched.add(key);
}
let changed = false;
const seenQueued = new Set();
const next = tasks.filter(task => {
if (task?.type !== 'download' || task.status !== 'queued') return true;
const key = _downloadDedupeKey(task);
if (!key) return true;
if (launched.has(key) || seenQueued.has(key)) {
changed = true;
return false;
}
seenQueued.add(key);
return true;
});
return changed ? next : tasks;
}
function _loadPrunedTasks() {
const tasks = _loadTasks();
const pruned = _pruneQueuedDownloadDuplicates(tasks);
if (pruned !== tasks) _saveTasks(pruned);
return pruned;
}
// Tombstones for removed tasks. Without these, removing a task only deletes it
// locally — but the server still has it (its own POST guard even re-preserves
// recently-added ones), so the next sync/poll merges it right back ("I removed
// it and it came back"). A tombstone makes the removal stick: merges skip any
// id the user removed, until the entry expires.
const _REMOVED_KEY = 'cookbook-removed-tasks';
const _TOMBSTONE_TTL_MS = 24 * 3600 * 1000;
function _loadTombstones() {
try { return JSON.parse(localStorage.getItem(_REMOVED_KEY)) || {}; }
catch { return {}; }
}
function _tombstoneTask(id) {
if (!id) return;
const tomb = _loadTombstones();
const now = Date.now();
tomb[id] = now;
for (const k in tomb) { if (now - tomb[k] > _TOMBSTONE_TTL_MS) delete tomb[k]; }
localStorage.setItem(_REMOVED_KEY, JSON.stringify(tomb));
}
function _isTombstoned(id) {
const ts = _loadTombstones()[id];
return ts != null && (Date.now() - ts) <= _TOMBSTONE_TTL_MS;
}
function _stripTaskSecrets(task) {
if (!task || typeof task !== 'object') return task;
const safe = { ...task };
if (safe.payload && typeof safe.payload === 'object') {
safe.payload = { ...safe.payload };
delete safe.payload.hf_token;
}
return safe;
}
function _stripStateSecrets(state) {
const safe = { ...state };
if (safe.env && typeof safe.env === 'object') {
const { hfToken, ...env } = safe.env;
if (hfToken) env.hfToken = hfToken;
safe.env = env;
}
if (Array.isArray(safe.tasks)) safe.tasks = safe.tasks.map(_stripTaskSecrets);
return safe;
}
export function _saveTasks(tasks) {
localStorage.setItem(TASKS_KEY, JSON.stringify((tasks || []).map(_stripTaskSecrets)));
_syncToServer();
}
export function _addTask(sessionId, name, type, payload) {
let tasks = _loadTasks();
const remoteHost = (payload && payload.remote_host) || _envState.remoteHost || '';
const sshPort = (payload && payload.ssh_port) || _getPort(remoteHost) || '';
const platform = (payload && payload.platform) || _getPlatform(remoteHost) || '';
// Serving a model supersedes its finished download — clear the matching
// finished download card (covers serving directly from the Serve tab, not just
// via the download card's "Serve →" button).
if (type === 'serve' && payload && payload.repo_id) {
const _repoId = payload.repo_id;
tasks = tasks.filter(t => !(t.type === 'download' && t.status === 'done' && t.payload && t.payload.repo_id === _repoId));
}
if (type === 'download' && payload && payload.repo_id) {
const key = _downloadDedupeKey({ type: 'download', payload, remoteHost });
tasks = tasks.filter(t => {
if (t.sessionId === sessionId) return false;
return !(key && t.type === 'download' && t.status === 'queued' && _downloadDedupeKey(t) === key);
});
}
const task = _stripTaskSecrets({ id: sessionId, sessionId, name, type, status: 'running', output: '', ts: Date.now(), payload: payload || null, remoteHost, sshPort, platform });
tasks.push(task);
_saveTasks(tasks);
// New action → collapse all other cards, leave only this one open.
_soloExpandTaskId = sessionId;
_renderRunningTab();
// Always start the background monitor when a task is added — works even
// when modal is closed and ensures the sidebar shows live status immediately
_startBackgroundMonitor();
// Switch to Running tab
const body = document.querySelector('#cookbook-modal .cookbook-body');
if (body) {
const tab = body.querySelector('.cookbook-tab[data-backend="Running"]');
if (tab) tab.click();
}
return task;
}
function _updateTask(sessionId, updates) {
const tasks = _loadTasks();
const task = tasks.find(t => t.sessionId === sessionId);
if (task) {
Object.assign(task, updates);
_saveTasks(tasks);
}
if ('status' in updates || '_unreachable' in updates) {
_refreshServerDots();
}
if (updates.status && updates.status !== 'running') {
const el = document.querySelector(`.cookbook-task[data-task-id="${sessionId}"]`);
if (el) {
if (el._uptimeInterval) { clearInterval(el._uptimeInterval); el._uptimeInterval = null; }
const wave = el.querySelector('.cookbook-task-wave');
if (wave) wave.style.display = 'none';
const uptime = el.querySelector('.cookbook-task-uptime');
if (uptime) uptime.style.display = 'none';
}
}
}
function _refreshDepsAfterInstall(task) {
if (!task || task.type !== 'download' || !task.payload?._dep) return;
try {
_refreshDependencies?.({ host: task.remoteHost || '', port: task.sshPort || '', venv: task.payload?.env_path || '' });
} catch {}
}
export function _removeTask(sessionId) {
_tombstoneTask(sessionId); // so sync/poll can't resurrect it
const tasks = _loadTasks().filter(t => t.sessionId !== sessionId);
_saveTasks(tasks);
_renderRunningTab();
}
// Fade/slide the task card out, then remove it — so the smooth exit is the same
// whether a task auto-stops or the user removes/kills it manually.
function _animateOutThenRemove(el, sessionId) {
if (!el || !el.style) { _removeTask(sessionId); return; }
if (el._abort) el._abort.abort();
el.style.transition = 'opacity 0.35s ease, transform 0.35s ease';
el.style.opacity = '0';
el.style.transform = 'translateX(-10px)';
setTimeout(() => _removeTask(sessionId), 360);
}
// ── tmux / Windows session commands ──
export function _tmuxCmd(task, tmuxArgs) {
if (_isWindows(task)) {
return _winSessionCmd(task, tmuxArgs);
}
if (task.remoteHost) {
return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} 'tmux ${tmuxArgs}' 2>/dev/null`;
}
return `tmux ${tmuxArgs} 2>/dev/null`;
}
function _winSessionCmd(task, tmuxArgs) {
const host = task.remoteHost;
const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux';
const sid = task.sessionId;
const pf = _sshPrefix(_getPort(task));
if (tmuxArgs.includes('capture-pane')) {
const lines = tmuxArgs.match(/-S\s*-?(\d+)/)?.[1] || '200';
const ps = host
? `Get-Content '${sd}\\${sid}.log' -Tail ${lines} -ErrorAction SilentlyContinue`
: `Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.log') -Tail ${lines} -ErrorAction SilentlyContinue`;
return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`;
}
if (tmuxArgs.includes('has-session')) {
const ps = host
? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Get-Process -Id $p -ErrorAction SilentlyContinue | Out-Null; if ($?) { exit 0 } else { exit 1 } } else { exit 1 }`
: `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Get-Process -Id $p -ErrorAction SilentlyContinue | Out-Null; if ($?) { exit 0 } else { exit 1 } } else { exit 1 }`;
return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`;
}
if (tmuxArgs.includes('kill-session')) {
const ps = host
? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item '${sd}\\${sid}.*' -Force -ErrorAction SilentlyContinue`
: `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.*') -Force -ErrorAction SilentlyContinue`;
return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`;
}
if (tmuxArgs.includes('send-keys') && tmuxArgs.includes('C-c')) {
const ps = host
? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -ErrorAction SilentlyContinue }`
: `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -ErrorAction SilentlyContinue }`;
return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`;
}
return host ? `ssh ${pf}${host} 'tmux ${tmuxArgs}' 2>/dev/null` : `tmux ${tmuxArgs} 2>/dev/null`;
}
function _tmuxGracefulKill(task) {
if (_isWindows(task)) {
const host = task.remoteHost;
const sd = host ? '$env:TEMP\\odysseus-sessions' : '$env:TEMP\\odysseus-tmux';
const sid = task.sessionId;
const pf = _sshPrefix(_getPort(task));
const ps = host
? `$p = Get-Content '${sd}\\${sid}.pid' -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item '${sd}\\${sid}.*' -Force -ErrorAction SilentlyContinue`
: `$p = Get-Content (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.pid') -ErrorAction SilentlyContinue; if ($p) { Stop-Process -Id $p -Force -ErrorAction SilentlyContinue }; Remove-Item (Join-Path $env:TEMP 'odysseus-tmux\\${sid}.*') -Force -ErrorAction SilentlyContinue`;
return host ? `ssh ${pf}${host} "powershell -Command \\"${ps}\\""` : `powershell -Command "${ps}"`;
}
if (task.remoteHost) {
return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} 'tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null'`;
}
return `tmux send-keys -t ${task.sessionId} C-c 2>/dev/null; sleep 2; tmux kill-session -t ${task.sessionId} 2>/dev/null`;
}
function _shQuote(value) {
return "'" + String(value ?? '').replace(/'/g, "'\\''") + "'";
}
function _taskLooksOllama(task, outputText = '') {
const haystack = `${task?.payload?.backend || ''} ${task?.payload?._cmd || ''} ${task?.payload?._fields?.backend || ''} ${outputText || ''}`;
return /\bollama\b/i.test(haystack) || /Ollama API ready on port\s+\d+/i.test(haystack);
}
function _ollamaBaseUrlForTask(task, outputText = '') {
const out = String(outputText || '');
const ready = out.match(/Ollama API ready on port\s+\d+:\s*(http:\/\/[^\s]+)/i);
if (ready) return ready[1].replace(/\/+$/, '');
const cmd = String(task?.payload?._cmd || '');
const host = cmd.match(/OLLAMA_HOST=([^\s]+)/)?.[1] || '';
const port = host.match(/:(\d+)$/)?.[1] || '11434';
return `http://127.0.0.1:${port}`;
}
function _ollamaModelForTask(task) {
return String(task?.payload?.model || task?.payload?.repo_id || task?.name || '').trim();
}
function _ollamaUnloadCommand(task, outputText = '') {
if (!_taskLooksOllama(task, outputText)) return '';
const model = _ollamaModelForTask(task);
if (!model) return '';
const base = _ollamaBaseUrlForTask(task, outputText);
const body = JSON.stringify({ model, prompt: '', keep_alive: 0, stream: false });
const inner = `curl -sf -X POST ${_shQuote(base + '/api/generate')} -H 'Content-Type: application/json' -d ${_shQuote(body)} >/dev/null 2>&1 || true`;
if (task.remoteHost) {
return `ssh ${_sshPrefix(_getPort(task))}${task.remoteHost} ${_shQuote(inner)}`;
}
return inner;
}
function _endpointUrlForTask(task, outputText = '') {
if (_taskLooksOllama(task, outputText)) {
return _ollamaBaseUrlForTask(task, outputText) + '/v1';
}
const host = _connectHostFromRemote(task.remoteHost);
const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
const port = portMatch ? portMatch[1] : '8000';
return `http://${host}:${port}/v1`;
}
// ── Wave animation ──
const _waveFrames = ['▁▂▃', '▂▃▄', '▃▄▅', '▄▅▆', '▅▆▅', '▆▅▄', '▅▄▃', '▄▃▂', '▃▂▁'];
let _waveIdx = 0;
let _waveTimer = null;
const _waveEls = new Set();
function _startWaveSync() {
if (_waveTimer) return;
_waveTimer = setInterval(() => {
_waveIdx = (_waveIdx + 1) % _waveFrames.length;
for (const el of _waveEls) {
if (!el.isConnected) { _waveEls.delete(el); continue; }
if (el.style.display !== 'none') el.textContent = _waveFrames[_waveIdx];
}
if (!_waveEls.size) { clearInterval(_waveTimer); _waveTimer = null; }
}, 200);
}
function _registerWaveEl(el) { _waveEls.add(el); _startWaveSync(); }
// ── Notifications ──
function _showCookbookNotif(isError = false) {
const dot = document.getElementById('cookbook-notif-dot');
if (dot) {
dot.style.display = '';
dot.classList.toggle('cookbook-notif-error', isError);
}
const btn = document.getElementById('tool-cookbook-btn');
if (btn) { btn.style.opacity = '1'; btn.classList.add('cookbook-notif-active'); }
const railBtn = document.getElementById('rail-cookbook');
if (railBtn) {
railBtn.classList.remove('rail-notify-success', 'rail-notify-error');
railBtn.classList.add('rail-notify', isError ? 'rail-notify-error' : 'rail-notify-success', 'cookbook-notif-active');
}
if (window._syncRailDynamic) window._syncRailDynamic();
}
export function _clearCookbookNotif() {
const dot = document.getElementById('cookbook-notif-dot');
if (dot) dot.style.display = 'none';
const btn = document.getElementById('tool-cookbook-btn');
if (btn) { btn.style.opacity = ''; btn.classList.remove('cookbook-notif-active'); }
const railBtn = document.getElementById('rail-cookbook');
if (railBtn) {
railBtn.classList.remove('rail-notify', 'rail-notify-success', 'cookbook-notif-active');
}
if (window._syncRailDynamic) window._syncRailDynamic();
}
// ── Presets helper (for save-from-task) ──
// A preset must carry the venv + activated GPUs, not just the command — without
// them a relaunch has no environment activated and no GPU pinning, so a config
// that worked when saved fails on reload. Pull them from the launch payload
// (_env/_envPath/_gpus, captured by _launchServeTask) and fold them into the
// serve-form `fields` the Serve panel restores from.
function _presetEnvFields(task) {
const p = task.payload || {};
const fields = { ...(p._fields || {}) };
// The Serve panel's venv field is a path; conda/venv both activate from it.
if (p._envPath && (p._env === 'venv' || p._env === 'conda')) fields.venv = fields.venv || p._envPath;
if (p._gpus) fields.gpus = p._gpus;
return {
fields: Object.keys(fields).length ? fields : undefined,
env: p._env || '',
envPath: p._envPath || '',
gpus: p._gpus || '',
};
}
function _saveTaskAsPreset(task, label) {
const host = task.remoteHost || 'localhost';
const portMatch = task.payload?._cmd?.match(/--port\s+(\d+)/);
const port = portMatch ? portMatch[1] : '8000';
const presets = _loadPresets();
if (presets.some(p => p.cmd === task.payload._cmd)) return false;
presets.push({ name: task.name, model: task.payload.repo_id, backend: 'vllm', host, port, cmd: task.payload._cmd, remoteHost: task.remoteHost || '', label: label || task.name, ..._presetEnvFields(task) });
_savePresets(presets);
return true;
}
// Same model-matching as cookbookServe's _presetsForModel, so the auto-save cap
// counts the exact slots the Serve tab shows for this model.
function _presetsForModelLocal(presets, repo) {
const short = (repo || '').split('/').pop();
return presets.filter(p => {
const pm = p.model || '', pn = p.name || '';
return pm === repo || pn === repo || pm.split('/').pop() === short || pn === short;
});
}
// Build a short auto-label from the launched command so an auto-saved config is
// recognizable in the Saved dropdown (e.g. "TP2 · 16k ctx · AWQ").
function _autoConfigLabel(task) {
const cmd = task.payload?._cmd || '';
const bits = [];
const tp = cmd.match(/--tensor-parallel-size[=\s]+(\d+)/);
if (tp && tp[1] !== '1') bits.push('TP' + tp[1]);
const ml = cmd.match(/--max-model-len[=\s]+(\d+)/);
if (ml) { const n = parseInt(ml[1]); bits.push((n >= 1024 ? Math.round(n / 1024) + 'k' : n) + ' ctx'); }
const q = (task.name || '').match(/AWQ|GPTQ|FP8|Q4|Q5|Q6|Q8|INT8|INT4/i);
if (q) bits.push(q[0].toUpperCase());
return bits.length ? bits.join(' · ') : 'working';
}
// Auto-save a serve config the moment its endpoint registers successfully, and
// flag it confirmed-working. Dedups by exact command: if the same settings are
// already saved we just upgrade that slot's badge instead of duplicating it.
// Runs at most once per task.
function _autoSaveWorkingConfig(task) {
if (!task || task.type !== 'serve' || !task.payload?._cmd) return;
if (task._autoSaved) return;
const cmd = task.payload._cmd;
// Diffusion/image servers aren't vLLM presets — skip them.
if (cmd.includes('diffusion_server')) { task._autoSaved = true; return; }
const model = task.payload.repo_id || task.name;
const presets = _loadPresets();
const existing = presets.find(p => p.cmd === cmd);
if (existing) {
task._autoSaved = true;
if (!existing.confirmedWorking) { existing.confirmedWorking = true; _savePresets(presets); }
return; // already saved → just confirm it, no duplicate, no toast
}
// Respect the per-model cap the manual save flow uses (max 5).
if (_presetsForModelLocal(presets, model).length >= 5) { task._autoSaved = true; return; }
const host = task.remoteHost || 'localhost';
const portMatch = cmd.match(/--port[=\s]+(\d+)/);
const port = portMatch ? portMatch[1] : '8000';
presets.push({
name: task.name, model, backend: 'vllm', host, port,
cmd, remoteHost: task.remoteHost || '',
label: _autoConfigLabel(task), confirmedWorking: true, autoSaved: true,
..._presetEnvFields(task),
});
_savePresets(presets);
task._autoSaved = true;
uiModule.showToast('Saved working config');
}
// ── Cross-device sync ──
let _syncTimer = null;
function _syncToServer() {
// Debounce to coalesce bursts of writes, but keep latency low so the server
// is effectively authoritative across devices
clearTimeout(_syncTimer);
_syncTimer = setTimeout(async () => {
try {
// Don't push a not-yet-hydrated state. A legit state always has at
// least the "Local" server, so an empty servers list means we loaded
// before GET /state populated _envState — syncing it would wipe the
// saved servers. (The server has an anti-wipe guard too; this avoids
// the needless round-trip.)
if (!_envState || !Array.isArray(_envState.servers) || _envState.servers.length === 0) return;
const state = {
tasks: _loadTasks(),
presets: _loadPresets(),
env: _envState,
serveState: null,
};
try { state.serveState = JSON.parse(localStorage.getItem(SERVE_STATE_KEY)); } catch {}
await fetch('/api/cookbook/state', {
method: 'POST', credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(_stripStateSecrets(state)),
});
} catch {}
}, 400);
}
// Normalize state from server: collapse legacy duplicate keys to canonical form.
// - server.modelDir (singular) → server.modelDirs[0] (canonical)
// - strip ✕/✖ pollution from modelDirs
// - dedupe modelDirs
function _normalizeState(state) {
if (!state || typeof state !== 'object') return state;
if (state.env && Array.isArray(state.env.servers)) {
for (const s of state.env.servers) {
// Collapse legacy modelDir → modelDirs
let dirs = Array.isArray(s.modelDirs) ? s.modelDirs : [];
if (s.modelDir && !dirs.includes(s.modelDir)) dirs.push(s.modelDir);
dirs = dirs
.map(d => (d || '').replaceAll('\u2715', '').replaceAll('\u2716', '').trim())
.filter(Boolean);
if (!dirs.includes('~/.cache/huggingface/hub')) dirs.unshift('~/.cache/huggingface/hub');
s.modelDirs = [...new Set(dirs)];
delete s.modelDir; // Drop the legacy singular form
// A download target that's no longer in the dir list falls back to the
// default HF cache (empty) so we never download into an unscanned dir.
if (s.downloadDir && !s.modelDirs.includes(s.downloadDir)) s.downloadDir = '';
}
}
return state;
}
export async function _syncFromServer() {
try {
const res = await fetch('/api/cookbook/state', { credentials: 'same-origin' });
if (!res.ok) return false;
const state = _normalizeState(await res.json());
if (!state || !state.env) return false;
const localTasks = _loadTasks();
const serverTasks = state.tasks || [];
const localIds = new Set(localTasks.map(t => t.sessionId));
const merged = [...localTasks];
for (const t of serverTasks) {
if (!localIds.has(t.sessionId) && !_isTombstoned(t.sessionId)) {
merged.push(t);
}
}
localStorage.setItem(TASKS_KEY, JSON.stringify(merged.map(_stripTaskSecrets)));
if (state.env) {
// The active server selection (remoteHost + its env/path/platform) is a
// per-device, live choice. NEVER let the server's stored copy overwrite
// it here — doing so silently snapped the active host back to whatever was
// saved server-side, so downloads/scans ignored what the user just
// picked. Sync only the shared non-secret settings (servers list, gpus, paths).
const { remoteHost: _rh, env: _e, envPath: _ep, platform: _pf, ...settings } = state.env;
delete settings.hfToken;
Object.assign(_envState, settings);
const { hfToken, ...safeState } = _envState;
localStorage.setItem('cookbook-last-state', JSON.stringify(safeState));
}
if (state.presets) {
localStorage.setItem(STORAGE_KEY, JSON.stringify(state.presets));
}
if (state.serveState) {
localStorage.setItem(SERVE_STATE_KEY, JSON.stringify(state.serveState));
}
return true;
} catch { return false; }
}
// ── Retry download ──
// Bounded auto-retry counter for downloads, keyed by model — network blips on
// big multi-file downloads are common and HF resumes from the .incomplete parts.
const _dlRetryCount = new Map();
const _DL_MAX_AUTO_RETRY = 2;
// Kill + relaunch a task (download or serve). Shared by the ⋮ → Restart action
// and the click-to-retry on a stalled download badge.
async function _retryTask(el, task) {
if (el && el._abort) el._abort.abort();
const badge = el?.querySelector('.cookbook-task-status');
if (badge) { badge.textContent = 'restarting...'; badge.className = 'cookbook-task-status'; }
try {
await fetch('/api/shell/exec', {
method: 'POST', credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ command: _tmuxGracefulKill(task) }),
});
} catch {}
if (task.payload) {
if (task.type === 'serve' && task.payload._cmd) {
_removeTask(task.sessionId);
_launchServeTask(task.name, task.payload.repo_id, task.payload._cmd, task.payload._fields, task.remoteHost || '');
} else {
uiModule.showToast('Retrying download — progress may look reset while HuggingFace checks cached files, then it should resume.', 7000);
_updateTask(task.sessionId, {
status: 'running',
output: `${task.output || ''}\n\n[odysseus] Retrying download. Progress may briefly look like a fresh download while HuggingFace checks cached/incomplete files; cached partial files will be reused when available.`.trim(),
_retrying: true,
});
_retryDownload(task.name, task.payload, task.sessionId);
}
}
}
async function _retryDownload(name, payload, replaceSessionId = '') {
try {
// A retry means the fast hf_transfer path already failed once — fall back to
// the plain, reliable downloader for this and any further attempt (it resumes
// from the cached .incomplete files, so no progress is lost).
const _payload = { ...(payload || {}), disable_hf_transfer: true };
const res = await fetch('/api/model/download', {
method: 'POST', credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(_payload),
});
if (!res.ok) {
uiModule.showToast('Download failed: HTTP ' + res.status);
if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
return;
}
const data = await res.json();
if (!data.ok) {
uiModule.showToast('Download failed: ' + (data.error || ''));
if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
return;
}
if (replaceSessionId) {
const tasks = _loadTasks();
const task = tasks.find(t => t.sessionId === replaceSessionId);
if (task) {
task.id = data.session_id;
task.sessionId = data.session_id;
task.status = 'running';
task.output = '';
task.ts = Date.now();
task.payload = _payload;
task._retrying = false;
_saveTasks(tasks);
_soloExpandTaskId = data.session_id;
_renderRunningTab();
_startBackgroundMonitor();
} else {
_addTask(data.session_id, name, 'download', _payload);
}
} else {
_addTask(data.session_id, name, 'download', _payload);
}
uiModule.showToast(`Downloading ${name}...`);
} catch (e) {
uiModule.showToast('Download failed: ' + e.message);
if (replaceSessionId) _updateTask(replaceSessionId, { status: 'crashed', _retrying: false });
}
}
// ── Serve auto-fix (kill + relaunch with env var) ──
// Block stacked retries: once any "Retry with X" is clicked for a task, ignore
// every further retry click for it. Each retry fires its own _launchServeTask,
// so clicking several options — or one repeatedly during the fade-out / while a
// relaunch was loading — used to stack up multiple servers (e.g. 6 launches).
// The flag rides on the card element (removed right after), so it can't re-arm.
function _guardServeRetry(panel, taskEl) {
if (!taskEl || taskEl.dataset.retrying) return false;
taskEl.dataset.retrying = '1';
panel.querySelectorAll('button').forEach(b => {
b.disabled = true;
b.style.opacity = '0.5';
b.style.pointerEvents = 'none';
});
return true;
}
export async function _serveAutoFix(panel, envVar) {
const taskEl = panel.closest('.cookbook-task');
if (!taskEl) return;
const taskId = taskEl.dataset.taskId;
const tasks = _loadTasks();
const task = tasks.find(t => t.sessionId === taskId);
if (!task || !task.payload) return;
if (!_guardServeRetry(panel, taskEl)) return;
const killCmd = _tmuxCmd(task, `kill-session -t ${taskId}`);
try {
await fetch('/api/shell/exec', {
method: 'POST', credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ command: killCmd }),
});
} catch {}
_animateOutThenRemove(taskEl, taskId);
const origCmd = task.payload._cmd || '';
const newCmd = `export ${envVar} && ${origCmd}`;
const origHost = _envState.remoteHost;
if (task.remoteHost) _envState.remoteHost = task.remoteHost;
try {
uiModule.showToast(`Retrying with ${envVar}...`);
await _launchServeTask(task.name, task.payload.repo_id, newCmd);
} finally {
// Always restore — otherwise a thrown launch leaves the global host stuck
// on this serve task, so later downloads/scans hit it.
_envState.remoteHost = origHost;
}
}
// Open the Serve panel pre-filled for a task — the same flow as the task's
// Edit button, but optionally with a modified command (used by the diagnosis
// "Retry with X" buttons so a retry lands in the editable Serve panel with the
// adjusted setting, instead of blindly relaunching).
async function _openServeEditForTask(task, cmdOverride, fieldOverrides = null) {
const repo = task.payload?.repo_id;
if (!repo) { uiModule.showToast('No model info on this task'); return; }
const cmd = cmdOverride || task.payload?._cmd;
// A modified cmd must be re-parsed; otherwise prefer the exact launch fields.
let fields = cmdOverride
? _parseServeCmdToFields(cmd)
: (task.payload?._fields || (cmd ? _parseServeCmdToFields(cmd) : null));
if (fieldOverrides && typeof fieldOverrides === 'object') {
fields = { ...(fields || {}), ...fieldOverrides };
}
// Switch the active server to the one this serve ran on (mirrors _openEdit).
const _tHost = task.remoteHost || '';
_envState.remoteHost = _tHost;
const _tSrv = _envState.servers.find(s => s.host === _tHost);
if (_tSrv) { _envState.env = _tSrv.env || 'none'; _envState.envPath = _tSrv.envPath || ''; _envState.platform = _tSrv.platform || ''; }
else if (!_tHost) { _envState.env = 'none'; _envState.envPath = ''; _envState.platform = ''; }
document.querySelectorAll('#hwfit-server-select, #hwfit-dl-server, #hwfit-cache-server, #hwfit-deps-server').forEach(sel => {
if (!sel || sel.tagName !== 'SELECT') return;
sel.value = _tHost || 'local';
});
try {
const { openServePanelForRepo } = await import('./cookbookServe.js');
await openServePanelForRepo(repo, fields);
} catch (err) {
console.error('[cookbook] open serve panel failed', err);
uiModule.showToast('Could not open serve panel');
}
}
export async function _serveAutoRetryReplace(panel, flag, value) {
const taskEl = panel.closest('.cookbook-task');
if (!taskEl) return;
const taskId = taskEl.dataset.taskId;
const tasks = _loadTasks();
const task = tasks.find(t => t.sessionId === taskId);
if (!task || !task.payload || !task.payload._cmd) return;
if (!_guardServeRetry(panel, taskEl)) return;
try {
await fetch('/api/shell/exec', {
method: 'POST', credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${taskId}`) }),
});
} catch {}
_animateOutThenRemove(taskEl, taskId);
let newCmd = task.payload._cmd;
const re = new RegExp(flag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\s+\\S+');
if (re.test(newCmd)) {
newCmd = newCmd.replace(re, `${flag} ${value}`);
} else {
newCmd += ` ${flag} ${value}`;
}
const origHost = _envState.remoteHost;
if (task.remoteHost) _envState.remoteHost = task.remoteHost;
try {
uiModule.showToast(`Retrying with ${flag} ${value}...`);
await _launchServeTask(task.name, task.payload.repo_id, newCmd);
} finally {
_envState.remoteHost = origHost;
}
}
export async function _serveAutoRetryRemove(panel, flag) {
const taskEl = panel.closest('.cookbook-task');
if (!taskEl) return;
const taskId = taskEl.dataset.taskId;
const tasks = _loadTasks();
const task = tasks.find(t => t.sessionId === taskId);
if (!task || !task.payload || !task.payload._cmd) return;
if (!_guardServeRetry(panel, taskEl)) return;
try {
await fetch('/api/shell/exec', {
method: 'POST', credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${taskId}`) }),
});
} catch {}
_animateOutThenRemove(taskEl, taskId);
let newCmd = task.payload._cmd;
const re = new RegExp(flag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '\\s+\\S+');
newCmd = newCmd.replace(re, '').replace(/\s{2,}/g, ' ').trim();
const origHost = _envState.remoteHost;
if (task.remoteHost) _envState.remoteHost = task.remoteHost;
try {
uiModule.showToast(`Retrying without ${flag}...`);
await _launchServeTask(task.name, task.payload.repo_id, newCmd);
} finally {
_envState.remoteHost = origHost;
}
}
export async function _serveAutoRetry(panel, flag) {
const taskEl = panel.closest('.cookbook-task');
if (!taskEl) return;
const taskId = taskEl.dataset.taskId;
const tasks = _loadTasks();
const task = tasks.find(t => t.sessionId === taskId);
if (!task || !task.payload || !task.payload._cmd) return;
if (!_guardServeRetry(panel, taskEl)) return;
try {
await fetch('/api/shell/exec', {
method: 'POST', credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ command: _tmuxCmd(task, `kill-session -t ${taskId}`) }),
});
} catch {}
_animateOutThenRemove(taskEl, taskId);
let newCmd = task.payload._cmd;
if (!newCmd.includes(flag)) {
newCmd += ' ' + flag;
}
const origHost = _envState.remoteHost;
if (task.remoteHost) _envState.remoteHost = task.remoteHost;
try {
uiModule.showToast(`Retrying with ${flag}...`);
await _launchServeTask(task.name, task.payload.repo_id, newCmd);
} finally {
_envState.remoteHost = origHost;
}
}
// ── Edit-command prompt ──
// Shows a small modal with a textarea pre-filled with the current serve cmd.
// Resolves to the edited string on Save, or null on Cancel.
function _promptEditServeCmd(currentCmd) {
return new Promise((resolve) => {
const overlay = document.createElement('div');
overlay.className = 'cookbook-edit-overlay';
overlay.innerHTML = `
Edit serve command
`;
const ta = overlay.querySelector('.cookbook-edit-textarea');
ta.value = currentCmd || '';
document.body.appendChild(overlay);
setTimeout(() => { ta.focus(); ta.setSelectionRange(ta.value.length, ta.value.length); }, 0);
const close = (result) => {
overlay.remove();
document.removeEventListener('keydown', onKey);
resolve(result);
};
const onKey = (e) => {
if (e.key === 'Escape') close(null);
else if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) close(ta.value.trim() || null);
};
overlay.querySelector('.cookbook-edit-cancel').addEventListener('click', () => close(null));
overlay.querySelector('.cookbook-edit-save').addEventListener('click', () => close(ta.value.trim() || null));
overlay.addEventListener('click', (e) => { if (e.target === overlay) close(null); });
document.addEventListener('keydown', onKey);
});
}
// ── Launch serve task ──
// Best-effort reconstruction of serve-form field values from a raw launch
// command. Fallback for tasks created before _fields capture existed.
// Mirrors the regex parser in cookbookServe.js's _loadSlotIntoPanel.
function _parseServeCmdToFields(cmd) {
if (!cmd) return null;
const ex = (re) => { const m = cmd.match(re); return m ? m[1] : ''; };
const fields = {
backend: cmd.includes('llama_cpp') || cmd.includes('llama-server') ? 'llamacpp'
: cmd.includes('diffusion_server') ? 'diffusers'
: cmd.includes('sglang') ? 'sglang'
: cmd.includes('ollama') ? 'ollama' : 'vllm',
port: ex(/--port\s+(\d+)/) || '8000',
tp: ex(/--tensor-parallel-size\s+(\d+)/) || '1',
ctx: ex(/--max-model-len\s+(\d+)/) || ex(/--n_ctx\s+(\d+)/) || ex(/-c\s+(\d+)/) || '8192',
gpu_mem: ex(/--gpu-memory-utilization\s+([\d.]+)/) || '0.90',
swap: ex(/--swap-space\s+(\d+)/) || '',
dtype: ex(/--dtype\s+(\w+)/) || 'auto',
vllm_kv_cache_dtype: ex(/--kv-cache-dtype\s+([\w.-]+)/) || 'auto',
max_seqs: ex(/--max-num-seqs\s+(\d+)/) || '',
gpus: ex(/CUDA_VISIBLE_DEVICES=(\S+)/) || '',
cache_type: ex(/(?:--cache-type-k|-ctk)\s+(\S+)/) || '',
llama_fit: ex(/(?:--fit|-fit)\s+(on|off)/) || '',
llama_split_mode: ex(/(?:--split-mode|-sm)\s+(none|layer|row|tensor)/) || '',
llama_tensor_split: ex(/(?:--tensor-split|-ts)\s+([0-9.,]+)/) || '',
llama_main_gpu: ex(/(?:--main-gpu|-mg)\s+(\d+)/) || '',
llama_parallel: ex(/(?:--parallel|-np)\s+(\d+)/) || '',
llama_batch_size: ex(/(?:--batch-size|-b)\s+(\d+)/) || '',
llama_ubatch_size: ex(/(?:--ubatch-size|-ub)\s+(\d+)/) || '',
llama_spec_tokens: ex(/--spec-draft-n-max\s+(\d+)/) || '3',
enforce_eager: cmd.includes('--enforce-eager'),
trust_remote: cmd.includes('--trust-remote-code'),
prefix_cache: cmd.includes('--enable-prefix-caching'),
auto_tool: cmd.includes('--enable-auto-tool-choice'),
flash_attn: /--flash-attn\s+on\b/.test(cmd),
unified_mem: /GGML_CUDA_ENABLE_UNIFIED_MEMORY=1/.test(cmd),
llama_no_mmap: /--no-mmap\b/.test(cmd),
llama_no_warmup: /--no-warmup\b/.test(cmd),
llama_speculative_mtp: /--spec-type\s+\S*draft-mtp/.test(cmd),
speculative: cmd.includes('--speculative-config'),
};
const spec = cmd.match(/--speculative-config\s+'?\{[^}]*"method"\s*:\s*"([^"]+)"[^}]*"num_speculative_tokens"\s*:\s*(\d+)/);
if (spec) { fields.spec_method = spec[1]; fields.spec_tokens = spec[2]; }
return fields;
}
export async function _launchServeTask(shortName, repo, cmd, fields, hostOverride) {
// Host resolution mirrors the download path: when the caller passes an explicit
// host (resolved from the dropdown the user actually picked), use it and look
// up that server's port/platform from the shared servers list. Only fall back
// to _envState.remoteHost for legacy callers (diagnosis/pip-update).
const _host = (hostOverride !== undefined) ? (hostOverride || '') : (_envState.remoteHost || '');
const _hsrv = _envState.servers.find(s => s.host === _host) || {};
const _hplatform = _host ? (_hsrv.platform || '') : (_envState.platform || '');
// Replace any serve already targeting this same host:port — you can't run two
// servers on one port, so re-serving (or retrying) should stop & remove the
// old one instead of leaving a dead duplicate behind. (The retry buttons
// already removed their own task, so this is a no-op for them.)
try {
const _pm = cmd.match(/--port[=\s]+(\d+)/) || cmd.match(/(?:^|\s)-p[=\s]+(\d+)/);
const _newPort = _pm ? _pm[1] : '';
if (_newPort) {
for (const _t of _loadTasks()) {
if (_t.type !== 'serve' || !_t.payload || !_t.payload._cmd) continue;
const _tm = _t.payload._cmd.match(/--port[=\s]+(\d+)/) || _t.payload._cmd.match(/(?:^|\s)-p[=\s]+(\d+)/);
if ((_tm ? _tm[1] : '') === _newPort && (_t.remoteHost || '') === _host) {
try {
await fetch('/api/shell/exec', {
method: 'POST', credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ command: _tmuxGracefulKill(_t) }),
});
} catch {}
_removeTask(_t.sessionId);
}
}
}
} catch {}
// Capture the env + GPU pin used for THIS launch BEFORE building the request.
// The serve panel sets _envState.env/envPath/gpus, calls us, then restores them
// synchronously — and our payload is built after an `await`, so reading
// _envState there would see the restored (wrong) values. Persisting these lets
// a saved preset relaunch with the same venv + GPUs (otherwise a confirmed
// working config fails: no venv activation, no GPU pinning).
const _usedEnv = _envState.env;
const _usedEnvPath = _envState.envPath;
const _usedGpus = _envState.gpus || '';
let envPrefix = '';
if (_isWindows()) {
if (_envState.env === 'venv' && _envState.envPath) {
envPrefix = '& ' + (_envState.envPath.endsWith('\\Scripts\\Activate.ps1') ? _envState.envPath : _envState.envPath + '\\Scripts\\Activate.ps1');
} else if (_envState.env === 'conda' && _envState.envPath) {
envPrefix = 'conda activate ' + _envState.envPath;
}
} else {
if (_envState.env === 'venv' && _envState.envPath) {
const p = _envState.envPath;
envPrefix = 'source ' + (p.endsWith('/bin/activate') ? p : p + '/bin/activate');
} else if (_envState.env === 'conda' && _envState.envPath) {
envPrefix = 'eval "$(conda shell.bash hook)" && conda activate ' + _envState.envPath;
}
}
const reqBody = {
repo_id: repo,
cmd: cmd,
remote_host: _host || undefined,
ssh_port: _getPort(_host) || undefined,
env_prefix: envPrefix || undefined,
hf_token: _envState.hfToken || undefined,
gpus: _envState.gpus || undefined,
platform: _hplatform || undefined,
};
try {
const res = await fetch('/api/model/serve', {
method: 'POST', credentials: 'same-origin',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(reqBody),
});
const data = await res.json();
if (!data.ok) {
// Two error shapes: `{ok:false, error}` (tmux launch failed) or
// `{detail}` (FastAPI HTTPException). Show whichever is present
// + log full payload so the user can copy the error.
const err = data.error || data.detail || res.statusText || 'unknown';
console.error('[cookbook] /api/model/serve failed', { status: res.status, body: data });
uiModule.showToast('Failed to start: ' + String(err).slice(0, 200), 9000);
return;
}
const _sp = _getPort(_host);
// _fields = the exact structured serve-form values used for this launch,
// so the "Edit / relaunch" button can re-open the Serve panel pre-filled
// with these precise settings (not just the last-used-for-repo state).
const payload = { repo_id: repo, remote_host: _host || undefined, ssh_port: _sp || undefined, _cmd: cmd, _fields: fields || undefined, _env: _usedEnv, _envPath: _usedEnvPath, _gpus: _usedGpus };
_addTask(data.session_id, shortName, 'serve', payload);
uiModule.showToast(`Serving ${shortName}...`);
} catch (e) {
uiModule.showToast('Failed: ' + e.message);
}
}
// ── Render Running tab ──
export function _renderRunningTab() {
// Auto-clear the sidebar notif (the bright-icon highlight) when no tasks
// are actively running or errored. _showCookbookNotif fires on each task
// event but the matching clear only ran on modal-open, so the highlight
// persisted indefinitely after tasks finished in the background.
try {
const _activeTasks = _loadPrunedTasks().filter(t => t.status === 'running' || t.status === 'queued' || t.status === 'error');
if (!_activeTasks.length) _clearCookbookNotif();
} catch {}
const body = document.querySelector('#cookbook-modal .cookbook-body');
if (!body) return;
// Capture expansion state so re-renders don't collapse whatever the user
// had open. Task output: presence of .cookbook-task-collapsed means collapsed.
// Section body: inline display:none means collapsed.
const _collapsedTaskIds = new Set();
const _expandedTaskIds = new Set(); // mobile: tasks the user explicitly opened
body.querySelectorAll('.cookbook-task').forEach(tEl => {
const id = tEl.dataset.taskId;
if (!id) return;
const wrap = tEl.querySelector('.cookbook-output-wrap');
if (!wrap) return;
if (wrap.classList.contains('cookbook-task-collapsed')) _collapsedTaskIds.add(id);
else _expandedTaskIds.add(id);
});
// A new action was just started — collapse every existing card and open only
// the new one (works on both desktop and the mobile collapse-by-default path).
if (_soloExpandTaskId) {
const _allIds = new Set([..._collapsedTaskIds, ..._expandedTaskIds]);
_collapsedTaskIds.clear();
_expandedTaskIds.clear();
_allIds.forEach(id => { if (id !== _soloExpandTaskId) _collapsedTaskIds.add(id); });
_expandedTaskIds.add(_soloExpandTaskId);
_soloExpandTaskId = null;
}
// On mobile, task outputs start COLLAPSED — having every running window
// expanded on entry meant a lot of tapping to collapse them. User-expanded
// ones are re-opened from _expandedTaskIds below.
const _mobileCollapseDefault = window.innerWidth <= 768;
const _collapsedSectionIds = new Set();
body.querySelectorAll('.cookbook-section-body').forEach(sb => {
if (sb.style.display === 'none' && sb.id) _collapsedSectionIds.add(sb.id);
});
const tasks = _loadTasks();
const hasContent = tasks.length > 0;
// Count anything that's really active: explicit 'running'/'queued' status,
// OR a download whose tmux output is still showing live shard progress.
// Without the output check, a task whose status got stuck at 'done' /
// 'crashed' (before auto-reconnect catches it) would read as "Running 0"
// even when the model is actively downloading on the host.
const activeCount = tasks.filter(t =>
t.status === 'running'
|| t.status === 'queued'
|| _downloadOutputLooksActive(t)
).length;
const activeCountHtml = activeCount ? ` ${activeCount}` : '';
let tabBar = body.querySelector('.cookbook-tabs');
if (!tabBar) return;
let runTab = tabBar.querySelector('.cookbook-tab[data-backend="Running"]');
if (hasContent && !runTab) {
runTab = document.createElement('button');
runTab.className = 'cookbook-tab';
runTab.dataset.backend = 'Running';
const _errCount = tasks.filter(t => t.status === 'error' || t.status === 'crashed').length;
runTab.innerHTML = `Running${activeCountHtml}${_errCount ? `` : ''}`;
tabBar.insertBefore(runTab, tabBar.firstChild);
runTab.addEventListener('click', () => {
tabBar.querySelectorAll('.cookbook-tab').forEach(t => t.classList.remove('active'));
runTab.classList.add('active');
body.querySelectorAll('.cookbook-group').forEach(g => {
g.classList.toggle('hidden', g.dataset.backendGroup !== 'Running');
});
});
} else if (runTab) {
const _errCount2 = tasks.filter(t => t.status === 'error' || t.status === 'crashed').length;
runTab.innerHTML = tasks.length ? `Running${activeCountHtml}${_errCount2 ? '' : ''}` : 'Running';
if (!hasContent) {
if (runTab.classList.contains('active')) {
const wfTab = tabBar.querySelector('.cookbook-tab[data-backend="Search"]');
if (wfTab) wfTab.click();
}
runTab.remove();
}
}
let group = body.querySelector('.cookbook-group[data-backend-group="Running"]');
if (hasContent && !group) {
group = document.createElement('div');
group.className = 'cookbook-group hidden';
group.dataset.backendGroup = 'Running';
group.innerHTML = '' +
'
' +
'
Running ' + activeCount + '
' +
'' +
'
Active downloads and serving processes.
' +
'
${esc(task.sessionId)}${(task.type === 'download') ? `Dir: ${esc(task.payload?.local_dir || '~/.cache/huggingface/hub')}` : ''}
${esc(task.output || '')}