diff --git a/static/js/cookbookRunning.js b/static/js/cookbookRunning.js index 5f75a49..1f225b7 100644 --- a/static/js/cookbookRunning.js +++ b/static/js/cookbookRunning.js @@ -66,6 +66,23 @@ function _clearPillLabel(task) { return 'clear'; } +// A pip dependency/driver install (payload._dep) reports success with the +// runner's "=== Process exited with code 0 ===" sentinel and pip's +// "Successfully installed" line — never the HuggingFace download markers +// (DONE / 100% / /snapshots/ / DOWNLOAD_OK) that the download heuristics look +// for. Without this, a clean install whose tmux pane has already gone away is +// misread as crashed/stopped even though pip exited 0. Prefer the authoritative +// exit-code sentinel; fall back to pip's success line when no sentinel was +// captured (and there's no install error in the same output). +function _depInstallSucceeded(output) { + const text = String(output || ''); + if (!text) return false; + const exitMatch = text.match(/=== Process exited with code (-?\d+) ===/); + if (exitMatch) return Number(exitMatch[1]) === 0; + return /\b(?:Successfully installed|Requirement already satisfied)\b/.test(text) + && !/\bERROR\b|No matching distribution|Could not find a version|Traceback \(most recent call last\)/.test(text); +} + function _shouldOfferCrashReport(task) { if (!task) return false; if (task._unreachable && task.type === 'serve') return true; @@ -2448,7 +2465,10 @@ async function _reconnectTask(el, task) { const downloadLooksSuccessful = !lastOutput.includes('DOWNLOAD_FAILED') && (lastOutput.includes('DONE') || lastOutput.includes('100%') || lastOutput.includes('/snapshots/') || lastOutput.includes('Download complete') || lastOutput.includes('DOWNLOAD_OK')); const serveLooksReady = task.type === 'serve' && _serveOutputLooksReady({ ...task, output: lastOutput }); - const looksSuccessful = task.type === 'download' ? downloadLooksSuccessful : serveLooksReady; + // Dependency installs are tracked as download tasks but finish with a + // pip exit-0 sentinel, not HF download markers — so check that too. + const depInstallSucceeded = !!task.payload?._dep && _depInstallSucceeded(lastOutput); + const looksSuccessful = depInstallSucceeded || (task.type === 'download' ? downloadLooksSuccessful : serveLooksReady); if (!lastOutput.trim() || !looksSuccessful) { _updateTask(task.sessionId, { status: 'crashed' }); el.dataset.status = 'crashed'; @@ -3309,11 +3329,18 @@ async function _pollBackgroundStatus() { const live = statusById.get(task.sessionId); if (!live) continue; const updates = {}; + // A finished dependency install whose tmux pane is gone is reported + // "stopped" by the backend (its pip package is never in the HF cache the + // dead-session check inspects). Recover "done" from the retained output's + // exit-0 sentinel so a clean install isn't downgraded to crashed. + const depDone = !!task.payload?._dep && _depInstallSucceeded(task.output); const nextStatus = live.status === 'completed' ? 'done' : (live.status === 'error' ? 'error' - : (live.status === 'stopped' ? (task.type === 'download' ? 'crashed' : 'stopped') : null)); + : (live.status === 'stopped' + ? (depDone ? 'done' : (task.type === 'download' ? 'crashed' : 'stopped')) + : null)); if (nextStatus && task.status !== nextStatus) { updates.status = nextStatus; if (nextStatus === 'done' && task.payload?._dep) completedDeps.push(task); diff --git a/tests/test_cookbook_dependency_completion_regression.py b/tests/test_cookbook_dependency_completion_regression.py index b47e9b2..4c0ab59 100644 --- a/tests/test_cookbook_dependency_completion_regression.py +++ b/tests/test_cookbook_dependency_completion_regression.py @@ -37,6 +37,42 @@ def test_local_windows_session_commands_use_local_powershell_log_dir(): assert ": `powershell -Command \"${ps}\"`;" in source +def test_dep_install_success_recognized_from_exit_sentinel(): + """A pip dependency install reports success via the runner's exit-0 + sentinel / pip's "Successfully installed" line, not the HuggingFace + download markers. The shared helper must key off those, so an install + whose tmux pane is gone isn't misread as crashed.""" + source = _read("static/js/cookbookRunning.js") + + assert "function _depInstallSucceeded(output) {" in source + assert "=== Process exited with code" in source + assert "Successfully installed" in source + + +def test_session_gone_heuristic_honors_dep_install_success(): + """The reconnect loop's session-gone branch (download tasks need an HF + marker to look successful) must also accept a finished dependency install, + otherwise a clean pip install with no HF markers is marked crashed.""" + source = _read("static/js/cookbookRunning.js") + + assert "const depInstallSucceeded = !!task.payload?._dep && _depInstallSucceeded(lastOutput);" in source + assert ( + "const looksSuccessful = depInstallSucceeded " + "|| (task.type === 'download' ? downloadLooksSuccessful : serveLooksReady);" + ) in source + + +def test_background_poll_recovers_done_for_stopped_dependency_install(): + """When the backend reports a finished dependency install as "stopped" + (its pip package is never in the HF cache the dead-session check inspects), + the reconciler must recover "done" from the retained output instead of + downgrading the card to crashed.""" + source = _read("static/js/cookbookRunning.js") + + assert "const depDone = !!task.payload?._dep && _depInstallSucceeded(task.output);" in source + assert "depDone ? 'done' : (task.type === 'download' ? 'crashed' : 'stopped')" in source + + def test_dependency_install_payload_keeps_env_path_for_refresh(): source = _read("static/js/cookbook.js")