diff --git a/src/servers/api/command-hq/watcher-revive.ts b/src/servers/api/command-hq/watcher-revive.ts index 39c1220e..acca3da0 100644 --- a/src/servers/api/command-hq/watcher-revive.ts +++ b/src/servers/api/command-hq/watcher-revive.ts @@ -2,9 +2,8 @@ import { listAllCommanders, getUserById } from 'officerdb'; import { getCommanderStateDir } from '@@/data-path'; import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; -import { startClaudeTurn, COMMANDER_DEFAULT_MODEL, collectRunningTasks } from '../chat/websocket'; -import { getChatEventsSince } from 'officerdb'; -import type { ServerMessage } from '../chat/types'; +import { startClaudeTurn, COMMANDER_DEFAULT_MODEL } from '../chat/websocket'; +import * as sidecar from '@@/sidecar-registry'; import { clearLoopPid, liveLoopPids } from './spawned'; import { getNotifyServerUrl, notifyProducerHeaders } from '../notify/router'; @@ -129,17 +128,33 @@ function clearAttempts(email: string, dirName: string): void { * answer to "is this watching", not that a fire was observed being misread. Either way the window is real: * between a watcher exiting and its agent re-arming, the file names a process that is gone. * - * Hence `isGenerating`: if a turn is running, the exit is being handled and the re-arm is the last thing - * that turn will do. Waking then is not just wasteful, it is wrong — it would arrive mid-work carrying an - * instruction the agent has no reason to obey. + * Hence the generating check: if a TURN is running, the exit is being handled and the re-arm is the last + * thing that turn will do. Waking then is not just wasteful, it is wrong — it would arrive mid-work + * carrying an instruction the agent has no reason to obey. + * + * ── "generating" is not "has an outstanding background task" ── + * + * This asked `collectRunningTasks` until 2026-08-26, and that was wrong in a way that disabled the whole + * feature. Those are BACKGROUND TASKS, and **a commander's watcher is itself a background task** — one that + * stays outstanding until it reports a terminal state, which a killed process never does. So every + * commander with an armed watcher had one outstanding by definition, the check returned "busy" forever, and + * the reviver could never wake the one thing it exists for. + * + * Found on the deploy commander: its loop was dead, its pid file said so, and the task + * "Platform Deployer watcher — polls origin for deploy/* tags" was still marked running from the revival + * six hours earlier. The guard was permanently true. + * + * The right question is whether the AGENT is mid-turn, which the sidecar answers directly. */ -async function watchDied(email: string, dirName: string, sessionId: string): Promise { +async function watchDied(email: string, dirName: string, sessionId: string, userId: number): Promise { if (!existsSync(join(getCommanderStateDir(email, dirName), PID_FILE))) return false; if (liveLoopPids(email, dirName).length > 0) return false; try { - const events = await getChatEventsSince(sessionId, 0); - if (collectRunningTasks(events.map((e) => e.event as ServerMessage)).length > 0) return false; + const live = await sidecar.listLiveClaudeSessions(userId); + const mine = live.find((s) => s.sessionKey === sessionId || s.claudeSessionId === sessionId); + // Absent means idle or collected, and both are fine — `fire` resumes by transcript id. + if (mine?.isGenerating) return false; } catch { // Cannot tell whether it is working, so assume it is. A missed revival is reported next tick; a wake // delivered into a live turn is not recoverable by waiting. @@ -271,7 +286,7 @@ export function startWatcherRevive(): void { for (const commander of await listAllCommanders()) { const user = await getUserById(commander.userId); if (!user) continue; - if (!(await watchDied(user.email, commander.dirName, commander.sessionId))) { + if (!(await watchDied(user.email, commander.dirName, commander.sessionId, commander.userId))) { // Healthy, stopped, or mid-turn. A live loop means the last wake worked, so the count resets. if (liveLoopPids(user.email, commander.dirName).length > 0) clearAttempts(user.email, commander.dirName); continue;