diff --git a/src/databases/officer_db/src/command-hq/index.ts b/src/databases/officer_db/src/command-hq/index.ts index ec2ad5c8..13020989 100644 --- a/src/databases/officer_db/src/command-hq/index.ts +++ b/src/databases/officer_db/src/command-hq/index.ts @@ -9,6 +9,7 @@ export { listCommandHqSessions, listCommandHqSessionIds, listCommandHqOwnedSessionIds, + listAllCommanders, getCommandHqSession, setCommandHqSessionCategory, deleteCommandHqSession, diff --git a/src/databases/officer_db/src/command-hq/queries.ts b/src/databases/officer_db/src/command-hq/queries.ts index 160d19f8..1639abba 100644 --- a/src/databases/officer_db/src/command-hq/queries.ts +++ b/src/databases/officer_db/src/command-hq/queries.ts @@ -213,6 +213,28 @@ export async function listCommandHqOwnedSessionIds(userId: number): Promise sql`${commandHqSessions.sessionId} is not null`; +/** + * Every commander on the machine, across every account — id, owner and directory only. + * + * The one query here that is not scoped to a user, and it exists for a caller that is not a request: + * `servers/api/command-hq/watcher-revive.ts` ticks over all of them looking for a watch whose process + * has gone. There is no `userId` to scope by at that point, and the reviver needs the owner's id in + * order to resolve their email and their home — so it is returned rather than looked up per row. + * + * Narrow on purpose. A background sweep has no business reading names, categories or cost, and the next + * caller that wants those should say so by widening deliberately. + */ +export async function listAllCommanders(): Promise<{ userId: number; sessionId: string; dirName: string }[]> { + const rows = await db + .select({ + userId: commandHqSessions.userId, + sessionId: commandHqSessions.sessionId, + dirName: commandHqSessions.dirName, + }) + .from(commandHqSessions); + return rows.filter((row): row is { userId: number; sessionId: string; dirName: string } => row.sessionId !== null); +} + export async function getCommandHqSession(userId: number, sessionId: string): Promise { const [row] = await db .select(SESSION_ROW_COLUMNS) diff --git a/src/server.tsx b/src/server.tsx index 22e65a19..949ca7eb 100644 --- a/src/server.tsx +++ b/src/server.tsx @@ -20,6 +20,7 @@ import { chatWebsocket } from './servers/api/chat/websocket'; import { chatNotifyWebsocket } from './servers/api/chat/notify-delivery'; import { registerChatPushNotifications } from './servers/api/chat/notify-push'; import { startNudger } from './servers/api/chat/nudger'; +import { startWatcherRevive } from './servers/api/command-hq/watcher-revive'; import { taskRunnerWebsocket } from './servers/api/tasks/task-executor'; import { pipelineWebsocket } from './servers/api/tasks/pipeline-executor'; import officerWeb from './apps/officer-web/index.gen.html'; @@ -226,6 +227,7 @@ registerChatPushNotifications(); // Nudger's tick. Here rather than on first use, because a schedule armed before a restart has to survive // one — the row is in Postgres and nothing else would ever look at it again. startNudger(); +startWatcherRevive(); // That the working directory is the repo, because every path derives from its parent. First of the three, // since a wrong answer here makes the other two check the wrong files. diff --git a/src/servers/api/command-hq/directory.ts b/src/servers/api/command-hq/directory.ts index 94af5d89..1045cd17 100644 --- a/src/servers/api/command-hq/directory.ts +++ b/src/servers/api/command-hq/directory.ts @@ -214,6 +214,31 @@ export function commanderClaudeMd(params: CommanderClaudeMdParams): string { 'The platform cannot find this process and cannot recover a spawned agent id after it exits, so this', 'record is the only one. Without it, deleting this commander leaves its loop running.', '', + '### `WATCHERS.md` — how to start your own watch again', + '', + 'A pid is not a watch. It is a number that stops meaning anything the moment the process ends, and it', + 'ends more often than you would think: the machine reboots, something kills the tree it was spawned', + 'under, the loop hits a bug and exits. When that happens the platform will notice, and it will hand', + 'the turn back to you asking you to start the watch again.', + '', + '**At that moment your only source is this file.** The transcript may be long past the turn that armed', + 'it, and the platform deliberately does not remember your command — it never starts a watch itself,', + 'because a process it spawned could not reach you when it fired.', + '', + 'So keep `WATCHERS.md` true, and write it the moment you arm, change or stop a watch. One entry each:', + '', + '- **what it watches** — in a sentence, so a person can tell whether it is still wanted.', + '- **the exact command**, verbatim, the way you ran it. Not a description of it.', + '- **how often** it checks, and how it decides something has happened.', + '- **what it does when it fires**, or which file says.', + '- **when you armed it**, and anything you learned since that would change how you start it.', + '', + 'The test: could you restore this watch from that entry alone, having read nothing else? If not, the', + 'entry is a note rather than a record, and the watch dies with the process.', + '', + 'Rewrite the entry when you re-arm. A stale command in this file is worse than no file, because it', + 'will be followed.', + '', ...credentialSection(params.credentialPath), '## The shape: a trigger, a ledger, and up to three steps between them', '', @@ -345,6 +370,8 @@ export function commanderClaudeMd(params: CommanderClaudeMdParams): string { ' the job is settled and kept true after that.', 'CLASSIFICATION.md the category rules, if it classifies. yours, and expected to change.', 'DISPATCHING.md what happens to a result beyond keeping it, if anything. yours.', + 'WATCHERS.md every watch you are running, exactly enough to start it again. yours,', + ' rewritten whenever you arm, change or stop one.', 'data/ everything this commander has produced. ALWAYS called data.', ' / only if it classifies — one directory per category, as they are agreed', ' unclassified/ only if it classifies — produced and recorded, placement still open', diff --git a/src/servers/api/command-hq/spawned.ts b/src/servers/api/command-hq/spawned.ts index e39b10e5..1a6e0e82 100644 --- a/src/servers/api/command-hq/spawned.ts +++ b/src/servers/api/command-hq/spawned.ts @@ -187,5 +187,26 @@ export function stopCommanderProcesses(email: string, dirName: string): number { // Gone between the walk and the signal. Not a failure — that is the outcome we wanted. } } + + // The record goes with the processes, and this is load-bearing rather than tidiness. + // + // `watcher-revive.ts` treats "a pid file whose pids are all dead" as a watch that died and needs the + // commander woken. A Stop that killed the loop and left the file behind is indistinguishable from a + // crash, so the reviver would hand the turn back and the commander would dutifully re-arm the thing + // the owner just stopped. Removing the file is what makes deliberate and accidental different. + clearLoopPid(email, dirName); return signalled; } + +/** + * Forget the recorded pid. "No file" means no watch, on purpose — which is exactly what a fresh + * commander looks like, and what a stopped one should look like too. + */ +export function clearLoopPid(email: string, dirName: string): void { + try { + rmSync(loopPidPath(email, dirName), { force: true }); + } catch { + // A file we cannot remove is not worth failing a Stop over. The reviver's own guard covers it: it + // wakes a commander once and clears the file itself, so the worst case is one wasted turn. + } +} diff --git a/src/servers/api/command-hq/watcher-revive.ts b/src/servers/api/command-hq/watcher-revive.ts new file mode 100644 index 00000000..dcde04b0 --- /dev/null +++ b/src/servers/api/command-hq/watcher-revive.ts @@ -0,0 +1,207 @@ +import { listAllCommanders, getUserById } from 'officerdb'; +import { getCommanderStateDir } from '@@/data-path'; +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { startClaudeTurn, COMMANDER_DEFAULT_MODEL } from '../chat/websocket'; +import { clearLoopPid, liveLoopPids } from './spawned'; +import { getNotifyServerUrl, notifyProducerHeaders } from '../notify/router'; + +// Waking a commander whose watch has died. +// +// ── The failure this exists for ── +// +// A watch is a background process the commander started, and nothing on this machine supervises it. So a +// reboot ends every one of them at once, and nothing says so: the card simply stops saying "watching", +// which looks identical to a commander that was never armed. +// +// That is not theoretical. On 2026-08-24 this machine rebooted at 12:40 and took the deploy commander's +// loop with it. A release was tagged the next day and sat on the remote with nobody looking at it, and +// the only evidence was a word missing from a card. `todo/agent-coordination/` carries the full account. +// +// ── Why the platform wakes the agent instead of restarting the process ── +// +// The obvious fix is for the platform to respawn the loop itself, and it is WRONG — this was settled on +// 2026-08-18 and is written down in `docs/command-hq.md` under "The watcher loop". The agent is only ever +// notified about a process it started and is itself tracking. A process the platform spawned sits outside +// that entirely, so it would watch correctly, fire, and reach nobody. +// +// A respawn would therefore trade a visible failure for an invisible one, which is the worse of the two: +// today the commander is plainly not watching, and that version would be watching and unable to tell +// anyone. So the platform's whole job here is to NOTICE and hand the turn back. The agent re-arms itself, +// exactly as it does after every ordinary fire, and the process it starts is tracked. +// +// The owner's two conditions, 2026-08-25, both kept by construction: the commander is still the only +// thing that starts a watch, and the indicator still reads the pid file rather than any stored claim. +// +// ── Why the pid file, and not `watcherArmed` ── +// +// `watcherArmed` is a claim only one path sets, and most commanders never take that path — they arm by +// backgrounding a process, which records a pid and nothing else. Both commanders on this machine read +// `watcher_armed = false` while one of them was demonstrably watching. `router.ts` already resolves this +// the same way for its cards: **the pid file is the fact.** +// +// So the signal is: a pid file EXISTS and nothing in it is alive. That means this commander had a watch +// and no longer does. A commander that never armed has no file, and a commander the owner deliberately +// stopped no longer has one either — `stopCommanderProcesses` removes it, which is what makes stopped and +// crashed distinguishable at all. + +const TICK_MS = 120_000; + +/** + * The first tick is delayed rather than immediate. + * + * A reboot is the case this is for, and at boot the platform comes up before much else — waking a + * commander into a machine still starting its own services would ask it to re-arm a watch against + * half-present dependencies. Two minutes is not a considered number; it is enough for the rest of PM2. + */ +const FIRST_TICK_MS = 120_000; + +const PID_FILE = 'loop-pid'; + +/** A watch that was running and is not. See the note above for why this is the pid file's question. */ +function watchDied(email: string, dirName: string): boolean { + if (!existsSync(join(getCommanderStateDir(email, dirName), PID_FILE))) return false; + return liveLoopPids(email, dirName).length === 0; +} + +/** + * What the commander is told. Deliberately short, and it points at the file rather than describing the + * watch — the platform does not know what was being watched and must not guess. + * + * `WATCHERS.md` is the commander's own record, required by its `CLAUDE.md`: what each watch is, the exact + * command, how often, and what happens when it fires. This message is the moment that file earns its keep. + * + * It also has to work for a commander that predates the file, which every commander on this machine does + * today — so reconstructing from `JOB.md`, a script in its own directory, or the turn that armed it is + * explicitly allowed. That is reading its own records, not guessing. Inventing a command it cannot find + * is the thing forbidden, because a wrong watch looks exactly like a working one. + */ +const WAKE_PROMPT = [ + 'Your watch is not running. The platform found a recorded pid with no live process behind it —', + 'usually the machine rebooted, or something killed the tree it was started under.', + '', + 'Read `WATCHERS.md` and start it again, exactly as it is written there: same command, same interval,', + 'same behaviour on firing. Then record the new pid as your `CLAUDE.md` describes, and update the entry', + 'if anything about it changed.', + '', + 'If `WATCHERS.md` does not exist, you may rebuild the watch from your own records — `JOB.md`, a script', + 'in your directory, the turn in this conversation that armed it. That is reading, not guessing, and', + 'write `WATCHERS.md` as you go so this is the last time it has to be reconstructed.', + '', + 'But if none of those say what the command actually was, do NOT invent one. Say what is missing and', + 'stop. A wrong watch is worse than none, because it looks like one that works.', +].join('\n'); + +async function wake(userId: number, sessionId: string, dirName: string, email: string): Promise { + // Cleared BEFORE the turn, not after. The file is the trigger, so leaving it in place while a turn runs + // means the next tick sees the same dead pid and wakes the commander a second time. Clearing first also + // gives the right outcome when the wake fails: one attempt, one notification, and silence — rather than + // a commander woken every two minutes for as long as the machine is up. + clearLoopPid(email, dirName); + + await new Promise((resolve) => { + let settled = false; + const done = () => { + if (!settled) { + settled = true; + resolve(); + } + }; + const timer = setTimeout(done, 60_000); + + void startClaudeTurn({ + email, + username: email, + userId, + // No socket, the same way the nudger drives a turn. Whoever has the commander open sees it arrive + // through their own connection. + ws: null, + sessionId, + model: COMMANDER_DEFAULT_MODEL, + msg: { prompt: WAKE_PROMPT, resumeSessionId: sessionId }, + effectivePrompt: WAKE_PROMPT, + onOutcome: (outcome) => { + clearTimeout(timer); + if (!outcome.ok) console.error(`[watcher-revive] ${dirName}: ${outcome.error}`); + done(); + }, + }).catch((err) => { + clearTimeout(timer); + console.error(`[watcher-revive] ${dirName} threw:`, err instanceof Error ? err.message : err); + done(); + }); + }); +} + +/** + * Tell the owner, separately from waking the commander. + * + * These are two different audiences and only one of them can act when the wake does not work. A + * commander that cannot find its command says so into a transcript nobody is reading at 4am; the + * notification is what makes "your watch has been off since the reboot" reach a person. + * + * Best-effort in both directions: the wake happens whether or not this lands, and a failure here is + * logged rather than raised. Null headers mean the notify sidecar has not registered yet, and sending + * unauthenticated is not the fallback — see `notifyProducerHeaders`. + */ +async function tellTheOwner(userId: number, dirName: string): Promise { + const base = getNotifyServerUrl(); + const producer = notifyProducerHeaders(); + if (!base || !producer) return; + + try { + await fetch(`${base}/_officer/notify`, { + method: 'POST', + headers: { 'content-type': 'application/json', ...producer }, + body: JSON.stringify({ + code: 'officer/core:message', + userId, + title: 'A commander stopped watching', + // Names the commander by its directory and says nothing about what it watches. The subject of a + // watch is the owner's business and this text goes through Apple and Google to a lock screen. + body: `Commander ${dirName}'s watch was not running. It has been asked to start it again.`, + }), + signal: AbortSignal.timeout(15_000), + }); + } catch (err) { + console.error(`[watcher-revive] could not notify about ${dirName}:`, err instanceof Error ? err.message : err); + } +} + +let timer: ReturnType | null = null; + +/** + * Start the sweep. Called once from `server.tsx`. + * + * `running` guards a slow tick against the next one, and `unref` keeps it from holding the process open. + * Both copied from the nudger, which has the same shape and the same failure to avoid. + */ +export function startWatcherRevive(): void { + if (timer) return; + let running = false; + + const tick = async () => { + if (running) return; + running = true; + try { + for (const commander of await listAllCommanders()) { + const user = await getUserById(commander.userId); + if (!user) continue; + if (!watchDied(user.email, commander.dirName)) continue; + + console.warn(`[watcher-revive] commander ${commander.dirName}: watch is dead — waking it`); + await tellTheOwner(commander.userId, commander.dirName); + await wake(commander.userId, commander.sessionId, commander.dirName, user.email); + } + } catch (err) { + console.error('[watcher-revive] tick failed:', err instanceof Error ? err.message : err); + } finally { + running = false; + } + }; + + setTimeout(() => void tick(), FIRST_TICK_MS).unref?.(); + timer = setInterval(() => void tick(), TICK_MS); + timer.unref?.(); + console.log('[watcher-revive] armed — checking every 120s for watches that have died'); +}