deleting a bot stops it, and a dead bot wakes up

a bot was deleted tonight and its watcher was still polling a website eleven minutes later. nothing in the product could find it: a bot's watch is a detached shell process, killing the harness ends the agent, and the loop it started is not its child. so a bot writes <sessionId>.bot.json beside its script — what it watches, the script, the state file, the pids. that record is the ONLY thing connecting a process to the bot that owns it, and the brief now says so in those words rather than as bookkeeping. the id is in the FILENAME because bots share a directory: deleting knows only an id, so this is a lookup rather than a scan, and a record whose contents are damaged still gets found and cleaned up. DELETE /chat/sessions/:id tears it down after forgetting the tables — children first then the recorded pid, same ordering as stopCommanderProcesses, so the loop cannot spawn a replacement between the two. every transcript in the returned chain, not just the id asked for. unconditional rather than gated on "was this a bot", because the record only exists in the bots directory and a branch on cwd is a second place to get it wrong. script and state go too, and only if they are inside the bots directory — an agent wrote that path. and the reviver sweeps bots on the same tick. a record with pids and none of them alive is a stopped watch; there is no armed flag and there should not be, since the record is written when it arms, so its existence IS the intent. the three-wake bound is shared, keyed bot-<sessionId>. the wake prompt forbids reconstructing a watch from memory when the record is gone: a watch subtly different from the one agreed is worse than none, because it looks like it is working. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Officer-Session: 656f5c85-e49a-4f1f-b9f3-042a04b4d043
2026-08-26 08:04:49 +00:00
parent 513a50f2e8
commit 84d9c9d2d8
4 changed files with 282 additions and 5 deletions
@@ -1,10 +1,11 @@
 import { listAllCommanders, getUserById } from 'officerdb';
-import { getCommanderStateDir } from '@@/data-path';
+import { getCommanderStateDir, getBotsDir } from '@@/data-path';
 import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { join } from 'node:path';
 import { startClaudeTurn, COMMANDER_DEFAULT_MODEL } from '../chat/websocket';
 import * as sidecar from '@@/sidecar-registry';
 import { clearLoopPid, liveLoopPids } from './spawned';
+import { listBotRecords, liveBotPids } from '../bots/teardown';
 import { getNotifyServerUrl, notifyProducerHeaders } from '../notify/router';

 // Waking a commander whose watch has died.
@@ -60,6 +61,18 @@ const FIRST_TICK_MS = 120_000;
 const PID_FILE = 'loop-pid';
 const ATTEMPTS_FILE = 'revive-attempts';

+/**
+ * Where a subject's wake attempts are counted.
+ *
+ * A commander keeps them in its own `state/`. A bot has no directory of its own, so its key is
+ * `bot-<sessionId>` and the file goes in the bots directory under that name — same bound, same window,
+ * one implementation.
+ */
+const attemptsPath = (email: string, key: string) =>
+  key.startsWith('bot-')
+    ? join(getBotsDir(email), `${key}.${ATTEMPTS_FILE}`)
+    : join(getCommanderStateDir(email, key), ATTEMPTS_FILE);
+
 /**
 * How many times this will wake one commander, and over what window.
 *
@@ -76,7 +89,7 @@ const ATTEMPT_WINDOW_MS = 6 * 60 * 60 * 1000;

 /** Wake timestamps inside the window. Anything older is dropped rather than kept — this is a rate, not a history. */
 function recentAttempts(email: string, dirName: string): number[] {
-  const path = join(getCommanderStateDir(email, dirName), ATTEMPTS_FILE);
+  const path = attemptsPath(email, dirName);
  if (!existsSync(path)) return [];
  const cutoff = Date.now() - ATTEMPT_WINDOW_MS;
  try {
@@ -93,7 +106,7 @@ function recentAttempts(email: string, dirName: string): number[] {
 }

 function recordAttempt(email: string, dirName: string): void {
-  const path = join(getCommanderStateDir(email, dirName), ATTEMPTS_FILE);
+  const path = attemptsPath(email, dirName);
  try {
    writeFileSync(path, [...recentAttempts(email, dirName), Date.now()].join('\n'));
  } catch (err) {
@@ -267,6 +280,82 @@ async function tellTheOwner(userId: number, dirName: string): Promise<void> {
  }
 }

+/**
+ * The same question, asked of bots.
+ *
+ * A bot's watch dies exactly as a commander's does — a reboot, a killed process tree — and with the same
+ * silence: the card still lists it and nothing is polling anything. The difference is only where the
+ * record lives. A commander writes a pid file into its own `state/`; a bot writes `<sessionId>.bot.json`
+ * beside its script, because bots share one directory and the id has to be in the filename.
+ *
+ * A record with no live pid is a watch that has stopped. There is no `armed` flag to consult and there
+ * should not be: the record is written when it arms, so its existence IS the intent, and the pids in it
+ * are the fact.
+ */
+async function reviveBots(userId: number, email: string): Promise<void> {
+  for (const { sessionId, record } of listBotRecords(email)) {
+    if (!record.pids?.length) continue; // never armed, or already torn down
+    if (liveBotPids(email, sessionId).length > 0) continue;
+
+    const attempts = recentAttempts(email, `bot-${sessionId}`);
+    if (attempts.length >= MAX_ATTEMPTS) continue;
+
+    console.warn(`[watcher-revive] bot ${sessionId}: watch is dead — waking it`);
+    recordAttempt(email, `bot-${sessionId}`);
+    await tellTheOwner(userId, `bot ${sessionId.slice(0, 8)}`);
+    await wakeBot(userId, sessionId, email);
+  }
+}
+
+/**
+ * What a bot is told. Shorter than a commander's, because a bot has one job and its record names it.
+ */
+const WAKE_BOT_PROMPT = [
+  'Your watch is not running — the recorded process is gone, usually a reboot or something that killed',
+  'the tree it was started under.',
+  '',
+  'Read your own `<session-id>.bot.json` in this directory, arm the script it names again exactly as',
+  'before, and rewrite the record with the new pid. Do not start a second watch beside it, and do not',
+  'change what it watches.',
+  '',
+  'If the record or the script is missing, say what is gone and stop. Do not reconstruct a watch from',
+  'memory of the conversation — a watch that is subtly different from the one the owner agreed to is',
+  'worse than none, because it looks like it is working.',
+].join('\n');
+
+async function wakeBot(userId: number, sessionId: string, email: string): Promise<void> {
+  await new Promise<void>((resolve) => {
+    let settled = false;
+    const done = () => {
+      if (!settled) {
+        settled = true;
+        resolve();
+      }
+    };
+    const timer = setTimeout(done, 60_000);
+
+    void startClaudeTurn({
+      email,
+      username: email,
+      userId,
+      ws: null,
+      sessionId,
+      model: COMMANDER_DEFAULT_MODEL,
+      msg: { prompt: WAKE_BOT_PROMPT, resumeSessionId: sessionId },
+      effectivePrompt: WAKE_BOT_PROMPT,
+      onOutcome: (outcome) => {
+        clearTimeout(timer);
+        if (!outcome.ok) console.error(`[watcher-revive] bot ${sessionId}: ${outcome.error}`);
+        done();
+      },
+    }).catch((err) => {
+      clearTimeout(timer);
+      console.error(`[watcher-revive] bot ${sessionId} threw:`, err instanceof Error ? err.message : err);
+      done();
+    });
+  });
+}
+
 let timer: ReturnType<typeof setInterval> | null = null;

 /**
@@ -305,6 +394,17 @@ export function startWatcherRevive(): void {
        await tellTheOwner(commander.userId, commander.dirName);
        await wake(commander.userId, commander.sessionId, commander.dirName, user.email);
      }
+
+      // Bots, same tick. Their records live per account, so this needs the set of owners rather than the
+      // commander list — an account with no commanders can still have bots.
+      const owners = new Map<number, string>();
+      for (const c of await listAllCommanders()) {
+        const u = await getUserById(c.userId);
+        if (u) owners.set(c.userId, u.email);
+      }
+      const me = await getUserById(1);
+      if (me) owners.set(1, me.email);
+      for (const [userId, email] of owners) await reviveBots(userId, email);
    } catch (err) {
      console.error('[watcher-revive] tick failed:', err instanceof Error ? err.message : err);
    } finally {