platform/src/servers/api/chat/task-restart.ts

import { closeSync, openSync } from 'node:fs';
import { basename, dirname, join } from 'node:path';
import { runAsArgv } from '../../os-user';
import { readTextAs } from '../../read-as-user';
import { findTaskOutput, pidsHoldingFile, type ChatIdentity } from './claude-sessions';

/**
 * Run a background job's command again, without asking the agent.
 *
 * ── What this owns, and what it does not ──
 *
 * A background task normally belongs entirely to the harness: it spawns the process, watches it, and emits
 * `task:started` and `task:notification`. Everything downstream — the tray, the log view, reconnects — is
 * built on exactly those two events.
 *
 * A restart cannot be that. The agent is not involved, so nothing would emit either event, and the harness
 * stops watching the moment its own child dies. So the platform takes over supervision of the process it
 * spawned: it emits the same two events for it, and watches for the exit itself.
 *
 * That is a second owner for the same kind of thing, and it is a real cost — accepted deliberately, so
 * that restarting a dev server is a button rather than a sentence typed at an agent that may be mid-turn.
 * The seam is contained here: nothing else learns that a task can have two kinds of parent.
 *
 * ── The output file is the anchor ──
 *
 * The respawned process appends to the SAME `<task-id>.output` the harness created. That is what keeps
 * everything else working untouched: the log view reads that file, and Stop finds the process by asking
 * which pids hold it open. A new file would have meant teaching both about a second location.
 */

/**
 * In-flight restarts, so a second click stops the first rather than racing it.
 *
 * A timer rather than a subprocess handle, because the process is deliberately no longer ours — see
 * `detachedArgv`.
 */
const restarted = new Map<string, { watch: ReturnType<typeof setInterval> }>();

/** How long to let a job open its output file before an empty holder list is read as "it ended". */
const WATCH_GRACE_MS = 2500;
const WATCH_INTERVAL_MS = 5000;

/**
 * The argv that survives `pm2 restart officer`.
 *
 * A restarted job used to be spawned as a plain child of the platform, and it died with the platform. That
 * was not a footnote: the tray showed two chips that looked identical — one the agent started, one this
 * module restarted — and only the second evaporated on a server restart, while the button's own help text
 * promised that background jobs survive.
 *
 * Measured rather than reasoned about, against a throwaway PM2 app. Three spawn shapes, one `pm2 restart`:
 *
 *   plain `sh -c`              killed
 *   `setsid sh -c`             killed      <- a new SESSION is not enough
 *   `setsid --fork sh -c`      survived
 *
 * PM2 walks the process tree by ppid, so changing session leaves the job a child and it is found anyway.
 * `--fork` is the part that matters: setsid forks and its parent exits at once, so the job is reparented
 * to init before any restart can enumerate it. It is not officer's descendant, so there is nothing to walk.
 *
 * The cost is paid in `watchForExit`: the handle `Bun.spawn` returns belongs to the setsid parent, which
 * exits immediately, so the process's real ending has to be noticed some other way.
 *
 * The wrapper passes the command as a POSITIONAL PARAMETER rather than interpolating it into the script,
 * so a command containing quotes stays one argument and cannot terminate the wrapper's own quoting. `$2`
 * is where the exit status is recorded — the only place it survives, now that nothing is waiting on the
 * process.
 */
const WRAPPER = 'rm -f "$2"; eval "$1"; echo $? > "$2"';

const detachedArgv = (command: string, exitFile: string): string[] => [
  'setsid',
  '--fork',
  'sh',
  '-c',
  WRAPPER,
  'sh',
  command,
  exitFile,
];

export type RestartResult = { ok: boolean; command?: string; reason?: string };

/** A command and the directory it meant something in. The second half is not optional in practice. */
export type TaskCommand = { command: string; cwd: string | null };

/**
 * The command that started a task, recovered by joining the transcript back to itself.
 *
 * The harness records the tool call (`tool_use`, carrying `command`) and, separately, its result — which
 * is the text `Command running in background with ID: <task id>`. Neither alone is enough; together they
 * are a join on `tool_use_id`, and that is the only place the command survives.
 *
 * The transcript is found from the OUTPUT PATH rather than by searching: a task file lives at
 * `<tmp>/claude-<uid>/<project-slug>/<session-uuid>/tasks/<id>.output`, so the slug and session name the
 * transcript exactly. Guessing among a member's transcripts would be both slower and occasionally wrong.
 */
export async function commandForTask(who: ChatIdentity, taskId: string): Promise<TaskCommand | null> {
  const output = findTaskOutput(who, taskId);
  if (!output || !output.endsWith('.output')) return null;

  const sessionDir = dirname(dirname(output));
  const transcript = join(
    who.home,
    '.claude',
    'projects',
    basename(dirname(sessionDir)),
    `${basename(sessionDir)}.jsonl`,
  );

  let text: string;
  try {
    text = readTextAs(who.osUser, transcript);
  } catch {
    return null;
  }

  // The cwd is read from the SAME entry as the tool call, not from anywhere else. `bun dev` means one
  // thing in a project and something entirely different in the platform's own directory — see the note on
  // `restartBackgroundTask`.
  const commands = new Map<string, TaskCommand>();
  let toolUseId: string | null = null;

  for (const line of text.split('\n')) {
    if (!line) continue;
    let entry: unknown;
    try {
      entry = JSON.parse(line);
    } catch {
      continue;
    }
    const record = entry as { message?: { content?: unknown }; cwd?: unknown };
    const content = record?.message?.content;
    if (!Array.isArray(content)) continue;

    for (const block of content as Record<string, unknown>[]) {
      if (block.type === 'tool_use' && block.name === 'Bash') {
        const command = (block.input as { command?: unknown } | undefined)?.command;
        if (typeof block.id === 'string' && typeof command === 'string') {
          commands.set(block.id, { command, cwd: typeof record.cwd === 'string' ? record.cwd : null });
        }
      }
      if (block.type === 'tool_result') {
        const body = typeof block.content === 'string' ? block.content : JSON.stringify(block.content ?? '');
        // Both phrasings the harness produces carry the id; this is the one that names a shell job.
        if (body.includes(`background with ID: ${taskId}`) && typeof block.tool_use_id === 'string') {
          toolUseId = block.tool_use_id;
        }
      }
    }
  }

  return toolUseId ? (commands.get(toolUseId) ?? null) : null;
}

/**
 * Stop whatever is running for this task and start its command again.
 *
 * `onExit` is how the caller emits `task:notification` — this module deliberately knows nothing about
 * websockets or sessions.
 */
export async function restartBackgroundTask(
  who: ChatIdentity,
  taskId: string,
  onExit: (code: number) => void,
): Promise<RestartResult> {
  const output = findTaskOutput(who, taskId);
  if (!output) return { ok: false, reason: 'no such task' };

  const recovered = await commandForTask(who, taskId);
  if (!recovered) {
    // Honest about which half failed: the file is there, the command is not recoverable. That happens when
    // the transcript has been pruned, and no amount of retrying will change it.
    return { ok: false, reason: 'cannot tell what command started this job — its transcript is gone' };
  }

  /**
   * No directory, no restart.
   *
   * This refuses rather than falling back to the server's own cwd, and the reason is that the fallback
   * already happened: a `bun dev` belonging to a project in `~/dockers/officer-atelier` was respawned
   * without a cwd, inherited the PLATFORM's directory, and ran the platform's own `bun dev`. A second
   * server bound port 9000 alongside the real one — Bun allows it through SO_REUSEPORT — and requests
   * were split between two instances at random.
   *
   * A command is only meaningful with the directory it was written for, and guessing produced the worst
   * outcome available: not a failure, but a different program running successfully.
   */
  const { command, cwd } = recovered;
  if (!cwd) {
    return { ok: false, reason: 'cannot tell which directory this job ran in — refusing to guess' };
  }

  // A second click must replace the first restart, not leave two dev servers fighting over a port. Only
  // the watcher is ours to cancel — the process itself is detached, and is stopped by the kill loop below
  // like any other holder of the file.
  const previous = restarted.get(taskId);
  if (previous) {
    clearInterval(previous.watch);
    restarted.delete(taskId);
  }

  for (const pid of await pidsHoldingFile(who, output)) {
    const argv = who.osUser ? runAsArgv(who.osUser, ['kill', '-TERM', String(pid)]) : null;
    try {
      if (argv) Bun.spawn(argv, { stdout: 'ignore', stderr: 'ignore' });
      else process.kill(pid, 'SIGTERM');
    } catch {
      /* gone between listing and killing */
    }
  }
  // A dev server holding a port needs a moment to release it, and the whole point of a restart is that the
  // new one binds. Not a fix for a process that ignores SIGTERM — that is what the second click is for.
  await Bun.sleep(300);

  // Appended, never truncated: the log view is reading this file, and a restart is a new chapter in a job's
  // life rather than a new job.
  // The fd is CLOSED immediately after the spawn, and that one line is load-bearing.
  //
  // Leaving it open made the platform a holder of the job's output file — and `pidsHoldingFile` asks
  // `fuser` who holds it. So the next Pause sent `SIGSTOP` to the platform itself: eleven threads
  // stopped, port 9000 still listening with connections queueing behind it and nobody accepting. The
  // server was not down, it was frozen, which looks identical from outside and is why it took a terminal
  // to see. `kill -CONT` brought it back untouched.
  //
  // The child gets its own duplicate of the descriptor at spawn, so closing here costs it nothing.
  const exitFile = `${output}.exit`;
  const fd = openSync(output, 'a');
  const inner = detachedArgv(command, exitFile);
  const argv = who.osUser ? runAsArgv(who.osUser, inner) : inner;
  const proc = Bun.spawn(argv, { cwd, stdout: fd, stderr: fd, stdin: 'ignore' });
  closeSync(fd);

  // This resolves immediately and says nothing about the job: it is setsid reporting that it forked. A
  // non-zero status is still worth catching, because it means nothing was started at all — and reporting
  // `ok` there would leave a chip running forever with no process behind it.
  const spawnStatus = await proc.exited;
  if (spawnStatus !== 0) {
    return { ok: false, reason: `could not start the job — setsid exited ${spawnStatus}` };
  }

  restarted.set(taskId, { watch: watchForExit(who, taskId, output, exitFile, onExit) });
  return { ok: true, command };
}

/**
 * Notice a detached job ending, since nothing is waiting on it any more.
 *
 * The question "is it still running" is answered the same way Stop and Pause answer it — who holds the
 * output file open — so this adds no new notion of liveness, it reuses the one the rest of the module is
 * already built on. A paused job still holds its file, so `SIGSTOP` reads as running, which is correct.
 *
 * The grace period is the only subtle part: between the fork and the job opening its output there is a
 * moment with no holders, and polling into it would announce a completion for a job that had not started.
 */
function watchForExit(
  who: ChatIdentity,
  taskId: string,
  output: string,
  exitFile: string,
  onExit: (code: number) => void,
): ReturnType<typeof setInterval> {
  const startedAt = Date.now();
  const timer: ReturnType<typeof setInterval> = setInterval(() => {
    void (async () => {
      if (Date.now() - startedAt < WATCH_GRACE_MS) return;
      if ((await pidsHoldingFile(who, output)).length > 0) return;

      // A later restart owns the task by now; its watcher will do the reporting.
      if (restarted.get(taskId)?.watch !== timer) {
        clearInterval(timer);
        return;
      }
      clearInterval(timer);
      restarted.delete(taskId);

      // Read as the owning account: for a member this file sits in a directory the platform's own uid
      // cannot even traverse.
      let code = 0;
      try {
        code = Number.parseInt(readTextAs(who.osUser, exitFile).trim(), 10);
      } catch {
        // No status file: the job was killed before its wrapper could record one, which is what Stop looks
        // like. Reported as a clean finish rather than a failure — the pill has to clear either way, and
        // "you stopped it" is nearer the truth than "it crashed".
      }
      onExit(Number.isInteger(code) ? code : 0);
    })();
  }, WATCH_INTERVAL_MS);
  return timer;
}

/** Whether the platform (rather than the harness) is currently supervising this task. */
export const isRestarted = (taskId: string): boolean => restarted.has(taskId);

/**
 * Freeze or thaw a job, without ending it.
 *
 * `SIGSTOP` and `SIGCONT`, which is the only pair that means "stop doing things and keep everything" — the
 * process keeps its memory, its open files and its bound ports, and picks up mid-instruction.
 *
 * Worth knowing before pressing it: a paused server does not REFUSE connections, it stops answering them.
 * A browser pointed at it hangs rather than erroring, and an HMR client may give up and reconnect on
 * resume. That is inherent to what pausing is, not a shortcoming of this implementation.
 *
 * Signals go to every pid holding the output file — for a shell job that is the shell and whatever it
 * spawned, and freezing only one of them would be worse than not freezing at all.
 */
export async function pauseBackgroundTask(
  who: ChatIdentity,
  taskId: string,
  paused: boolean,
): Promise<{ ok: boolean; reason?: string }> {
  const output = findTaskOutput(who, taskId);
  if (!output) return { ok: false, reason: 'no such task' };

  const pids = await pidsHoldingFile(who, output);
  if (pids.length === 0) return { ok: false, reason: 'nothing is running for this job' };

  const signal = paused ? '-STOP' : '-CONT';
  for (const pid of pids) {
    const argv = who.osUser ? runAsArgv(who.osUser, ['kill', signal, String(pid)]) : null;
    try {
      if (argv) Bun.spawn(argv, { stdout: 'ignore', stderr: 'ignore' });
      else process.kill(pid, paused ? 'SIGSTOP' : 'SIGCONT');
    } catch {
      // Gone between listing and signalling. Nothing to do, and nothing to report: the caller asked for a
      // state, and "not running" satisfies "paused" closely enough to not be worth an error.
    }
  }
  return { ok: true };
}