stop orphaning opencode turns and serves on restart
B8, both halves. They share index.ts, so they share a commit. In-flight turns: `opencode run` is spawned, not supervised, so pm2 restart officer-opencode left every turn ALIVE — reparented, still spending tokens, still writing files as the agent, with the only reader of its stdout gone. The transcript stopped mid-tool-call, which reads as the agent hanging. stopAllOpenCodeTurns kills them and settles each synchronously, because the caller is about to process.exit and nothing waiting on proc.exited would ever run. Settling writes a reason, so a reload after a restart explains itself instead of trailing off. Turns are stopped BEFORE the connection is destroyed — that write travels over it — and the flush is bounded, since losing the explanation is bad but hanging the restart is worse. Stale serves: the sweep read /proc, so it was a no-op on macOS and orphaned serves piled up, one per unclean exit, each holding a port. Added a pidfile sweep alongside it. A pid we wrote ourselves needs no cwd guard to prove it is ours, which is the part ps cannot answer portably (macOS would need lsof), and a serve started by hand is never in the file. The guard checks command AND subcommand: matching the word serve anywhere in the line would sweep a running turn whose prompt merely mentioned it. Fixtures are real ps output from both machines, not invented. Split into serve-sweep.ts because index.ts spawns a serve at module scope, so a test importing it would start one. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,11 +1,12 @@
|
||||
import { mkdirSync, readdirSync, readFileSync, realpathSync } from 'node:fs';
|
||||
import { mkdirSync, readdirSync, readFileSync, realpathSync, unlinkSync, writeFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { DATA_PATH } from '../../data-path';
|
||||
import { createSidecarConnector } from '../connect';
|
||||
import { sweepRecordedServe } from './serve-sweep';
|
||||
import { createSessionLogStore } from '../claude/session-log';
|
||||
import type { SidecarCommand, SidecarEvent } from '../protocol';
|
||||
import { runOpenCodeTurn, killOpenCodeTurn, listRunningOpenCodeTurns } from './runner';
|
||||
import { runOpenCodeTurn, killOpenCodeTurn, listRunningOpenCodeTurns, stopAllOpenCodeTurns } from './runner';
|
||||
|
||||
// The OpenCode sidecar (officer-opencode). Same philosophy as officer-claude: a singleton process that
|
||||
// OWNS its runtime — here, an `opencode serve` — registers with the API server, and answers commands. It
|
||||
@@ -95,8 +96,11 @@ mkdirSync(SERVE_CWD, { recursive: true });
|
||||
// anchors the agent's own file operations, not merely the process cwd, and the anchor survives a
|
||||
// multi-step turn including a write (docs/opencode-phase0-review.md). Nothing replaces this.
|
||||
|
||||
// Sweep any orphaned serve from a previous unclean exit (scoped strictly to SERVE_CWD) so we never
|
||||
// end up with two serves for this directory.
|
||||
// Sweep any orphaned serve from a previous unclean exit so we never end up with two for this directory.
|
||||
// Two mechanisms, deliberately: the pidfile is portable and precise, the /proc scan is the Linux-only
|
||||
// backstop for a serve whose pidfile was lost (killed -9 mid-write, or predating the pidfile entirely).
|
||||
const SERVE_PID_FILE = join(SERVE_CWD, 'serve.pid');
|
||||
sweepRecordedServe(SERVE_PID_FILE);
|
||||
sweepStaleServes(SERVE_CWD);
|
||||
|
||||
const port = getFreePort();
|
||||
@@ -109,6 +113,14 @@ const serve = Bun.spawn([OPENCODE_BIN, 'serve', '--port', String(port), '--hostn
|
||||
stderr: 'inherit',
|
||||
});
|
||||
|
||||
// Recorded before the health check, not after: an unhealthy serve is exactly the kind that gets left
|
||||
// behind, and it still needs sweeping next time.
|
||||
try {
|
||||
writeFileSync(SERVE_PID_FILE, String(serve.pid));
|
||||
} catch (err) {
|
||||
console.error('[opencode] could not record the serve pid; sweeping will fall back to /proc', err);
|
||||
}
|
||||
|
||||
if (!(await waitHealthy(baseUrl, HEALTH_TIMEOUT_MS))) {
|
||||
console.error('[opencode] serve failed its health check');
|
||||
try {
|
||||
@@ -189,7 +201,26 @@ const sessionLog = createSessionLogStore((d) =>
|
||||
|
||||
// ── Graceful shutdown ──
|
||||
|
||||
function shutdown(signal: string) {
|
||||
// PM2 sends SIGTERM and follows with SIGKILL shortly after, so everything below is on a budget. The
|
||||
// flush is bounded rather than awaited outright: losing the explanation is bad, hanging the restart is
|
||||
// worse, and an unbounded await on a wedged Postgres would do exactly that.
|
||||
const SHUTDOWN_FLUSH_MS = 1_000;
|
||||
|
||||
let shuttingDown = false;
|
||||
|
||||
async function shutdown(signal: string) {
|
||||
if (shuttingDown) return; // SIGINT after SIGTERM must not re-enter and cut the flush short
|
||||
shuttingDown = true;
|
||||
|
||||
// Before the connection goes: killing a turn produces an event, and that event's durable write travels
|
||||
// over this socket. Tearing it down first would stop every turn silently — the exact outcome this is
|
||||
// here to prevent.
|
||||
const stopped = stopAllOpenCodeTurns(`The OpenCode sidecar restarted (${signal}), so this turn stopped.`);
|
||||
if (stopped > 0) {
|
||||
console.log(`[opencode] ${signal} received, stopping ${stopped} in-flight turn(s)...`);
|
||||
await Promise.race([sessionLog.flush(), Bun.sleep(SHUTDOWN_FLUSH_MS)]);
|
||||
}
|
||||
|
||||
console.log(`[opencode] ${signal} received, stopping serve...`);
|
||||
connection.destroy();
|
||||
try {
|
||||
@@ -197,8 +228,15 @@ function shutdown(signal: string) {
|
||||
} catch {
|
||||
/* already gone */
|
||||
}
|
||||
// We killed it ourselves, so the record has done its job. Leaving it would make the next start check a
|
||||
// pid that is either gone or, worse, reused.
|
||||
try {
|
||||
unlinkSync(SERVE_PID_FILE);
|
||||
} catch {
|
||||
/* never written, or already gone */
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||
process.on('SIGTERM', () => void shutdown('SIGTERM'));
|
||||
process.on('SIGINT', () => void shutdown('SIGINT'));
|
||||
|
||||
Reference in New Issue
Block a user