From 33d96f19eefeea32ad7f44947f35943d7dd76ff9 Mon Sep 17 00:00:00 2001 From: Andre Padez Date: Sat, 8 Aug 2026 10:01:10 +0100 Subject: [PATCH] end a turn that has gone silent, instead of generating forever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A turn could stop producing events and stay `isGenerating` indefinitely. Nothing covered it: the idle timer answers the opposite question — how long a session with NO turn in flight may sit before collection — and every client showed a spinner with no timeout of its own, so a wedged turn presented as a chat that was still thinking. On 2026-08-08 one ran for seventeen minutes inside an auto-compaction, reached over the socket to an iPad, and was indistinguishable there from a dead app. The compaction is silent by design (the PreCompact hook is the only announcement, and the code's own comment allows 2.5 minutes), so there was nothing to distinguish it from. A stall watchdog now rides every emitted event: any sign of life pushes the deadline back, and expiry ends the turn the way a real failure would — isGenerating off, idle re-armed, and an `error` the client can render. The agent process is deliberately left alive, since it may still be working and the next turn resumes it; what this guarantees is that the client is TOLD, which is the part that was missing. The budgets are generous rather than tight — ten minutes of silence normally, twenty while compacting, re-armed from the PreCompact hook because that hook fires as the long silence begins and the deadline the turn is holding was sized for ordinary work. Killing a turn that was about to succeed is worse than the hang this prevents. Co-Authored-By: Claude Opus 5 (1M context) --- src/servers/sidecar/claude/claude-manager.ts | 71 ++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/src/servers/sidecar/claude/claude-manager.ts b/src/servers/sidecar/claude/claude-manager.ts index 27a97cd5..f3c050e3 100644 --- a/src/servers/sidecar/claude/claude-manager.ts +++ b/src/servers/sidecar/claude/claude-manager.ts @@ -186,8 +186,30 @@ type PersistentSession = { */ compactStartedAt?: number; idleTimer?: ReturnType; + /** + * Fires when a turn that claims to be generating has emitted nothing for too long. + * + * The idle timer above answers the opposite question — how long a session with NO turn in flight may + * sit before it is collected — so neither one covers a turn that is wedged. Nothing did: a turn could + * stop producing events and stay `isGenerating` forever, and every client showed a spinner with no + * timeout of its own. On 2026-08-08 that ran for seventeen minutes inside a compaction and was + * indistinguishable, from the phone, from a dead chat. + */ + stallTimer?: ReturnType; }; +/* + How long a generating turn may say nothing before we call it stalled. + + Generous on purpose, because the legitimate silences here are long: compaction narrates nothing for + as long as it takes (minutes on a large conversation), and a single deep tool call can be quiet for a + while too. This is a backstop against turns that will never speak again, not a latency budget — too + tight and it would kill work that was about to succeed, which is worse than the hang it prevents. +*/ +const STALL_TIMEOUT_MS = 10 * 60 * 1000; +/** Compaction gets longer still: it is the known-slowest silent phase, and the one that stalled. */ +const COMPACT_STALL_TIMEOUT_MS = 20 * 60 * 1000; + const sessions = new Map(); /** A hand-rolled async iterable we can push turns onto and close on teardown. */ @@ -222,6 +244,43 @@ function makeInputQueue() { }; } +/** + * (Re)arm the stall watchdog. Called on every emitted event, so any sign of life pushes it back. + * + * On expiry it ends the turn the same way a real failure would — `isGenerating` off, idle re-armed, + * and an `error` the client can render — rather than tearing the session down. The agent process is + * left alive deliberately: it may still be working, and the next turn resumes it. What this guarantees + * is that the CLIENT is told, which is the part that was missing. + */ +function armStall(session: PersistentSession): void { + if (session.stallTimer) clearTimeout(session.stallTimer); + if (!session.isGenerating) { + session.stallTimer = undefined; + return; + } + const compacting = session.compactStartedAt !== undefined; + session.stallTimer = setTimeout( + () => { + if (!session.isGenerating) return; + const waited = Math.round((compacting ? COMPACT_STALL_TIMEOUT_MS : STALL_TIMEOUT_MS) / 60000); + console.error( + `[claude:stall:${session.sessionKey}] no events for ${waited}m${compacting ? ' (compacting)' : ''} — ending the turn`, + ); + session.isGenerating = false; + session.compactStartedAt = undefined; + session.interrupted = false; + if (session.pendingTasks.size === 0) armIdle(session); + session.emit({ + type: 'error', + message: compacting + ? `Compaction has produced nothing for ${waited} minutes and has been given up on. Your conversation is safe — send again to continue.` + : `The agent has produced nothing for ${waited} minutes and has been given up on. Your conversation is safe — send again to continue.`, + }); + }, + compacting ? COMPACT_STALL_TIMEOUT_MS : STALL_TIMEOUT_MS, + ); +} + function armIdle(session: PersistentSession): void { if (session.idleTimer) clearTimeout(session.idleTimer); session.idleTimer = setTimeout(() => { @@ -278,6 +337,9 @@ function createSession(params: ClaudeSpawnStreamingParams, onEvent: (event: Chat session.compactStartedAt = Date.now(); const trigger = 'trigger' in input && input.trigger === 'manual' ? 'manual' : 'auto'; session.emit({ type: 'compact:start', trigger }); + // Re-arm on the compaction budget: this hook fires as the long silence BEGINS, so the + // deadline the turn is holding was sized for ordinary work and is about to be wrong. + armStall(session); return { continue: true }; }, ], @@ -301,6 +363,8 @@ function createSession(params: ClaudeSpawnStreamingParams, onEvent: (event: Chat session.pushTurn = (prompt: string, images?: PromptImage[]) => { if (session.idleTimer) clearTimeout(session.idleTimer); session.isGenerating = true; + // A turn that dies before its FIRST event is the case the emit-path arming cannot reach. + armStall(session); // Images first, then the text: the model reads what it is looking at before what to do about it. const content: string | ContentBlock[] = images?.length ? [ @@ -354,6 +418,9 @@ function createSession(params: ClaudeSpawnStreamingParams, onEvent: (event: Chat session.interrupted = false; if (session.pendingTasks.size === 0) armIdle(session); } + // Any event at all is a sign of life, so push the stall deadline back. Placed after the branch + // above so a terminal event disarms rather than re-arms it. + armStall(session); session.emit(event); }; try { @@ -369,6 +436,7 @@ function createSession(params: ClaudeSpawnStreamingParams, onEvent: (event: Chat } } finally { if (session.idleTimer) clearTimeout(session.idleTimer); + if (session.stallTimer) clearTimeout(session.stallTimer); sessions.delete(sessionKey); } })(); @@ -400,6 +468,8 @@ export async function interruptClaudeSession(sessionKey: string): Promise