chat: durable Postgres event queue + cursor replay on reconnect (Phase 2+3)

Completes the turn/session decoupling so nothing is lost across disconnects.

Phase 2 (durability):
- New chat_session_events table (global monotonic id = cursor) + queries
  appendChatEvent / getChatEventsSince / pruneChatEventsOlderThan.
- Every durable outbound ServerMessage now goes through emitToSession: appended
  to the queue (even while the client is disconnected) and delivered live with
  its seq. Streaming deltas stay ephemeral (live-only, never persisted).

Phase 3 (resilient transport):
- New 'resume-cursor' client message → handleResumeCursor re-binds the socket to
  the (still-live) session (cancels idle-GC via attachWs) and replays every event
  since the client's cursor.
- useChatWebSocket already auto-reconnects; added an onOpen hook. useChat tracks
  the max seq and, on every (re)connect with an established session, sends
  resume-cursor — so a dropped connection self-heals with no manual navigate
  away/back, and background task notifications that landed while offline replay.

Verified end-to-end: disconnect after a turn's result but before a background
task finishes, reconnect with the cursor → the missed task:notification is
replayed from Postgres, no duplicates.

Note: the DB is managed via drizzle push/direct DDL (no __drizzle_migrations
table), so 0001 was applied directly; the generated migration is committed for
the record.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-27 00:12:46 +00:00
co-authored by Claude Opus 4.8
parent 449f28b1e5
commit 6b3eb247a3
11 changed files with 1792 additions and 19 deletions
@@ -0,0 +1,8 @@
CREATE TABLE "chat_session_events" (
"id" bigserial PRIMARY KEY NOT NULL,
"session_id" text NOT NULL,
"event" jsonb NOT NULL,
"created_at" timestamp with time zone DEFAULT now() NOT NULL
);
--> statement-breakpoint
CREATE INDEX "idx_chat_session_events_session_id" ON "chat_session_events" USING btree ("session_id","id");
File diff suppressed because it is too large Load Diff
@@ -8,6 +8,13 @@
"when": 1785018112373,
"tag": "0000_eager_sir_ram",
"breakpoints": true
},
{
"idx": 1,
"version": "7",
"when": 1785110579165,
"tag": "0001_omniscient_photon",
"breakpoints": true
}
]
}
+2
View File
@@ -77,5 +77,7 @@ export {
markInterruptedJobs,
} from './queries/pipeline-jobs';
export { appendChatEvent, getChatEventsSince, pruneChatEventsOlderThan } from './queries/chat-events';
export { db } from './db';
export * as schema from './schema';
@@ -0,0 +1,29 @@
import { eq, and, gt, asc, lt } from 'drizzle-orm';
import { db } from '../db';
import { chatSessionEvents } from '../schema';
/** Append one outbound event to a session's durable log; returns its global cursor id. */
export async function appendChatEvent(sessionId: string, event: unknown): Promise<number> {
const [row] = await db
.insert(chatSessionEvents)
.values({ sessionId, event })
.returning({ id: chatSessionEvents.id });
return row!.id;
}
/** Replay everything a (re)connecting client missed: events for this session with id > cursor. */
export async function getChatEventsSince(
sessionId: string,
cursor: number,
): Promise<Array<{ id: number; event: unknown }>> {
return db
.select({ id: chatSessionEvents.id, event: chatSessionEvents.event })
.from(chatSessionEvents)
.where(and(eq(chatSessionEvents.sessionId, sessionId), gt(chatSessionEvents.id, cursor)))
.orderBy(asc(chatSessionEvents.id));
}
/** Retention: drop events older than the cutoff (called periodically). */
export async function pruneChatEventsOlderThan(cutoff: Date): Promise<void> {
await db.delete(chatSessionEvents).where(lt(chatSessionEvents.createdAt, cutoff));
}
@@ -0,0 +1,16 @@
import { pgTable, bigserial, text, jsonb, timestamp, index } from 'drizzle-orm/pg-core';
// Durable per-session event log. Every outbound chat ServerMessage is appended here with a global
// monotonic `id` (the cursor). A (re)connecting client sends its last-seen cursor and the server
// replays everything since — so background task notifications and turn output that landed while the
// client was disconnected are never lost (the durability half of the turn/session decoupling).
export const chatSessionEvents = pgTable(
'chat_session_events',
{
id: bigserial('id', { mode: 'number' }).primaryKey(),
sessionId: text('session_id').notNull(),
event: jsonb('event').notNull(),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
},
(t) => [index('idx_chat_session_events_session_id').on(t.sessionId, t.id)],
);
@@ -5,3 +5,4 @@ export * from './operations';
export * from './server';
export * from './email';
export * from './pipeline-jobs';
export * from './chat-events';
+8
View File
@@ -72,6 +72,14 @@ export type ClientMessage =
// Tear down the whole session (kill any in-flight turn + drop the in-memory session), not just
// the current turn. Frees the session so its transcript can be resumed elsewhere.
type: 'disconnect';
}
| {
// Sent on (re)connect: re-bind this socket to the session and replay every durable event queued
// since `cursor` (the last seq the client saw). Powers transparent reconnect without losing
// background task notifications that landed while disconnected.
type: 'resume-cursor';
sessionId: string;
cursor: number;
};
export type ServerMessage =
+59 -16
View File
@@ -8,7 +8,7 @@ import { ensureGeneralChatSessionsCwd } from './claude-sessions';
import * as sidecar from '@@/sidecar-registry';
import { join } from 'path';
import { getOwnerHomeDir, getEmailAccountsDir } from '../../../servers/data-path';
import { getUserSettings, getEmailAccounts } from 'officerdb';
import { getUserSettings, getEmailAccounts, appendChatEvent, getChatEventsSince } from 'officerdb';
import { mkdirSync } from 'node:fs';
import { logger } from './logger';
@@ -90,12 +90,31 @@ const wsToSessionMap = new WeakMap<any, string>();
const pingTimers = new WeakMap<ServerWebSocket<WSData>, ReturnType<typeof setInterval>>();
const PING_INTERVAL_MS = 25_000;
function sendToClient(ws: ServerWebSocket<WSData> | null, msg: ServerMessage): void {
function sendToClient(ws: ServerWebSocket<WSData> | null, msg: ServerMessage, seq?: number): void {
if (ws?.readyState === 1) {
ws.send(JSON.stringify(msg));
ws.send(JSON.stringify(seq === undefined ? msg : { ...msg, seq }));
}
}
// Persist a durable session event to the Postgres queue (for replay across reconnects) and deliver it
// live to the attached socket with its cursor `seq`. Events are queued even while the client is
// DISCONNECTED (ws null) — that's what lets a reconnecting client replay what it missed (e.g. a
// background task:notification). Streaming deltas are ephemeral: delivered live, never persisted.
async function emitToSession(sessionId: string, msg: ServerMessage): Promise<void> {
const ws = (sessionManager.getSession(sessionId)?.ws ?? null) as ServerWebSocket<WSData> | null;
if (msg.type === 'assistant:delta') {
sendToClient(ws, msg);
return;
}
let seq: number | undefined;
try {
seq = await appendChatEvent(sessionId, msg);
} catch (err) {
logger.error('Failed to persist chat event', { sessionId, error: String(err) });
}
sendToClient(ws, msg, seq);
}
export async function open(ws: ServerWebSocket<WSData>): Promise<void> {
const timer = setInterval(() => {
try {
@@ -122,6 +141,8 @@ export function message(ws: ServerWebSocket<WSData>, raw: string | Buffer): void
await handleStop(ws);
} else if (clientMsg.type === 'disconnect') {
await handleDisconnect(ws);
} else if (clientMsg.type === 'resume-cursor') {
await handleResumeCursor(ws, clientMsg);
}
} catch (err) {
logger.error('Error handling WebSocket message', { email: ws.data.email, error: String(err) });
@@ -149,11 +170,10 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
const session = sessionManager.getSession(sessionId);
if (!session) return;
const ws = session.ws as ServerWebSocket<WSData> | null;
// All durable sends go through emitToSession (persist + deliver with cursor seq). Deltas stay live-only.
switch (event.type) {
case 'delta': {
sendToClient(ws, { type: 'assistant:delta', text: event.text });
await emitToSession(sessionId, { type: 'assistant:delta', text: event.text });
session.streamBuffer += event.text;
break;
}
@@ -162,7 +182,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
// Flush streaming buffer as complete text
const text = event.text || session.streamBuffer;
if (text) {
sendToClient(ws, { type: 'assistant:text', text });
await emitToSession(sessionId, { type: 'assistant:text', text });
const assistantMsg: Message = {
id: randomUUID(),
@@ -181,7 +201,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
case 'tool:start': {
// Flush any pending streaming text first
if (session.streamBuffer) {
sendToClient(ws, { type: 'assistant:text', text: session.streamBuffer });
await emitToSession(sessionId, { type: 'assistant:text', text: session.streamBuffer });
const assistantMsg: Message = {
id: randomUUID(),
@@ -195,7 +215,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
session.streamBuffer = '';
}
sendToClient(ws, {
await emitToSession(sessionId, {
type: 'tool:start',
toolCallId: event.toolCallId,
toolName: event.toolName,
@@ -216,7 +236,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
}
case 'tool:result': {
sendToClient(ws, {
await emitToSession(sessionId, {
type: 'tool:result',
toolCallId: event.toolCallId,
output: event.output,
@@ -238,7 +258,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
case 'result': {
// Flush any remaining streaming buffer
if (session.streamBuffer) {
sendToClient(ws, { type: 'assistant:text', text: session.streamBuffer });
await emitToSession(sessionId, { type: 'assistant:text', text: session.streamBuffer });
const assistantMsg: Message = {
id: randomUUID(),
@@ -253,7 +273,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
session.streamBuffer = '';
}
sendToClient(ws, { type: 'result', sessionId, cost: event.cost });
await emitToSession(sessionId, { type: 'result', sessionId, cost: event.cost });
session.isGenerating = false;
session.meta.cost.inputTokens += event.cost.inputTokens;
@@ -265,26 +285,26 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
}
case 'error': {
sendToClient(ws, { type: 'error', message: event.message });
await emitToSession(sessionId, { type: 'error', message: event.message });
session.isGenerating = false;
break;
}
case 'stopped': {
sendToClient(ws, { type: 'stopped' });
await emitToSession(sessionId, { type: 'stopped' });
session.isGenerating = false;
break;
}
case 'task:started': {
// Background task launched (run_in_background / Monitor). Independent of turn state.
sendToClient(ws, { type: 'task:started', taskId: event.taskId, description: event.description, taskType: event.taskType });
await emitToSession(sessionId, { type: 'task:started', taskId: event.taskId, description: event.description, taskType: event.taskType });
break;
}
case 'task:notification': {
// Background task finished — arrives AFTER the turn's 'result'; the whole point of the fix.
sendToClient(ws, { type: 'task:notification', taskId: event.taskId, status: event.status, summary: event.summary });
await emitToSession(sessionId, { type: 'task:notification', taskId: event.taskId, status: event.status, summary: event.summary });
break;
}
}
@@ -585,6 +605,29 @@ async function handleDisconnect(ws: ServerWebSocket<WSData>): Promise<void> {
sendToClient(ws, { type: 'disconnected' });
}
// Reconnect: re-bind this socket to the (possibly still-live) session and replay every durable event
// queued since the client's cursor — so a brief disconnect never loses turn output or a background
// task:notification. attachWs cancels the pending idle-GC. If the in-memory session was already
// GC'd, we still replay history from Postgres (new turns will respawn the session).
async function handleResumeCursor(
ws: ServerWebSocket<WSData>,
msg: { sessionId: string; cursor: number },
): Promise<void> {
const { sessionId, cursor } = msg;
if (sessionManager.getSession(sessionId)) {
sessionManager.attachWs(sessionId, ws);
wsToSessionMap.set(ws as any, sessionId);
}
try {
const events = await getChatEventsSince(sessionId, cursor ?? 0);
for (const { id, event } of events) {
sendToClient(ws, event as ServerMessage, id);
}
} catch (err) {
logger.error('Failed to replay chat events on resume-cursor', { sessionId, error: String(err) });
}
}
export const chatWebsocket = {
open,
message,
+7 -1
View File
@@ -3,9 +3,12 @@ import { useState, useEffect, useRef } from 'react';
type UseChatWebSocketParams = {
url: string;
onMessage: (data: unknown) => void;
// Called on every (re)connect once the socket is OPEN — used to send a resume-cursor so a reconnect
// re-binds to the session and replays missed events (no manual navigate-away/back).
onOpen?: () => void;
};
export const useChatWebSocket = ({ url, onMessage }: UseChatWebSocketParams) => {
export const useChatWebSocket = ({ url, onMessage, onOpen }: UseChatWebSocketParams) => {
const [isConnected, setIsConnected] = useState(false);
const socketRef = useRef<WebSocket | null>(null);
@@ -14,6 +17,8 @@ export const useChatWebSocket = ({ url, onMessage }: UseChatWebSocketParams) =>
const isCleaningUpRef = useRef(false);
const onMessageRef = useRef(onMessage);
onMessageRef.current = onMessage;
const onOpenRef = useRef(onOpen);
onOpenRef.current = onOpen;
const connect = () => {
if (isCleaningUpRef.current) return;
@@ -26,6 +31,7 @@ export const useChatWebSocket = ({ url, onMessage }: UseChatWebSocketParams) =>
if (socketRef.current !== socket) return;
setIsConnected(true);
retryRef.current = 0;
onOpenRef.current?.();
});
socket.addEventListener('message', (ev) => {
+20 -2
View File
@@ -1,4 +1,4 @@
import { useState, useEffect, useRef } from 'react';
import { useState, useEffect, useRef, useCallback } from 'react';
import { useChatWebSocket } from 'hooks/useChatWebSocket';
import { useClient } from 'hooks/useClient';
import { useSettings } from 'state/useSettings';
@@ -48,6 +48,12 @@ export function useChat(initialSessionId?: string, initialModel?: string | null,
const [cwd, setCwd] = useState<string | null>(null);
const [thinkingLevel, setThinkingLevel] = useState<string | null>(null);
// Resilient-transport cursor: the max durable-event `seq` seen. On (re)connect we send it as a
// resume-cursor so the server replays anything we missed (e.g. a background task:notification that
// landed while briefly disconnected). sendRef breaks the hook ↔ onOpen circular dependency.
const cursorRef = useRef(0);
const sendRef = useRef<(d: Record<string, unknown>) => void>(() => {});
// Track if session has started (first message sent)
const [hasStarted, setHasStarted] = useState(!!preloadedMessages?.length);
@@ -101,6 +107,10 @@ export function useChat(initialSessionId?: string, initialModel?: string | null,
function handleMessage(data: unknown) {
const msg = data as ServerMessage;
// Advance the resume cursor for any durable (seq-carrying) event.
const seq = (data as { seq?: number }).seq;
if (typeof seq === 'number' && seq > cursorRef.current) cursorRef.current = seq;
switch (msg.type) {
case 'session:init':
sessionIdRef.current = msg.sessionId;
@@ -223,7 +233,15 @@ export function useChat(initialSessionId?: string, initialModel?: string | null,
}
}
const { isConnected, send } = useChatWebSocket({ url: wsUrl, onMessage: handleMessage });
// On every (re)connect, if a session is already established, re-bind + replay via resume-cursor.
// The first connect (no session yet) no-ops; the first turn establishes the session via session:init.
const onOpen = useCallback(() => {
const sid = sessionIdRef.current;
if (sid) sendRef.current({ type: 'resume-cursor', sessionId: sid, cursor: cursorRef.current });
}, []);
const { isConnected, send } = useChatWebSocket({ url: wsUrl, onMessage: handleMessage, onOpen });
sendRef.current = send;
// Clean up RAF on unmount
useEffect(() => {