chat: durable Postgres event queue + cursor replay on reconnect (Phase 2+3)
Completes the turn/session decoupling so nothing is lost across disconnects. Phase 2 (durability): - New chat_session_events table (global monotonic id = cursor) + queries appendChatEvent / getChatEventsSince / pruneChatEventsOlderThan. - Every durable outbound ServerMessage now goes through emitToSession: appended to the queue (even while the client is disconnected) and delivered live with its seq. Streaming deltas stay ephemeral (live-only, never persisted). Phase 3 (resilient transport): - New 'resume-cursor' client message → handleResumeCursor re-binds the socket to the (still-live) session (cancels idle-GC via attachWs) and replays every event since the client's cursor. - useChatWebSocket already auto-reconnects; added an onOpen hook. useChat tracks the max seq and, on every (re)connect with an established session, sends resume-cursor — so a dropped connection self-heals with no manual navigate away/back, and background task notifications that landed while offline replay. Verified end-to-end: disconnect after a turn's result but before a background task finishes, reconnect with the cursor → the missed task:notification is replayed from Postgres, no duplicates. Note: the DB is managed via drizzle push/direct DDL (no __drizzle_migrations table), so 0001 was applied directly; the generated migration is committed for the record. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,8 @@
|
||||
CREATE TABLE "chat_session_events" (
|
||||
"id" bigserial PRIMARY KEY NOT NULL,
|
||||
"session_id" text NOT NULL,
|
||||
"event" jsonb NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT now() NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE INDEX "idx_chat_session_events_session_id" ON "chat_session_events" USING btree ("session_id","id");
|
||||
File diff suppressed because it is too large
Load Diff
@@ -8,6 +8,13 @@
|
||||
"when": 1785018112373,
|
||||
"tag": "0000_eager_sir_ram",
|
||||
"breakpoints": true
|
||||
},
|
||||
{
|
||||
"idx": 1,
|
||||
"version": "7",
|
||||
"when": 1785110579165,
|
||||
"tag": "0001_omniscient_photon",
|
||||
"breakpoints": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -77,5 +77,7 @@ export {
|
||||
markInterruptedJobs,
|
||||
} from './queries/pipeline-jobs';
|
||||
|
||||
export { appendChatEvent, getChatEventsSince, pruneChatEventsOlderThan } from './queries/chat-events';
|
||||
|
||||
export { db } from './db';
|
||||
export * as schema from './schema';
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import { eq, and, gt, asc, lt } from 'drizzle-orm';
|
||||
import { db } from '../db';
|
||||
import { chatSessionEvents } from '../schema';
|
||||
|
||||
/** Append one outbound event to a session's durable log; returns its global cursor id. */
|
||||
export async function appendChatEvent(sessionId: string, event: unknown): Promise<number> {
|
||||
const [row] = await db
|
||||
.insert(chatSessionEvents)
|
||||
.values({ sessionId, event })
|
||||
.returning({ id: chatSessionEvents.id });
|
||||
return row!.id;
|
||||
}
|
||||
|
||||
/** Replay everything a (re)connecting client missed: events for this session with id > cursor. */
|
||||
export async function getChatEventsSince(
|
||||
sessionId: string,
|
||||
cursor: number,
|
||||
): Promise<Array<{ id: number; event: unknown }>> {
|
||||
return db
|
||||
.select({ id: chatSessionEvents.id, event: chatSessionEvents.event })
|
||||
.from(chatSessionEvents)
|
||||
.where(and(eq(chatSessionEvents.sessionId, sessionId), gt(chatSessionEvents.id, cursor)))
|
||||
.orderBy(asc(chatSessionEvents.id));
|
||||
}
|
||||
|
||||
/** Retention: drop events older than the cutoff (called periodically). */
|
||||
export async function pruneChatEventsOlderThan(cutoff: Date): Promise<void> {
|
||||
await db.delete(chatSessionEvents).where(lt(chatSessionEvents.createdAt, cutoff));
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { pgTable, bigserial, text, jsonb, timestamp, index } from 'drizzle-orm/pg-core';
|
||||
|
||||
// Durable per-session event log. Every outbound chat ServerMessage is appended here with a global
|
||||
// monotonic `id` (the cursor). A (re)connecting client sends its last-seen cursor and the server
|
||||
// replays everything since — so background task notifications and turn output that landed while the
|
||||
// client was disconnected are never lost (the durability half of the turn/session decoupling).
|
||||
export const chatSessionEvents = pgTable(
|
||||
'chat_session_events',
|
||||
{
|
||||
id: bigserial('id', { mode: 'number' }).primaryKey(),
|
||||
sessionId: text('session_id').notNull(),
|
||||
event: jsonb('event').notNull(),
|
||||
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
|
||||
},
|
||||
(t) => [index('idx_chat_session_events_session_id').on(t.sessionId, t.id)],
|
||||
);
|
||||
@@ -5,3 +5,4 @@ export * from './operations';
|
||||
export * from './server';
|
||||
export * from './email';
|
||||
export * from './pipeline-jobs';
|
||||
export * from './chat-events';
|
||||
|
||||
@@ -72,6 +72,14 @@ export type ClientMessage =
|
||||
// Tear down the whole session (kill any in-flight turn + drop the in-memory session), not just
|
||||
// the current turn. Frees the session so its transcript can be resumed elsewhere.
|
||||
type: 'disconnect';
|
||||
}
|
||||
| {
|
||||
// Sent on (re)connect: re-bind this socket to the session and replay every durable event queued
|
||||
// since `cursor` (the last seq the client saw). Powers transparent reconnect without losing
|
||||
// background task notifications that landed while disconnected.
|
||||
type: 'resume-cursor';
|
||||
sessionId: string;
|
||||
cursor: number;
|
||||
};
|
||||
|
||||
export type ServerMessage =
|
||||
|
||||
@@ -8,7 +8,7 @@ import { ensureGeneralChatSessionsCwd } from './claude-sessions';
|
||||
import * as sidecar from '@@/sidecar-registry';
|
||||
import { join } from 'path';
|
||||
import { getOwnerHomeDir, getEmailAccountsDir } from '../../../servers/data-path';
|
||||
import { getUserSettings, getEmailAccounts } from 'officerdb';
|
||||
import { getUserSettings, getEmailAccounts, appendChatEvent, getChatEventsSince } from 'officerdb';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import { logger } from './logger';
|
||||
|
||||
@@ -90,12 +90,31 @@ const wsToSessionMap = new WeakMap<any, string>();
|
||||
const pingTimers = new WeakMap<ServerWebSocket<WSData>, ReturnType<typeof setInterval>>();
|
||||
const PING_INTERVAL_MS = 25_000;
|
||||
|
||||
function sendToClient(ws: ServerWebSocket<WSData> | null, msg: ServerMessage): void {
|
||||
function sendToClient(ws: ServerWebSocket<WSData> | null, msg: ServerMessage, seq?: number): void {
|
||||
if (ws?.readyState === 1) {
|
||||
ws.send(JSON.stringify(msg));
|
||||
ws.send(JSON.stringify(seq === undefined ? msg : { ...msg, seq }));
|
||||
}
|
||||
}
|
||||
|
||||
// Persist a durable session event to the Postgres queue (for replay across reconnects) and deliver it
|
||||
// live to the attached socket with its cursor `seq`. Events are queued even while the client is
|
||||
// DISCONNECTED (ws null) — that's what lets a reconnecting client replay what it missed (e.g. a
|
||||
// background task:notification). Streaming deltas are ephemeral: delivered live, never persisted.
|
||||
async function emitToSession(sessionId: string, msg: ServerMessage): Promise<void> {
|
||||
const ws = (sessionManager.getSession(sessionId)?.ws ?? null) as ServerWebSocket<WSData> | null;
|
||||
if (msg.type === 'assistant:delta') {
|
||||
sendToClient(ws, msg);
|
||||
return;
|
||||
}
|
||||
let seq: number | undefined;
|
||||
try {
|
||||
seq = await appendChatEvent(sessionId, msg);
|
||||
} catch (err) {
|
||||
logger.error('Failed to persist chat event', { sessionId, error: String(err) });
|
||||
}
|
||||
sendToClient(ws, msg, seq);
|
||||
}
|
||||
|
||||
export async function open(ws: ServerWebSocket<WSData>): Promise<void> {
|
||||
const timer = setInterval(() => {
|
||||
try {
|
||||
@@ -122,6 +141,8 @@ export function message(ws: ServerWebSocket<WSData>, raw: string | Buffer): void
|
||||
await handleStop(ws);
|
||||
} else if (clientMsg.type === 'disconnect') {
|
||||
await handleDisconnect(ws);
|
||||
} else if (clientMsg.type === 'resume-cursor') {
|
||||
await handleResumeCursor(ws, clientMsg);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error('Error handling WebSocket message', { email: ws.data.email, error: String(err) });
|
||||
@@ -149,11 +170,10 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
|
||||
const session = sessionManager.getSession(sessionId);
|
||||
if (!session) return;
|
||||
|
||||
const ws = session.ws as ServerWebSocket<WSData> | null;
|
||||
|
||||
// All durable sends go through emitToSession (persist + deliver with cursor seq). Deltas stay live-only.
|
||||
switch (event.type) {
|
||||
case 'delta': {
|
||||
sendToClient(ws, { type: 'assistant:delta', text: event.text });
|
||||
await emitToSession(sessionId, { type: 'assistant:delta', text: event.text });
|
||||
session.streamBuffer += event.text;
|
||||
break;
|
||||
}
|
||||
@@ -162,7 +182,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
|
||||
// Flush streaming buffer as complete text
|
||||
const text = event.text || session.streamBuffer;
|
||||
if (text) {
|
||||
sendToClient(ws, { type: 'assistant:text', text });
|
||||
await emitToSession(sessionId, { type: 'assistant:text', text });
|
||||
|
||||
const assistantMsg: Message = {
|
||||
id: randomUUID(),
|
||||
@@ -181,7 +201,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
|
||||
case 'tool:start': {
|
||||
// Flush any pending streaming text first
|
||||
if (session.streamBuffer) {
|
||||
sendToClient(ws, { type: 'assistant:text', text: session.streamBuffer });
|
||||
await emitToSession(sessionId, { type: 'assistant:text', text: session.streamBuffer });
|
||||
|
||||
const assistantMsg: Message = {
|
||||
id: randomUUID(),
|
||||
@@ -195,7 +215,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
|
||||
session.streamBuffer = '';
|
||||
}
|
||||
|
||||
sendToClient(ws, {
|
||||
await emitToSession(sessionId, {
|
||||
type: 'tool:start',
|
||||
toolCallId: event.toolCallId,
|
||||
toolName: event.toolName,
|
||||
@@ -216,7 +236,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
|
||||
}
|
||||
|
||||
case 'tool:result': {
|
||||
sendToClient(ws, {
|
||||
await emitToSession(sessionId, {
|
||||
type: 'tool:result',
|
||||
toolCallId: event.toolCallId,
|
||||
output: event.output,
|
||||
@@ -238,7 +258,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
|
||||
case 'result': {
|
||||
// Flush any remaining streaming buffer
|
||||
if (session.streamBuffer) {
|
||||
sendToClient(ws, { type: 'assistant:text', text: session.streamBuffer });
|
||||
await emitToSession(sessionId, { type: 'assistant:text', text: session.streamBuffer });
|
||||
|
||||
const assistantMsg: Message = {
|
||||
id: randomUUID(),
|
||||
@@ -253,7 +273,7 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
|
||||
session.streamBuffer = '';
|
||||
}
|
||||
|
||||
sendToClient(ws, { type: 'result', sessionId, cost: event.cost });
|
||||
await emitToSession(sessionId, { type: 'result', sessionId, cost: event.cost });
|
||||
|
||||
session.isGenerating = false;
|
||||
session.meta.cost.inputTokens += event.cost.inputTokens;
|
||||
@@ -265,26 +285,26 @@ function createEventHandler(sessionId: string, model: string, cwd: string) {
|
||||
}
|
||||
|
||||
case 'error': {
|
||||
sendToClient(ws, { type: 'error', message: event.message });
|
||||
await emitToSession(sessionId, { type: 'error', message: event.message });
|
||||
session.isGenerating = false;
|
||||
break;
|
||||
}
|
||||
|
||||
case 'stopped': {
|
||||
sendToClient(ws, { type: 'stopped' });
|
||||
await emitToSession(sessionId, { type: 'stopped' });
|
||||
session.isGenerating = false;
|
||||
break;
|
||||
}
|
||||
|
||||
case 'task:started': {
|
||||
// Background task launched (run_in_background / Monitor). Independent of turn state.
|
||||
sendToClient(ws, { type: 'task:started', taskId: event.taskId, description: event.description, taskType: event.taskType });
|
||||
await emitToSession(sessionId, { type: 'task:started', taskId: event.taskId, description: event.description, taskType: event.taskType });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'task:notification': {
|
||||
// Background task finished — arrives AFTER the turn's 'result'; the whole point of the fix.
|
||||
sendToClient(ws, { type: 'task:notification', taskId: event.taskId, status: event.status, summary: event.summary });
|
||||
await emitToSession(sessionId, { type: 'task:notification', taskId: event.taskId, status: event.status, summary: event.summary });
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -585,6 +605,29 @@ async function handleDisconnect(ws: ServerWebSocket<WSData>): Promise<void> {
|
||||
sendToClient(ws, { type: 'disconnected' });
|
||||
}
|
||||
|
||||
// Reconnect: re-bind this socket to the (possibly still-live) session and replay every durable event
|
||||
// queued since the client's cursor — so a brief disconnect never loses turn output or a background
|
||||
// task:notification. attachWs cancels the pending idle-GC. If the in-memory session was already
|
||||
// GC'd, we still replay history from Postgres (new turns will respawn the session).
|
||||
async function handleResumeCursor(
|
||||
ws: ServerWebSocket<WSData>,
|
||||
msg: { sessionId: string; cursor: number },
|
||||
): Promise<void> {
|
||||
const { sessionId, cursor } = msg;
|
||||
if (sessionManager.getSession(sessionId)) {
|
||||
sessionManager.attachWs(sessionId, ws);
|
||||
wsToSessionMap.set(ws as any, sessionId);
|
||||
}
|
||||
try {
|
||||
const events = await getChatEventsSince(sessionId, cursor ?? 0);
|
||||
for (const { id, event } of events) {
|
||||
sendToClient(ws, event as ServerMessage, id);
|
||||
}
|
||||
} catch (err) {
|
||||
logger.error('Failed to replay chat events on resume-cursor', { sessionId, error: String(err) });
|
||||
}
|
||||
}
|
||||
|
||||
export const chatWebsocket = {
|
||||
open,
|
||||
message,
|
||||
|
||||
@@ -3,9 +3,12 @@ import { useState, useEffect, useRef } from 'react';
|
||||
type UseChatWebSocketParams = {
|
||||
url: string;
|
||||
onMessage: (data: unknown) => void;
|
||||
// Called on every (re)connect once the socket is OPEN — used to send a resume-cursor so a reconnect
|
||||
// re-binds to the session and replays missed events (no manual navigate-away/back).
|
||||
onOpen?: () => void;
|
||||
};
|
||||
|
||||
export const useChatWebSocket = ({ url, onMessage }: UseChatWebSocketParams) => {
|
||||
export const useChatWebSocket = ({ url, onMessage, onOpen }: UseChatWebSocketParams) => {
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
|
||||
const socketRef = useRef<WebSocket | null>(null);
|
||||
@@ -14,6 +17,8 @@ export const useChatWebSocket = ({ url, onMessage }: UseChatWebSocketParams) =>
|
||||
const isCleaningUpRef = useRef(false);
|
||||
const onMessageRef = useRef(onMessage);
|
||||
onMessageRef.current = onMessage;
|
||||
const onOpenRef = useRef(onOpen);
|
||||
onOpenRef.current = onOpen;
|
||||
|
||||
const connect = () => {
|
||||
if (isCleaningUpRef.current) return;
|
||||
@@ -26,6 +31,7 @@ export const useChatWebSocket = ({ url, onMessage }: UseChatWebSocketParams) =>
|
||||
if (socketRef.current !== socket) return;
|
||||
setIsConnected(true);
|
||||
retryRef.current = 0;
|
||||
onOpenRef.current?.();
|
||||
});
|
||||
|
||||
socket.addEventListener('message', (ev) => {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
import { useState, useEffect, useRef, useCallback } from 'react';
|
||||
import { useChatWebSocket } from 'hooks/useChatWebSocket';
|
||||
import { useClient } from 'hooks/useClient';
|
||||
import { useSettings } from 'state/useSettings';
|
||||
@@ -48,6 +48,12 @@ export function useChat(initialSessionId?: string, initialModel?: string | null,
|
||||
const [cwd, setCwd] = useState<string | null>(null);
|
||||
const [thinkingLevel, setThinkingLevel] = useState<string | null>(null);
|
||||
|
||||
// Resilient-transport cursor: the max durable-event `seq` seen. On (re)connect we send it as a
|
||||
// resume-cursor so the server replays anything we missed (e.g. a background task:notification that
|
||||
// landed while briefly disconnected). sendRef breaks the hook ↔ onOpen circular dependency.
|
||||
const cursorRef = useRef(0);
|
||||
const sendRef = useRef<(d: Record<string, unknown>) => void>(() => {});
|
||||
|
||||
// Track if session has started (first message sent)
|
||||
const [hasStarted, setHasStarted] = useState(!!preloadedMessages?.length);
|
||||
|
||||
@@ -101,6 +107,10 @@ export function useChat(initialSessionId?: string, initialModel?: string | null,
|
||||
function handleMessage(data: unknown) {
|
||||
const msg = data as ServerMessage;
|
||||
|
||||
// Advance the resume cursor for any durable (seq-carrying) event.
|
||||
const seq = (data as { seq?: number }).seq;
|
||||
if (typeof seq === 'number' && seq > cursorRef.current) cursorRef.current = seq;
|
||||
|
||||
switch (msg.type) {
|
||||
case 'session:init':
|
||||
sessionIdRef.current = msg.sessionId;
|
||||
@@ -223,7 +233,15 @@ export function useChat(initialSessionId?: string, initialModel?: string | null,
|
||||
}
|
||||
}
|
||||
|
||||
const { isConnected, send } = useChatWebSocket({ url: wsUrl, onMessage: handleMessage });
|
||||
// On every (re)connect, if a session is already established, re-bind + replay via resume-cursor.
|
||||
// The first connect (no session yet) no-ops; the first turn establishes the session via session:init.
|
||||
const onOpen = useCallback(() => {
|
||||
const sid = sessionIdRef.current;
|
||||
if (sid) sendRef.current({ type: 'resume-cursor', sessionId: sid, cursor: cursorRef.current });
|
||||
}, []);
|
||||
|
||||
const { isConnected, send } = useChatWebSocket({ url: wsUrl, onMessage: handleMessage, onOpen });
|
||||
sendRef.current = send;
|
||||
|
||||
// Clean up RAF on unmount
|
||||
useEffect(() => {
|
||||
|
||||
Reference in New Issue
Block a user