task logs: migrate from filesystem to postgresql; refactor sidecars into submodules

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/servers/sidecar/claude/claude-manager.ts

@@ -1,7 +1,7 @@
 import { join } from 'node:path';
 import type { Subprocess } from 'bun';
-import type { PiEvent, MessageCost } from '../api/pi/types';
-import type { ClaudeSpawnParams, ClaudeSpawnStreamingParams, ClaudeCodeResult } from './protocol';
+import type { PiEvent, MessageCost } from '../../api/pi/types';
+import type { ClaudeSpawnParams, ClaudeSpawnStreamingParams, ClaudeCodeResult } from '../protocol';
 import { getState, setClaudeSession, clearClaudeSession, getClaudeSession } from './state';
 import { getProxySecret } from './proxy';
 
@@ -13,7 +13,13 @@ const getHomeDir = (email: string) => join(DATA_PATH, email, 'home');
 
 const toShellUsername = (username: string, email: string): string => {
   const raw = username || email.split('@')[0]!;
-  return raw.replace(/@.*$/, '').replace(/[^a-zA-Z0-9._-]/g, '_').toLowerCase().slice(0, 32) || 'officer';
+  return (
+    raw
+      .replace(/@.*$/, '')
+      .replace(/[^a-zA-Z0-9._-]/g, '_')
+      .toLowerCase()
+      .slice(0, 32) || 'officer'
+  );
 };
 
 async function hasOwnCredentials(homeDir: string): Promise<boolean> {
@@ -33,7 +39,12 @@ const CLAUDE_BIN = (() => {
 // Active streaming processes
 const activeProcs = new Map<string, Subprocess>();
 
-function buildAuthEnv(shellUsername: string, homeDir: string, isServiceUser: boolean, userHasCredentials: boolean): Record<string, string> {
+function buildAuthEnv(
+  shellUsername: string,
+  homeDir: string,
+  isServiceUser: boolean,
+  userHasCredentials: boolean,
+): Record<string, string> {
   if (isServiceUser) return { HOME: process.env.HOME ?? '' };
   if (userHasCredentials) return { HOME: homeDir };
   return { ANTHROPIC_BASE_URL: `http://127.0.0.1:${PROXY_PORT}`, ANTHROPIC_API_KEY: getProxySecret() };
@@ -83,7 +94,11 @@ export async function spawnClaude(params: ClaudeSpawnParams): Promise<ClaudeCode
       );
 
   const timeout = setTimeout(() => {
-    try { proc.kill(); } catch { /* already dead */ }
+    try {
+      proc.kill();
+    } catch {
+      /* already dead */
+    }
   }, SEND_TIMEOUT_MS);
 
   try {
@@ -143,10 +158,14 @@ export async function spawnClaudeStreaming(
   const workDir = cwd ?? homeDir;
 
   const claudeArgs = [
-    CLAUDE_BIN, '-p', prompt,
+    CLAUDE_BIN,
+    '-p',
+    prompt,
     '--dangerously-skip-permissions',
-    '--output-format', 'stream-json',
-    '--verbose', '--include-partial-messages',
+    '--output-format',
+    'stream-json',
+    '--verbose',
+    '--include-partial-messages',
   ];
 
   const subModel = params.model?.split('/')[1];
@@ -170,7 +189,13 @@ export async function spawnClaudeStreaming(
   };
 
   const proc = isServiceUser
-    ? Bun.spawn(claudeArgs, { cwd: workDir, stdin: 'ignore', stdout: 'pipe', stderr: 'pipe', env: { ...cleanEnv, ...env } })
+    ? Bun.spawn(claudeArgs, {
+        cwd: workDir,
+        stdin: 'ignore',
+        stdout: 'pipe',
+        stderr: 'pipe',
+        env: { ...cleanEnv, ...env },
+      })
     : Bun.spawn(
         ['sudo', '-u', shellUsername, 'env', ...Object.entries(env).map(([k, v]) => `${k}=${v}`), ...claudeArgs],
         { cwd: workDir, stdin: 'ignore', stdout: 'pipe', stderr: 'pipe' },
@@ -179,7 +204,11 @@ export async function spawnClaudeStreaming(
   activeProcs.set(sessionKey, proc);
 
   const timeout = setTimeout(() => {
-    try { proc.kill(); } catch { /* already dead */ }
+    try {
+      proc.kill();
+    } catch {
+      /* already dead */
+    }
     onEvent({ type: 'error', message: 'Claude Code timed out after 5 minutes' });
   }, SEND_TIMEOUT_MS);
 
@@ -339,7 +368,11 @@ export async function spawnClaudeStreaming(
 export function killClaudeSession(sessionKey: string): boolean {
   const proc = activeProcs.get(sessionKey);
   if (proc) {
-    try { proc.kill(); } catch { /* already dead */ }
+    try {
+      proc.kill();
+    } catch {
+      /* already dead */
+    }
     activeProcs.delete(sessionKey);
     return true;
   }

src/servers/sidecar/claude/index.ts

@@ -0,0 +1,119 @@
+import type { SidecarCommand, SidecarEvent } from '../protocol';
+import { loadState, flushAndSave, acquireLock, releaseLock, getState } from './state';
+import { startAnthropicProxy, getProxySecret, ensureProxySecret } from './proxy';
+import * as claudeManager from './claude-manager';
+import { createSidecarConnector } from '../connect';
+
+const API_URL = process.env.API_URL ?? `ws://127.0.0.1:${process.env.PORT ?? '5000'}`;
+
+// ── Startup ──
+
+if (!acquireLock()) {
+  console.error('[claude] another instance is already running (lock file exists with live PID)');
+  process.exit(1);
+}
+
+loadState();
+ensureProxySecret();
+
+// Start Anthropic proxy
+try {
+  startAnthropicProxy();
+} catch (err) {
+  console.error('[claude] failed to start Anthropic proxy:', err instanceof Error ? err.message : err);
+}
+
+// ── Command handlers ──
+
+type ReplyFn = (msg: SidecarEvent) => void;
+
+async function handleCommand(cmd: SidecarCommand, reply: ReplyFn) {
+  switch (cmd.type) {
+    case 'ping':
+      reply({ type: 'pong', id: cmd.id });
+      break;
+
+    case 'state:sync':
+      reply({
+        type: 'state:sync',
+        id: cmd.id,
+        state: {
+          proxySecret: getProxySecret(),
+          claudeSessions: { ...getState().claudeSessions },
+        },
+      });
+      break;
+
+    case 'proxy:secret':
+      reply({ type: 'proxy:secret', id: cmd.id, secret: getProxySecret() });
+      break;
+
+    case 'claude:spawn': {
+      try {
+        const result = await claudeManager.spawnClaude(cmd.params);
+        reply({ type: 'claude:result', id: cmd.id, result });
+      } catch (err) {
+        reply({ type: 'claude:error', id: cmd.id, error: err instanceof Error ? err.message : String(err) });
+      }
+      break;
+    }
+
+    case 'claude:spawn-streaming': {
+      reply({ type: 'claude:spawned', id: cmd.id, sessionKey: cmd.params.sessionKey });
+
+      const onEvent = (event: import('../../api/pi/types').PiEvent) => {
+        connection.send({ type: 'claude:event', sessionKey: cmd.params.sessionKey, event });
+      };
+
+      claudeManager.spawnClaudeStreaming(cmd.params, onEvent).catch((err) => {
+        connection.send({
+          type: 'claude:event',
+          sessionKey: cmd.params.sessionKey,
+          event: { type: 'error', message: err instanceof Error ? err.message : String(err) },
+        });
+      });
+      break;
+    }
+
+    case 'claude:kill':
+      claudeManager.killClaudeSession(cmd.sessionKey);
+      reply({ type: 'claude:killed', id: cmd.id });
+      break;
+
+    case 'claude:clear-session':
+      claudeManager.clearSession(cmd.sessionKey);
+      reply({ type: 'claude:session-cleared', id: cmd.id });
+      break;
+
+    default:
+      reply({
+        type: 'error',
+        id: (cmd as SidecarCommand).id,
+        error: `Unknown command type: ${(cmd as Record<string, unknown>).type}`,
+      });
+  }
+}
+
+// ── Connect to API server ──
+
+const connection = createSidecarConnector({
+  apiUrl: `${API_URL}/api/sidecar/register`,
+  name: 'claude',
+  capabilities: ['claude', 'proxy'],
+  onCommand(cmd, reply) {
+    handleCommand(cmd as SidecarCommand, reply as ReplyFn);
+  },
+});
+
+// ── Graceful shutdown ──
+
+async function shutdown(signal: string) {
+  console.log(`[claude] ${signal} received, saving state...`);
+  connection.destroy();
+  await flushAndSave();
+  releaseLock();
+  process.exit(0);
+}
+
+process.on('SIGTERM', () => shutdown('SIGTERM'));
+process.on('SIGINT', () => shutdown('SIGINT'));

src/servers/sidecar/claude/proxy.ts

@@ -100,5 +100,5 @@ export function startAnthropicProxy() {
     },
   });
 
-  console.log(`[sidecar:proxy] listening on 127.0.0.1:${PROXY_PORT}`);
+  console.log(`[claude:proxy] listening on 127.0.0.1:${PROXY_PORT}`);
 }

src/servers/sidecar/claude/state.ts

@@ -3,26 +3,17 @@ import { mkdirSync, existsSync } from 'node:fs';
 
 const DATA_PATH = process.env.DATA_PATH ?? join(process.cwd(), 'data');
 const STATE_DIR = join(DATA_PATH, 'sidecar');
-const STATE_FILE = join(STATE_DIR, 'state.json');
-const LOCK_FILE = join(STATE_DIR, 'sidecar.lock');
+const STATE_FILE = join(STATE_DIR, 'claude-state.json');
+const LOCK_FILE = join(STATE_DIR, 'claude.lock');
 
 export type PersistedState = {
   proxySecret: string;
   claudeSessions: Record<string, string>; // sessionKey → Claude Code session_id
-  piSessions: Array<{
-    sessionId: string;
-    email: string;
-    userId: number;
-    model: string;
-    cwd: string;
-    pid: number;
-  }>;
 };
 
 const DEFAULT_STATE: PersistedState = {
   proxySecret: '',
   claudeSessions: {},
-  piSessions: [],
 };
 
 let currentState: PersistedState = { ...DEFAULT_STATE };
@@ -37,13 +28,10 @@ function ensureDir() {
 export function loadState(): PersistedState {
   ensureDir();
   try {
-    const raw = Bun.file(STATE_FILE);
-    // Synchronous check — Bun.file doesn't have sync exists, use fs
     if (!existsSync(STATE_FILE)) {
       currentState = { ...DEFAULT_STATE };
       return currentState;
     }
-    // We need to read synchronously at startup
     const text = require('node:fs').readFileSync(STATE_FILE, 'utf-8');
     currentState = { ...DEFAULT_STATE, ...JSON.parse(text) };
     return currentState;
@@ -106,9 +94,8 @@ export function acquireLock(): boolean {
       const pidStr = require('node:fs').readFileSync(LOCK_FILE, 'utf-8').trim();
       const pid = Number(pidStr);
       if (pid && isProcessAlive(pid)) {
-        return false; // another sidecar is running
+        return false;
       }
-      // Stale lock — remove it
     }
     require('node:fs').writeFileSync(LOCK_FILE, String(process.pid));
     return true;
@@ -135,7 +122,3 @@ function isProcessAlive(pid: number): boolean {
     return false;
   }
 }
-
-export function isPidAlive(pid: number): boolean {
-  return isProcessAlive(pid);
-}

src/servers/sidecar/email/email-cron.ts

@@ -1,16 +1,20 @@
+import type { Job, EnqueueParams } from '../../queue/types';
 import { getAllSyncedAccounts, getUserById, getUserIntegration } from 'officerdb';
-import * as queueRunner from './queue-runner';
 
 const INTERVAL_MS = 10 * 60 * 1000; // 10 minutes
 
 let timer: ReturnType<typeof setInterval> | null = null;
+let enqueueFn: ((params: EnqueueParams) => Promise<Job>) | null = null;
+let listJobsFn: (() => Promise<Job[]>) | null = null;
 
 async function tick() {
+  if (!enqueueFn || !listJobsFn) return;
+
   try {
     const accounts = await getAllSyncedAccounts();
     if (accounts.length === 0) return;
 
-    const allJobs = await queueRunner.listAllJobs();
+    const allJobs = await listJobsFn();
     const activeEmailSyncIds = new Set(
       allJobs
         .filter((j) => j.type === 'email-sync' && (j.status === 'queued' || j.status === 'running'))
@@ -40,7 +44,7 @@ async function tick() {
       }
 
       try {
-        await queueRunner.enqueue({
+        await enqueueFn({
           lane: 'email',
           type: 'email-sync',
           userId: user.email,
@@ -63,7 +67,10 @@ async function tick() {
         });
         console.log(`[email-cron] Enqueued incremental sync for ${account.email}`);
       } catch (err) {
-        console.error(`[email-cron] Failed to enqueue sync for ${account.email}:`, err instanceof Error ? err.message : err);
+        console.error(
+          `[email-cron] Failed to enqueue sync for ${account.email}:`,
+          err instanceof Error ? err.message : err,
+        );
       }
     }
   } catch (err) {
@@ -71,11 +78,18 @@ async function tick() {
   }
 }
 
-export function initEmailCron() {
+type EmailCronDeps = {
+  enqueue: (params: EnqueueParams) => Promise<Job>;
+  listJobs: () => Promise<Job[]>;
+};
+
+export function initEmailCron(deps: EmailCronDeps) {
   if (timer) return;
+  enqueueFn = deps.enqueue;
+  listJobsFn = deps.listJobs;
   console.log(`[email-cron] Starting email sync cron (every ${INTERVAL_MS / 60_000} min)`);
   timer = setInterval(tick, INTERVAL_MS);
-  // Run first tick after a short delay to let the queue initialize
+  // Run first tick after a short delay
   setTimeout(tick, 30_000);
 }
 

src/servers/sidecar/email/index.ts

@@ -0,0 +1,100 @@
+import type { SidecarEvent } from '../protocol';
+import type { Job, EnqueueParams } from '../../queue/types';
+import { initEmailCron, stopEmailCron } from './email-cron';
+import { createSidecarConnector } from '../connect';
+
+const API_URL = process.env.API_URL ?? `ws://127.0.0.1:${process.env.PORT ?? '5000'}`;
+
+// ── Queue access via WS ──
+
+let reqCounter = 0;
+const pendingQueue = new Map<string, { resolve: (v: unknown) => void; reject: (e: Error) => void; timer: Timer }>();
+
+function nextQueueId(): string {
+  return `eq_${Date.now()}_${++reqCounter}`;
+}
+
+function sendQueueCommand(cmd: Record<string, unknown>): Promise<unknown> {
+  return new Promise((resolve, reject) => {
+    const id = cmd.id as string;
+    const timer = setTimeout(() => {
+      pendingQueue.delete(id);
+      reject(new Error(`Queue command ${cmd.type} timed out`));
+    }, 30_000);
+    pendingQueue.set(id, { resolve, reject, timer });
+    connection.send(cmd as SidecarEvent);
+  });
+}
+
+async function enqueueViaWs(params: EnqueueParams): Promise<Job> {
+  const res = (await sendQueueCommand({ type: 'queue:enqueue', id: nextQueueId(), params })) as Record<string, unknown>;
+  if (res.type === 'queue:enqueued') return res.job as Job;
+  if (res.type === 'queue:error') throw new Error(res.error as string);
+  throw new Error('Unexpected response');
+}
+
+async function listJobsViaWs(): Promise<Job[]> {
+  const res = (await sendQueueCommand({ type: 'queue:list', id: nextQueueId() })) as Record<string, unknown>;
+  if (res.type === 'queue:list') return res.jobs as Job[];
+  throw new Error('Unexpected response');
+}
+
+// ── Command handlers ──
+
+type ReplyFn = (msg: SidecarEvent) => void;
+
+function handleCommand(cmd: Record<string, unknown>, reply: ReplyFn) {
+  switch (cmd.type) {
+    case 'ping':
+      reply({ type: 'pong', id: cmd.id as string });
+      break;
+
+    default:
+      // Check if this is a queue response (from API server responding to our queue commands)
+      if (
+        typeof cmd.type === 'string' &&
+        cmd.type.startsWith('queue:') &&
+        cmd.id &&
+        pendingQueue.has(cmd.id as string)
+      ) {
+        const pending = pendingQueue.get(cmd.id as string)!;
+        pendingQueue.delete(cmd.id as string);
+        clearTimeout(pending.timer);
+        pending.resolve(cmd);
+        return;
+      }
+
+      reply({
+        type: 'error',
+        id: cmd.id as string,
+        error: `Unknown command type: ${cmd.type}`,
+      });
+  }
+}
+
+// ── Connect to API server ──
+
+const connection = createSidecarConnector({
+  apiUrl: `${API_URL}/api/sidecar/register`,
+  name: 'email',
+  capabilities: ['email'],
+  onCommand(cmd, reply) {
+    handleCommand(cmd as Record<string, unknown>, reply as ReplyFn);
+  },
+  onConnected() {
+    // Start email cron once connected (so queue commands can reach API server)
+    // initEmailCron({ enqueue: enqueueViaWs, listJobs: listJobsViaWs }); // TODO: re-enable after testing
+  },
+});
+
+// ── Graceful shutdown ──
+
+function shutdown(signal: string) {
+  console.log(`[email] ${signal} received, shutting down...`);
+  stopEmailCron();
+  connection.destroy();
+  process.exit(0);
+}
+
+process.on('SIGTERM', () => shutdown('SIGTERM'));
+process.on('SIGINT', () => shutdown('SIGINT'));

src/servers/sidecar/index.ts

@@ -1,216 +0,0 @@
-import type { SidecarCommand, SidecarEvent, SidecarState } from './protocol';
-import { loadState, flushAndSave, acquireLock, releaseLock, getState } from './state';
-import { startAnthropicProxy, getProxySecret, ensureProxySecret } from './proxy';
-import * as claudeManager from './claude-manager';
-import * as piManager from './pi-manager';
-import * as queueRunner from './queue-runner';
-import { initEmailCron, stopEmailCron } from './email-cron';
-import { createSidecarConnector } from './connect';
-
-const API_URL = process.env.API_URL ?? `ws://127.0.0.1:${process.env.PORT ?? '5000'}`;
-const startedAt = Date.now();
-
-// ── Startup ──
-
-if (!acquireLock()) {
-  console.error('[sidecar] another instance is already running (lock file exists with live PID)');
-  process.exit(1);
-}
-
-loadState();
-ensureProxySecret();
-
-// Start Anthropic proxy
-try {
-  startAnthropicProxy();
-} catch (err) {
-  console.error('[sidecar] failed to start Anthropic proxy:', err instanceof Error ? err.message : err);
-}
-
-// Initialize queue
-queueRunner.initQueue().catch((err) => {
-  console.error('[sidecar] failed to initialize queue:', err);
-});
-
-// Start email sync cron
-// initEmailCron(); // TODO: re-enable after initial sync testing
-
-// ── State ──
-
-function buildState(): SidecarState {
-  return {
-    proxySecret: getProxySecret(),
-    claudeSessions: { ...getState().claudeSessions },
-    piSessions: piManager.getAllSessions(),
-    uptime: Date.now() - startedAt,
-  };
-}
-
-// ── Command handlers ──
-
-type ReplyFn = (msg: SidecarEvent) => void;
-
-async function handleCommand(cmd: SidecarCommand, reply: ReplyFn) {
-  switch (cmd.type) {
-    case 'ping':
-      reply({ type: 'pong', id: cmd.id });
-      break;
-
-    case 'state:sync':
-      reply({ type: 'state:sync', id: cmd.id, state: buildState() });
-      break;
-
-    case 'proxy:secret':
-      reply({ type: 'proxy:secret', id: cmd.id, secret: getProxySecret() });
-      break;
-
-    // ── Claude Code ──
-
-    case 'claude:spawn': {
-      try {
-        const result = await claudeManager.spawnClaude(cmd.params);
-        reply({ type: 'claude:result', id: cmd.id, result });
-      } catch (err) {
-        reply({ type: 'claude:error', id: cmd.id, error: err instanceof Error ? err.message : String(err) });
-      }
-      break;
-    }
-
-    case 'claude:spawn-streaming': {
-      reply({ type: 'claude:spawned', id: cmd.id, sessionKey: cmd.params.sessionKey });
-
-      const onEvent = (event: import('../api/pi/types').PiEvent) => {
-        connection.send({ type: 'claude:event', sessionKey: cmd.params.sessionKey, event });
-      };
-
-      claudeManager.spawnClaudeStreaming(cmd.params, onEvent).catch((err) => {
-        connection.send({
-          type: 'claude:event',
-          sessionKey: cmd.params.sessionKey,
-          event: { type: 'error', message: err instanceof Error ? err.message : String(err) },
-        });
-      });
-      break;
-    }
-
-    case 'claude:kill':
-      claudeManager.killClaudeSession(cmd.sessionKey);
-      reply({ type: 'claude:killed', id: cmd.id });
-      break;
-
-    case 'claude:clear-session':
-      claudeManager.clearSession(cmd.sessionKey);
-      reply({ type: 'claude:session-cleared', id: cmd.id });
-      break;
-
-    // ── Pi ──
-
-    case 'pi:spawn': {
-      try {
-        const onEvent = (event: import('../api/pi/types').PiEvent) => {
-          connection.send({ type: 'pi:event', sessionId: cmd.params.sessionId, event });
-        };
-
-        await piManager.spawnPi({
-          sessionId: cmd.params.sessionId,
-          email: cmd.params.email,
-          userId: cmd.params.userId,
-          username: cmd.params.username,
-          role: cmd.params.role,
-          cwd: cmd.params.cwd,
-          model: cmd.params.model,
-          sessionFile: cmd.params.sessionFile,
-          onEvent,
-        });
-
-        reply({ type: 'pi:spawned', id: cmd.id, sessionId: cmd.params.sessionId });
-      } catch (err) {
-        reply({ type: 'pi:error', id: cmd.id, error: err instanceof Error ? err.message : String(err) });
-      }
-      break;
-    }
-
-    case 'pi:prompt':
-      piManager.sendPrompt(cmd.sessionId, cmd.prompt, cmd.requestId);
-      break;
-
-    case 'pi:abort':
-      piManager.abort(cmd.sessionId, cmd.requestId);
-      break;
-
-    case 'pi:kill':
-      piManager.killPiSession(cmd.sessionId);
-      reply({ type: 'pi:killed', id: cmd.id });
-      break;
-
-    case 'pi:set-thinking':
-      piManager.setThinkingLevel(cmd.sessionId, cmd.level);
-      break;
-
-    // ── Queue ──
-
-    case 'queue:enqueue': {
-      try {
-        const job = await queueRunner.enqueue(cmd.params);
-        reply({ type: 'queue:enqueued', id: cmd.id, job });
-      } catch (err) {
-        reply({ type: 'queue:error', id: cmd.id, error: err instanceof Error ? err.message : String(err) });
-      }
-      break;
-    }
-
-    case 'queue:cancel': {
-      try {
-        const job = await queueRunner.cancelJob(cmd.jobId);
-        reply({ type: 'queue:cancelled', id: cmd.id, job });
-      } catch (err) {
-        reply({ type: 'queue:error', id: cmd.id, error: err instanceof Error ? err.message : String(err) });
-      }
-      break;
-    }
-
-    case 'queue:list': {
-      const jobs = await queueRunner.listAllJobs();
-      reply({ type: 'queue:list', id: cmd.id, jobs });
-      break;
-    }
-
-    case 'queue:get': {
-      const job = await queueRunner.readJob(cmd.jobId);
-      reply({ type: 'queue:get', id: cmd.id, job });
-      break;
-    }
-
-    default:
-      reply({
-        type: 'error',
-        id: (cmd as SidecarCommand).id,
-        error: `Unknown command type: ${(cmd as Record<string, unknown>).type}`,
-      });
-  }
-}
-
-// ── Connect to API server ──
-
-const connection = createSidecarConnector({
-  apiUrl: `${API_URL}/api/sidecar/register`,
-  name: 'process',
-  capabilities: ['claude', 'pi', 'queue', 'proxy'],
-  onCommand(cmd, reply) {
-    handleCommand(cmd as SidecarCommand, reply as ReplyFn);
-  },
-});
-
-// ── Graceful shutdown ──
-
-async function shutdown(signal: string) {
-  console.log(`[sidecar] ${signal} received, saving state...`);
-  stopEmailCron();
-  connection.destroy();
-  await flushAndSave();
-  releaseLock();
-  process.exit(0);
-}
-
-process.on('SIGTERM', () => shutdown('SIGTERM'));
-process.on('SIGINT', () => shutdown('SIGINT'));

src/servers/sidecar/pi/index.ts

@@ -0,0 +1,88 @@
+import type { SidecarCommand, SidecarEvent } from '../protocol';
+import * as piManager from './pi-manager';
+import { createSidecarConnector } from '../connect';
+
+const API_URL = process.env.API_URL ?? `ws://127.0.0.1:${process.env.PORT ?? '5000'}`;
+
+// ── Command handlers ──
+
+type ReplyFn = (msg: SidecarEvent) => void;
+
+async function handleCommand(cmd: SidecarCommand, reply: ReplyFn) {
+  switch (cmd.type) {
+    case 'ping':
+      reply({ type: 'pong', id: cmd.id });
+      break;
+
+    case 'pi:spawn': {
+      try {
+        const onEvent = (event: import('../../api/pi/types').PiEvent) => {
+          connection.send({ type: 'pi:event', sessionId: cmd.params.sessionId, event });
+        };
+
+        await piManager.spawnPi({
+          sessionId: cmd.params.sessionId,
+          email: cmd.params.email,
+          userId: cmd.params.userId,
+          username: cmd.params.username,
+          role: cmd.params.role,
+          cwd: cmd.params.cwd,
+          model: cmd.params.model,
+          sessionFile: cmd.params.sessionFile,
+          onEvent,
+        });
+
+        reply({ type: 'pi:spawned', id: cmd.id, sessionId: cmd.params.sessionId });
+      } catch (err) {
+        reply({ type: 'pi:error', id: cmd.id, error: err instanceof Error ? err.message : String(err) });
+      }
+      break;
+    }
+
+    case 'pi:prompt':
+      piManager.sendPrompt(cmd.sessionId, cmd.prompt, cmd.requestId);
+      break;
+
+    case 'pi:abort':
+      piManager.abort(cmd.sessionId, cmd.requestId);
+      break;
+
+    case 'pi:kill':
+      piManager.killPiSession(cmd.sessionId);
+      reply({ type: 'pi:killed', id: cmd.id });
+      break;
+
+    case 'pi:set-thinking':
+      piManager.setThinkingLevel(cmd.sessionId, cmd.level);
+      break;
+
+    default:
+      reply({
+        type: 'error',
+        id: (cmd as SidecarCommand).id,
+        error: `Unknown command type: ${(cmd as Record<string, unknown>).type}`,
+      });
+  }
+}
+
+// ── Connect to API server ──
+
+const connection = createSidecarConnector({
+  apiUrl: `${API_URL}/api/sidecar/register`,
+  name: 'pi',
+  capabilities: ['pi'],
+  onCommand(cmd, reply) {
+    handleCommand(cmd as SidecarCommand, reply as ReplyFn);
+  },
+});
+
+// ── Graceful shutdown ──
+
+function shutdown(signal: string) {
+  console.log(`[pi] ${signal} received, shutting down...`);
+  connection.destroy();
+  process.exit(0);
+}
+
+process.on('SIGTERM', () => shutdown('SIGTERM'));
+process.on('SIGINT', () => shutdown('SIGINT'));

src/servers/sidecar/pi/pi-manager.ts

@@ -1,9 +1,8 @@
 import { join } from 'node:path';
 import { readdirSync, existsSync, mkdirSync } from 'node:fs';
 import type { Subprocess } from 'bun';
-import type { PiEvent, MessageCost } from '../api/pi/types';
-import type { PiSpawnParams, PiSessionInfo } from './protocol';
-import { isPidAlive } from './state';
+import type { PiEvent, MessageCost } from '../../api/pi/types';
+import type { PiSpawnParams, PiSessionInfo } from '../protocol';
 
 const DATA_PATH = process.env.DATA_PATH ?? join(process.cwd(), 'data');
 const PI_CONFIG_DIR = join(require('node:os').homedir(), '.pi', 'agent');
@@ -27,6 +26,15 @@ const toShellUsername = (username: string, email: string): string => {
   );
 };
 
+function isPidAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 // Resolve pi as [node, cli.js]
 const PI_CMD = (() => {
   const whichResult = Bun.spawnSync({ cmd: ['which', 'pi'], stdout: 'pipe', stderr: 'ignore' });
@@ -216,7 +224,7 @@ function writeRpcCommand(proc: Subprocess, command: Record<string, unknown>): vo
     writer.write(JSON.stringify(command) + '\n');
     writer.flush();
   } catch (err) {
-    console.error('[sidecar:pi] writeRpcCommand error:', err);
+    console.error('[pi] writeRpcCommand error:', err);
   }
 }
 
@@ -288,7 +296,7 @@ export async function spawnPi(options: PiSpawnOptions): Promise<void> {
   const session: PiSession = { sessionId, email, userId, model, cwd, proc, onEvent };
   sessions.set(sessionId, session);
 
-  console.log(`[sidecar:pi] spawned Pi for session ${sessionId} (model=${model}, pid=${proc.pid})`);
+  console.log(`[pi] spawned Pi for session ${sessionId} (model=${model}, pid=${proc.pid})`);
 
   // Read stdout JSON event stream
   const stdout = proc.stdout as ReadableStream<Uint8Array>;
@@ -338,7 +346,7 @@ export async function spawnPi(options: PiSpawnOptions): Promise<void> {
         const { done, value } = await stderrReader.read();
         if (done) break;
         const text = stderrDecoder.decode(value, { stream: true });
-        if (text.trim()) console.log(`[sidecar:pi:stderr] ${text.trim()}`);
+        if (text.trim()) console.log(`[pi:stderr] ${text.trim()}`);
       }
     } catch {
       /* process ended */
@@ -349,7 +357,7 @@ export async function spawnPi(options: PiSpawnOptions): Promise<void> {
   proc.exited.then((code) => {
     sessions.delete(sessionId);
     if (code !== 0) {
-      console.error(`[sidecar:pi] Pi process ${sessionId} exited with code ${code}`);
+      console.error(`[pi] Pi process ${sessionId} exited with code ${code}`);
     }
   });
 }

src/servers/sidecar/queue-runner.ts

@@ -1,285 +0,0 @@
-import { type Job, type JobProgress, type EnqueueParams, type StepContext, PermanentError } from '../queue/types';
-import { readJob, writeJob, listAllJobs, ensureQueueDir } from '../queue/storage';
-import { getHandler } from '../queue/handler-registry';
-
-// Import handlers to register them
-import '../queue/handlers';
-
-function formatDuration(ms: number): string {
-  const s = Math.floor(ms / 1000);
-  if (s < 60) return `${s}s`;
-  const m = Math.floor(s / 60);
-  const rem = s % 60;
-  if (m < 60) return rem > 0 ? `${m}m${rem}s` : `${m}m`;
-  const h = Math.floor(m / 60);
-  const remM = m % 60;
-  return remM > 0 ? `${h}h${remM}m` : `${h}h`;
-}
-
-const activeLanes = new Map<string, boolean>();
-const PROGRESS_THROTTLE_MS = 1000;
-
-export async function initQueue() {
-  await ensureQueueDir();
-  await resumeInterruptedJobs();
-  console.log('[sidecar:queue] initialized');
-}
-
-export async function enqueue(params: EnqueueParams): Promise<Job> {
-  const handler = getHandler(params.type);
-  if (!handler) throw new Error(`No handler registered for job type: ${params.type}`);
-
-  const job: Job = {
-    id: crypto.randomUUID(),
-    lane: params.lane,
-    type: params.type,
-    userId: params.userId,
-    status: 'queued',
-    steps: handler.steps.map((s) => ({ name: s.name, status: 'pending' as const })),
-    currentStep: 0,
-    createdAt: Date.now(),
-    meta: params.meta,
-    notify: params.notify,
-  };
-
-  await writeJob(job);
-  console.log(`[sidecar:queue] enqueued job ${job.id} (${job.type}) in lane ${job.lane}`);
-  kickLane(job.lane);
-  return job;
-}
-
-export async function cancelJob(id: string): Promise<Job | null> {
-  const job = await readJob(id);
-  if (!job) return null;
-  if (job.status === 'completed' || job.status === 'failed' || job.status === 'cancelled') return job;
-
-  job.status = 'cancelled';
-  job.completedAt = Date.now();
-  for (const step of job.steps) {
-    if (step.status === 'pending' || step.status === 'running') {
-      step.status = 'failed';
-      step.error = 'Cancelled';
-    }
-  }
-  await writeJob(job);
-  console.log(`[sidecar:queue] cancelled job ${job.id}`);
-  return job;
-}
-
-async function resumeInterruptedJobs() {
-  const jobs = await listAllJobs();
-  const lanesToKick = new Set<string>();
-
-  for (const job of jobs) {
-    if (job.status === 'running') {
-      job.status = 'queued';
-      job.startedAt = undefined;
-      for (const step of job.steps) {
-        if (step.status === 'running') {
-          step.status = 'pending';
-          step.startedAt = undefined;
-        }
-      }
-      await writeJob(job);
-      console.log(`[sidecar:queue] reset interrupted job ${job.id} back to queued`);
-      lanesToKick.add(job.lane);
-    } else if (job.status === 'queued') {
-      // Clear retry delay on restart — no reason to wait after a sidecar restart
-      if (job.retryAt) {
-        job.retryAt = undefined;
-        await writeJob(job);
-        console.log(`[sidecar:queue] cleared retry delay for job ${job.id}`);
-      }
-      lanesToKick.add(job.lane);
-    }
-  }
-
-  for (const lane of lanesToKick) {
-    kickLane(lane);
-  }
-}
-
-function kickLane(lane: string) {
-  if (activeLanes.get(lane)) return;
-  activeLanes.set(lane, true);
-  processNextInLane(lane);
-}
-
-function scheduleRetry(lane: string, delayMs: number) {
-  setTimeout(() => kickLane(lane), delayMs);
-}
-
-async function processNextInLane(lane: string) {
-  try {
-    const jobs = await listAllJobs();
-    const now = Date.now();
-    const next = jobs
-      .filter((j) => j.lane === lane && j.status === 'queued' && (!j.retryAt || j.retryAt <= now))
-      .sort((a, b) => a.createdAt - b.createdAt)[0];
-
-    if (!next) {
-      activeLanes.set(lane, false);
-      return;
-    }
-
-    await runJob(next);
-  } catch (err) {
-    console.error(`[sidecar:queue] lane ${lane} processing error:`, err);
-  } finally {
-    const jobs = await listAllJobs();
-    const hasMore = jobs.some(
-      (j) => j.lane === lane && j.status === 'queued' && (!j.retryAt || j.retryAt <= Date.now()),
-    );
-    if (hasMore) {
-      processNextInLane(lane);
-    } else {
-      activeLanes.set(lane, false);
-    }
-  }
-}
-
-async function runJob(job: Job) {
-  const handler = getHandler(job.type);
-  if (!handler) {
-    job.status = 'failed';
-    job.error = `No handler for type: ${job.type}`;
-    job.completedAt = Date.now();
-    await writeJob(job);
-    return;
-  }
-
-  job.status = 'running';
-  job.startedAt = Date.now();
-  job.retryAt = undefined;
-  await writeJob(job);
-  const isRetry = (job.retries ?? 0) > 0;
-  const startTime = Date.now();
-  console.log(
-    `[sidecar:queue] ▶ ${isRetry ? 'resuming' : 'running'} job ${job.id} (${job.type})${isRetry ? ` retry ${job.retries}` : ''}`,
-  );
-
-  const sharedMeta: Record<string, unknown> = { ...(job.meta ?? {}) };
-
-  for (let i = 0; i < handler.steps.length; i++) {
-    const fresh = await readJob(job.id);
-    if (!fresh || fresh.status === 'cancelled') {
-      console.log(`[sidecar:queue] job ${job.id} was cancelled, stopping`);
-      return;
-    }
-
-    const handlerStep = handler.steps[i]!;
-    const step = fresh.steps[i]!;
-
-    if (step.status === 'completed') continue;
-
-    fresh.currentStep = i;
-    step.status = 'running';
-    step.startedAt = Date.now();
-    await writeJob(fresh);
-
-    let lastProgressWrite = 0;
-    let pendingProgress: JobProgress | null = null;
-
-    const updateProgress = async (progress: JobProgress) => {
-      step.progress = progress;
-      const now = Date.now();
-      if (now - lastProgressWrite >= PROGRESS_THROTTLE_MS) {
-        lastProgressWrite = now;
-        pendingProgress = null;
-        await writeJob(fresh);
-      } else {
-        pendingProgress = progress;
-      }
-    };
-
-    const ctx: StepContext = { job: fresh, step, updateProgress, meta: sharedMeta };
-
-    try {
-      await handlerStep.run(ctx);
-
-      if (pendingProgress) {
-        step.progress = pendingProgress;
-      }
-      step.status = 'completed';
-      step.completedAt = Date.now();
-      await writeJob(fresh);
-    } catch (err) {
-      const errorMessage = err instanceof Error ? err.message : String(err);
-      console.error(`[sidecar:queue] step "${step.name}" failed: ${errorMessage}`);
-      step.status = 'failed';
-      step.error = errorMessage;
-      step.completedAt = Date.now();
-
-      const isPermanent = err instanceof PermanentError;
-      const retries = (fresh.retries ?? 0) + 1;
-      if (!isPermanent && handler.retry && retries <= handler.retry.maxRetries) {
-        step.status = 'pending';
-        step.error = undefined;
-        step.startedAt = undefined;
-        step.completedAt = undefined;
-        step.progress = undefined;
-        fresh.status = 'queued';
-        fresh.error = undefined;
-        fresh.completedAt = undefined;
-        fresh.startedAt = undefined;
-        fresh.retries = retries;
-        fresh.retryAt = Date.now() + handler.retry.delayMs;
-        await writeJob(fresh);
-        console.log(
-          `[sidecar:queue] job ${fresh.id} will retry (${retries}/${handler.retry.maxRetries}) in ${handler.retry.delayMs / 1000}s`,
-        );
-        scheduleRetry(fresh.lane, handler.retry.delayMs);
-        return;
-      }
-
-      fresh.status = 'failed';
-      fresh.error = `Step "${step.name}" failed: ${errorMessage}`;
-      fresh.completedAt = Date.now();
-      fresh.meta = { ...fresh.meta, ...sharedMeta };
-      await writeJob(fresh);
-      console.error(`[sidecar:queue] ✗ job ${fresh.id} failed at step "${step.name}" in ${formatDuration(Date.now() - startTime)}:`, errorMessage);
-      await notifyFailure(fresh);
-      return;
-    }
-  }
-
-  const final = await readJob(job.id);
-  if (final && final.status === 'running') {
-    final.status = 'completed';
-    final.completedAt = Date.now();
-    final.meta = { ...final.meta, ...sharedMeta };
-    await writeJob(final);
-    console.log(`[sidecar:queue] ✓ job ${final.id} completed in ${formatDuration(Date.now() - startTime)}`);
-    await notifyCompletion(final);
-  }
-}
-
-async function notifyCompletion(job: Job) {
-  try {
-    const { sendMail } = await import('emailer');
-    await sendMail({
-      template: 'JobCompleted',
-      subject: `Job completed: ${job.type}`,
-      to: job.userId,
-      data: { job },
-    });
-  } catch {
-    // SMTP might not be configured — non-fatal
-  }
-}
-
-async function notifyFailure(job: Job) {
-  try {
-    const { sendMail } = await import('emailer');
-    await sendMail({
-      template: 'JobFailed',
-      subject: `Job failed: ${job.type}`,
-      to: job.userId,
-      data: { job },
-    });
-  } catch {
-    // SMTP might not be configured — non-fatal
-  }
-}
-
-export { readJob, listAllJobs };