sidecar self-registration: sidecars connect to API server instead of vice versa

Flips the connection model so sidecars register themselves with the API
server via WebSocket at /api/sidecar/register, enabling dynamic discovery,
location independence, and automatic reconnection from either side.

- Add registration protocol types and PTY command/event types
- Create sidecar-registry.ts (replaces sidecar-client.ts) as passive registry
- Create sidecar connector (connect.ts) with exponential backoff reconnect
- Convert process sidecar from WS server to WS client
- Convert PTY sidecar from WS server to multiplexed WS client
- Simplify terminal bridge to thin adapter using registry
- Add PTY sidecar as PM2-managed process
- Update all consumer imports

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/servers/api/terminal/websocket.ts

@@ -1,8 +1,9 @@
 import type { ServerWebSocket } from 'bun';
 import { mkdirSync } from 'node:fs';
 import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
 import { getHomeDir } from '@@/data-path';
+import { sendPtyCommand, sendPtyCommandAsync, on, isTerminalConnected } from '@@/sidecar-registry';
+import type { PtyInitConfig } from '../../sidecar/protocol';
 
 type WSData = {
   userId: number;
@@ -15,17 +16,20 @@ type WSData = {
   cols?: number;
   rows?: number;
 };
+
 type BridgeSession = {
   client: ServerWebSocket<WSData>;
-  sidecar: WebSocket | null;
-  pendingMessages: string[];
+  sessionId: string;
+  unsubOutput: (() => void) | null;
+  unsubExit: (() => void) | null;
 };
 
-const HOST_SIDECAR_PORT = 5338;
-
 const sessions = new Map<ServerWebSocket<WSData>, BridgeSession>();
 
-let hostSidecarProcess: ReturnType<typeof import('bun').spawn> | null = null;
+let idCounter = 0;
+function nextId(): string {
+  return `pty_${Date.now()}_${++idCounter}`;
+}
 
 const sendOutput = (ws: ServerWebSocket<WSData>, data: string) => {
   try {
@@ -35,105 +39,6 @@ const sendOutput = (ws: ServerWebSocket<WSData>, data: string) => {
   }
 };
 
-const connectSidecar = async (): Promise<WebSocket> => {
-  const delays = [200, 300, 500, 800, 1200, 1600, 2000];
-  let lastError: Error | null = null;
-
-  for (const delay of delays) {
-    try {
-      const ws = await new Promise<WebSocket>((resolve, reject) => {
-        const socket = new WebSocket(`ws://127.0.0.1:${HOST_SIDECAR_PORT}`);
-        const timeout = setTimeout(() => {
-          try {
-            socket.close();
-          } catch {
-            // ignore
-          }
-          reject(new Error('Terminal sidecar timeout'));
-        }, 2000);
-
-        socket.addEventListener('open', () => {
-          clearTimeout(timeout);
-          resolve(socket);
-        });
-        socket.addEventListener('error', () => {
-          clearTimeout(timeout);
-          reject(new Error('Terminal sidecar connection failed'));
-        });
-      });
-
-      return ws;
-    } catch (err) {
-      lastError = err instanceof Error ? err : new Error('Terminal sidecar connection failed');
-      await new Promise((resolve) => setTimeout(resolve, delay));
-    }
-  }
-
-  throw lastError ?? new Error('Terminal sidecar connection failed');
-};
-
-const sidecarAlive = async (): Promise<boolean> => {
-  try {
-    const res = await fetch(`http://127.0.0.1:${HOST_SIDECAR_PORT}`, { signal: AbortSignal.timeout(500) });
-    return res.ok;
-  } catch {
-    return false;
-  }
-};
-
-const killSidecarOnPort = (port: number) => {
-  try {
-    const result = Bun.spawnSync({ cmd: ['fuser', '-k', `${port}/tcp`], stdout: 'ignore', stderr: 'ignore' });
-    if (result.exitCode === 0) console.log(`[terminal] killed stale sidecar on port ${port}`);
-  } catch {
-    // fuser not available or failed
-  }
-};
-
-const startHostSidecar = async () => {
-  if (hostSidecarProcess) {
-    hostSidecarProcess.kill();
-    await hostSidecarProcess.exited.catch(() => {});
-    hostSidecarProcess = null;
-  }
-
-  killSidecarOnPort(HOST_SIDECAR_PORT);
-  await new Promise((resolve) => setTimeout(resolve, 200));
-
-  const sidecarPath = fileURLToPath(new URL('./pty-sidecar.mjs', import.meta.url));
-  hostSidecarProcess = Bun.spawn({
-    cmd: ['node', sidecarPath],
-    env: { ...process.env, TERMINAL_PTY_PORT: String(HOST_SIDECAR_PORT) },
-    stdout: 'inherit',
-    stderr: 'inherit',
-  });
-  console.log(`[terminal] host sidecar started on port ${HOST_SIDECAR_PORT}`);
-};
-
-export const ensureHostSidecar = async () => {
-  const alive = await sidecarAlive();
-  if (!alive) {
-    await startHostSidecar();
-    for (let i = 0; i < 10; i++) {
-      await new Promise((r) => setTimeout(r, 300));
-      if (await sidecarAlive()) return;
-    }
-    throw new Error('Host sidecar failed to start');
-  }
-};
-
-export const initTerminalSidecars = async () => {
-  // Sidecar is managed by pm2 — wait for it to be available
-  for (let i = 0; i < 15; i++) {
-    if (await sidecarAlive()) {
-      console.log(`[terminal] host sidecar already running on port ${HOST_SIDECAR_PORT}`);
-      return;
-    }
-    await new Promise((r) => setTimeout(r, 500));
-  }
-  console.warn(`[terminal] host sidecar not detected on port ${HOST_SIDECAR_PORT} — terminals will retry on connect`);
-};
-
 const resolveCwd = (home: string, cwd?: string) => {
   if (!cwd || cwd === '~') return home;
   if (cwd.startsWith('~/')) return join(home, cwd.slice(2));
@@ -150,102 +55,126 @@ export const terminalWebsocket = {
       `[terminal] open: email=${email} username=${username} role=${role} sandboxed=${sandboxed} isHost=${isHost}`,
     );
 
-    const session: BridgeSession = { client: ws, sidecar: null, pendingMessages: [] };
-    sessions.set(ws, session);
-
-    let sidecar: WebSocket | null = null;
-    try {
-      sidecar = await connectSidecar();
-      console.log('[terminal] sidecar connected');
-    } catch (err) {
-      const message = err instanceof Error ? err.message : 'Failed to connect terminal sidecar';
-      console.error('[terminal] sidecar connection failed:', message);
-      sendOutput(ws, `\r\n[Terminal error] ${message}\r\n`);
-      sessions.delete(ws);
+    if (!isTerminalConnected()) {
+      sendOutput(ws, '\r\n[Terminal error] PTY sidecar is not connected\r\n');
       return;
     }
 
-    session.sidecar = sidecar;
+    const sessionId = ws.data.sessionId ?? (isHost ? `host-${ws.data.userId}` : `default-${ws.data.userId}`);
 
-    sidecar.addEventListener('message', (ev) => {
-      try {
-        if (typeof ev.data === 'string') {
-          ws.send(ev.data);
-        } else {
-          ws.send(new TextDecoder().decode(ev.data));
-        }
-      } catch {
-        // ws already closed
-      }
-    });
+    // Build PTY init config
+    let config: PtyInitConfig;
 
     if (isHost) {
-      // Super Admin host terminal — spawn as the service user directly
-      sidecar.send(
-        JSON.stringify({
-          type: 'init',
-          host: true,
-          sessionId: ws.data.sessionId ?? `host-${ws.data.userId}`,
-          shell: { command: process.env.SHELL ?? '/bin/zsh', args: ['-i'] },
-          cwd: resolveCwd(process.env.HOME!, ws.data.cwd),
-          homeDir: process.env.HOME,
-          userLabel: email,
-          cols: ws.data.cols,
-          rows: ws.data.rows,
-        }),
-      );
+      config = {
+        sessionId,
+        host: true,
+        shell: { command: process.env.SHELL ?? '/bin/zsh', args: ['-i'] },
+        cwd: resolveCwd(process.env.HOME!, ws.data.cwd),
+        homeDir: process.env.HOME!,
+        userLabel: email,
+        cols: ws.data.cols,
+        rows: ws.data.rows,
+      };
     } else {
-      // User terminal — spawn as the target Linux user via sudo -u
       const homeDir = getHomeDir(email);
       mkdirSync(dirname(homeDir), { recursive: true });
       mkdirSync(homeDir, { recursive: true });
 
-      sidecar.send(
-        JSON.stringify({
-          type: 'init',
-          username,
-          sessionId: ws.data.sessionId ?? `default-${ws.data.userId}`,
-          cwd: resolveCwd(homeDir, ws.data.cwd),
-          homeDir,
-          userLabel: email,
-          cols: ws.data.cols,
-          rows: ws.data.rows,
-        }),
-      );
+      config = {
+        sessionId,
+        username,
+        cwd: resolveCwd(homeDir, ws.data.cwd),
+        homeDir,
+        userLabel: email,
+        cols: ws.data.cols,
+        rows: ws.data.rows,
+      };
     }
 
-    for (const msg of session.pendingMessages) sidecar.send(msg);
-    session.pendingMessages = [];
+    // Subscribe to events for this session
+    const unsubOutput = on('pty:output', (msg) => {
+      if (msg.type === 'pty:output' && msg.sessionId === sessionId) {
+        try {
+          ws.send(JSON.stringify({ type: 'output', data: msg.data }));
+        } catch {
+          // ws already closed
+        }
+      }
+    });
+
+    const unsubExit = on('pty:exit', (msg) => {
+      if (msg.type === 'pty:exit' && msg.sessionId === sessionId) {
+        try {
+          ws.send(JSON.stringify({ type: 'exit' }));
+        } catch {
+          // ws already closed
+        }
+      }
+    });
+
+    const session: BridgeSession = { client: ws, sessionId, unsubOutput, unsubExit };
+    sessions.set(ws, session);
+
+    // Send init command to PTY sidecar
+    try {
+      await sendPtyCommandAsync({ type: 'pty:init', id: nextId(), sessionId, config });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : 'Failed to initialize terminal';
+      sendOutput(ws, `\r\n[Terminal error] ${message}\r\n`);
+      unsubOutput();
+      unsubExit();
+      sessions.delete(ws);
+    }
   },
 
   message(ws: ServerWebSocket<WSData>, raw: string | Buffer) {
     const session = sessions.get(ws);
     if (!session) return;
 
-    const payload = typeof raw === 'string' ? raw : raw.toString();
-
-    if (!session.sidecar || session.sidecar.readyState !== WebSocket.OPEN) {
-      session.pendingMessages.push(payload);
-      return;
-    }
-
     try {
-      session.sidecar.send(payload);
+      const payload = typeof raw === 'string' ? raw : raw.toString();
+      const msg = JSON.parse(payload);
+
+      switch (msg.type) {
+        case 'input':
+          sendPtyCommand({ type: 'pty:input', id: nextId(), sessionId: session.sessionId, data: msg.data ?? '' });
+          break;
+        case 'resize':
+          if (msg.cols > 0 && msg.rows > 0) {
+            sendPtyCommand({
+              type: 'pty:resize',
+              id: nextId(),
+              sessionId: session.sessionId,
+              cols: msg.cols,
+              rows: msg.rows,
+            });
+          }
+          break;
+        case 'cwd':
+          if (msg.path) {
+            sendPtyCommand({
+              type: 'pty:input',
+              id: nextId(),
+              sessionId: session.sessionId,
+              data: `cd ${JSON.stringify(msg.path)}\r`,
+            });
+          }
+          break;
+      }
     } catch {
-      // ignore
+      // ignore malformed messages
     }
   },
 
   close(ws: ServerWebSocket<WSData>) {
     const session = sessions.get(ws);
-    if (session?.sidecar) {
-      try {
-        session.sidecar.close();
-      } catch {
-        // ignore
-      }
+    if (session) {
+      session.unsubOutput?.();
+      session.unsubExit?.();
+      // Don't kill PTY — it can be reattached
+      sessions.delete(ws);
     }
-    sessions.delete(ws);
   },
 
   drain() {},
@@ -253,8 +182,8 @@ export const terminalWebsocket = {
 
 export const broadcastPanelRefresh = (email: string) => {
   const msg = JSON.stringify({ type: 'panel-refresh' });
-  for (const [ws, session] of sessions) {
-    if (ws.data.email === email && session.sidecar) {
+  for (const [ws] of sessions) {
+    if (ws.data.email === email) {
       try {
         ws.send(msg);
       } catch {
@@ -263,12 +192,3 @@ export const broadcastPanelRefresh = (email: string) => {
     }
   }
 };
-
-export const stopAllSidecars = async () => {
-  if (hostSidecarProcess) {
-    hostSidecarProcess.kill();
-    await hostSidecarProcess.exited.catch(() => {});
-    hostSidecarProcess = null;
-    console.log('[terminal] host sidecar stopped');
-  }
-};