From af56eb36ff052d336183e79239c23e5f26cf4891 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20Padez?= <andre.padez@gmail.com>
Date: Fri, 31 Jul 2026 12:10:35 +0000
Subject: [PATCH] email: move the mail store and every route into the sidecar
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Email was the one sidecar built inside out. The platform held ~1,800 lines — the per-account
SQLite store, all 14 HTTP routes, account CRUD, resync, IMAP validation — while the 314-line
sidecar was a scheduler that reached BACK into the platform to do anything
(`import { performResync } from '../../api/email/resync'`).

The sidecar now serves its own HTTP listener and announces `email:server`, and
/api/email/* on the platform is createSidecarProxy like every other one: 1,801 lines down
to 22, with no mail knowledge left in it — not a message, not a folder, not a credential.

The routes moved verbatim, Hono and all. http.ts only reconstructs what the platform's
middleware used to provide: `user` on the context, from the X-Officer-User header the proxy
injects (trusted because this server binds loopback), and an error handler that turns
custom-errors into status codes.

The /email/events SSE stream went with them, which removes a whole round trip: the IDLE
watcher used to send `email:new` over the registration socket so the platform could push to
its SSE clients. Those clients are here now, so it calls broadcastEmailNew in-process and
`email:new` is gone from the wire protocol.

DELIBERATELY NOT DONE YET, and left backwards on purpose rather than half-moved:

- The two sync handlers (email-sync 381 lines, gmail-sync 712) still run in the platform's
  queue and now import the store from its new home — a platform → sidecar import, which is
  the wrong direction and is temporary. Moving them is option (A) from the plan: the sidecar
  schedules its own syncs, independent of the platform Jobs list.
- accounts.ts still imports queue/init to enqueue a sync and to report sync status, and
  index.ts still carries the queue-over-WS shim that inversion needs.
- The three channel handlers still open the mail store directly rather than asking over HTTP.

Two things worth knowing while testing: a from-scratch sync holds a proxied request open
well past the 60s idle default, hence timeoutSeconds on the proxy; and `gmail-sync` is
hardcoded in all three channel handlers even though the only account is provider=gmail with
auth_type=password, which routes to IMAP — so "sync emails" from a chat channel is
almost certainly already broken, and folds into the next stage.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 scripts/migrate-emails-to-sqlite.ts           |  2 +-
 scripts/seed-imap-uids.ts                     |  2 +-
 src/server.tsx                                |  7 ---
 src/servers/api/email/router.ts               | 22 +++++++++
 src/servers/channels/discord/handler.ts       |  2 +-
 src/servers/channels/telegram/handler.ts      |  2 +-
 src/servers/channels/whatsapp/handler.ts      |  2 +-
 src/servers/hono.ts                           |  2 +-
 src/servers/queue/handlers/email-sync.ts      |  2 +-
 src/servers/queue/handlers/gmail-sync.ts      |  2 +-
 .../{api => sidecar}/email/accounts.ts        |  4 +-
 src/servers/sidecar/email/email-cron.ts       |  2 +-
 src/servers/sidecar/email/email-idle.ts       |  2 +-
 src/servers/sidecar/email/http.ts             | 48 +++++++++++++++++++
 .../{api => sidecar}/email/imap-validate.ts   |  0
 src/servers/sidecar/email/index.ts            | 15 +++++-
 src/servers/{api => sidecar}/email/resync.ts  |  2 +-
 .../email.ts => sidecar/email/routes.ts}      |  2 +-
 .../email-db.ts => sidecar/email/store.ts}    |  0
 src/servers/sidecar/protocol.ts               |  5 +-
 20 files changed, 101 insertions(+), 24 deletions(-)
 create mode 100644 src/servers/api/email/router.ts
 rename src/servers/{api => sidecar}/email/accounts.ts (98%)
 create mode 100644 src/servers/sidecar/email/http.ts
 rename src/servers/{api => sidecar}/email/imap-validate.ts (100%)
 rename src/servers/{api => sidecar}/email/resync.ts (99%)
 rename src/servers/{api/email/email.ts => sidecar/email/routes.ts} (99%)
 rename src/servers/{api/email/email-db.ts => sidecar/email/store.ts} (100%)

diff --git a/scripts/migrate-emails-to-sqlite.ts b/scripts/migrate-emails-to-sqlite.ts
index 9fc5f0a9..aa1a1bc2 100644
--- a/scripts/migrate-emails-to-sqlite.ts
+++ b/scripts/migrate-emails-to-sqlite.ts
@@ -1,6 +1,6 @@
 import { readdirSync, readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import { openEmailDb, upsertFromRawEml, setSyncMeta } from '../src/servers/api/email/email-db';
+import { openEmailDb, upsertFromRawEml, setSyncMeta } from '../src/servers/sidecar/email/store';
 
 const DATA_PATH = process.env.DATA_PATH ?? join(process.cwd(), 'data');
 
diff --git a/scripts/seed-imap-uids.ts b/scripts/seed-imap-uids.ts
index 8e22ae7e..a23a9c2c 100644
--- a/scripts/seed-imap-uids.ts
+++ b/scripts/seed-imap-uids.ts
@@ -1,6 +1,6 @@
 import { ImapFlow } from 'imapflow';
 import { getUserByEmail, getUserIntegration, getServerIntegration } from 'officerdb';
-import { openEmailDb, setSyncMeta } from '../src/servers/api/email/email-db';
+import { openEmailDb, setSyncMeta } from '../src/servers/sidecar/email/store';
 
 const userEmail = process.argv[2];
 if (!userEmail) {
diff --git a/src/server.tsx b/src/server.tsx
index 76758295..b7c36825 100644
--- a/src/server.tsx
+++ b/src/server.tsx
@@ -15,7 +15,6 @@ import officerWeb from './apps/officer-web/index.gen.html';
 import { startBrowserRelay } from './servers/api/browser/relay';
 import { registerSidecar, unregisterSidecar, handleSidecarMessage } from './servers/sidecar-registry';
 import './servers/api/chat/opencode/sidecar-server'; // subscribe to the opencode sidecar's port report
-import { broadcastEmailNew } from './servers/api/email/email';
 import type { SidecarRegistration } from './servers/sidecar/registration-protocol';
 import { toShellUsername } from './servers/data-path';
 
@@ -75,12 +74,6 @@ const sidecarWebsocket = {
         return;
       }
 
-      // Email IDLE watcher reporting new mail → push to that user's /email SSE clients.
-      if (msg.type === 'email:new' && typeof msg.userEmail === 'string') {
-        broadcastEmailNew(msg.userEmail);
-        return;
-      }
-
       const id = sidecarConnections.get(ws);
       if (id) {
         handleSidecarMessage(id, msg);
diff --git a/src/servers/api/email/router.ts b/src/servers/api/email/router.ts
new file mode 100644
index 00000000..34546c5e
--- /dev/null
+++ b/src/servers/api/email/router.ts
@@ -0,0 +1,22 @@
+import { createSidecarProxy } from '../../sidecar/create-proxy';
+
+// /api/email/* — auth, then forward to officer-email. No routes of its own and no mail knowledge: not a
+// message, not a folder, not an IMAP or SMTP credential.
+//
+// The mail store, the 14 HTTP routes, account CRUD, resync and IMAP validation all used to live in this
+// directory — ~1,800 lines against a 314-line sidecar, with the sidecar importing BACK into the platform
+// to reach them. They are the sidecar's now.
+//
+// A from-scratch mailbox sync holds a proxied request open well past the 60s idle default, hence the
+// timeout; it applies to the whole prefix because the proxy must not know which routes are slow.
+
+const proxy = createSidecarProxy({
+  name: 'email',
+  prefix: '/api/email',
+  timeoutSeconds: 1800,
+});
+
+export const emailRouter = proxy.router;
+
+/** Base URL of the sidecar's HTTP server, or null if it hasn't reported in yet. */
+export const getEmailServerUrl = proxy.getHttpUrl;
diff --git a/src/servers/channels/discord/handler.ts b/src/servers/channels/discord/handler.ts
index a10bb628..5a574d58 100644
--- a/src/servers/channels/discord/handler.ts
+++ b/src/servers/channels/discord/handler.ts
@@ -6,7 +6,7 @@ import { chunkMessage } from './chunker';
 import { listChatModels } from '@@/api/chat/list-models';
 import { enqueueJob } from '../../queue/init';
 import { readJob } from '@@/queue/storage';
-import { openUserEmailDb } from '@@/api/email/email-db';
+import { openUserEmailDb } from '@@/sidecar/email/store';
 import type { ModelInfo } from '@@/api/chat/types';
 import { toShellUsername } from '@@/data-path';
 
diff --git a/src/servers/channels/telegram/handler.ts b/src/servers/channels/telegram/handler.ts
index 9851b517..95e72ee3 100644
--- a/src/servers/channels/telegram/handler.ts
+++ b/src/servers/channels/telegram/handler.ts
@@ -7,7 +7,7 @@ import { getTelegramBot } from './bot';
 import { listChatModels } from '@@/api/chat/list-models';
 import { enqueueJob } from '../../queue/init';
 import { readJob } from '@@/queue/storage';
-import { openUserEmailDb } from '@@/api/email/email-db';
+import { openUserEmailDb } from '@@/sidecar/email/store';
 import type { ModelInfo } from '@@/api/chat/types';
 import { toShellUsername } from '@@/data-path';
 
diff --git a/src/servers/channels/whatsapp/handler.ts b/src/servers/channels/whatsapp/handler.ts
index f9d0d6ac..57e1dd09 100644
--- a/src/servers/channels/whatsapp/handler.ts
+++ b/src/servers/channels/whatsapp/handler.ts
@@ -6,7 +6,7 @@ import { getWhatsAppClient } from './bot';
 import { listChatModels } from '@@/api/chat/list-models';
 import { enqueueJob } from '../../queue/init';
 import { readJob } from '@@/queue/storage';
-import { openUserEmailDb } from '@@/api/email/email-db';
+import { openUserEmailDb } from '@@/sidecar/email/store';
 import type { ModelInfo } from '@@/api/chat/types';
 import { toShellUsername } from '@@/data-path';
 
diff --git a/src/servers/hono.ts b/src/servers/hono.ts
index 0b121fa4..9ada939b 100644
--- a/src/servers/hono.ts
+++ b/src/servers/hono.ts
@@ -36,7 +36,7 @@ import './api/vault/sidecar-server'; // side-effect: capture the officer-vault r
 import { dockRouter } from './api/dock/dock';
 import { integrationsRouter, googleCallbackHandler } from './api/integrations/integrations';
 import { queueRouter } from './api/queue/queue';
-import { emailRouter } from './api/email/email';
+import { emailRouter } from './api/email/router';
 import { channelsRouter } from './channels/routes';
 import { browserRouter } from './api/browser/router';
 import { desktopRouter } from './api/desktop/rest';
diff --git a/src/servers/queue/handlers/email-sync.ts b/src/servers/queue/handlers/email-sync.ts
index 2a37e70a..cdbef18b 100644
--- a/src/servers/queue/handlers/email-sync.ts
+++ b/src/servers/queue/handlers/email-sync.ts
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto';
 import type { JobHandler } from '../types';
 import { PermanentError } from '../types';
 import { registerHandler } from '../handler-registry';
-import { openEmailDb, upsertFromRawEml } from '../../api/email/email-db';
+import { openEmailDb, upsertFromRawEml } from '../../sidecar/email/store';
 import { refreshGoogleAccessToken } from '../../api/integrations/google-auth';
 import {
   getEmailAccount,
diff --git a/src/servers/queue/handlers/gmail-sync.ts b/src/servers/queue/handlers/gmail-sync.ts
index d43e7bec..028cc582 100644
--- a/src/servers/queue/handlers/gmail-sync.ts
+++ b/src/servers/queue/handlers/gmail-sync.ts
@@ -2,7 +2,7 @@ import type { Database } from 'bun:sqlite';
 import { createHash } from 'node:crypto';
 import { type JobHandler, PermanentError } from '../types';
 import { registerHandler } from '../handler-registry';
-import { openEmailDb, upsertFromRawEml, setSyncMeta, getSyncMeta } from '../../api/email/email-db';
+import { openEmailDb, upsertFromRawEml, setSyncMeta, getSyncMeta } from '../../sidecar/email/store';
 import {
   getUserByEmail,
   getUserIntegration,
diff --git a/src/servers/api/email/accounts.ts b/src/servers/sidecar/email/accounts.ts
similarity index 98%
rename from src/servers/api/email/accounts.ts
rename to src/servers/sidecar/email/accounts.ts
index 0d00cd3f..47696f1b 100644
--- a/src/servers/api/email/accounts.ts
+++ b/src/servers/sidecar/email/accounts.ts
@@ -7,10 +7,10 @@ import {
   deleteEmailAccount,
   updateEmailAccountStatus,
 } from 'officerdb';
-import { getValidGoogleAccessToken } from '../integrations/google-auth';
+import { getValidGoogleAccessToken } from '../../api/integrations/google-auth';
 import { validateImapConnection } from './imap-validate';
 import { enqueueJob, listAllJobs } from '../../queue/init';
-import { openEmailDb, getSyncMeta } from './email-db';
+import { openEmailDb, getSyncMeta } from './store';
 import { performResync } from './resync';
 
 type CreateAccountBody = {
diff --git a/src/servers/sidecar/email/email-cron.ts b/src/servers/sidecar/email/email-cron.ts
index c2f4463d..4b9a0ceb 100644
--- a/src/servers/sidecar/email/email-cron.ts
+++ b/src/servers/sidecar/email/email-cron.ts
@@ -1,5 +1,5 @@
 import { getAllSyncedAccounts, getUserById } from 'officerdb';
-import { performResync } from '../../api/email/resync';
+import { performResync } from './resync';
 
 const INTERVAL_MS = 10 * 60 * 1000; // 10 minutes
 
diff --git a/src/servers/sidecar/email/email-idle.ts b/src/servers/sidecar/email/email-idle.ts
index d2438879..48ac6b74 100644
--- a/src/servers/sidecar/email/email-idle.ts
+++ b/src/servers/sidecar/email/email-idle.ts
@@ -1,6 +1,6 @@
 import type { ImapFlow } from 'imapflow';
 import { getAllSyncedAccounts, getUserById } from 'officerdb';
-import { performResync } from '../../api/email/resync';
+import { performResync } from './resync';
 import { getValidGoogleAccessToken } from '../../api/integrations/google-auth';
 
 // Real-time email via IMAP IDLE: one persistent connection per account. imapflow auto-enters IDLE
diff --git a/src/servers/sidecar/email/http.ts b/src/servers/sidecar/email/http.ts
new file mode 100644
index 00000000..a41b5d68
--- /dev/null
+++ b/src/servers/sidecar/email/http.ts
@@ -0,0 +1,48 @@
+import { Hono } from 'hono';
+import { emailRouter } from './routes';
+
+// The email sidecar's own listener. `/api/email/*` on the platform is a proxy onto this — the platform
+// authenticates the owner, injects X-Officer-User, and forwards without reading the body.
+//
+// The routes moved here verbatim, Hono and all, so this file's only real job is to reconstruct the two
+// things the platform's middleware used to provide: the authenticated user on the context, and an error
+// handler that turns thrown custom-errors into status codes.
+
+type HonoVariables = { user: { id: number }; body: Record<string, unknown>; origin: string };
+
+const app = new Hono<{ Variables: HonoVariables }>();
+
+// The platform's userMiddleware set `user` from the JWT. Here it comes from the header the proxy injects —
+// trusted because this server binds loopback only and nothing else can reach it.
+app.use('*', async (ctx, next) => {
+  const id = Number(ctx.req.header('X-Officer-User'));
+  if (!Number.isFinite(id) || id <= 0) return ctx.json({ error: 'missing X-Officer-User' }, 401);
+  ctx.set('user', { id } as never);
+  await next();
+});
+
+app.route('/', emailRouter as never);
+
+// custom-errors carry a `status`; anything else is a 500 with no detail leaked to the caller.
+app.onError((err, ctx) => {
+  const status = (err as { status?: number }).status;
+  if (typeof status === 'number' && status >= 400 && status < 600) {
+    return ctx.json({ error: err.message }, status as 400);
+  }
+  console.error('[email] unhandled error', err);
+  return ctx.json({ error: 'internal error' }, 500);
+});
+
+export function startEmailServer(): number {
+  const server = Bun.serve({
+    port: 0,
+    hostname: '127.0.0.1',
+    // A from-scratch mailbox sync answers slowly; the platform proxy extends its own side to match.
+    idleTimeout: 255,
+    fetch: app.fetch,
+  });
+  const port = server.port;
+  if (port == null) throw new Error('[email] failed to acquire a port');
+  console.log(`[email] http server listening on http://127.0.0.1:${port}`);
+  return port;
+}
diff --git a/src/servers/api/email/imap-validate.ts b/src/servers/sidecar/email/imap-validate.ts
similarity index 100%
rename from src/servers/api/email/imap-validate.ts
rename to src/servers/sidecar/email/imap-validate.ts
diff --git a/src/servers/sidecar/email/index.ts b/src/servers/sidecar/email/index.ts
index ef40d73a..f83a9b82 100644
--- a/src/servers/sidecar/email/index.ts
+++ b/src/servers/sidecar/email/index.ts
@@ -2,6 +2,8 @@ import type { SidecarEvent } from '../protocol';
 import type { Job, EnqueueParams } from '../../queue/types';
 import { initEmailCron, stopEmailCron } from './email-cron';
 import { initEmailIdle, stopEmailIdle } from './email-idle';
+import { broadcastEmailNew } from './routes';
+import { startEmailServer } from './http';
 import { createSidecarConnector } from '../connect';
 
 const API_URL = process.env.API_URL ?? `ws://127.0.0.1:${process.env.PORT ?? '5000'}`;
@@ -73,6 +75,12 @@ function handleCommand(cmd: Record<string, unknown>, reply: ReplyFn) {
   }
 }
 
+// ── HTTP server ──
+//
+// Started before the registration socket so the port is known by the time we announce it. `/api/email/*`
+// on the platform is a proxy onto this.
+const serverPort = startEmailServer();
+
 // ── Connect to API server ──
 
 const connection = createSidecarConnector({
@@ -83,11 +91,16 @@ const connection = createSidecarConnector({
     handleCommand(cmd as Record<string, unknown>, reply as ReplyFn);
   },
   onConnected() {
+    // The platform forgets the port when the socket drops, and this listener outlives an officer restart,
+    // so re-announce on every reconnect.
+    connection.send({ type: 'email:server', port: serverPort });
     // Start email cron once connected (so queue commands can reach API server)
     initEmailCron();
     // Real-time push via IMAP IDLE; the cron above is the slow backstop. On new mail, tell the API
     // server so it can push an SSE event to that user's open /email page.
-    initEmailIdle((userEmail) => connection.send({ type: 'email:new', userEmail }));
+    // Straight to this process's own SSE clients — the /email/events stream lives here now, so the
+    // round trip out to officer and back (the `email:new` wire event) is gone.
+    initEmailIdle((userEmail) => broadcastEmailNew(userEmail));
   },
 });
 
diff --git a/src/servers/api/email/resync.ts b/src/servers/sidecar/email/resync.ts
similarity index 99%
rename from src/servers/api/email/resync.ts
rename to src/servers/sidecar/email/resync.ts
index 32433915..45eb7a2a 100644
--- a/src/servers/api/email/resync.ts
+++ b/src/servers/sidecar/email/resync.ts
@@ -7,7 +7,7 @@ import {
   updateEmailAccountSyncMeta,
 } from 'officerdb';
 import { refreshGoogleAccessToken } from '@@/api/integrations/google-auth';
-import { openEmailDb, getSyncMeta, setSyncMeta, upsertFromRawEml } from './email-db';
+import { openEmailDb, getSyncMeta, setSyncMeta, upsertFromRawEml } from './store';
 import {
   type GmailCredentials,
   loadGmailCredentials,
diff --git a/src/servers/api/email/email.ts b/src/servers/sidecar/email/routes.ts
similarity index 99%
rename from src/servers/api/email/email.ts
rename to src/servers/sidecar/email/routes.ts
index 3066fa4e..667fb077 100644
--- a/src/servers/api/email/email.ts
+++ b/src/servers/sidecar/email/routes.ts
@@ -6,7 +6,7 @@ import { createRouter } from '../../create-router';
 import * as errors from '@@/custom-errors';
 import { getEmailAttachmentCacheDir } from '@@/data-path';
 import { getEmailAccounts } from 'officerdb';
-import { openEmailDb, openUserEmailDb, rowToSummary, getSyncMeta, searchEmails } from './email-db';
+import { openEmailDb, openUserEmailDb, rowToSummary, getSyncMeta, searchEmails } from './store';
 import { accountsRouter } from './accounts';
 
 export const emailRouter = createRouter();
diff --git a/src/servers/api/email/email-db.ts b/src/servers/sidecar/email/store.ts
similarity index 100%
rename from src/servers/api/email/email-db.ts
rename to src/servers/sidecar/email/store.ts
diff --git a/src/servers/sidecar/protocol.ts b/src/servers/sidecar/protocol.ts
index c21dd01c..8ae6ff26 100644
--- a/src/servers/sidecar/protocol.ts
+++ b/src/servers/sidecar/protocol.ts
@@ -50,8 +50,7 @@ export type SidecarEvent =
   | { type: 'vnc:stopped'; id: string }
   | { type: 'vnc:status'; id: string; session: VncSessionInfo | null }
   | { type: 'vnc:error'; id: string; error: string }
-  // Email
-  | { type: 'email:new'; userEmail: string }
+  // Email — new mail no longer crosses this socket; the sidecar owns the SSE stream and pushes directly
   // OpenCode — the sidecar reports where its `opencode serve` is listening (random port) on connect
   | { type: 'opencode:server'; port: number }
   // OpenCode turn streaming (analog of claude:*): spawned ack, per-message stream, session id report
@@ -78,6 +77,8 @@ export type SidecarEvent =
   | { type: 'wallet:server'; port: number }
   // PTY — the sidecar reports where its terminal HTTP/WS server is listening (random port) on connect
   | { type: 'pty:server'; port: number }
+  // Email — the sidecar reports where its mail HTTP server is listening (random port) on connect
+  | { type: 'email:server'; port: number }
   // Generic
   | { type: 'error'; id?: string; error: string };