email: move the mail store and every route into the sidecar

Email was the one sidecar built inside out. The platform held ~1,800 lines — the per-account SQLite store, all 14 HTTP routes, account CRUD, resync, IMAP validation — while the 314-line sidecar was a scheduler that reached BACK into the platform to do anything (`import { performResync } from '../../api/email/resync'`). The sidecar now serves its own HTTP listener and announces `email:server`, and /api/email/* on the platform is createSidecarProxy like every other one: 1,801 lines down to 22, with no mail knowledge left in it — not a message, not a folder, not a credential. The routes moved verbatim, Hono and all. http.ts only reconstructs what the platform's middleware used to provide: `user` on the context, from the X-Officer-User header the proxy injects (trusted because this server binds loopback), and an error handler that turns custom-errors into status codes. The /email/events SSE stream went with them, which removes a whole round trip: the IDLE watcher used to send `email:new` over the registration socket so the platform could push to its SSE clients. Those clients are here now, so it calls broadcastEmailNew in-process and `email:new` is gone from the wire protocol. DELIBERATELY NOT DONE YET, and left backwards on purpose rather than half-moved: - The two sync handlers (email-sync 381 lines, gmail-sync 712) still run in the platform's queue and now import the store from its new home — a platform → sidecar import, which is the wrong direction and is temporary. Moving them is option (A) from the plan: the sidecar schedules its own syncs, independent of the platform Jobs list. - accounts.ts still imports queue/init to enqueue a sync and to report sync status, and index.ts still carries the queue-over-WS shim that inversion needs. - The three channel handlers still open the mail store directly rather than asking over HTTP. Two things worth knowing while testing: a from-scratch sync holds a proxied request open well past the 60s idle default, hence timeoutSeconds on the proxy; and `gmail-sync` is hardcoded in all three channel handlers even though the only account is provider=gmail with auth_type=password, which routes to IMAP — so "sync emails" from a chat channel is almost certainly already broken, and folds into the next stage. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-31 12:10:35 +00:00
parent aaf0161620
commit af56eb36ff
20 changed files with 101 additions and 24 deletions
@@ -0,0 +1,309 @@
+import { createHash } from 'node:crypto';
+import {
+  getUserIntegration,
+  upsertUserIntegration,
+  getServerIntegration,
+  getEmailAccount,
+  updateEmailAccountSyncMeta,
+} from 'officerdb';
+import { refreshGoogleAccessToken } from '@@/api/integrations/google-auth';
+import { openEmailDb, getSyncMeta, setSyncMeta, upsertFromRawEml } from './store';
+import {
+  type GmailCredentials,
+  loadGmailCredentials,
+  gmailApiSync,
+} from '../../queue/handlers/gmail-sync';
+
+export type ResyncResult = { saved: number; skipped: number; errors: number };
+
+// ── Gmail resync (REST API, history-based) ──
+
+async function refreshCredentials(creds: GmailCredentials): Promise<GmailCredentials> {
+  if (!creds.accessToken || !creds.refreshToken) return creds;
+
+  const tokenExpired = !creds.expiresAt || creds.expiresAt < Date.now() + 60_000;
+  if (!tokenExpired) return creds;
+
+  console.log('[resync] Refreshing OAuth token');
+  const refreshed = await refreshGoogleAccessToken(creds.refreshToken);
+
+  const userGoogle = await getUserIntegration(creds.userId, 'google');
+  const existingConfig = (userGoogle?.config as Record<string, unknown>) ?? {};
+  const serverGoogle = await getServerIntegration('google');
+  await upsertUserIntegration({
+    userId: creds.userId,
+    provider: 'google',
+    serverIntegrationId: serverGoogle?.id,
+    config: { ...existingConfig, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
+  });
+
+  return { ...creds, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt };
+}
+
+async function gmailResync(userEmail: string, accountEmail: string): Promise<ResyncResult> {
+  let creds = await loadGmailCredentials(userEmail);
+  if (!creds.accessToken) {
+    throw new Error('OAuth not configured — connect Google in Settings → Integrations for resyncs');
+  }
+
+  creds = await refreshCredentials(creds);
+
+  const db = openEmailDb(userEmail, accountEmail);
+  try {
+    const result = await gmailApiSync({ creds, db });
+
+    setSyncMeta(db, 'last_sync_date', new Date().toISOString().split('T')[0]!);
+    setSyncMeta(db, 'last_sync_at', new Date().toISOString());
+
+    console.log(`[resync] Gmail done: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`);
+    return result;
+  } finally {
+    db.close();
+  }
+}
+
+// ── Generic IMAP resync ──
+
+type ImapAccountInfo = {
+  id: number;
+  userId: number;
+  email: string;
+  imapHost: string;
+  imapPort: number;
+  imapSecure: boolean;
+  provider: string;
+  authType: string;
+  credentials: Record<string, unknown>;
+};
+
+type FolderInfo = {
+  specialUse?: string;
+  path: string;
+  flags: Set<string>;
+  status?: { uidNext?: number; uidValidity?: number };
+};
+
+const SPECIAL_USE_LABEL_MAP: Record<string, string> = {
+  '\\Inbox': 'inbox',
+  '\\Sent': 'sent',
+  '\\Drafts': 'draft',
+  '\\Flagged': 'starred',
+  '\\Trash': 'trash',
+  '\\Junk': 'spam',
+  '\\All': 'archive',
+};
+
+const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
+
+function shouldSkipFolder(folder: FolderInfo): boolean {
+  if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
+  if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
+  return false;
+}
+
+function folderToLabel(folder: FolderInfo): string {
+  if (folder.specialUse && SPECIAL_USE_LABEL_MAP[folder.specialUse]) {
+    return SPECIAL_USE_LABEL_MAP[folder.specialUse]!;
+  }
+  if (folder.path === 'INBOX') return 'inbox';
+  return folder.path.toLowerCase();
+}
+
+function messageIdToStableId(raw: string): string | null {
+  const match = raw.match(/^Message-Id:\s*<?([^>\s]+)>?/im);
+  if (!match?.[1]) return null;
+  return createHash('sha1').update(match[1]).digest('hex').slice(0, 16);
+}
+
+async function resolveImapAuth(
+  account: ImapAccountInfo,
+): Promise<{ user: string; pass?: string; accessToken?: string }> {
+  if (account.authType !== 'oauth') {
+    return { user: account.email, pass: account.credentials.password as string };
+  }
+
+  const userGoogle = await getUserIntegration(account.userId, 'google');
+  const config = userGoogle?.config as Record<string, unknown> | undefined;
+  if (!config?.refreshToken) throw new Error('Google OAuth not configured — reconnect your Google account');
+
+  const expiresAt = config.expiresAt as number | undefined;
+  const tokenExpired = !expiresAt || expiresAt < Date.now() + 60_000;
+
+  if (!tokenExpired && config.accessToken) {
+    return { user: account.email, accessToken: config.accessToken as string };
+  }
+
+  const refreshed = await refreshGoogleAccessToken(config.refreshToken as string);
+  const serverGoogle = await getServerIntegration('google');
+  await upsertUserIntegration({
+    userId: account.userId,
+    provider: 'google',
+    serverIntegrationId: serverGoogle?.id,
+    config: { ...config, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
+  });
+
+  return { user: account.email, accessToken: refreshed.accessToken };
+}
+
+async function imapResync(account: ImapAccountInfo, userEmail: string): Promise<ResyncResult> {
+  const { ImapFlow } = await import('imapflow');
+
+  const imapAuth = await resolveImapAuth(account);
+
+  const freshAccount = await getEmailAccount(account.id);
+  if (!freshAccount) throw new Error(`Email account ${account.id} not found`);
+  const syncMeta = (freshAccount.syncMeta ?? {}) as Record<string, unknown>;
+
+  const db = openEmailDb(userEmail, account.email);
+  const existingIds = new Set<string>();
+  const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
+  for (const row of rows) existingIds.add(row.id);
+
+  let saved = 0;
+  let skipped = 0;
+  let errors = 0;
+
+  const client = new ImapFlow({
+    host: account.imapHost,
+    port: account.imapPort,
+    secure: account.imapSecure,
+    auth: imapAuth,
+    logger: false,
+    socketTimeout: 5 * 60 * 1000,
+  });
+
+  try {
+    await client.connect();
+    console.log('[resync] IMAP connected');
+
+    const folders = (await client.list({ statusQuery: { uidNext: true, uidValidity: true } })) as FolderInfo[];
+
+    for (const folder of folders) {
+      if (shouldSkipFolder(folder)) continue;
+
+      const uidValidityKey = `imap_uidvalidity:${folder.path}`;
+      const lastUidKey = `imap_lastuid:${folder.path}`;
+      const storedUidValidity = syncMeta[uidValidityKey] as string | undefined;
+      const storedLastUid = syncMeta[lastUidKey] as string | undefined;
+      const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
+      const uidNext = folder.status?.uidNext ?? 0;
+      const lastUid = storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
+
+      if (lastUid > 0 && uidNext <= lastUid + 1) continue;
+
+      let lock;
+      try {
+        lock = await client.getMailboxLock(folder.path);
+      } catch {
+        continue;
+      }
+
+      try {
+        const mailbox = client.mailbox;
+        if (!mailbox) continue;
+
+        const mbUidValidity = String(mailbox.uidValidity);
+        const effectiveLastUid = storedUidValidity === mbUidValidity ? lastUid : 0;
+        let maxUid = effectiveLastUid;
+        const label = folderToLabel(folder);
+        const range = effectiveLastUid > 0 ? `${effectiveLastUid + 1}:*` : '1:*';
+
+        try {
+          const fetchOpts: Record<string, unknown> = { source: true, uid: true };
+          if (account.provider === 'gmail') fetchOpts.labels = true;
+
+          for await (const msg of client.fetch(range, fetchOpts, { uid: true })) {
+            if (msg.uid <= effectiveLastUid) continue;
+            maxUid = Math.max(maxUid, msg.uid);
+
+            if (!msg.source) { errors++; continue; }
+
+            const raw = msg.source.toString('utf-8');
+            const id = messageIdToStableId(raw);
+            if (!id) { errors++; continue; }
+            if (existingIds.has(id)) { skipped++; continue; }
+
+            try {
+              upsertFromRawEml({ db, id, raw, integration: account.provider, emailAccount: account.email, labels: [label] });
+              existingIds.add(id);
+              saved++;
+            } catch {
+              errors++;
+            }
+          }
+        } catch (fetchErr) {
+          const errMsg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
+          if (!errMsg.includes('Nothing to fetch')) {
+            console.log(`[resync] Fetch error in ${folder.path}: ${errMsg}`);
+            errors++;
+          }
+        }
+
+        syncMeta[uidValidityKey] = mbUidValidity;
+        if (maxUid > effectiveLastUid) {
+          syncMeta[lastUidKey] = String(maxUid);
+        }
+      } finally {
+        lock.release();
+      }
+    }
+
+    await client.logout().catch(() => {});
+  } catch (err) {
+    await client.logout().catch(() => {});
+    throw err;
+  } finally {
+    db.close();
+  }
+
+  syncMeta.last_sync_at = new Date().toISOString();
+  await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
+
+  console.log(`[resync] IMAP done: saved ${saved}, skipped ${skipped}, errors ${errors}`);
+  return { saved, skipped, errors };
+}
+
+// ── Public API ──
+
+type ResyncParams = {
+  accountId: number;
+  userEmail: string;
+  userId: number;
+};
+
+// Coalesce concurrent resyncs of the same account within this process (IDLE + cron + manual can all
+// fire) — a caller arriving mid-resync just awaits the one already running.
+const resyncInFlight = new Map<number, Promise<ResyncResult>>();
+
+export function performResync(params: ResyncParams): Promise<ResyncResult> {
+  const running = resyncInFlight.get(params.accountId);
+  if (running) return running;
+  const p = doResync(params).finally(() => resyncInFlight.delete(params.accountId));
+  resyncInFlight.set(params.accountId, p);
+  return p;
+}
+
+async function doResync({ accountId, userEmail, userId }: ResyncParams): Promise<ResyncResult> {
+  const account = await getEmailAccount(accountId);
+  if (!account || account.userId !== userId) throw new Error('Account not found');
+
+  // Gmail API resync needs OAuth; a gmail account authed with an app password resyncs over IMAP.
+  if (account.provider === 'gmail' && account.authType === 'oauth') {
+    return gmailResync(userEmail, account.email);
+  }
+
+  return imapResync(
+    {
+      id: account.id,
+      userId: account.userId,
+      email: account.email,
+      imapHost: account.imapHost,
+      imapPort: account.imapPort,
+      imapSecure: account.imapSecure,
+      provider: account.provider,
+      authType: account.authType,
+      credentials: account.credentials as Record<string, unknown>,
+    },
+    userEmail,
+  );
+}