inline email resync instead of job queue, refresh list on completion

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-22 17:07:06 +00:00
parent ea8113e61d
commit b3f1d10bc1
7 changed files with 367 additions and 83 deletions
@@ -10,6 +10,8 @@ import {
 } from 'officerdb';
 import { validateImapConnection } from './imap-validate';
 import { enqueueJob, listAllJobs } from '../../queue/init';
+import { openEmailDb, getSyncMeta } from './email-db';
+import { performResync } from './resync';

 type CreateAccountBody = {
  provider: string;
@@ -136,7 +138,34 @@ accountsRouter.post('/:id/sync', async (ctx) => {
  if (account.status === 'queued') throw BAD_REQUEST('Sync is already queued');
  if (account.status === 'syncing') throw BAD_REQUEST('Account is already syncing');

-  // Resolve auth before enqueueing
+  // Determine if this is a first sync or resync
+  let isFirstSync = true;
+  if (account.provider === 'gmail') {
+    const db = openEmailDb(user.email);
+    try {
+      isFirstSync = !getSyncMeta(db, 'last_sync_at');
+    } finally {
+      db.close();
+    }
+  } else {
+    const syncMeta = (account.syncMeta ?? {}) as Record<string, unknown>;
+    isFirstSync = !syncMeta.last_sync_at;
+  }
+
+  // Resync: call directly, no job queue
+  if (!isFirstSync) {
+    await updateEmailAccountStatus(id, 'syncing');
+    try {
+      const result = await performResync({ accountId: id, userEmail: user.email, userId: user.id });
+      await updateEmailAccountStatus(id, 'synced');
+      return ctx.json({ ok: true, saved: result.saved });
+    } catch (err) {
+      await updateEmailAccountStatus(id, 'synced').catch(() => {});
+      throw err;
+    }
+  }
+
+  // First sync: enqueue a job
  const authResult = await resolveAuth(
    user.id,
    account.authType,
@@ -145,7 +174,6 @@ accountsRouter.post('/:id/sync', async (ctx) => {
  );
  if (!authResult.ok) throw BAD_REQUEST(authResult.error);

-  // Set status immediately so the UI reflects the queued state
  await updateEmailAccountStatus(id, 'queued');

  const jobType = account.provider === 'gmail' ? 'gmail-sync' : 'email-sync';
@@ -0,0 +1,296 @@
+import { createHash } from 'node:crypto';
+import {
+  getUserIntegration,
+  upsertUserIntegration,
+  getServerIntegration,
+  getEmailAccount,
+  updateEmailAccountSyncMeta,
+} from 'officerdb';
+import { refreshGoogleAccessToken } from '@@/api/integrations/google-auth';
+import { openEmailDb, getSyncMeta, setSyncMeta, upsertFromRawEml } from './email-db';
+import {
+  type GmailCredentials,
+  loadGmailCredentials,
+  gmailApiSync,
+} from '../../queue/handlers/gmail-sync';
+
+export type ResyncResult = { saved: number; skipped: number; errors: number };
+
+// ── Gmail resync (REST API, history-based) ──
+
+async function refreshCredentials(creds: GmailCredentials): Promise<GmailCredentials> {
+  if (!creds.accessToken || !creds.refreshToken) return creds;
+
+  const tokenExpired = !creds.expiresAt || creds.expiresAt < Date.now() + 60_000;
+  if (!tokenExpired) return creds;
+
+  console.log('[resync] Refreshing OAuth token');
+  const refreshed = await refreshGoogleAccessToken(creds.refreshToken);
+
+  const userGoogle = await getUserIntegration(creds.userId, 'google');
+  const existingConfig = (userGoogle?.config as Record<string, unknown>) ?? {};
+  const serverGoogle = await getServerIntegration('google');
+  await upsertUserIntegration({
+    userId: creds.userId,
+    provider: 'google',
+    serverIntegrationId: serverGoogle?.id,
+    config: { ...existingConfig, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
+  });
+
+  return { ...creds, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt };
+}
+
+async function gmailResync(userEmail: string): Promise<ResyncResult> {
+  let creds = await loadGmailCredentials(userEmail);
+  if (!creds.accessToken) {
+    throw new Error('OAuth not configured — connect Google in Settings → Integrations for resyncs');
+  }
+
+  creds = await refreshCredentials(creds);
+
+  const db = openEmailDb(userEmail);
+  try {
+    const result = await gmailApiSync({ creds, db });
+
+    setSyncMeta(db, 'last_sync_date', new Date().toISOString().split('T')[0]!);
+    setSyncMeta(db, 'last_sync_at', new Date().toISOString());
+
+    console.log(`[resync] Gmail done: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`);
+    return result;
+  } finally {
+    db.close();
+  }
+}
+
+// ── Generic IMAP resync ──
+
+type ImapAccountInfo = {
+  id: number;
+  userId: number;
+  email: string;
+  imapHost: string;
+  imapPort: number;
+  imapSecure: boolean;
+  provider: string;
+  authType: string;
+  credentials: Record<string, unknown>;
+};
+
+type FolderInfo = {
+  specialUse?: string;
+  path: string;
+  flags: Set<string>;
+  status?: { uidNext?: number; uidValidity?: number };
+};
+
+const SPECIAL_USE_LABEL_MAP: Record<string, string> = {
+  '\\Inbox': 'inbox',
+  '\\Sent': 'sent',
+  '\\Drafts': 'draft',
+  '\\Flagged': 'starred',
+  '\\Trash': 'trash',
+  '\\Junk': 'spam',
+  '\\All': 'archive',
+};
+
+const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
+
+function shouldSkipFolder(folder: FolderInfo): boolean {
+  if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
+  if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
+  return false;
+}
+
+function folderToLabel(folder: FolderInfo): string {
+  if (folder.specialUse && SPECIAL_USE_LABEL_MAP[folder.specialUse]) {
+    return SPECIAL_USE_LABEL_MAP[folder.specialUse]!;
+  }
+  if (folder.path === 'INBOX') return 'inbox';
+  return folder.path.toLowerCase();
+}
+
+function messageIdToStableId(raw: string): string | null {
+  const match = raw.match(/^Message-Id:\s*<?([^>\s]+)>?/im);
+  if (!match?.[1]) return null;
+  return createHash('sha1').update(match[1]).digest('hex').slice(0, 16);
+}
+
+async function resolveImapAuth(
+  account: ImapAccountInfo,
+): Promise<{ user: string; pass?: string; accessToken?: string }> {
+  if (account.authType !== 'oauth') {
+    return { user: account.email, pass: account.credentials.password as string };
+  }
+
+  const userGoogle = await getUserIntegration(account.userId, 'google');
+  const config = userGoogle?.config as Record<string, unknown> | undefined;
+  if (!config?.refreshToken) throw new Error('Google OAuth not configured — reconnect your Google account');
+
+  const expiresAt = config.expiresAt as number | undefined;
+  const tokenExpired = !expiresAt || expiresAt < Date.now() + 60_000;
+
+  if (!tokenExpired && config.accessToken) {
+    return { user: account.email, accessToken: config.accessToken as string };
+  }
+
+  const refreshed = await refreshGoogleAccessToken(config.refreshToken as string);
+  const serverGoogle = await getServerIntegration('google');
+  await upsertUserIntegration({
+    userId: account.userId,
+    provider: 'google',
+    serverIntegrationId: serverGoogle?.id,
+    config: { ...config, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
+  });
+
+  return { user: account.email, accessToken: refreshed.accessToken };
+}
+
+async function imapResync(account: ImapAccountInfo, userEmail: string): Promise<ResyncResult> {
+  const { ImapFlow } = await import('imapflow');
+
+  const imapAuth = await resolveImapAuth(account);
+
+  const freshAccount = await getEmailAccount(account.id);
+  if (!freshAccount) throw new Error(`Email account ${account.id} not found`);
+  const syncMeta = (freshAccount.syncMeta ?? {}) as Record<string, unknown>;
+
+  const db = openEmailDb(userEmail);
+  const existingIds = new Set<string>();
+  const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
+  for (const row of rows) existingIds.add(row.id);
+
+  let saved = 0;
+  let skipped = 0;
+  let errors = 0;
+
+  const client = new ImapFlow({
+    host: account.imapHost,
+    port: account.imapPort,
+    secure: account.imapSecure,
+    auth: imapAuth,
+    logger: false,
+    socketTimeout: 5 * 60 * 1000,
+  });
+
+  try {
+    await client.connect();
+    console.log('[resync] IMAP connected');
+
+    const folders = (await client.list({ statusQuery: { uidNext: true, uidValidity: true } })) as FolderInfo[];
+
+    for (const folder of folders) {
+      if (shouldSkipFolder(folder)) continue;
+
+      const uidValidityKey = `imap_uidvalidity:${folder.path}`;
+      const lastUidKey = `imap_lastuid:${folder.path}`;
+      const storedUidValidity = syncMeta[uidValidityKey] as string | undefined;
+      const storedLastUid = syncMeta[lastUidKey] as string | undefined;
+      const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
+      const uidNext = folder.status?.uidNext ?? 0;
+      const lastUid = storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
+
+      if (lastUid > 0 && uidNext <= lastUid + 1) continue;
+
+      let lock;
+      try {
+        lock = await client.getMailboxLock(folder.path);
+      } catch {
+        continue;
+      }
+
+      try {
+        const mailbox = client.mailbox;
+        if (!mailbox) continue;
+
+        const mbUidValidity = String(mailbox.uidValidity);
+        const effectiveLastUid = storedUidValidity === mbUidValidity ? lastUid : 0;
+        let maxUid = effectiveLastUid;
+        const label = folderToLabel(folder);
+        const range = effectiveLastUid > 0 ? `${effectiveLastUid + 1}:*` : '1:*';
+
+        try {
+          const fetchOpts: Record<string, unknown> = { source: true, uid: true };
+          if (account.provider === 'gmail') fetchOpts.labels = true;
+
+          for await (const msg of client.fetch(range, fetchOpts, { uid: true })) {
+            if (msg.uid <= effectiveLastUid) continue;
+            maxUid = Math.max(maxUid, msg.uid);
+
+            if (!msg.source) { errors++; continue; }
+
+            const raw = msg.source.toString('utf-8');
+            const id = messageIdToStableId(raw);
+            if (!id) { errors++; continue; }
+            if (existingIds.has(id)) { skipped++; continue; }
+
+            try {
+              upsertFromRawEml({ db, id, raw, integration: account.provider, emailAccount: account.email, labels: [label] });
+              existingIds.add(id);
+              saved++;
+            } catch {
+              errors++;
+            }
+          }
+        } catch (fetchErr) {
+          const errMsg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
+          if (!errMsg.includes('Nothing to fetch')) {
+            console.log(`[resync] Fetch error in ${folder.path}: ${errMsg}`);
+            errors++;
+          }
+        }
+
+        syncMeta[uidValidityKey] = mbUidValidity;
+        if (maxUid > effectiveLastUid) {
+          syncMeta[lastUidKey] = String(maxUid);
+        }
+      } finally {
+        lock.release();
+      }
+    }
+
+    await client.logout().catch(() => {});
+  } catch (err) {
+    await client.logout().catch(() => {});
+    throw err;
+  } finally {
+    db.close();
+  }
+
+  syncMeta.last_sync_at = new Date().toISOString();
+  await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
+
+  console.log(`[resync] IMAP done: saved ${saved}, skipped ${skipped}, errors ${errors}`);
+  return { saved, skipped, errors };
+}
+
+// ── Public API ──
+
+type ResyncParams = {
+  accountId: number;
+  userEmail: string;
+  userId: number;
+};
+
+export async function performResync({ accountId, userEmail, userId }: ResyncParams): Promise<ResyncResult> {
+  const account = await getEmailAccount(accountId);
+  if (!account || account.userId !== userId) throw new Error('Account not found');
+
+  if (account.provider === 'gmail') {
+    return gmailResync(userEmail);
+  }
+
+  return imapResync(
+    {
+      id: account.id,
+      userId: account.userId,
+      email: account.email,
+      imapHost: account.imapHost,
+      imapPort: account.imapPort,
+      imapSecure: account.imapSecure,
+      provider: account.provider,
+      authType: account.authType,
+      credentials: account.credentials as Record<string, unknown>,
+    },
+    userEmail,
+  );
+}