wip: email sync via imap with status tracking and auto cron

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/servers/queue/handlers/email-sync.ts

@@ -0,0 +1,356 @@
+import { createHash } from 'node:crypto';
+import type { JobHandler } from '../types';
+import { PermanentError } from '../types';
+import { registerHandler } from '../handler-registry';
+import { openEmailDb, upsertFromRawEml, setSyncMeta, getSyncMeta } from '../../api/email/email-db';
+import { refreshGoogleAccessToken } from '../../api/integrations/google-auth';
+import {
+  getEmailAccount,
+  getUserById,
+  getUserIntegration,
+  upsertUserIntegration,
+  getServerIntegration,
+  updateEmailAccountStatus,
+  updateEmailAccountSyncMeta,
+  getDockPaths,
+  setDockPaths,
+} from 'officerdb';
+
+// ── Stable ID from Message-Id header ──
+
+function messageIdToStableId(raw: string): string | null {
+  const match = raw.match(/^Message-Id:\s*<?([^>\s]+)>?/im);
+  if (!match?.[1]) return null;
+  return createHash('sha1').update(match[1]).digest('hex').slice(0, 16);
+}
+
+// ── IMAP folder → label mapping ──
+
+const SPECIAL_USE_LABEL_MAP: Record<string, string> = {
+  '\\Inbox': 'inbox',
+  '\\Sent': 'sent',
+  '\\Drafts': 'draft',
+  '\\Flagged': 'starred',
+  '\\Trash': 'trash',
+  '\\Junk': 'spam',
+  '\\All': 'archive',
+};
+
+const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
+
+type FolderInfo = { specialUse?: string; path: string; flags: Set<string>; status?: { uidNext?: number; uidValidity?: number } };
+
+function shouldSkipFolder(folder: FolderInfo): boolean {
+  if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
+  if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
+  return false;
+}
+
+function folderToLabel(folder: FolderInfo): string {
+  if (folder.specialUse && SPECIAL_USE_LABEL_MAP[folder.specialUse]) {
+    return SPECIAL_USE_LABEL_MAP[folder.specialUse]!;
+  }
+  if (folder.path === 'INBOX') return 'inbox';
+  return folder.path.toLowerCase();
+}
+
+// ── Handler ──
+
+const emailSyncHandler: JobHandler = {
+  type: 'email-sync',
+  retry: { delayMs: 15 * 60 * 1000, maxRetries: 5 },
+  steps: [
+    {
+      name: 'Load account',
+      run: async (ctx) => {
+        const emailAccountId = ctx.meta.emailAccountId as number;
+        if (!emailAccountId) throw new PermanentError('Missing emailAccountId in job meta');
+
+        const account = await getEmailAccount(emailAccountId);
+        if (!account) throw new PermanentError(`Email account ${emailAccountId} not found`);
+
+        // Look up user email for openEmailDb
+        const user = await getUserById(account.userId);
+        if (!user) throw new PermanentError(`User ${account.userId} not found`);
+
+        ctx.meta.account = account;
+        ctx.meta.userEmail = user.email;
+
+        // Resolve IMAP credentials
+        if (account.authType === 'oauth') {
+          const userGoogle = await getUserIntegration(account.userId, 'google');
+          const config = userGoogle?.config as Record<string, unknown> | undefined;
+          if (!config?.refreshToken) {
+            throw new PermanentError('Google OAuth not configured — reconnect your Google account');
+          }
+
+          let accessToken = config.accessToken as string | undefined;
+          const expiresAt = config.expiresAt as number | undefined;
+          const tokenExpired = !expiresAt || expiresAt < Date.now() + 60_000;
+
+          if (tokenExpired) {
+            console.log('[email-sync] Refreshing OAuth token');
+            const refreshed = await refreshGoogleAccessToken(config.refreshToken as string);
+            accessToken = refreshed.accessToken;
+
+            // Persist refreshed token
+            const serverGoogle = await getServerIntegration('google');
+            await upsertUserIntegration({
+              userId: account.userId,
+              provider: 'google',
+              serverIntegrationId: serverGoogle?.id,
+              config: { ...config, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
+            });
+          }
+
+          ctx.meta.imapAuth = { user: account.email, accessToken };
+        } else {
+          const creds = account.credentials as Record<string, unknown>;
+          const password = creds.password as string | undefined;
+          if (!password) throw new PermanentError('No password stored for this account');
+          ctx.meta.imapAuth = { user: account.email, pass: password };
+        }
+
+        // Check if initial or incremental
+        const syncMeta = account.syncMeta as Record<string, unknown>;
+        ctx.meta.isIncremental = !!syncMeta.last_sync_at;
+        ctx.meta.syncMeta = syncMeta;
+
+        // Set status to syncing
+        await updateEmailAccountStatus(emailAccountId, 'syncing');
+
+        console.log(`[email-sync] ${ctx.meta.isIncremental ? 'Incremental' : 'Initial'} sync for ${account.email}`);
+      },
+    },
+    {
+      name: 'Sync emails',
+      run: async (ctx) => {
+        const { ImapFlow } = await import('imapflow');
+        const account = ctx.meta.account as { id: number; email: string; imapHost: string; imapPort: number; imapSecure: boolean; provider: string };
+        const imapAuth = ctx.meta.imapAuth as { user: string; pass?: string; accessToken?: string };
+        const userEmail = ctx.meta.userEmail as string;
+        const syncMeta = ctx.meta.syncMeta as Record<string, unknown>;
+
+        await ctx.updateProgress({ current: 0, total: 0, label: 'Connecting...' });
+
+        const client = new ImapFlow({
+          host: account.imapHost,
+          port: account.imapPort,
+          secure: account.imapSecure,
+          auth: imapAuth,
+          logger: false,
+          socketTimeout: 30 * 60 * 1000, // 30 min — large mailboxes need time
+        });
+
+        // Prevent unhandled 'error' event from crashing the process
+        const connState = { error: null as Error | null };
+        client.on('error', (err: Error) => {
+          console.log(`[email-sync] IMAP connection error: ${err.message}`);
+          connState.error = err;
+        });
+
+        const db = openEmailDb(userEmail);
+        let saved = 0;
+        let skipped = 0;
+        let errors = 0;
+
+        try {
+          await client.connect();
+          console.log('[email-sync] IMAP connected');
+
+          // Get all folders with status
+          const folders = await client.list({ statusQuery: { uidNext: true, uidValidity: true } }) as FolderInfo[];
+
+          // Load existing IDs for dedup
+          const existingIds = new Set<string>();
+          const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
+          for (const row of rows) existingIds.add(row.id);
+
+          // Filter to syncable folders and count total messages to fetch
+          const foldersToSync: Array<{ folder: FolderInfo; lastUid: number }> = [];
+          for (const folder of folders) {
+            if (shouldSkipFolder(folder)) continue;
+
+            const uidValidityKey = `imap_uidvalidity:${folder.path}`;
+            const lastUidKey = `imap_lastuid:${folder.path}`;
+            const storedUidValidity = syncMeta[uidValidityKey] as string | undefined;
+            const storedLastUid = syncMeta[lastUidKey] as string | undefined;
+
+            const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
+            const uidNext = folder.status?.uidNext ?? 0;
+            let lastUid = storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
+
+            if (storedUidValidity && uidValidity && storedUidValidity !== uidValidity) {
+              console.log(`[email-sync] UIDVALIDITY changed for ${folder.path} — will re-scan`);
+              lastUid = 0;
+            }
+
+            // Skip if no new messages
+            if (lastUid > 0 && uidNext <= lastUid + 1) continue;
+
+            foldersToSync.push({ folder, lastUid });
+          }
+
+          console.log(`[email-sync] ${foldersToSync.length} folder(s) to sync`);
+
+          for (let fi = 0; fi < foldersToSync.length; fi++) {
+            const { folder, lastUid } = foldersToSync[fi]!;
+
+            // If connection is already dead, stop trying more folders
+            if (connState.error) break;
+
+            console.log(`[email-sync] Syncing folder ${fi + 1}/${foldersToSync.length}: ${folder.path}`);
+
+            let lock;
+            try {
+              lock = await client.getMailboxLock(folder.path);
+            } catch (err) {
+              const errMsg = err instanceof Error ? err.message : String(err);
+              if (errMsg.includes('not available') || errMsg.includes('timeout') || errMsg.includes('closed')) {
+                console.log(`[email-sync] Connection lost at folder ${folder.path}`);
+                connState.error = connState.error ?? new Error(errMsg);
+                break;
+              }
+              console.log(`[email-sync] Skipping folder ${folder.path}: ${errMsg}`);
+              continue;
+            }
+
+            try {
+              const mailbox = client.mailbox;
+              if (!mailbox) continue;
+
+              const uidValidity = String(mailbox.uidValidity);
+              const effectiveLastUid = syncMeta[`imap_uidvalidity:${folder.path}`] === uidValidity ? lastUid : 0;
+              let maxUid = effectiveLastUid;
+              const label = folderToLabel(folder);
+              const range = effectiveLastUid > 0 ? `${effectiveLastUid + 1}:*` : '1:*';
+
+              try {
+                const fetchOpts: Record<string, unknown> = { source: true, uid: true };
+                if (account.provider === 'gmail') fetchOpts.labels = true;
+
+                for await (const msg of client.fetch(range, fetchOpts, { uid: true })) {
+                  if (msg.uid <= effectiveLastUid) continue;
+
+                  maxUid = Math.max(maxUid, msg.uid);
+
+                  if (!msg.source) {
+                    errors++;
+                    continue;
+                  }
+
+                  const raw = msg.source.toString('utf-8');
+                  const id = messageIdToStableId(raw);
+                  if (!id) {
+                    errors++;
+                    continue;
+                  }
+
+                  if (existingIds.has(id)) {
+                    skipped++;
+                    continue;
+                  }
+
+                  const labels = [label];
+                  try {
+                    upsertFromRawEml({ db, id, raw, integration: account.provider, emailAccount: account.email, labels });
+                    existingIds.add(id);
+                    saved++;
+                  } catch {
+                    errors++;
+                  }
+
+                  if ((saved + skipped) % 50 === 0) {
+                    const progressLabel = `${folder.path} — ${saved.toLocaleString()} saved, ${skipped.toLocaleString()} skipped`;
+                    console.log(`[email-sync] ${progressLabel}`);
+                    await ctx.updateProgress({ current: saved + skipped, total: 0, label: progressLabel });
+
+                    // Save progress mid-folder so retries resume from here
+                    if (maxUid > effectiveLastUid) {
+                      syncMeta[`imap_lastuid:${folder.path}`] = String(maxUid);
+                      await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
+                    }
+                  }
+                }
+              } catch (fetchErr) {
+                const errMsg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
+                if (errMsg.includes('Nothing to fetch')) {
+                  // No messages in range — normal
+                } else if (errMsg.includes('not available') || errMsg.includes('timeout') || errMsg.includes('closed')) {
+                  console.log(`[email-sync] Connection lost during fetch in ${folder.path}: ${errMsg}`);
+                  connState.error = connState.error ?? new Error(errMsg);
+                } else {
+                  console.log(`[email-sync] Fetch error in ${folder.path}: ${errMsg}`);
+                  errors++;
+                }
+              }
+
+              syncMeta[`imap_uidvalidity:${folder.path}`] = uidValidity;
+              if (maxUid > effectiveLastUid) {
+                syncMeta[`imap_lastuid:${folder.path}`] = String(maxUid);
+              }
+
+              console.log(`[email-sync] ${folder.path} done: ${saved} saved, ${skipped} skipped, ${errors} errors`);
+              await ctx.updateProgress({ current: saved + skipped, total: 0, label: `${folder.path}: ${saved} saved` });
+
+              // Save sync meta after each folder
+              await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
+            } finally {
+              lock.release();
+            }
+          }
+
+          // If connection died, throw to trigger retry (progress is already saved)
+          if (connState.error) {
+            console.log(`[email-sync] Connection lost after saving ${saved} emails — will retry remaining folders`);
+            throw new Error(`IMAP connection lost: ${connState.error.message}`);
+          }
+
+          // Mark sync complete
+          syncMeta.last_sync_at = new Date().toISOString();
+          await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
+
+          await client.logout().catch(() => {});
+        } catch (err) {
+          await client.logout().catch(() => {});
+          throw err;
+        } finally {
+          db.close();
+        }
+
+        console.log(`[email-sync] Done: saved ${saved}, skipped ${skipped}, errors ${errors}`);
+        ctx.meta.saved = saved;
+        await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${saved} new emails` });
+      },
+    },
+    {
+      name: 'Finalize',
+      run: async (ctx) => {
+        const account = ctx.meta.account as { id: number; userId: number; email: string };
+        const saved = ctx.meta.saved as number;
+
+        // Update account status to synced
+        await updateEmailAccountStatus(account.id, 'synced');
+
+        // Auto-add /email to dock
+        if (saved > 0) {
+          try {
+            const paths = await getDockPaths(account.userId);
+            if (!paths) {
+              const defaults = ['/', '/files', '/automation', '/projects', '/dashboards', '/chat'];
+              await setDockPaths(account.userId, [...defaults, '/email']);
+            } else if (!paths.includes('/email')) {
+              await setDockPaths(account.userId, [...paths, '/email']);
+            }
+          } catch {
+            // Non-fatal
+          }
+        }
+
+        console.log(`[email-sync] ${account.email} status set to synced`);
+      },
+    },
+  ],
+};
+
+registerHandler(emailSyncHandler);

src/servers/queue/handlers/gmail-sync.ts

@@ -4,28 +4,59 @@ import { join } from 'node:path';
 import { readdir, readFile, writeFile, unlink, mkdir, stat } from 'node:fs/promises';
 import { type JobHandler, PermanentError } from '../types';
 import { registerHandler } from '../handler-registry';
-import { openEmailDb, upsertFromRawEml, setSyncMeta } from '../../api/email/email-db';
-import { getUserByEmail, getUserIntegration, getDockPaths, setDockPaths } from 'officerdb';
+import { openEmailDb, upsertFromRawEml, setSyncMeta, getSyncMeta } from '../../api/email/email-db';
+import {
+  getUserByEmail,
+  getUserIntegration,
+  upsertUserIntegration,
+  getServerIntegration,
+  getDockPaths,
+  setDockPaths,
+} from 'officerdb';
 import { getMaildirPath } from '@@/data-path';
+import { refreshGoogleAccessToken } from '@@/api/integrations/google-auth';
 
 // ── Credentials ──
 
-type ImapCredentials = { email: string; appPassword: string };
+type GmailCredentials = {
+  email: string;
+  userId: number;
+  appPassword?: string;
+  accessToken?: string;
+  refreshToken?: string;
+  expiresAt?: number;
+};
 
-async function loadImapCredentials(userEmail: string): Promise<ImapCredentials> {
+async function loadGmailCredentials(userEmail: string): Promise<GmailCredentials> {
   const dbUser = await getUserByEmail(userEmail);
   if (!dbUser) throw new PermanentError('User not found');
 
   const userGoogle = await getUserIntegration(dbUser.id, 'google');
   const config = userGoogle?.config as Record<string, unknown> | undefined;
+
+  const gmailEmail = (config?.email as string) ?? userEmail;
+
+  // Prefer OAuth tokens, fall back to app password
+  if (config?.accessToken && config?.refreshToken) {
+    return {
+      email: gmailEmail,
+      userId: dbUser.id,
+      accessToken: config.accessToken as string,
+      refreshToken: config.refreshToken as string,
+      expiresAt: config.expiresAt as number | undefined,
+      appPassword: config.imapAppPassword as string | undefined,
+    };
+  }
+
   if (!config?.imapAppPassword) {
     throw new PermanentError('Gmail App Password not configured — set it in Settings → Integrations');
   }
 
-  const gmailEmail = (config.email as string) ?? userEmail;
-  return { email: gmailEmail, appPassword: config.imapAppPassword as string };
+  return { email: gmailEmail, userId: dbUser.id, appPassword: config.imapAppPassword as string };
 }
 
+// refreshGoogleAccessToken is imported from @@/api/integrations/google-auth
+
 // ── mbsync config ──
 
 function buildMbsyncConfig(email: string, appPassword: string, maildirPath: string): string {
@@ -48,7 +79,7 @@ SubFolders Verbatim
 Channel gmail
 Far :gmail-remote:
 Near :gmail-local:
-Patterns * ![Gmail]/Trash ![Gmail]/Spam ![Gmail]/Bin ![Google Mail]/Trash ![Google Mail]/Spam ![Google Mail]/Bin
+Patterns * ![Gmail]/Trash ![Gmail]/Spam ![Gmail]/Bin !"[Gmail]/All Mail" ![Google Mail]/Trash ![Google Mail]/Spam ![Google Mail]/Bin !"[Google Mail]/All Mail"
 Create Near
 Expunge None
 SyncState *
@@ -99,21 +130,40 @@ function messageIdToStableId(raw: string): string | null {
 
 type ImportResult = { saved: number; skipped: number; errors: number };
 
-export async function importMaildir(
-  maildirPath: string,
-  emailAccount: string,
-  db: Database,
-  onProgress?: (saved: number, skipped: number) => void,
-): Promise<ImportResult> {
+type ImportMaildirParams = {
+  maildirPath: string;
+  emailAccount: string;
+  db: Database;
+  lastSyncAt?: string | null;
+  onProgress?: (saved: number, skipped: number) => void;
+};
+
+export async function importMaildir({
+  maildirPath,
+  emailAccount,
+  db,
+  lastSyncAt,
+  onProgress,
+}: ImportMaildirParams): Promise<ImportResult> {
   let saved = 0;
   let skipped = 0;
   let errors = 0;
 
+  // For incremental syncs, skip files older than last sync (with 60s buffer for clock skew)
+  const mtimeCutoff = lastSyncAt ? new Date(lastSyncAt).getTime() - 60_000 : 0;
+  const isIncremental = mtimeCutoff > 0;
+
   // Load existing IDs for fast dedup
   const existingIds = new Set<string>();
   const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
   for (const row of rows) existingIds.add(row.id);
 
+  if (isIncremental) {
+    console.log(
+      `[gmail-sync] Incremental import — only reading files newer than ${new Date(mtimeCutoff).toISOString()}`,
+    );
+  }
+
   // First pass: collect all message files with their folders to build label map
   const messageIdLabels = new Map<string, Set<string>>();
   const messageFiles = new Map<string, string>(); // id → first file path
@@ -162,6 +212,12 @@ export async function importMaildir(
       for (const file of files) {
         const filePath = join(dirPath, file);
         try {
+          // Skip files older than last sync for incremental imports
+          if (isIncremental) {
+            const fileStat = await stat(filePath);
+            if (fileStat.mtimeMs < mtimeCutoff) continue;
+          }
+
           const raw = await readFile(filePath, 'utf-8');
           const id = messageIdToStableId(raw);
           if (!id) {
@@ -238,6 +294,220 @@ async function countMaildirFiles(maildirPath: string): Promise<number> {
   return count;
 }
 
+// ── Gmail IMAP label → our label mapping ──
+
+const GMAIL_LABEL_MAP: Record<string, string> = {
+  '\\Inbox': 'inbox',
+  '\\Sent': 'sent',
+  '\\Drafts': 'draft',
+  '\\Starred': 'starred',
+  '\\Important': 'important',
+  '\\All': 'archive',
+  '\\Trash': 'trash',
+  '\\Junk': 'spam',
+};
+
+function gmailLabelsToLabels(gmailLabels: Set<string>): string[] {
+  const labels: string[] = [];
+  for (const gl of gmailLabels) {
+    const mapped = GMAIL_LABEL_MAP[gl];
+    if (mapped) {
+      labels.push(mapped);
+    } else if (!gl.startsWith('\\')) {
+      // Custom label — lowercase it
+      labels.push(gl.toLowerCase());
+    }
+  }
+  // If only "archive" and no specific folder, keep it; otherwise drop "archive"
+  if (labels.length > 1 && labels.includes('archive')) {
+    return labels.filter((l) => l !== 'archive');
+  }
+  return labels;
+}
+
+// ── IMAP special-use folders to skip ──
+
+const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
+
+function shouldSkipFolder(folder: { specialUse?: string; path: string; flags: Set<string> }): boolean {
+  if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
+  if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
+  const norm = normalizeGmailFolder(folder.path);
+  return norm === 'All Mail' || norm === 'Trash' || norm === 'Spam' || norm === 'Bin';
+}
+
+// ── Incremental IMAP sync ──
+
+type IncrementalSyncParams = {
+  creds: GmailCredentials;
+  db: Database;
+  onProgress?: (fetched: number, folder: string) => void;
+};
+
+type IncrementalSyncResult = { saved: number; skipped: number; errors: number };
+
+async function incrementalImapSync({ creds, db, onProgress }: IncrementalSyncParams): Promise<IncrementalSyncResult> {
+  const { ImapFlow } = await import('imapflow');
+
+  // Determine auth method: prefer OAuth, fall back to app password
+  const auth: { user: string; pass?: string; accessToken?: string } = { user: creds.email };
+  if (creds.accessToken) {
+    auth.accessToken = creds.accessToken;
+  } else if (creds.appPassword) {
+    auth.pass = creds.appPassword;
+  } else {
+    throw new PermanentError('No authentication method available for IMAP');
+  }
+
+  const client = new ImapFlow({
+    host: 'imap.gmail.com',
+    port: 993,
+    secure: true,
+    auth,
+    logger: false,
+  });
+
+  let saved = 0;
+  let skipped = 0;
+  let errors = 0;
+
+  try {
+    await client.connect();
+    console.log('[gmail-sync] Incremental IMAP connected');
+
+    // Get all folders with status in a single LIST command
+    const folders = await client.list({ statusQuery: { uidNext: true, uidValidity: true } });
+
+    // Load existing IDs for dedup
+    const existingIds = new Set<string>();
+    const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
+    for (const row of rows) existingIds.add(row.id);
+
+    // Filter to only folders with new messages
+    const foldersToSync: typeof folders = [];
+    for (const folder of folders) {
+      if (shouldSkipFolder(folder)) continue;
+
+      const folderPath = folder.path;
+      const uidValidityKey = `imap_uidvalidity:${folderPath}`;
+      const lastUidKey = `imap_lastuid:${folderPath}`;
+      const storedUidValidity = getSyncMeta(db, uidValidityKey);
+      const storedLastUid = getSyncMeta(db, lastUidKey);
+
+      const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
+      const uidNext = folder.status?.uidNext ?? 0;
+      const lastUid =
+        storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
+
+      if (uidValidity) setSyncMeta(db, uidValidityKey, uidValidity);
+
+      if (lastUid > 0 && uidNext <= lastUid + 1) continue; // no new messages
+
+      if (storedUidValidity && uidValidity && storedUidValidity !== uidValidity) {
+        console.log(`[gmail-sync] UIDVALIDITY changed for ${folderPath} — will re-scan`);
+      }
+
+      foldersToSync.push(folder);
+    }
+
+    console.log(`[gmail-sync] ${foldersToSync.length} folder(s) with new messages`);
+
+    for (const folder of foldersToSync) {
+      const folderPath = folder.path;
+      const uidValidityKey = `imap_uidvalidity:${folderPath}`;
+      const lastUidKey = `imap_lastuid:${folderPath}`;
+      const storedUidValidity = getSyncMeta(db, uidValidityKey);
+      const storedLastUid = getSyncMeta(db, lastUidKey);
+
+      // Open folder and fetch new messages
+      let lock;
+      try {
+        lock = await client.getMailboxLock(folderPath);
+      } catch (err) {
+        console.log(`[gmail-sync] Skipping folder ${folderPath}: ${err instanceof Error ? err.message : err}`);
+        continue;
+      }
+
+      try {
+        const mailbox = client.mailbox;
+        if (!mailbox) continue;
+
+        const uidValidity = String(mailbox.uidValidity);
+        let lastUid = 0;
+        if (storedUidValidity === uidValidity && storedLastUid) {
+          lastUid = parseInt(storedLastUid, 10);
+        }
+
+        const range = lastUid > 0 ? `${lastUid + 1}:*` : '1:*';
+        let maxUid = lastUid;
+        let folderFetched = 0;
+
+        try {
+          for await (const msg of client.fetch(range, { source: true, labels: true, uid: true }, { uid: true })) {
+            if (msg.uid <= lastUid) continue;
+
+            maxUid = Math.max(maxUid, msg.uid);
+            folderFetched++;
+
+            if (!msg.source) {
+              errors++;
+              continue;
+            }
+
+            const raw = msg.source.toString('utf-8');
+            const id = messageIdToStableId(raw);
+            if (!id) {
+              errors++;
+              continue;
+            }
+
+            if (existingIds.has(id)) {
+              skipped++;
+              continue;
+            }
+
+            // Gmail X-GM-EXT-1 labels don't include folder membership (e.g. \Inbox),
+            // so always use the folder we're fetching from as the base label
+            const folderLabel = folderToLabel(folderPath);
+            const gmailLabels = msg.labels ? gmailLabelsToLabels(msg.labels) : [];
+            const labels = folderLabel ? [...new Set([folderLabel, ...gmailLabels])] : gmailLabels;
+
+            try {
+              upsertFromRawEml({ db, id, raw, integration: 'gmail', emailAccount: creds.email, labels });
+              existingIds.add(id);
+              saved++;
+            } catch {
+              errors++;
+            }
+          }
+        } catch (fetchErr) {
+          const msg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
+          if (!msg.includes('Nothing to fetch')) {
+            console.log(`[gmail-sync] Fetch error in ${folderPath}: ${msg}`);
+            errors++;
+          }
+        }
+
+        if (maxUid > lastUid) {
+          setSyncMeta(db, lastUidKey, String(maxUid));
+        }
+        setSyncMeta(db, uidValidityKey, uidValidity);
+
+        if (folderFetched > 0) {
+          console.log(`[gmail-sync] ${folderPath}: fetched ${folderFetched}, saved ${saved}, skipped ${skipped}`);
+          onProgress?.(saved + skipped, folderPath);
+        }
+      } finally {
+        lock.release();
+      }
+    }
+  } finally {
+    await client.logout().catch(() => {});
+  }
+
+  return { saved, skipped, errors };
+}
+
 // ── Handler ──
 
 const gmailSyncHandler: JobHandler = {
@@ -247,119 +517,219 @@ const gmailSyncHandler: JobHandler = {
     {
       name: 'Verify credentials',
       run: async (ctx) => {
-        const creds = await loadImapCredentials(ctx.job.userId);
+        const creds = await loadGmailCredentials(ctx.job.userId);
+
+        // Determine sync mode: incremental if we have a previous sync
+        const db = openEmailDb(ctx.job.userId);
+        try {
+          const lastSyncAt = getSyncMeta(db, 'last_sync_at');
+          ctx.meta.isIncremental = !!lastSyncAt;
+        } finally {
+          db.close();
+        }
+
+        // For incremental sync with OAuth, refresh token if expired
+        if (ctx.meta.isIncremental && creds.accessToken && creds.refreshToken) {
+          const tokenExpired = !creds.expiresAt || creds.expiresAt < Date.now() + 60_000;
+          if (tokenExpired) {
+            console.log('[gmail-sync] Refreshing OAuth token for incremental sync');
+            try {
+              const refreshed = await refreshGoogleAccessToken(creds.refreshToken);
+              creds.accessToken = refreshed.accessToken;
+              creds.expiresAt = refreshed.expiresAt;
+
+              // Persist refreshed token
+              const userGoogle = await getUserIntegration(creds.userId, 'google');
+              const existingConfig = (userGoogle?.config as Record<string, unknown>) ?? {};
+              const serverGoogle = await getServerIntegration('google');
+              await upsertUserIntegration({
+                userId: creds.userId,
+                provider: 'google',
+                serverIntegrationId: serverGoogle?.id,
+                config: { ...existingConfig, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
+              });
+            } catch (err) {
+              console.log(`[gmail-sync] OAuth refresh failed, will fall back to app password: ${err}`);
+              // Clear OAuth so we fall back to app password
+              creds.accessToken = undefined;
+              creds.refreshToken = undefined;
+            }
+          }
+        }
+
+        ctx.meta.creds = creds;
         ctx.meta.email = creds.email;
         ctx.meta.appPassword = creds.appPassword;
+
+        if (ctx.meta.isIncremental) {
+          console.log(`[gmail-sync] Incremental sync mode (${creds.accessToken ? 'OAuth' : 'App Password'})`);
+        } else {
+          console.log('[gmail-sync] Full sync mode (mbsync)');
+          if (!creds.appPassword) {
+            throw new PermanentError('Gmail App Password required for initial sync');
+          }
+        }
       },
     },
     {
-      name: 'Sync via mbsync',
+      name: 'Sync emails',
       run: async (ctx) => {
-        const email = ctx.meta.email as string;
-        const appPassword = ctx.meta.appPassword as string;
-        const maildirPath = getMaildirPath(ctx.job.userId);
+        if (ctx.meta.isIncremental) {
+          // ── Incremental: direct IMAP via imapflow ──
+          const creds = ctx.meta.creds as GmailCredentials;
+          await ctx.updateProgress({ current: 0, total: 0, label: 'Connecting to Gmail...' });
 
-        // Ensure Maildir root exists
-        await mkdir(maildirPath, { recursive: true });
-
-        // Write temp config
-        const configPath = join(maildirPath, '.mbsyncrc');
-        const config = buildMbsyncConfig(email, appPassword, maildirPath);
-        await writeFile(configPath, config, { mode: 0o600 });
-
-        await ctx.updateProgress({ current: 0, total: 0, label: 'Running mbsync...' });
-
-        try {
-          let proc: ReturnType<typeof Bun.spawn>;
+          const db = openEmailDb(ctx.job.userId);
           try {
-            proc = Bun.spawn(['mbsync', '-c', configPath, '-a', '-V'], {
-              stdout: 'pipe',
-              stderr: 'pipe',
+            const result = await incrementalImapSync({
+              creds,
+              db,
+              onProgress: (fetched, folder) => {
+                ctx.updateProgress({ current: fetched, total: 0, label: `Syncing ${folder}...` });
+              },
             });
-          } catch (spawnErr) {
-            const msg = spawnErr instanceof Error ? spawnErr.message : String(spawnErr);
-            throw new PermanentError(`Failed to start mbsync: ${msg}`);
-          }
 
-          // Periodically count downloaded emails and update progress
-          let emailCount = 0;
-          let counting = true;
-          const countLoop = (async () => {
-            while (counting) {
-              await new Promise((r) => setTimeout(r, 3000));
-              if (!counting) break;
+            console.log(
+              `[gmail-sync] Incremental sync done: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`,
+            );
+
+            setSyncMeta(db, 'last_sync_date', new Date().toISOString().split('T')[0]!);
+            setSyncMeta(db, 'last_sync_at', new Date().toISOString());
+
+            ctx.meta.syncResult = result;
+            await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${result.saved} new emails` });
+          } finally {
+            db.close();
+          }
+        } else {
+          // ── Full sync: mbsync ──
+          const email = ctx.meta.email as string;
+          const appPassword = ctx.meta.appPassword as string;
+          const maildirPath = getMaildirPath(ctx.job.userId);
+
+          await mkdir(maildirPath, { recursive: true });
+
+          const configPath = join(maildirPath, '.mbsyncrc');
+          const config = buildMbsyncConfig(email, appPassword, maildirPath);
+          await writeFile(configPath, config, { mode: 0o600 });
+
+          await ctx.updateProgress({ current: 0, total: 0, label: 'Running mbsync...' });
+
+          try {
+            let proc: ReturnType<typeof Bun.spawn>;
+            try {
+              proc = Bun.spawn(['mbsync', '-c', configPath, '-a'], {
+                stdout: 'pipe',
+                stderr: 'pipe',
+              });
+            } catch (spawnErr) {
+              const msg = spawnErr instanceof Error ? spawnErr.message : String(spawnErr);
+              throw new PermanentError(`Failed to start mbsync: ${msg}`);
+            }
+
+            let emailCount = 0;
+            let counting = true;
+            const countLoop = (async () => {
+              while (counting) {
+                await new Promise((r) => setTimeout(r, 3000));
+                if (!counting) break;
+                emailCount = await countMaildirFiles(maildirPath);
+                ctx.updateProgress({
+                  current: emailCount,
+                  total: 0,
+                  label: `Downloading — ${emailCount.toLocaleString()} emails`,
+                });
+              }
+            })();
+
+            let stderrBuf = '';
+            const reader = (proc.stderr as ReadableStream<Uint8Array>).getReader();
+            const decoder = new TextDecoder();
+            const readLoop = (async () => {
+              while (true) {
+                const { done, value } = await reader.read();
+                if (done) break;
+                const chunk = decoder.decode(value, { stream: true });
+                stderrBuf += chunk;
+                const lines = chunk.split('\n');
+                for (const line of lines) {
+                  const trimmed = line.trim();
+                  if (trimmed) console.log(`[gmail-sync] mbsync: ${trimmed}`);
+                }
+              }
+            })();
+
+            const exitCode = await proc.exited;
+            counting = false;
+            await readLoop;
+            await countLoop;
+
+            if (exitCode !== 0) {
+              const isOverquota = stderrBuf.includes('OVERQUOTA');
+              const isAuthFail =
+                stderrBuf.includes('AUTHENTICATIONFAILED') || stderrBuf.includes('Invalid credentials');
+
+              if (isOverquota) {
+                const emailCount = await countMaildirFiles(maildirPath);
+                console.log(`[gmail-sync] Gmail OVERQUOTA — proceeding to import ${emailCount} downloaded emails`);
+                ctx.meta.gmailSyncPartial = true;
+                ctx.meta.gmailSyncEmailCount = emailCount;
+              } else {
+                console.error(`[gmail-sync] mbsync failed`);
+                if (isAuthFail) {
+                  const emailCount = await countMaildirFiles(maildirPath);
+                  ctx.meta.gmailSyncRecoverable = true;
+                  ctx.meta.gmailSyncEmailCount = emailCount;
+                  ctx.meta.gmailSyncIsAuthFail = true;
+                  throw new PermanentError(`Authentication failed — check your App Password`);
+                }
+                throw new Error(`mbsync exited with code ${exitCode}: ${stderrBuf.slice(-500)}`);
+              }
+            } else {
               emailCount = await countMaildirFiles(maildirPath);
-              ctx.updateProgress({
+              console.log(`[gmail-sync] mbsync completed successfully — ${emailCount} emails`);
+              await ctx.updateProgress({
                 current: emailCount,
-                total: 0,
-                label: `Downloading — ${emailCount.toLocaleString()} emails`,
+                total: emailCount,
+                label: `Download complete — ${emailCount.toLocaleString()} emails`,
               });
             }
-          })();
-
-          // Stream stderr for logging
-          let stderrBuf = '';
-          const reader = (proc.stderr as ReadableStream<Uint8Array>).getReader();
-          const decoder = new TextDecoder();
-          const readLoop = (async () => {
-            while (true) {
-              const { done, value } = await reader.read();
-              if (done) break;
-              const chunk = decoder.decode(value, { stream: true });
-              stderrBuf += chunk;
-              const lines = chunk.split('\n');
-              for (const line of lines) {
-                const trimmed = line.trim();
-                if (trimmed) console.log(`[gmail-sync] mbsync: ${trimmed}`);
-              }
-            }
-          })();
-
-          const exitCode = await proc.exited;
-          counting = false;
-          await readLoop;
-          await countLoop;
-
-          if (exitCode !== 0) {
-            const isOverquota = stderrBuf.includes('OVERQUOTA');
-            const isAuthFail = stderrBuf.includes('AUTHENTICATIONFAILED') || stderrBuf.includes('Invalid credentials');
-
-            if (isOverquota) {
-              // Gmail throttled us — don't retry, just import what we have
-              const emailCount = await countMaildirFiles(maildirPath);
-              console.log(`[gmail-sync] Gmail OVERQUOTA — proceeding to import ${emailCount} downloaded emails`);
-              ctx.meta.gmailSyncPartial = true;
-              ctx.meta.gmailSyncEmailCount = emailCount;
-              // Fall through to import step
-            } else {
-              console.error(`[gmail-sync] mbsync failed`);
-              if (isAuthFail) {
-                const emailCount = await countMaildirFiles(maildirPath);
-                ctx.meta.gmailSyncRecoverable = true;
-                ctx.meta.gmailSyncEmailCount = emailCount;
-                ctx.meta.gmailSyncIsAuthFail = true;
-                throw new PermanentError(`Authentication failed — check your App Password`);
-              }
-              throw new Error(`mbsync exited with code ${exitCode}: ${stderrBuf.slice(-500)}`);
-            }
-          } else {
-            emailCount = await countMaildirFiles(maildirPath);
-            console.log(`[gmail-sync] mbsync completed successfully — ${emailCount} emails`);
-            await ctx.updateProgress({
-              current: emailCount,
-              total: emailCount,
-              label: `Download complete — ${emailCount.toLocaleString()} emails`,
-            });
+          } finally {
+            await unlink(configPath).catch(() => {});
           }
-        } finally {
-          // Always clean up config (contains password)
-          await unlink(configPath).catch(() => {});
         }
       },
     },
     {
       name: 'Import to database',
       run: async (ctx) => {
+        // Incremental sync already imported in the previous step
+        if (ctx.meta.isIncremental) {
+          const result = ctx.meta.syncResult as IncrementalSyncResult | undefined;
+
+          // Auto-add /email to dock
+          if (result && result.saved > 0) {
+            try {
+              const dbUser = await getUserByEmail(ctx.job.userId);
+              if (dbUser) {
+                const paths = await getDockPaths(dbUser.id);
+                if (!paths) {
+                  const defaults = ['/', '/files', '/automation', '/projects', '/dashboards', '/chat'];
+                  await setDockPaths(dbUser.id, [...defaults, '/email']);
+                } else if (!paths.includes('/email')) {
+                  await setDockPaths(dbUser.id, [...paths, '/email']);
+                }
+              }
+            } catch {
+              // Non-fatal
+            }
+          }
+
+          await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${result?.saved ?? 0} new emails` });
+          return;
+        }
+
+        // Full sync: import from Maildir
         const emailAccount = ctx.meta.email as string;
         const maildirPath = getMaildirPath(ctx.job.userId);
 
@@ -367,12 +737,19 @@ const gmailSyncHandler: JobHandler = {
 
         const db = openEmailDb(ctx.job.userId);
         try {
-          const result = await importMaildir(maildirPath, emailAccount, db, (saved, skipped) => {
-            ctx.updateProgress({
-              current: saved + skipped,
-              total: 0,
-              label: `Saved ${saved.toLocaleString()}, skipped ${skipped.toLocaleString()}`,
-            });
+          const lastSyncAt = getSyncMeta(db, 'last_sync_at');
+          const result = await importMaildir({
+            maildirPath,
+            emailAccount,
+            db,
+            lastSyncAt,
+            onProgress: (saved, skipped) => {
+              ctx.updateProgress({
+                current: saved + skipped,
+                total: 0,
+                label: `Saved ${saved.toLocaleString()}, skipped ${skipped.toLocaleString()}`,
+              });
+            },
           });
 
           console.log(
@@ -389,7 +766,6 @@ const gmailSyncHandler: JobHandler = {
               if (dbUser) {
                 const paths = await getDockPaths(dbUser.id);
                 if (!paths) {
-                  // User hasn't customized dock — initialize with defaults + /email
                   const defaults = ['/', '/files', '/automation', '/projects', '/dashboards', '/chat'];
                   await setDockPaths(dbUser.id, [...defaults, '/email']);
                 } else if (!paths.includes('/email')) {

src/servers/queue/handlers/index.ts

@@ -1 +1,2 @@
 import './gmail-sync';
+import './email-sync';