platform/src/servers/sidecar/email/sync-meta.ts

import type { Database } from 'bun:sqlite';
import { getSyncMeta, setSyncMeta } from './store';

// Where an account's sync position lives: IN ITS OWN emails.db, not in Postgres.
//
// ── Why this moved ──
//
// The messages are in `emails.db` and the position — `last_sync_at`, and per-folder `uidvalidity` +
// `lastuid` — used to be a jsonb column on `email_accounts` in Postgres. Two stores for one fact, and
// the split had a sharp edge that only appears when you try to move a mailbox to another machine.
//
// The expensive thing about an email account is the initial sync: hours of IMAP for a large mailbox. So
// "copy emails.db to the new server" is the obvious way to bring an account across. With the position in
// Postgres that silently does not work — the new server's `sync_meta` is empty, `!last_sync_at` says
// first sync, and it downloads the whole mailbox again on top of the one you just restored.
//
// The worse direction is quieter. Restore an OLDER emails.db while Postgres still holds a NEWER
// position, and the sidecar skips every message between the two — permanently, because nothing ever
// looks below `lastuid` again. Re-syncing is slow; skipping mail is data loss you do not notice.
//
// Keeping both in one file makes the file self-describing: it says what it holds and where it got to,
// and the two cannot disagree because they travel together.
//
// ── Why this is a generalisation, not a new idea ──
//
// The Gmail path already did exactly this (`gmail-api.ts`): read SQLite, fall back to the Postgres
// column, backfill SQLite so the fallback is only ever taken once. That was the right shape and only
// the IMAP path had not followed. This is that pattern, extracted so both use one copy of it.
//
// ── The Postgres column stays ──
//
// `email_accounts.sync_meta` is not dropped. It is the backfill source for every account that exists
// today, and it costs nothing to leave. It is no longer written to; see `readSyncMeta` for the one-way
// migration, which happens the first time an account syncs after this change.

/**
 * The whole sync position for this account, from SQLite — migrating it out of Postgres once if this is
 * the first read since the change.
 *
 * `pgFallback` is the Postgres jsonb as it stands. Pass it; the migration cannot happen without it, and
 * an account whose position never migrates re-syncs its entire mailbox.
 */
export function readSyncMeta(db: Database, pgFallback?: Record<string, unknown> | null): Record<string, string> {
  const rows = db.query('SELECT key, value FROM sync_meta').all() as Array<{ key: string; value: string }>;
  const meta: Record<string, string> = {};
  for (const row of rows) meta[row.key] = row.value;

  // Merged PER KEY, and the file always wins a conflict.
  //
  // This started life as all-or-nothing — migrate only when SQLite is completely empty — on the
  // assumption that a non-empty file is an authoritative one. A real account disproved it immediately.
  // The older Gmail backfill wrote SOME keys into SQLite (`last_sync_at`, the `uidvalidity` set) and
  // never the `imap_lastuid:*` ones, so the file was non-empty and half-migrated at the same time.
  // All-or-nothing therefore skipped the migration, leaving no lastuid for any folder — and a missing
  // lastuid means the next sync refetches that folder from UID 1. On the mailbox this was found on,
  // that was 18,755 messages and 6.9 GB.
  //
  // Per-key with file-wins keeps the property that made all-or-nothing attractive: a restored older
  // emails.db still overrides a newer Postgres row for every key it actually has, so it cannot be
  // advanced past mail it does not contain. What it adds is the keys the file never had, which are
  // exactly the ones whose absence is expensive.
  if (pgFallback) {
    for (const [key, value] of Object.entries(pgFallback)) {
      if (typeof value !== 'string') continue;
      if (key in meta) continue; // the file's own answer stands
      meta[key] = value;
      setSyncMeta(db, key, value);
    }
  }
  return meta;
}

/**
 * Persist the position. SQLite only — Postgres is no longer written.
 *
 * Called after the messages are already stored, which keeps the safe ordering the previous code had: if
 * this never runs, the position stays behind and those messages are fetched again. Re-fetching is
 * wasteful; advancing past mail that was never stored would lose it.
 */
export function writeSyncMeta(db: Database, meta: Record<string, unknown>): void {
  for (const [key, value] of Object.entries(meta)) {
    if (value === undefined || value === null) continue;
    setSyncMeta(db, key, String(value));
  }
}

/** Has this account ever completed a sync? The question `isFirstSync` is really asking. */
export function hasSyncedBefore(db: Database, pgFallback?: Record<string, unknown> | null): boolean {
  if (getSyncMeta(db, 'last_sync_at')) return true;
  // Not yet migrated: an account that synced before this change has its answer in Postgres, and
  // treating it as a first sync would re-download the whole mailbox.
  const pgValue = pgFallback?.last_sync_at;
  return typeof pgValue === 'string' && pgValue.length > 0;
}