From 977d14922ffd636f15ee0c0fa616211d48266209 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Mon, 10 Aug 2026 13:42:30 +0000 Subject: [PATCH] email: migrate the sync position per key, not all-or-nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit migrated only when SQLite was completely empty, on the assumption that a non-empty file is an authoritative one. A real account disproved that within minutes of it landing. The older Gmail backfill had already written SOME keys into SQLite — last_sync_at and the uidvalidity set — and never the imap_lastuid ones. So the file was non-empty and half-migrated at the same time, all-or-nothing skipped the migration, and nine imap_lastuid keys stayed only in Postgres. A missing lastuid makes the next sync refetch that folder from UID 1: on the mailbox this was found on, 18,755 messages and 6.9 GB. Now merged per key with the file always winning a conflict. That keeps the property all-or-nothing was protecting — a restored older emails.db still overrides a newer Postgres row for every key it has, so it cannot be advanced past mail it does not contain — and adds the keys the file never had, which are exactly the ones whose absence is expensive. Verified against the live account: all 22 Postgres keys present afterwards, imap_lastuid:INBOX restored to 208407, and last_sync_at left at the file's older value, so it re-checks a fortnight rather than skipping it. Co-Authored-By: Claude Opus 5 --- src/servers/sidecar/email/sync-meta.test.ts | 25 +++++++++++++++++++-- src/servers/sidecar/email/sync-meta.ts | 19 ++++++++++++---- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/servers/sidecar/email/sync-meta.test.ts b/src/servers/sidecar/email/sync-meta.test.ts index 89b1201b..30a30ee1 100644 --- a/src/servers/sidecar/email/sync-meta.test.ts +++ b/src/servers/sidecar/email/sync-meta.test.ts @@ -22,6 +22,26 @@ describe('readSyncMeta', () => { expect(readSyncMeta(db)).toEqual({ last_sync_at: '2026-01-01T00:00:00Z', 'imap_lastuid:INBOX': '4200' }); }); + it('migrates keys the file is MISSING, even when it already has others', () => { + // The case that broke a live 6.9 GB mailbox. The older Gmail backfill wrote last_sync_at and the + // uidvalidity set into SQLite and never the imap_lastuid ones, so the file was non-empty and + // half-migrated at once. All-or-nothing skipped it, leaving no lastuid for any folder — and a + // missing lastuid refetches that folder from UID 1. + const db = freshDb(); + writeSyncMeta(db, { last_sync_at: '2026-07-23T00:00:00Z', 'imap_uidvalidity:INBOX': '2' }); + + const meta = readSyncMeta(db, { + last_sync_at: '2026-08-10T00:00:00Z', // newer, but the file's own answer stands + 'imap_uidvalidity:INBOX': '2', + 'imap_lastuid:INBOX': '4200', // absent from the file — this is what must arrive + }); + + expect(meta['imap_lastuid:INBOX']).toBe('4200'); + expect(meta.last_sync_at).toBe('2026-07-23T00:00:00Z'); + // Persisted, so the fallback is not needed next time. + expect(readSyncMeta(db)['imap_lastuid:INBOX']).toBe('4200'); + }); + it('migrates from Postgres exactly once, when the file has nothing', () => { // The account existed before the move. Without this it looks like a first sync and re-downloads // the entire mailbox. @@ -40,10 +60,11 @@ describe('readSyncMeta', () => { const db = freshDb(); writeSyncMeta(db, { 'imap_lastuid:INBOX': '100' }); - const meta = readSyncMeta(db, { 'imap_lastuid:INBOX': '9999', last_sync_at: '2026-06-01T00:00:00Z' }); + const meta = readSyncMeta(db, { 'imap_lastuid:INBOX': '9999' }); + // The key the file has is never overwritten — that is what stops a restored older mailbox being + // advanced past mail it does not contain. expect(meta['imap_lastuid:INBOX']).toBe('100'); - expect(meta.last_sync_at).toBeUndefined(); }); it('ignores non-string values rather than stringifying them', () => { diff --git a/src/servers/sidecar/email/sync-meta.ts b/src/servers/sidecar/email/sync-meta.ts index eb74e6bb..e0596092 100644 --- a/src/servers/sidecar/email/sync-meta.ts +++ b/src/servers/sidecar/email/sync-meta.ts @@ -45,13 +45,24 @@ export function readSyncMeta(db: Database, pgFallback?: Record const meta: Record = {}; for (const row of rows) meta[row.key] = row.value; - // Only when SQLite has nothing at all. A partially-written position is still this file's own, and - // topping it up from Postgres could reintroduce exactly the divergence this move exists to prevent. - if (Object.keys(meta).length > 0) return meta; - + // Merged PER KEY, and the file always wins a conflict. + // + // This started life as all-or-nothing — migrate only when SQLite is completely empty — on the + // assumption that a non-empty file is an authoritative one. A real account disproved it immediately. + // The older Gmail backfill wrote SOME keys into SQLite (`last_sync_at`, the `uidvalidity` set) and + // never the `imap_lastuid:*` ones, so the file was non-empty and half-migrated at the same time. + // All-or-nothing therefore skipped the migration, leaving no lastuid for any folder — and a missing + // lastuid means the next sync refetches that folder from UID 1. On the mailbox this was found on, + // that was 18,755 messages and 6.9 GB. + // + // Per-key with file-wins keeps the property that made all-or-nothing attractive: a restored older + // emails.db still overrides a newer Postgres row for every key it actually has, so it cannot be + // advanced past mail it does not contain. What it adds is the keys the file never had, which are + // exactly the ones whose absence is expensive. if (pgFallback) { for (const [key, value] of Object.entries(pgFallback)) { if (typeof value !== 'string') continue; + if (key in meta) continue; // the file's own answer stands meta[key] = value; setSyncMeta(db, key, value); }