Stage 2, and the end of the inversion. The two sync handlers (1,093 lines) ran in the platform's queue, which meant the sidecar reached back over its registration socket to ask the platform to enqueue work, and the credentials travelled through Postgres job metadata to get there. Option (A) from the plan: they run here now, and the Jobs screen is left to the things it actually describes. The handlers moved almost unedited. Their bodies were already a list of steps taking a context, so sync-runner.ts synthesizes that context and runs them; what went away is the JobHandler wrapper and the registration. `job.userId` is the OWNER'S EMAIL rather than a numeric id — the queue's naming — and it resolves the mail store path, so it is called out in the type. That is the same field whose absence made the mailbox read as empty two commits ago; it is set from user.email and checked this time. Deliberately not a queue: one run per account, no persistence, no retry. A failure is picked up by the ten-minute cron like any other, and a sync interrupted by a restart resumes from the stored cursor rather than the beginning. PermanentError survives as a local class — it signalled "do not retry" to the queue and now just carries its message to the sync state. accounts.ts asks the runner whether an account is syncing instead of scanning job rows, and the queue-over-WS shim in index.ts is gone: enqueueViaWs, listJobsViaWs, the pending-response map and the queue branch in the command handler. Nothing but a port crosses that socket now. The three chat channels stop opening the mail store directly. They each carried their own copy of count-rows / enqueue / poll / count-again, coupling three chat bridges to the mail schema — and they enqueued `gmail-sync` unconditionally, the OAuth path, for an app-password account that syncs over IMAP, so the command was already broken. One shared helper calls a new POST /sync-now on the sidecar, which syncs and reports what arrived. queue/handlers/ is now empty; both handlers there were email. The queue is untouched and still serves the Jobs screen. Not moved, and fine where they are: scripts/migrate-emails-to-sqlite.ts and scripts/seed-imap-uids.ts are one-off maintenance scripts that open the store directly. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
310 lines
10 KiB
TypeScript
310 lines
10 KiB
TypeScript
import { createHash } from 'node:crypto';
|
|
import {
|
|
getUserIntegration,
|
|
upsertUserIntegration,
|
|
getServerIntegration,
|
|
getEmailAccount,
|
|
updateEmailAccountSyncMeta,
|
|
} from 'officerdb';
|
|
import { refreshGoogleAccessToken } from '@@/api/integrations/google-auth';
|
|
import { openEmailDb, getSyncMeta, setSyncMeta, upsertFromRawEml } from './store';
|
|
import {
|
|
type GmailCredentials,
|
|
loadGmailCredentials,
|
|
gmailApiSync,
|
|
} from './gmail-api';
|
|
|
|
export type ResyncResult = { saved: number; skipped: number; errors: number };
|
|
|
|
// ── Gmail resync (REST API, history-based) ──
|
|
|
|
async function refreshCredentials(creds: GmailCredentials): Promise<GmailCredentials> {
|
|
if (!creds.accessToken || !creds.refreshToken) return creds;
|
|
|
|
const tokenExpired = !creds.expiresAt || creds.expiresAt < Date.now() + 60_000;
|
|
if (!tokenExpired) return creds;
|
|
|
|
console.log('[resync] Refreshing OAuth token');
|
|
const refreshed = await refreshGoogleAccessToken(creds.refreshToken);
|
|
|
|
const userGoogle = await getUserIntegration(creds.userId, 'google');
|
|
const existingConfig = (userGoogle?.config as Record<string, unknown>) ?? {};
|
|
const serverGoogle = await getServerIntegration('google');
|
|
await upsertUserIntegration({
|
|
userId: creds.userId,
|
|
provider: 'google',
|
|
serverIntegrationId: serverGoogle?.id,
|
|
config: { ...existingConfig, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
|
|
});
|
|
|
|
return { ...creds, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt };
|
|
}
|
|
|
|
async function gmailResync(userEmail: string, accountEmail: string): Promise<ResyncResult> {
|
|
let creds = await loadGmailCredentials(userEmail);
|
|
if (!creds.accessToken) {
|
|
throw new Error('OAuth not configured — connect Google in Settings → Integrations for resyncs');
|
|
}
|
|
|
|
creds = await refreshCredentials(creds);
|
|
|
|
const db = openEmailDb(userEmail, accountEmail);
|
|
try {
|
|
const result = await gmailApiSync({ creds, db });
|
|
|
|
setSyncMeta(db, 'last_sync_date', new Date().toISOString().split('T')[0]!);
|
|
setSyncMeta(db, 'last_sync_at', new Date().toISOString());
|
|
|
|
console.log(`[resync] Gmail done: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`);
|
|
return result;
|
|
} finally {
|
|
db.close();
|
|
}
|
|
}
|
|
|
|
// ── Generic IMAP resync ──
|
|
|
|
type ImapAccountInfo = {
|
|
id: number;
|
|
userId: number;
|
|
email: string;
|
|
imapHost: string;
|
|
imapPort: number;
|
|
imapSecure: boolean;
|
|
provider: string;
|
|
authType: string;
|
|
credentials: Record<string, unknown>;
|
|
};
|
|
|
|
type FolderInfo = {
|
|
specialUse?: string;
|
|
path: string;
|
|
flags: Set<string>;
|
|
status?: { uidNext?: number; uidValidity?: number };
|
|
};
|
|
|
|
const SPECIAL_USE_LABEL_MAP: Record<string, string> = {
|
|
'\\Inbox': 'inbox',
|
|
'\\Sent': 'sent',
|
|
'\\Drafts': 'draft',
|
|
'\\Flagged': 'starred',
|
|
'\\Trash': 'trash',
|
|
'\\Junk': 'spam',
|
|
'\\All': 'archive',
|
|
};
|
|
|
|
const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
|
|
|
|
function shouldSkipFolder(folder: FolderInfo): boolean {
|
|
if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
|
|
if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
|
|
return false;
|
|
}
|
|
|
|
function folderToLabel(folder: FolderInfo): string {
|
|
if (folder.specialUse && SPECIAL_USE_LABEL_MAP[folder.specialUse]) {
|
|
return SPECIAL_USE_LABEL_MAP[folder.specialUse]!;
|
|
}
|
|
if (folder.path === 'INBOX') return 'inbox';
|
|
return folder.path.toLowerCase();
|
|
}
|
|
|
|
function messageIdToStableId(raw: string): string | null {
|
|
const match = raw.match(/^Message-Id:\s*<?([^>\s]+)>?/im);
|
|
if (!match?.[1]) return null;
|
|
return createHash('sha1').update(match[1]).digest('hex').slice(0, 16);
|
|
}
|
|
|
|
async function resolveImapAuth(
|
|
account: ImapAccountInfo,
|
|
): Promise<{ user: string; pass?: string; accessToken?: string }> {
|
|
if (account.authType !== 'oauth') {
|
|
return { user: account.email, pass: account.credentials.password as string };
|
|
}
|
|
|
|
const userGoogle = await getUserIntegration(account.userId, 'google');
|
|
const config = userGoogle?.config as Record<string, unknown> | undefined;
|
|
if (!config?.refreshToken) throw new Error('Google OAuth not configured — reconnect your Google account');
|
|
|
|
const expiresAt = config.expiresAt as number | undefined;
|
|
const tokenExpired = !expiresAt || expiresAt < Date.now() + 60_000;
|
|
|
|
if (!tokenExpired && config.accessToken) {
|
|
return { user: account.email, accessToken: config.accessToken as string };
|
|
}
|
|
|
|
const refreshed = await refreshGoogleAccessToken(config.refreshToken as string);
|
|
const serverGoogle = await getServerIntegration('google');
|
|
await upsertUserIntegration({
|
|
userId: account.userId,
|
|
provider: 'google',
|
|
serverIntegrationId: serverGoogle?.id,
|
|
config: { ...config, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
|
|
});
|
|
|
|
return { user: account.email, accessToken: refreshed.accessToken };
|
|
}
|
|
|
|
async function imapResync(account: ImapAccountInfo, userEmail: string): Promise<ResyncResult> {
|
|
const { ImapFlow } = await import('imapflow');
|
|
|
|
const imapAuth = await resolveImapAuth(account);
|
|
|
|
const freshAccount = await getEmailAccount(account.id);
|
|
if (!freshAccount) throw new Error(`Email account ${account.id} not found`);
|
|
const syncMeta = (freshAccount.syncMeta ?? {}) as Record<string, unknown>;
|
|
|
|
const db = openEmailDb(userEmail, account.email);
|
|
const existingIds = new Set<string>();
|
|
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
|
for (const row of rows) existingIds.add(row.id);
|
|
|
|
let saved = 0;
|
|
let skipped = 0;
|
|
let errors = 0;
|
|
|
|
const client = new ImapFlow({
|
|
host: account.imapHost,
|
|
port: account.imapPort,
|
|
secure: account.imapSecure,
|
|
auth: imapAuth,
|
|
logger: false,
|
|
socketTimeout: 5 * 60 * 1000,
|
|
});
|
|
|
|
try {
|
|
await client.connect();
|
|
console.log('[resync] IMAP connected');
|
|
|
|
const folders = (await client.list({ statusQuery: { uidNext: true, uidValidity: true } })) as FolderInfo[];
|
|
|
|
for (const folder of folders) {
|
|
if (shouldSkipFolder(folder)) continue;
|
|
|
|
const uidValidityKey = `imap_uidvalidity:${folder.path}`;
|
|
const lastUidKey = `imap_lastuid:${folder.path}`;
|
|
const storedUidValidity = syncMeta[uidValidityKey] as string | undefined;
|
|
const storedLastUid = syncMeta[lastUidKey] as string | undefined;
|
|
const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
|
|
const uidNext = folder.status?.uidNext ?? 0;
|
|
const lastUid = storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
|
|
|
|
if (lastUid > 0 && uidNext <= lastUid + 1) continue;
|
|
|
|
let lock;
|
|
try {
|
|
lock = await client.getMailboxLock(folder.path);
|
|
} catch {
|
|
continue;
|
|
}
|
|
|
|
try {
|
|
const mailbox = client.mailbox;
|
|
if (!mailbox) continue;
|
|
|
|
const mbUidValidity = String(mailbox.uidValidity);
|
|
const effectiveLastUid = storedUidValidity === mbUidValidity ? lastUid : 0;
|
|
let maxUid = effectiveLastUid;
|
|
const label = folderToLabel(folder);
|
|
const range = effectiveLastUid > 0 ? `${effectiveLastUid + 1}:*` : '1:*';
|
|
|
|
try {
|
|
const fetchOpts: Record<string, unknown> = { source: true, uid: true };
|
|
if (account.provider === 'gmail') fetchOpts.labels = true;
|
|
|
|
for await (const msg of client.fetch(range, fetchOpts, { uid: true })) {
|
|
if (msg.uid <= effectiveLastUid) continue;
|
|
maxUid = Math.max(maxUid, msg.uid);
|
|
|
|
if (!msg.source) { errors++; continue; }
|
|
|
|
const raw = msg.source.toString('utf-8');
|
|
const id = messageIdToStableId(raw);
|
|
if (!id) { errors++; continue; }
|
|
if (existingIds.has(id)) { skipped++; continue; }
|
|
|
|
try {
|
|
upsertFromRawEml({ db, id, raw, integration: account.provider, emailAccount: account.email, labels: [label] });
|
|
existingIds.add(id);
|
|
saved++;
|
|
} catch {
|
|
errors++;
|
|
}
|
|
}
|
|
} catch (fetchErr) {
|
|
const errMsg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
|
|
if (!errMsg.includes('Nothing to fetch')) {
|
|
console.log(`[resync] Fetch error in ${folder.path}: ${errMsg}`);
|
|
errors++;
|
|
}
|
|
}
|
|
|
|
syncMeta[uidValidityKey] = mbUidValidity;
|
|
if (maxUid > effectiveLastUid) {
|
|
syncMeta[lastUidKey] = String(maxUid);
|
|
}
|
|
} finally {
|
|
lock.release();
|
|
}
|
|
}
|
|
|
|
await client.logout().catch(() => {});
|
|
} catch (err) {
|
|
await client.logout().catch(() => {});
|
|
throw err;
|
|
} finally {
|
|
db.close();
|
|
}
|
|
|
|
syncMeta.last_sync_at = new Date().toISOString();
|
|
await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
|
|
|
|
console.log(`[resync] IMAP done: saved ${saved}, skipped ${skipped}, errors ${errors}`);
|
|
return { saved, skipped, errors };
|
|
}
|
|
|
|
// ── Public API ──
|
|
|
|
type ResyncParams = {
|
|
accountId: number;
|
|
userEmail: string;
|
|
userId: number;
|
|
};
|
|
|
|
// Coalesce concurrent resyncs of the same account within this process (IDLE + cron + manual can all
|
|
// fire) — a caller arriving mid-resync just awaits the one already running.
|
|
const resyncInFlight = new Map<number, Promise<ResyncResult>>();
|
|
|
|
export function performResync(params: ResyncParams): Promise<ResyncResult> {
|
|
const running = resyncInFlight.get(params.accountId);
|
|
if (running) return running;
|
|
const p = doResync(params).finally(() => resyncInFlight.delete(params.accountId));
|
|
resyncInFlight.set(params.accountId, p);
|
|
return p;
|
|
}
|
|
|
|
async function doResync({ accountId, userEmail, userId }: ResyncParams): Promise<ResyncResult> {
|
|
const account = await getEmailAccount(accountId);
|
|
if (!account || account.userId !== userId) throw new Error('Account not found');
|
|
|
|
// Gmail API resync needs OAuth; a gmail account authed with an app password resyncs over IMAP.
|
|
if (account.provider === 'gmail' && account.authType === 'oauth') {
|
|
return gmailResync(userEmail, account.email);
|
|
}
|
|
|
|
return imapResync(
|
|
{
|
|
id: account.id,
|
|
userId: account.userId,
|
|
email: account.email,
|
|
imapHost: account.imapHost,
|
|
imapPort: account.imapPort,
|
|
imapSecure: account.imapSecure,
|
|
provider: account.provider,
|
|
authType: account.authType,
|
|
credentials: account.credentials as Record<string, unknown>,
|
|
},
|
|
userEmail,
|
|
);
|
|
}
|