email: the sidecar schedules its own syncs
Stage 2, and the end of the inversion. The two sync handlers (1,093 lines) ran in the platform's queue, which meant the sidecar reached back over its registration socket to ask the platform to enqueue work, and the credentials travelled through Postgres job metadata to get there. Option (A) from the plan: they run here now, and the Jobs screen is left to the things it actually describes. The handlers moved almost unedited. Their bodies were already a list of steps taking a context, so sync-runner.ts synthesizes that context and runs them; what went away is the JobHandler wrapper and the registration. `job.userId` is the OWNER'S EMAIL rather than a numeric id — the queue's naming — and it resolves the mail store path, so it is called out in the type. That is the same field whose absence made the mailbox read as empty two commits ago; it is set from user.email and checked this time. Deliberately not a queue: one run per account, no persistence, no retry. A failure is picked up by the ten-minute cron like any other, and a sync interrupted by a restart resumes from the stored cursor rather than the beginning. PermanentError survives as a local class — it signalled "do not retry" to the queue and now just carries its message to the sync state. accounts.ts asks the runner whether an account is syncing instead of scanning job rows, and the queue-over-WS shim in index.ts is gone: enqueueViaWs, listJobsViaWs, the pending-response map and the queue branch in the command handler. Nothing but a port crosses that socket now. The three chat channels stop opening the mail store directly. They each carried their own copy of count-rows / enqueue / poll / count-again, coupling three chat bridges to the mail schema — and they enqueued `gmail-sync` unconditionally, the OAuth path, for an app-password account that syncs over IMAP, so the command was already broken. One shared helper calls a new POST /sync-now on the sidecar, which syncs and reports what arrived. queue/handlers/ is now empty; both handlers there were email. The queue is untouched and still serves the Jobs screen. Not moved, and fine where they are: scripts/migrate-emails-to-sqlite.ts and scripts/seed-imap-uids.ts are one-off maintenance scripts that open the store directly. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -9,7 +9,9 @@ import {
|
||||
} from 'officerdb';
|
||||
import { getValidGoogleAccessToken } from '../../api/integrations/google-auth';
|
||||
import { validateImapConnection } from './imap-validate';
|
||||
import { enqueueJob, listAllJobs } from '../../queue/init';
|
||||
import { startSync, isSyncing, getSyncStates } from './sync-runner';
|
||||
import { imapSyncSteps } from './imap-sync';
|
||||
import { gmailSyncSteps } from './gmail-api';
|
||||
import { openEmailDb, getSyncMeta } from './store';
|
||||
import { performResync } from './resync';
|
||||
|
||||
@@ -45,17 +47,9 @@ accountsRouter.get('/', async (ctx) => {
|
||||
let activeJobAccountIds = new Set<number>();
|
||||
|
||||
if (hasActiveAccounts) {
|
||||
try {
|
||||
const jobs = await listAllJobs();
|
||||
activeJobAccountIds = new Set(
|
||||
jobs
|
||||
.filter((j) => (j.type === 'email-sync' || j.type === 'gmail-sync') && (j.status === 'queued' || j.status === 'running'))
|
||||
.map((j) => (j.meta as Record<string, unknown> | undefined)?.emailAccountId as number)
|
||||
.filter(Boolean),
|
||||
);
|
||||
} catch {
|
||||
// Sidecar unavailable — all syncing/queued accounts are stale
|
||||
}
|
||||
// In-process now: a running sync is one this process started, not a queued job row. An account marked
|
||||
// syncing with nothing running is stale — the usual cause is a restart mid-sync.
|
||||
activeJobAccountIds = new Set(getSyncStates().filter((s) => s.status === 'running').map((s) => s.accountId));
|
||||
|
||||
for (const a of accounts) {
|
||||
if ((a.status === 'syncing' || a.status === 'queued') && !activeJobAccountIds.has(a.id)) {
|
||||
@@ -177,12 +171,16 @@ accountsRouter.post('/:id/sync', async (ctx) => {
|
||||
await updateEmailAccountStatus(id, 'queued');
|
||||
|
||||
// Gmail API sync needs OAuth; a gmail account with an app password syncs over IMAP instead.
|
||||
const jobType = account.provider === 'gmail' && account.authType === 'oauth' ? 'gmail-sync' : 'email-sync';
|
||||
const useGmailApi = account.provider === 'gmail' && account.authType === 'oauth';
|
||||
|
||||
const job = await enqueueJob({
|
||||
lane: 'email',
|
||||
type: jobType,
|
||||
userId: user.email,
|
||||
if (isSyncing(id)) return ctx.json({ ok: true, alreadyRunning: true });
|
||||
|
||||
startSync({
|
||||
accountId: id,
|
||||
// The OWNER's email, not the account's — the store path is keyed by it.
|
||||
ownerEmail: user.email,
|
||||
steps: useGmailApi ? gmailSyncSteps : imapSyncSteps,
|
||||
onDone: (ok) => updateEmailAccountStatus(id, ok ? 'synced' : 'error').then(() => undefined),
|
||||
meta: {
|
||||
emailAccountId: id,
|
||||
userEmail: user.email,
|
||||
@@ -201,7 +199,7 @@ accountsRouter.post('/:id/sync', async (ctx) => {
|
||||
},
|
||||
});
|
||||
|
||||
return ctx.json({ ok: true, jobId: job.id }, 201);
|
||||
return ctx.json({ ok: true }, 201);
|
||||
});
|
||||
|
||||
accountsRouter.post('/validate', async (ctx) => {
|
||||
|
||||
@@ -0,0 +1,717 @@
|
||||
import type { Database } from 'bun:sqlite';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { openEmailDb, upsertFromRawEml, setSyncMeta, getSyncMeta } from './store';
|
||||
import {
|
||||
getUserByEmail,
|
||||
getUserIntegration,
|
||||
upsertUserIntegration,
|
||||
getServerIntegration,
|
||||
getDockPaths,
|
||||
setDockPaths,
|
||||
updateEmailAccountStatus,
|
||||
getEmailAccount,
|
||||
} from 'officerdb';
|
||||
import { refreshGoogleAccessToken } from '@@/api/integrations/google-auth';
|
||||
|
||||
import type { SyncStep, SyncCtx } from './sync-runner';
|
||||
|
||||
// Thrown by the handlers for failures that a retry cannot fix (missing credentials, deleted account).
|
||||
// The queue used it to skip retrying; here it is an ordinary error whose message reaches the sync state.
|
||||
export class PermanentError extends Error {}
|
||||
|
||||
// ── Credentials ──
|
||||
|
||||
export type GmailCredentials = {
|
||||
email: string;
|
||||
userId: number;
|
||||
appPassword?: string;
|
||||
accessToken?: string;
|
||||
refreshToken?: string;
|
||||
expiresAt?: number;
|
||||
};
|
||||
|
||||
export async function loadGmailCredentials(userEmail: string): Promise<GmailCredentials> {
|
||||
const dbUser = await getUserByEmail(userEmail);
|
||||
if (!dbUser) throw new PermanentError('User not found');
|
||||
|
||||
const userGoogle = await getUserIntegration(dbUser.id, 'google');
|
||||
const config = userGoogle?.config as Record<string, unknown> | undefined;
|
||||
|
||||
const gmailEmail = (config?.email as string) ?? userEmail;
|
||||
|
||||
// Prefer OAuth tokens, fall back to app password
|
||||
if (config?.accessToken && config?.refreshToken) {
|
||||
return {
|
||||
email: gmailEmail,
|
||||
userId: dbUser.id,
|
||||
accessToken: config.accessToken as string,
|
||||
refreshToken: config.refreshToken as string,
|
||||
expiresAt: config.expiresAt as number | undefined,
|
||||
appPassword: config.imapAppPassword as string | undefined,
|
||||
};
|
||||
}
|
||||
|
||||
if (!config?.imapAppPassword) {
|
||||
throw new PermanentError('Gmail App Password not configured — set it in Settings → Integrations');
|
||||
}
|
||||
|
||||
return { email: gmailEmail, userId: dbUser.id, appPassword: config.imapAppPassword as string };
|
||||
}
|
||||
|
||||
// ── Stable ID from Message-Id header ──
|
||||
|
||||
function messageIdToStableId(raw: string): string | null {
|
||||
const match = raw.match(/^Message-Id:\s*<?([^>\s]+)>?/im);
|
||||
if (!match?.[1]) return null;
|
||||
return createHash('sha1').update(match[1]).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
// ── Gmail API label → our label mapping ──
|
||||
|
||||
const GMAIL_API_LABEL_MAP: Record<string, string> = {
|
||||
INBOX: 'inbox',
|
||||
SENT: 'sent',
|
||||
DRAFT: 'draft',
|
||||
STARRED: 'starred',
|
||||
IMPORTANT: 'important',
|
||||
TRASH: 'trash',
|
||||
SPAM: 'spam',
|
||||
CATEGORY_PROMOTIONS: 'promotions',
|
||||
CATEGORY_SOCIAL: 'social',
|
||||
CATEGORY_UPDATES: 'updates',
|
||||
CATEGORY_FORUMS: 'forums',
|
||||
};
|
||||
|
||||
const SKIP_LABELS = new Set(['TRASH', 'SPAM', 'DRAFT']);
|
||||
|
||||
function gmailApiLabelsToLabels(labelIds: string[]): string[] {
|
||||
const labels: string[] = [];
|
||||
for (const id of labelIds) {
|
||||
const mapped = GMAIL_API_LABEL_MAP[id];
|
||||
if (mapped) {
|
||||
labels.push(mapped);
|
||||
} else if (!id.startsWith('CATEGORY_') && id !== 'UNREAD' && id !== 'IMPORTANT') {
|
||||
// Custom label — use as-is (Gmail API returns label IDs, not names, for custom labels)
|
||||
labels.push(id.toLowerCase());
|
||||
}
|
||||
}
|
||||
return labels;
|
||||
}
|
||||
|
||||
function shouldSkipMessage(labelIds: string[]): boolean {
|
||||
return labelIds.some((id) => SKIP_LABELS.has(id));
|
||||
}
|
||||
|
||||
// ── Gmail API helpers ──
|
||||
|
||||
const GMAIL_API = 'https://gmail.googleapis.com/gmail/v1/users/me';
|
||||
|
||||
export type GmailApiResult = { saved: number; skipped: number; errors: number };
|
||||
|
||||
async function gmailApiFetch(accessToken: string, path: string, params?: Record<string, string>): Promise<Response> {
|
||||
const url = new URL(`${GMAIL_API}${path}`);
|
||||
if (params) {
|
||||
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
|
||||
}
|
||||
const res = await fetch(url, { headers: { Authorization: `Bearer ${accessToken}` } });
|
||||
return res;
|
||||
}
|
||||
|
||||
async function getGmailProfile(accessToken: string): Promise<{ historyId: string }> {
|
||||
const res = await gmailApiFetch(accessToken, '/profile');
|
||||
if (!res.ok) throw new Error(`Gmail profile request failed: ${res.status} ${await res.text()}`);
|
||||
const data = (await res.json()) as { historyId: string };
|
||||
return data;
|
||||
}
|
||||
|
||||
type HistoryMessage = { id: string; threadId: string };
|
||||
type HistoryEntry = { messagesAdded?: Array<{ message: HistoryMessage }> };
|
||||
type HistoryResponse = { history?: HistoryEntry[]; historyId: string; nextPageToken?: string };
|
||||
|
||||
async function getHistoryChanges(
|
||||
accessToken: string,
|
||||
startHistoryId: string,
|
||||
): Promise<{ messageIds: string[]; historyId: string }> {
|
||||
const messageIds = new Set<string>();
|
||||
let pageToken: string | undefined;
|
||||
let latestHistoryId = startHistoryId;
|
||||
|
||||
while (true) {
|
||||
const params: Record<string, string> = {
|
||||
startHistoryId,
|
||||
historyTypes: 'messageAdded',
|
||||
maxResults: '500',
|
||||
};
|
||||
if (pageToken) params.pageToken = pageToken;
|
||||
|
||||
const res = await gmailApiFetch(accessToken, '/history', params);
|
||||
|
||||
if (res.status === 404 || res.status === 410) {
|
||||
// historyId too old or invalid — caller should fall back to date-based listing
|
||||
throw new HistoryExpiredError();
|
||||
}
|
||||
|
||||
if (!res.ok) throw new Error(`Gmail history request failed: ${res.status} ${await res.text()}`);
|
||||
|
||||
const data = (await res.json()) as HistoryResponse;
|
||||
latestHistoryId = data.historyId;
|
||||
|
||||
if (data.history) {
|
||||
for (const entry of data.history) {
|
||||
if (entry.messagesAdded) {
|
||||
for (const added of entry.messagesAdded) {
|
||||
messageIds.add(added.message.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!data.nextPageToken) break;
|
||||
pageToken = data.nextPageToken;
|
||||
}
|
||||
|
||||
return { messageIds: [...messageIds], historyId: latestHistoryId };
|
||||
}
|
||||
|
||||
class HistoryExpiredError extends Error {
|
||||
constructor() {
|
||||
super('Gmail historyId expired');
|
||||
}
|
||||
}
|
||||
|
||||
type MessageListResponse = { messages?: Array<{ id: string }>; nextPageToken?: string };
|
||||
|
||||
async function listMessagesSince(accessToken: string, sinceDate: string): Promise<string[]> {
|
||||
const messageIds: string[] = [];
|
||||
let pageToken: string | undefined;
|
||||
|
||||
while (true) {
|
||||
const params: Record<string, string> = {
|
||||
q: `after:${sinceDate}`,
|
||||
maxResults: '500',
|
||||
};
|
||||
if (pageToken) params.pageToken = pageToken;
|
||||
|
||||
const res = await gmailApiFetch(accessToken, '/messages', params);
|
||||
if (!res.ok) throw new Error(`Gmail messages.list failed: ${res.status} ${await res.text()}`);
|
||||
|
||||
const data = (await res.json()) as MessageListResponse;
|
||||
if (data.messages) {
|
||||
for (const msg of data.messages) messageIds.push(msg.id);
|
||||
}
|
||||
|
||||
if (!data.nextPageToken) break;
|
||||
pageToken = data.nextPageToken;
|
||||
}
|
||||
|
||||
return messageIds;
|
||||
}
|
||||
|
||||
type GmailMessage = { id: string; labelIds: string[]; raw: string; historyId: string };
|
||||
|
||||
async function fetchRawMessage(accessToken: string, messageId: string): Promise<GmailMessage | null> {
|
||||
const res = await gmailApiFetch(accessToken, `/messages/${messageId}`, { format: 'raw' });
|
||||
if (res.status === 404) return null;
|
||||
if (!res.ok) throw new Error(`Gmail messages.get failed: ${res.status} ${await res.text()}`);
|
||||
return (await res.json()) as GmailMessage;
|
||||
}
|
||||
|
||||
// ── Gmail API sync (for resyncs) ──
|
||||
|
||||
export type GmailApiSyncParams = {
|
||||
creds: GmailCredentials;
|
||||
db: Database;
|
||||
onProgress?: (saved: number, total: number) => void;
|
||||
};
|
||||
|
||||
export async function gmailApiSync({ creds, db, onProgress }: GmailApiSyncParams): Promise<GmailApiResult> {
|
||||
if (!creds.accessToken) throw new PermanentError('OAuth access token required for Gmail API sync');
|
||||
|
||||
let saved = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
// Load existing IDs for dedup
|
||||
const existingIds = new Set<string>();
|
||||
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
||||
for (const row of rows) existingIds.add(row.id);
|
||||
|
||||
const storedHistoryId = getSyncMeta(db, 'gmail_history_id');
|
||||
|
||||
let messageIds: string[];
|
||||
let newHistoryId: string;
|
||||
|
||||
if (storedHistoryId) {
|
||||
// Try history-based sync first
|
||||
try {
|
||||
const result = await getHistoryChanges(creds.accessToken, storedHistoryId);
|
||||
messageIds = result.messageIds;
|
||||
newHistoryId = result.historyId;
|
||||
console.log(`[gmail-sync] History returned ${messageIds.length} new message(s)`);
|
||||
} catch (err) {
|
||||
if (err instanceof HistoryExpiredError) {
|
||||
// historyId too old — fall back to date-based listing
|
||||
const lastSyncDate = getSyncMeta(db, 'last_sync_date');
|
||||
const sinceDate = lastSyncDate ?? new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString().split('T')[0]!;
|
||||
console.log(`[gmail-sync] History expired, falling back to messages since ${sinceDate}`);
|
||||
messageIds = await listMessagesSince(creds.accessToken, sinceDate);
|
||||
const profile = await getGmailProfile(creds.accessToken);
|
||||
newHistoryId = profile.historyId;
|
||||
console.log(`[gmail-sync] Date query returned ${messageIds.length} message(s)`);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// No historyId stored — use date-based listing
|
||||
const lastSyncDate = getSyncMeta(db, 'last_sync_date');
|
||||
const sinceDate = lastSyncDate ?? new Date(Date.now() - 7 * 24 * 60 * 60 * 1000).toISOString().split('T')[0]!;
|
||||
console.log(`[gmail-sync] No historyId, listing messages since ${sinceDate}`);
|
||||
messageIds = await listMessagesSince(creds.accessToken, sinceDate);
|
||||
const profile = await getGmailProfile(creds.accessToken);
|
||||
newHistoryId = profile.historyId;
|
||||
console.log(`[gmail-sync] Date query returned ${messageIds.length} message(s)`);
|
||||
}
|
||||
|
||||
// Fetch and store each new message
|
||||
for (let i = 0; i < messageIds.length; i++) {
|
||||
const gmailId = messageIds[i]!;
|
||||
|
||||
try {
|
||||
const msg = await fetchRawMessage(creds.accessToken, gmailId);
|
||||
if (!msg) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (shouldSkipMessage(msg.labelIds ?? [])) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Gmail API returns base64url-encoded raw RFC822
|
||||
const rawBytes = Buffer.from(msg.raw, 'base64url');
|
||||
const raw = rawBytes.toString('utf-8');
|
||||
|
||||
const id = messageIdToStableId(raw);
|
||||
if (!id) {
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existingIds.has(id)) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const labels = gmailApiLabelsToLabels(msg.labelIds ?? []);
|
||||
|
||||
upsertFromRawEml({ db, id, raw, integration: 'gmail', emailAccount: creds.email, labels });
|
||||
existingIds.add(id);
|
||||
saved++;
|
||||
|
||||
if ((saved + skipped) % 50 === 0) {
|
||||
onProgress?.(saved, messageIds.length);
|
||||
}
|
||||
} catch (err) {
|
||||
console.log(`[gmail-sync] Error fetching message ${gmailId}: ${err instanceof Error ? err.message : err}`);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
// Persist new historyId
|
||||
setSyncMeta(db, 'gmail_history_id', newHistoryId);
|
||||
|
||||
return { saved, skipped, errors };
|
||||
}
|
||||
|
||||
// ── IMAP sync (for first sync) ──
|
||||
|
||||
// Folder → label mapping for IMAP
|
||||
const GMAIL_PREFIX_RE = /^\[(?:Gmail|Google Mail)\]\//;
|
||||
|
||||
function normalizeGmailFolder(folder: string): string {
|
||||
return folder.replace(GMAIL_PREFIX_RE, '');
|
||||
}
|
||||
|
||||
const SUFFIX_LABEL_MAP: Record<string, string> = {
|
||||
'Sent Mail': 'sent',
|
||||
Drafts: 'draft',
|
||||
Starred: 'starred',
|
||||
Important: 'important',
|
||||
'All Mail': 'archive',
|
||||
Trash: 'trash',
|
||||
Bin: 'trash',
|
||||
Spam: 'spam',
|
||||
};
|
||||
|
||||
function folderToLabel(folder: string): string | null {
|
||||
const suffix = normalizeGmailFolder(folder);
|
||||
if (suffix !== folder) {
|
||||
if (SUFFIX_LABEL_MAP[suffix]) return SUFFIX_LABEL_MAP[suffix]!;
|
||||
return suffix.toLowerCase();
|
||||
}
|
||||
if (folder === 'INBOX') return 'inbox';
|
||||
return folder.toLowerCase();
|
||||
}
|
||||
|
||||
const IMAP_LABEL_MAP: Record<string, string> = {
|
||||
'\\Inbox': 'inbox',
|
||||
'\\Sent': 'sent',
|
||||
'\\Drafts': 'draft',
|
||||
'\\Starred': 'starred',
|
||||
'\\Important': 'important',
|
||||
'\\All': 'archive',
|
||||
'\\Trash': 'trash',
|
||||
'\\Junk': 'spam',
|
||||
};
|
||||
|
||||
function imapLabelsToLabels(gmailLabels: Set<string>): string[] {
|
||||
const labels: string[] = [];
|
||||
for (const gl of gmailLabels) {
|
||||
const mapped = IMAP_LABEL_MAP[gl];
|
||||
if (mapped) {
|
||||
labels.push(mapped);
|
||||
} else if (!gl.startsWith('\\')) {
|
||||
labels.push(gl.toLowerCase());
|
||||
}
|
||||
}
|
||||
if (labels.length > 1 && labels.includes('archive')) {
|
||||
return labels.filter((l) => l !== 'archive');
|
||||
}
|
||||
return labels;
|
||||
}
|
||||
|
||||
const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
|
||||
|
||||
function shouldSkipFolder(folder: { specialUse?: string; path: string; flags: Set<string> }): boolean {
|
||||
if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
|
||||
if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
|
||||
const norm = normalizeGmailFolder(folder.path);
|
||||
return norm === 'All Mail' || norm === 'Trash' || norm === 'Spam' || norm === 'Bin';
|
||||
}
|
||||
|
||||
type ImapSyncParams = {
|
||||
creds: GmailCredentials;
|
||||
db: Database;
|
||||
onProgress?: (fetched: number, folder: string) => void;
|
||||
};
|
||||
|
||||
type SyncResult = { saved: number; skipped: number; errors: number };
|
||||
|
||||
async function imapFullSync({ creds, db, onProgress }: ImapSyncParams): Promise<SyncResult> {
|
||||
const { ImapFlow } = await import('imapflow');
|
||||
|
||||
const auth: { user: string; pass?: string; accessToken?: string } = { user: creds.email };
|
||||
if (creds.appPassword) {
|
||||
auth.pass = creds.appPassword;
|
||||
} else if (creds.accessToken) {
|
||||
auth.accessToken = creds.accessToken;
|
||||
} else {
|
||||
throw new PermanentError('No authentication method available for IMAP');
|
||||
}
|
||||
|
||||
const client = new ImapFlow({
|
||||
host: 'imap.gmail.com',
|
||||
port: 993,
|
||||
secure: true,
|
||||
auth,
|
||||
logger: false,
|
||||
});
|
||||
|
||||
let saved = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
try {
|
||||
await client.connect();
|
||||
console.log('[gmail-sync] IMAP connected for full sync');
|
||||
|
||||
const folders = await client.list({ statusQuery: { uidNext: true, uidValidity: true } });
|
||||
|
||||
const existingIds = new Set<string>();
|
||||
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
||||
for (const row of rows) existingIds.add(row.id);
|
||||
|
||||
const foldersToSync: typeof folders = [];
|
||||
for (const folder of folders) {
|
||||
if (shouldSkipFolder(folder)) continue;
|
||||
|
||||
const folderPath = folder.path;
|
||||
const uidValidityKey = `imap_uidvalidity:${folderPath}`;
|
||||
const lastUidKey = `imap_lastuid:${folderPath}`;
|
||||
const storedUidValidity = getSyncMeta(db, uidValidityKey);
|
||||
const storedLastUid = getSyncMeta(db, lastUidKey);
|
||||
|
||||
const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
|
||||
const uidNext = folder.status?.uidNext ?? 0;
|
||||
const lastUid =
|
||||
storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
|
||||
|
||||
if (uidValidity) setSyncMeta(db, uidValidityKey, uidValidity);
|
||||
if (lastUid > 0 && uidNext <= lastUid + 1) continue;
|
||||
|
||||
foldersToSync.push(folder);
|
||||
}
|
||||
|
||||
console.log(`[gmail-sync] ${foldersToSync.length} folder(s) to sync`);
|
||||
|
||||
for (const folder of foldersToSync) {
|
||||
const folderPath = folder.path;
|
||||
const uidValidityKey = `imap_uidvalidity:${folderPath}`;
|
||||
const lastUidKey = `imap_lastuid:${folderPath}`;
|
||||
const storedUidValidity = getSyncMeta(db, uidValidityKey);
|
||||
const storedLastUid = getSyncMeta(db, lastUidKey);
|
||||
|
||||
let lock;
|
||||
try {
|
||||
lock = await client.getMailboxLock(folderPath);
|
||||
} catch (err) {
|
||||
console.log(`[gmail-sync] Skipping folder ${folderPath}: ${err instanceof Error ? err.message : err}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const mailbox = client.mailbox;
|
||||
if (!mailbox) continue;
|
||||
|
||||
const uidValidity = String(mailbox.uidValidity);
|
||||
let lastUid = 0;
|
||||
if (storedUidValidity === uidValidity && storedLastUid) {
|
||||
lastUid = parseInt(storedLastUid, 10);
|
||||
}
|
||||
|
||||
const range = lastUid > 0 ? `${lastUid + 1}:*` : '1:*';
|
||||
let maxUid = lastUid;
|
||||
let folderFetched = 0;
|
||||
|
||||
try {
|
||||
for await (const msg of client.fetch(range, { source: true, labels: true, uid: true }, { uid: true })) {
|
||||
if (msg.uid <= lastUid) continue;
|
||||
|
||||
maxUid = Math.max(maxUid, msg.uid);
|
||||
folderFetched++;
|
||||
|
||||
if (!msg.source) {
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const raw = msg.source.toString('utf-8');
|
||||
const id = messageIdToStableId(raw);
|
||||
if (!id) {
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (existingIds.has(id)) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const folderLabel = folderToLabel(folderPath);
|
||||
const gmailLabels = msg.labels ? imapLabelsToLabels(msg.labels) : [];
|
||||
const labels = folderLabel ? [...new Set([folderLabel, ...gmailLabels])] : gmailLabels;
|
||||
|
||||
try {
|
||||
upsertFromRawEml({ db, id, raw, integration: 'gmail', emailAccount: creds.email, labels });
|
||||
existingIds.add(id);
|
||||
saved++;
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
} catch (fetchErr) {
|
||||
const msg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
|
||||
if (!msg.includes('Nothing to fetch')) {
|
||||
console.log(`[gmail-sync] Fetch error in ${folderPath}: ${msg}`);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
if (maxUid > lastUid) {
|
||||
setSyncMeta(db, lastUidKey, String(maxUid));
|
||||
}
|
||||
setSyncMeta(db, uidValidityKey, uidValidity);
|
||||
|
||||
if (folderFetched > 0) {
|
||||
console.log(`[gmail-sync] ${folderPath}: fetched ${folderFetched}, saved ${saved}, skipped ${skipped}`);
|
||||
onProgress?.(saved + skipped, folderPath);
|
||||
}
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await client.logout().catch(() => {});
|
||||
}
|
||||
|
||||
return { saved, skipped, errors };
|
||||
}
|
||||
|
||||
// ── Handler ──
|
||||
|
||||
const gmailSyncHandler = {
|
||||
type: 'gmail-sync',
|
||||
retry: { delayMs: 15 * 60 * 1000, maxRetries: 10 },
|
||||
steps: [
|
||||
{
|
||||
name: 'Verify credentials',
|
||||
run: async (ctx: SyncCtx) => {
|
||||
const creds = await loadGmailCredentials(ctx.job.userId);
|
||||
const emailAccountId = (ctx.meta as Record<string, unknown>).emailAccountId as number | undefined;
|
||||
const syncAccount = emailAccountId ? await getEmailAccount(emailAccountId) : undefined;
|
||||
|
||||
// Determine sync mode — check SQLite first, fall back to PostgreSQL syncMeta
|
||||
const db = openEmailDb(ctx.job.userId, syncAccount?.email ?? ctx.job.userId);
|
||||
let lastSyncAt: string | null = null;
|
||||
try {
|
||||
lastSyncAt = getSyncMeta(db, 'last_sync_at');
|
||||
if (!lastSyncAt && emailAccountId) {
|
||||
const pgAccount = await getEmailAccount(emailAccountId);
|
||||
const pgMeta = pgAccount?.syncMeta as Record<string, string> | null;
|
||||
if (pgMeta?.last_sync_at) {
|
||||
lastSyncAt = pgMeta.last_sync_at;
|
||||
// Backfill SQLite so future checks don't need PostgreSQL
|
||||
setSyncMeta(db, 'last_sync_at', lastSyncAt);
|
||||
if (pgMeta.last_sync_date) setSyncMeta(db, 'last_sync_date', pgMeta.last_sync_date);
|
||||
}
|
||||
}
|
||||
ctx.meta.isFirstSync = !lastSyncAt;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
if (emailAccountId) {
|
||||
await updateEmailAccountStatus(emailAccountId, 'syncing');
|
||||
ctx.meta.emailAccountId = emailAccountId;
|
||||
}
|
||||
|
||||
// For API resyncs, refresh OAuth token if expired
|
||||
if (!ctx.meta.isFirstSync && creds.accessToken && creds.refreshToken) {
|
||||
const tokenExpired = !creds.expiresAt || creds.expiresAt < Date.now() + 60_000;
|
||||
if (tokenExpired) {
|
||||
console.log('[gmail-sync] Refreshing OAuth token');
|
||||
try {
|
||||
const refreshed = await refreshGoogleAccessToken(creds.refreshToken);
|
||||
creds.accessToken = refreshed.accessToken;
|
||||
creds.expiresAt = refreshed.expiresAt;
|
||||
|
||||
const userGoogle = await getUserIntegration(creds.userId, 'google');
|
||||
const existingConfig = (userGoogle?.config as Record<string, unknown>) ?? {};
|
||||
const serverGoogle = await getServerIntegration('google');
|
||||
await upsertUserIntegration({
|
||||
userId: creds.userId,
|
||||
provider: 'google',
|
||||
serverIntegrationId: serverGoogle?.id,
|
||||
config: { ...existingConfig, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
|
||||
});
|
||||
} catch (err) {
|
||||
throw new PermanentError(`OAuth refresh failed — reconnect Google in Settings: ${err}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!ctx.meta.isFirstSync && !creds.accessToken) {
|
||||
throw new PermanentError('OAuth not configured — connect Google in Settings → Integrations for resyncs');
|
||||
}
|
||||
|
||||
ctx.meta.creds = creds;
|
||||
console.log(
|
||||
`[gmail-sync] ${ctx.meta.isFirstSync ? 'First sync (IMAP)' : 'Resync (Gmail API)'} — ${creds.accessToken ? 'OAuth' : 'App Password'}`,
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Sync emails',
|
||||
run: async (ctx: SyncCtx) => {
|
||||
const creds = ctx.meta.creds as GmailCredentials;
|
||||
|
||||
const emailAccountId = (ctx.meta as Record<string, unknown>).emailAccountId as number | undefined;
|
||||
const syncAccount = emailAccountId ? await getEmailAccount(emailAccountId) : undefined;
|
||||
const db = openEmailDb(ctx.job.userId, syncAccount?.email ?? ctx.job.userId);
|
||||
try {
|
||||
let result: SyncResult;
|
||||
|
||||
if (ctx.meta.isFirstSync) {
|
||||
// First sync: IMAP with app password
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Connecting to Gmail (IMAP)...' });
|
||||
|
||||
result = await imapFullSync({
|
||||
creds,
|
||||
db,
|
||||
onProgress: (fetched, folder) => {
|
||||
ctx.updateProgress({ current: fetched, total: 0, label: `Syncing ${folder}...` });
|
||||
},
|
||||
});
|
||||
|
||||
// Seed historyId for future API syncs
|
||||
if (creds.accessToken) {
|
||||
try {
|
||||
const profile = await getGmailProfile(creds.accessToken);
|
||||
setSyncMeta(db, 'gmail_history_id', profile.historyId);
|
||||
console.log(`[gmail-sync] Seeded historyId: ${profile.historyId}`);
|
||||
} catch (err) {
|
||||
console.log(`[gmail-sync] Could not seed historyId: ${err}`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Resync: Gmail API with OAuth
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Checking for new emails...' });
|
||||
|
||||
result = await gmailApiSync({
|
||||
creds,
|
||||
db,
|
||||
onProgress: (saved, total) => {
|
||||
ctx.updateProgress({ current: saved, total, label: `Fetching new emails (${saved}/${total})...` });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
console.log(
|
||||
`[gmail-sync] Done: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`,
|
||||
);
|
||||
|
||||
setSyncMeta(db, 'last_sync_date', new Date().toISOString().split('T')[0]!);
|
||||
setSyncMeta(db, 'last_sync_at', new Date().toISOString());
|
||||
|
||||
ctx.meta.syncResult = result;
|
||||
await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${result.saved} new emails` });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Finalize',
|
||||
run: async (ctx: SyncCtx) => {
|
||||
const result = ctx.meta.syncResult as SyncResult | undefined;
|
||||
const emailAccountId = ctx.meta.emailAccountId as number | undefined;
|
||||
|
||||
if (emailAccountId) {
|
||||
await updateEmailAccountStatus(emailAccountId, 'synced');
|
||||
}
|
||||
|
||||
if (result && result.saved > 0) {
|
||||
try {
|
||||
const dbUser = await getUserByEmail(ctx.job.userId);
|
||||
if (dbUser) {
|
||||
const paths = await getDockPaths(dbUser.id);
|
||||
if (!paths) {
|
||||
const defaults = ['/', '/files', '/automation', '/dashboards', '/chat'];
|
||||
await setDockPaths(dbUser.id, [...defaults, '/email']);
|
||||
} else if (!paths.includes('/email')) {
|
||||
await setDockPaths(dbUser.id, [...paths, '/email']);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Non-fatal
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
/** The handler's steps, for the sidecar's own runner. */
|
||||
export const gmailSyncSteps: SyncStep[] = gmailSyncHandler.steps as SyncStep[];
|
||||
@@ -0,0 +1,385 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { openEmailDb, upsertFromRawEml } from './store';
|
||||
import { refreshGoogleAccessToken } from '../../api/integrations/google-auth';
|
||||
import {
|
||||
getEmailAccount,
|
||||
getUserIntegration,
|
||||
upsertUserIntegration,
|
||||
getServerIntegration,
|
||||
updateEmailAccountStatus,
|
||||
updateEmailAccountSyncMeta,
|
||||
getDockPaths,
|
||||
setDockPaths,
|
||||
} from 'officerdb';
|
||||
|
||||
import type { SyncStep, SyncCtx } from './sync-runner';
|
||||
|
||||
// Thrown by the handlers for failures that a retry cannot fix (missing credentials, deleted account).
|
||||
// The queue used it to skip retrying; here it is an ordinary error whose message reaches the sync state.
|
||||
export class PermanentError extends Error {}
|
||||
|
||||
// ── Types for job meta (passed by the API server at enqueue time) ──
|
||||
|
||||
type EmailSyncMeta = {
|
||||
emailAccountId: number;
|
||||
userEmail: string;
|
||||
account: {
|
||||
id: number;
|
||||
userId: number;
|
||||
email: string;
|
||||
imapHost: string;
|
||||
imapPort: number;
|
||||
imapSecure: boolean;
|
||||
provider: string;
|
||||
authType: string;
|
||||
credentials: Record<string, unknown>;
|
||||
};
|
||||
imapAuth: { user: string; pass?: string; accessToken?: string };
|
||||
saved?: number;
|
||||
};
|
||||
|
||||
// ── Stable ID from Message-Id header ──
|
||||
|
||||
function messageIdToStableId(raw: string): string | null {
|
||||
const match = raw.match(/^Message-Id:\s*<?([^>\s]+)>?/im);
|
||||
if (!match?.[1]) return null;
|
||||
return createHash('sha1').update(match[1]).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
// ── IMAP folder → label mapping ──
|
||||
|
||||
const SPECIAL_USE_LABEL_MAP: Record<string, string> = {
|
||||
'\\Inbox': 'inbox',
|
||||
'\\Sent': 'sent',
|
||||
'\\Drafts': 'draft',
|
||||
'\\Flagged': 'starred',
|
||||
'\\Trash': 'trash',
|
||||
'\\Junk': 'spam',
|
||||
'\\All': 'archive',
|
||||
};
|
||||
|
||||
const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
|
||||
|
||||
type FolderInfo = { specialUse?: string; path: string; flags: Set<string>; status?: { uidNext?: number; uidValidity?: number } };
|
||||
|
||||
function shouldSkipFolder(folder: FolderInfo): boolean {
|
||||
if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
|
||||
if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function folderToLabel(folder: FolderInfo): string {
|
||||
if (folder.specialUse && SPECIAL_USE_LABEL_MAP[folder.specialUse]) {
|
||||
return SPECIAL_USE_LABEL_MAP[folder.specialUse]!;
|
||||
}
|
||||
if (folder.path === 'INBOX') return 'inbox';
|
||||
return folder.path.toLowerCase();
|
||||
}
|
||||
|
||||
// ── OAuth token refresh ──
|
||||
|
||||
async function resolveImapAuth(meta: EmailSyncMeta): Promise<{ user: string; pass?: string; accessToken?: string }> {
|
||||
if (meta.account.authType !== 'oauth') return meta.imapAuth;
|
||||
|
||||
// Refresh OAuth token if expired
|
||||
const userGoogle = await getUserIntegration(meta.account.userId, 'google');
|
||||
const config = userGoogle?.config as Record<string, unknown> | undefined;
|
||||
if (!config?.refreshToken) {
|
||||
throw new PermanentError('Google OAuth not configured — reconnect your Google account');
|
||||
}
|
||||
|
||||
const expiresAt = config.expiresAt as number | undefined;
|
||||
const tokenExpired = !expiresAt || expiresAt < Date.now() + 60_000;
|
||||
|
||||
if (!tokenExpired && config.accessToken) {
|
||||
return { user: meta.account.email, accessToken: config.accessToken as string };
|
||||
}
|
||||
|
||||
console.log('[email-sync] Refreshing OAuth token');
|
||||
const refreshed = await refreshGoogleAccessToken(config.refreshToken as string);
|
||||
|
||||
const serverGoogle = await getServerIntegration('google');
|
||||
await upsertUserIntegration({
|
||||
userId: meta.account.userId,
|
||||
provider: 'google',
|
||||
serverIntegrationId: serverGoogle?.id,
|
||||
config: { ...config, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
|
||||
});
|
||||
|
||||
return { user: meta.account.email, accessToken: refreshed.accessToken };
|
||||
}
|
||||
|
||||
// ── Handler ──
|
||||
|
||||
const emailSyncHandler = {
|
||||
type: 'email-sync',
|
||||
retry: { delayMs: 15 * 60 * 1000, maxRetries: 5 },
|
||||
steps: [
|
||||
{
|
||||
name: 'Sync emails',
|
||||
run: async (ctx: SyncCtx) => {
|
||||
const { ImapFlow } = await import('imapflow');
|
||||
const meta = ctx.meta as unknown as EmailSyncMeta;
|
||||
const { account, userEmail } = meta;
|
||||
|
||||
// Resolve auth (refreshes OAuth token if needed)
|
||||
const imapAuth = await resolveImapAuth(meta);
|
||||
|
||||
// Load syncMeta from DB (always fresh, not from job meta)
|
||||
const freshAccount = await getEmailAccount(account.id);
|
||||
if (!freshAccount) throw new PermanentError(`Email account ${account.id} not found`);
|
||||
const syncMeta = (freshAccount.syncMeta ?? {}) as Record<string, unknown>;
|
||||
const isIncremental = !!syncMeta.last_sync_at;
|
||||
|
||||
await updateEmailAccountStatus(account.id, 'syncing');
|
||||
console.log(`[email-sync] ${isIncremental ? 'Incremental' : 'Initial'} sync for ${account.email}`);
|
||||
|
||||
const MAX_RECONNECTS = 10;
|
||||
const RECONNECT_DELAY_MS = 5_000;
|
||||
let reconnects = 0;
|
||||
let saved = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
let allDone = false;
|
||||
|
||||
const db = openEmailDb(userEmail, account.email);
|
||||
|
||||
// Load existing IDs for dedup (once, shared across reconnections)
|
||||
const existingIds = new Set<string>();
|
||||
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
||||
for (const row of rows) existingIds.add(row.id);
|
||||
|
||||
try {
|
||||
while (!allDone && reconnects <= MAX_RECONNECTS) {
|
||||
if (reconnects > 0) {
|
||||
console.log(`[email-sync] Reconnecting (${reconnects}/${MAX_RECONNECTS}) after ${RECONNECT_DELAY_MS / 1000}s...`);
|
||||
await ctx.updateProgress({ current: saved + skipped, total: 0, label: `Reconnecting (${reconnects}/${MAX_RECONNECTS})...` });
|
||||
await new Promise((r) => setTimeout(r, RECONNECT_DELAY_MS));
|
||||
|
||||
// Re-read syncMeta from DB to get latest saved UIDs
|
||||
const updated = await getEmailAccount(account.id);
|
||||
if (updated?.syncMeta) {
|
||||
Object.assign(syncMeta, updated.syncMeta as Record<string, unknown>);
|
||||
}
|
||||
} else {
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Connecting...' });
|
||||
}
|
||||
|
||||
const client = new ImapFlow({
|
||||
host: account.imapHost,
|
||||
port: account.imapPort,
|
||||
secure: account.imapSecure,
|
||||
auth: imapAuth,
|
||||
logger: false,
|
||||
socketTimeout: 30 * 60 * 1000,
|
||||
});
|
||||
|
||||
const connState = { error: null as Error | null };
|
||||
client.on('error', (err: Error) => {
|
||||
console.log(`[email-sync] IMAP connection error: ${err.message}`);
|
||||
connState.error = err;
|
||||
});
|
||||
|
||||
try {
|
||||
await client.connect();
|
||||
console.log(`[email-sync] IMAP connected${reconnects > 0 ? ` (reconnect ${reconnects})` : ''}`);
|
||||
|
||||
const folders = await client.list({ statusQuery: { uidNext: true, uidValidity: true } }) as FolderInfo[];
|
||||
|
||||
// Filter to folders that still need syncing
|
||||
const foldersToSync: Array<{ folder: FolderInfo; lastUid: number }> = [];
|
||||
for (const folder of folders) {
|
||||
if (shouldSkipFolder(folder)) continue;
|
||||
|
||||
const uidValidityKey = `imap_uidvalidity:${folder.path}`;
|
||||
const lastUidKey = `imap_lastuid:${folder.path}`;
|
||||
const storedUidValidity = syncMeta[uidValidityKey] as string | undefined;
|
||||
const storedLastUid = syncMeta[lastUidKey] as string | undefined;
|
||||
|
||||
const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
|
||||
const uidNext = folder.status?.uidNext ?? 0;
|
||||
let lastUid = storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
|
||||
|
||||
if (storedUidValidity && uidValidity && storedUidValidity !== uidValidity) {
|
||||
console.log(`[email-sync] UIDVALIDITY changed for ${folder.path} — will re-scan`);
|
||||
lastUid = 0;
|
||||
}
|
||||
|
||||
if (lastUid > 0 && uidNext <= lastUid + 1) continue;
|
||||
|
||||
foldersToSync.push({ folder, lastUid });
|
||||
}
|
||||
|
||||
if (foldersToSync.length === 0) {
|
||||
console.log('[email-sync] All folders synced');
|
||||
allDone = true;
|
||||
await client.logout().catch(() => {});
|
||||
break;
|
||||
}
|
||||
|
||||
console.log(`[email-sync] ${foldersToSync.length} folder(s) to sync`);
|
||||
|
||||
let connectionLost = false;
|
||||
|
||||
for (let fi = 0; fi < foldersToSync.length; fi++) {
|
||||
const { folder, lastUid } = foldersToSync[fi]!;
|
||||
|
||||
if (connState.error) { connectionLost = true; break; }
|
||||
|
||||
console.log(`[email-sync] Syncing folder ${fi + 1}/${foldersToSync.length}: ${folder.path}`);
|
||||
|
||||
let lock;
|
||||
try {
|
||||
lock = await client.getMailboxLock(folder.path);
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
if (errMsg.includes('not available') || errMsg.includes('timeout') || errMsg.includes('closed')) {
|
||||
console.log(`[email-sync] Connection lost at folder ${folder.path}`);
|
||||
connectionLost = true;
|
||||
break;
|
||||
}
|
||||
console.log(`[email-sync] Skipping folder ${folder.path}: ${errMsg}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const mailbox = client.mailbox;
|
||||
if (!mailbox) continue;
|
||||
|
||||
const uidValidity = String(mailbox.uidValidity);
|
||||
const effectiveLastUid = syncMeta[`imap_uidvalidity:${folder.path}`] === uidValidity ? lastUid : 0;
|
||||
let maxUid = effectiveLastUid;
|
||||
const label = folderToLabel(folder);
|
||||
const range = effectiveLastUid > 0 ? `${effectiveLastUid + 1}:*` : '1:*';
|
||||
|
||||
try {
|
||||
const fetchOpts: Record<string, unknown> = { source: true, uid: true };
|
||||
if (account.provider === 'gmail') fetchOpts.labels = true;
|
||||
|
||||
for await (const msg of client.fetch(range, fetchOpts, { uid: true })) {
|
||||
if (msg.uid <= effectiveLastUid) continue;
|
||||
|
||||
maxUid = Math.max(maxUid, msg.uid);
|
||||
|
||||
if (!msg.source) { errors++; continue; }
|
||||
|
||||
const raw = msg.source.toString('utf-8');
|
||||
const id = messageIdToStableId(raw);
|
||||
if (!id) { errors++; continue; }
|
||||
|
||||
if (existingIds.has(id)) { skipped++; continue; }
|
||||
|
||||
const labels = [label];
|
||||
try {
|
||||
upsertFromRawEml({ db, id, raw, integration: account.provider, emailAccount: account.email, labels });
|
||||
existingIds.add(id);
|
||||
saved++;
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
|
||||
if ((saved + skipped) % 50 === 0) {
|
||||
const progressLabel = `${folder.path} — ${saved.toLocaleString()} saved, ${skipped.toLocaleString()} skipped`;
|
||||
console.log(`[email-sync] ${progressLabel}`);
|
||||
await ctx.updateProgress({ current: saved + skipped, total: 0, label: progressLabel });
|
||||
|
||||
if (maxUid > effectiveLastUid) {
|
||||
syncMeta[`imap_lastuid:${folder.path}`] = String(maxUid);
|
||||
await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (fetchErr) {
|
||||
const errMsg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
|
||||
if (errMsg.includes('Nothing to fetch')) {
|
||||
// No messages in range — normal
|
||||
} else if (errMsg.includes('not available') || errMsg.includes('timeout') || errMsg.includes('closed')) {
|
||||
console.log(`[email-sync] Connection lost during fetch in ${folder.path}: ${errMsg}`);
|
||||
connectionLost = true;
|
||||
} else {
|
||||
console.log(`[email-sync] Fetch error in ${folder.path}: ${errMsg}`);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
syncMeta[`imap_uidvalidity:${folder.path}`] = uidValidity;
|
||||
if (maxUid > effectiveLastUid) {
|
||||
syncMeta[`imap_lastuid:${folder.path}`] = String(maxUid);
|
||||
}
|
||||
|
||||
console.log(`[email-sync] ${folder.path} done: ${saved} saved, ${skipped} skipped, ${errors} errors`);
|
||||
await ctx.updateProgress({ current: saved + skipped, total: 0, label: `${folder.path}: ${saved} saved` });
|
||||
await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
|
||||
if (connectionLost) break;
|
||||
}
|
||||
|
||||
await client.logout().catch(() => {});
|
||||
|
||||
if (connectionLost) {
|
||||
console.log(`[email-sync] Connection lost after saving ${saved} emails — will reconnect`);
|
||||
reconnects++;
|
||||
continue;
|
||||
}
|
||||
|
||||
allDone = true;
|
||||
} catch (err) {
|
||||
await client.logout().catch(() => {});
|
||||
if (reconnects < MAX_RECONNECTS) {
|
||||
console.log(`[email-sync] Error: ${err instanceof Error ? err.message : String(err)} — will reconnect`);
|
||||
reconnects++;
|
||||
continue;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
if (!allDone) {
|
||||
throw new Error(`IMAP sync incomplete after ${MAX_RECONNECTS} reconnection attempts`);
|
||||
}
|
||||
|
||||
// Mark sync complete
|
||||
syncMeta.last_sync_at = new Date().toISOString();
|
||||
await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
console.log(`[email-sync] Done: saved ${saved}, skipped ${skipped}, errors ${errors}, reconnects ${reconnects}`);
|
||||
ctx.meta.saved = saved;
|
||||
await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${saved} new emails` });
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Finalize',
|
||||
run: async (ctx: SyncCtx) => {
|
||||
const meta = ctx.meta as unknown as EmailSyncMeta;
|
||||
const saved = meta.saved ?? 0;
|
||||
|
||||
await updateEmailAccountStatus(meta.account.id, 'synced');
|
||||
|
||||
if (saved > 0) {
|
||||
try {
|
||||
const paths = await getDockPaths(meta.account.userId);
|
||||
if (!paths) {
|
||||
const defaults = ['/', '/files', '/automation', '/dashboards', '/chat'];
|
||||
await setDockPaths(meta.account.userId, [...defaults, '/email']);
|
||||
} else if (!paths.includes('/email')) {
|
||||
await setDockPaths(meta.account.userId, [...paths, '/email']);
|
||||
}
|
||||
} catch {
|
||||
// Non-fatal
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[email-sync] ${meta.account.email} status set to synced`);
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
/** The handler's steps, for the sidecar's own runner. */
|
||||
export const imapSyncSteps: SyncStep[] = emailSyncHandler.steps as SyncStep[];
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { SidecarEvent } from '../protocol';
|
||||
import type { Job, EnqueueParams } from '../../queue/types';
|
||||
import { initEmailCron, stopEmailCron } from './email-cron';
|
||||
import { initEmailIdle, stopEmailIdle } from './email-idle';
|
||||
import { broadcastEmailNew } from './routes';
|
||||
@@ -8,39 +7,9 @@ import { createSidecarConnector } from '../connect';
|
||||
|
||||
const API_URL = process.env.API_URL ?? `ws://127.0.0.1:${process.env.PORT ?? '5000'}`;
|
||||
|
||||
// ── Queue access via WS ──
|
||||
|
||||
let reqCounter = 0;
|
||||
const pendingQueue = new Map<string, { resolve: (v: unknown) => void; reject: (e: Error) => void; timer: Timer }>();
|
||||
|
||||
function nextQueueId(): string {
|
||||
return `eq_${Date.now()}_${++reqCounter}`;
|
||||
}
|
||||
|
||||
function sendQueueCommand(cmd: Record<string, unknown>): Promise<unknown> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = cmd.id as string;
|
||||
const timer = setTimeout(() => {
|
||||
pendingQueue.delete(id);
|
||||
reject(new Error(`Queue command ${cmd.type} timed out`));
|
||||
}, 30_000);
|
||||
pendingQueue.set(id, { resolve, reject, timer });
|
||||
connection.send(cmd as SidecarEvent);
|
||||
});
|
||||
}
|
||||
|
||||
async function enqueueViaWs(params: EnqueueParams): Promise<Job> {
|
||||
const res = (await sendQueueCommand({ type: 'queue:enqueue', id: nextQueueId(), params })) as Record<string, unknown>;
|
||||
if (res.type === 'queue:enqueued') return res.job as Job;
|
||||
if (res.type === 'queue:error') throw new Error(res.error as string);
|
||||
throw new Error('Unexpected response');
|
||||
}
|
||||
|
||||
async function listJobsViaWs(): Promise<Job[]> {
|
||||
const res = (await sendQueueCommand({ type: 'queue:list', id: nextQueueId() })) as Record<string, unknown>;
|
||||
if (res.type === 'queue:list') return res.jobs as Job[];
|
||||
throw new Error('Unexpected response');
|
||||
}
|
||||
// The sidecar used to reach BACK into the platform's queue over this socket to get a sync run —
|
||||
// enqueueViaWs / listJobsViaWs and a pending-response map. Syncs run in this process now
|
||||
// (sync-runner.ts), so the shim is gone and nothing but a port crosses the socket at startup.
|
||||
|
||||
// ── Command handlers ──
|
||||
|
||||
@@ -53,20 +22,6 @@ function handleCommand(cmd: Record<string, unknown>, reply: ReplyFn) {
|
||||
break;
|
||||
|
||||
default:
|
||||
// Check if this is a queue response (from API server responding to our queue commands)
|
||||
if (
|
||||
typeof cmd.type === 'string' &&
|
||||
cmd.type.startsWith('queue:') &&
|
||||
cmd.id &&
|
||||
pendingQueue.has(cmd.id as string)
|
||||
) {
|
||||
const pending = pendingQueue.get(cmd.id as string)!;
|
||||
pendingQueue.delete(cmd.id as string);
|
||||
clearTimeout(pending.timer);
|
||||
pending.resolve(cmd);
|
||||
return;
|
||||
}
|
||||
|
||||
reply({
|
||||
type: 'error',
|
||||
id: cmd.id as string,
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
type GmailCredentials,
|
||||
loadGmailCredentials,
|
||||
gmailApiSync,
|
||||
} from '../../queue/handlers/gmail-sync';
|
||||
} from './gmail-api';
|
||||
|
||||
export type ResyncResult = { saved: number; skipped: number; errors: number };
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import { getEmailAttachmentCacheDir } from '@@/data-path';
|
||||
import { getEmailAccounts } from 'officerdb';
|
||||
import { openEmailDb, openUserEmailDb, rowToSummary, getSyncMeta, searchEmails } from './store';
|
||||
import { accountsRouter } from './accounts';
|
||||
import { performResync } from './resync';
|
||||
|
||||
export const emailRouter = createRouter();
|
||||
|
||||
@@ -364,6 +365,34 @@ emailRouter.delete('/messages/:id', async (ctx) => {
|
||||
}
|
||||
});
|
||||
|
||||
// POST /sync-now — run a resync and report what arrived.
|
||||
//
|
||||
// For the chat channels ("sync my email" from Telegram/Discord/WhatsApp). They used to open the mail
|
||||
// store directly and enqueue a `gmail-sync` job, which stopped existing when sync moved in here; and the
|
||||
// job type was hardcoded to the OAuth path even though an app-password account syncs over IMAP, so that
|
||||
// command had been failing regardless. One call now: sync, then say what is new.
|
||||
emailRouter.post('/sync-now', async (ctx) => {
|
||||
const user = ctx.get('user');
|
||||
const accounts = await getEmailAccounts(user.id);
|
||||
const account = accounts.find((a) => a.enabled) ?? accounts[0];
|
||||
if (!account) return ctx.json({ saved: 0, newest: [], error: 'No email account configured' });
|
||||
|
||||
const result = await performResync({ accountId: account.id, userEmail: user.email, userId: user.id });
|
||||
|
||||
const db = openEmailDb(user.email, account.email);
|
||||
try {
|
||||
const newest =
|
||||
result.saved > 0
|
||||
? (db
|
||||
.query('SELECT from_name, from_address, subject FROM emails WHERE deleted = 0 ORDER BY date DESC LIMIT ?')
|
||||
.all(Math.min(result.saved, 20)) as Array<{ from_name: string | null; from_address: string; subject: string }>)
|
||||
: [];
|
||||
return ctx.json({ saved: result.saved, skipped: result.skipped, errors: result.errors, newest });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
emailRouter.get('/sync-status', async (ctx) => {
|
||||
const user = ctx.get('user');
|
||||
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
// Sync scheduling, owned by the sidecar.
|
||||
//
|
||||
// These syncs used to be platform queue jobs, which gave them retries and a row in the Jobs list. That was
|
||||
// the wrong home twice over: the credentials travelled through Postgres job metadata to get there, and a
|
||||
// mailbox sync has nothing to do with what the Jobs screen is for. They run here now, tracked in memory,
|
||||
// and the Jobs list is left to the things it actually describes.
|
||||
//
|
||||
// Deliberately NOT a queue: one run per account at a time, no persistence, no retry. A failed sync is
|
||||
// retried by the ten-minute cron like any other, and a sync interrupted by a restart resumes from the
|
||||
// stored sync cursor rather than from the beginning.
|
||||
|
||||
/** What a sync reports as it goes. `total: 0` means "unknown" — IMAP does not tell us up front. */
|
||||
export type SyncProgress = { current: number; total: number; label: string };
|
||||
|
||||
export type SyncState = {
|
||||
accountId: number;
|
||||
status: 'running' | 'completed' | 'failed';
|
||||
progress: SyncProgress | null;
|
||||
startedAt: number;
|
||||
finishedAt?: number;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
const states = new Map<number, SyncState>();
|
||||
|
||||
/** The steps of a converted job handler: each is run in order with a synthesized context. */
|
||||
export type SyncStep = { name: string; run: (ctx: SyncCtx) => Promise<void> | void };
|
||||
|
||||
/**
|
||||
* What the moved handlers expect on their context. `job.userId` is the OWNER'S EMAIL, not a numeric id —
|
||||
* the queue called it userId and the handlers use it to resolve the mail store path
|
||||
* (DATA_PATH/<owner>/email_accounts/…). Getting this wrong reads as an empty mailbox rather than an error.
|
||||
*/
|
||||
export type SyncCtx = {
|
||||
/** Same shape the queue's StepContext used, so the moved handlers read it unchanged. */
|
||||
meta: Record<string, unknown>;
|
||||
job: { userId: string };
|
||||
updateProgress: (p: SyncProgress) => Promise<void> | void;
|
||||
};
|
||||
|
||||
export function isSyncing(accountId: number): boolean {
|
||||
return states.get(accountId)?.status === 'running';
|
||||
}
|
||||
|
||||
export function getSyncStates(): SyncState[] {
|
||||
return [...states.values()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `steps` in the background for one account. Returns immediately — an initial mailbox sync takes
|
||||
* minutes to hours, so nothing waits on it. Re-entrant calls for an account already syncing are ignored.
|
||||
*/
|
||||
export function startSync(params: {
|
||||
accountId: number;
|
||||
ownerEmail: string;
|
||||
meta: Record<string, unknown>;
|
||||
steps: SyncStep[];
|
||||
onDone?: (ok: boolean) => Promise<void> | void;
|
||||
}): SyncState {
|
||||
const existing = states.get(params.accountId);
|
||||
if (existing?.status === 'running') return existing;
|
||||
|
||||
const state: SyncState = {
|
||||
accountId: params.accountId,
|
||||
status: 'running',
|
||||
progress: null,
|
||||
startedAt: Date.now(),
|
||||
};
|
||||
states.set(params.accountId, state);
|
||||
|
||||
const ctx: SyncCtx = {
|
||||
meta: params.meta,
|
||||
job: { userId: params.ownerEmail },
|
||||
updateProgress: (p) => {
|
||||
state.progress = p;
|
||||
},
|
||||
};
|
||||
|
||||
void (async () => {
|
||||
try {
|
||||
for (const step of params.steps) await step.run(ctx);
|
||||
state.status = 'completed';
|
||||
} catch (err) {
|
||||
state.status = 'failed';
|
||||
state.error = err instanceof Error ? err.message : String(err);
|
||||
console.error(`[email-sync] account ${params.accountId} failed:`, state.error);
|
||||
} finally {
|
||||
state.finishedAt = Date.now();
|
||||
await params.onDone?.(state.status === 'completed');
|
||||
}
|
||||
})();
|
||||
|
||||
return state;
|
||||
}
|
||||
Reference in New Issue
Block a user