replace imap gmail sync with mbsync + maildir import, app password UI
- rewrite gmail-sync handler: mbsync downloads to local Maildir, then import to sqlite - add app password field to google integration config and API - gmail sync section independent from oauth in settings UI - live mbsync progress streaming to job status - recoverable failure email with instructions for overquota/auth errors - sync meta persisted to job on failure for richer notifications Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
+124
-50
@@ -10,6 +10,7 @@ type GoogleStatus = {
|
||||
email: string | null;
|
||||
picture: string | null;
|
||||
configured: boolean;
|
||||
hasAppPassword: boolean;
|
||||
};
|
||||
|
||||
const formatTime = (ts: number | string) => {
|
||||
@@ -20,7 +21,10 @@ const formatTime = (ts: number | string) => {
|
||||
export const GoogleAccount = () => {
|
||||
const client = useClient();
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [status, setStatus] = useState<GoogleStatus>({ connected: false, email: null, picture: null, configured: false });
|
||||
const [status, setStatus] = useState<GoogleStatus>({ connected: false, email: null, picture: null, configured: false, hasAppPassword: false });
|
||||
const [appPassword, setAppPassword] = useState('');
|
||||
const [showPasswordInput, setShowPasswordInput] = useState(false);
|
||||
const [savingPassword, setSavingPassword] = useState(false);
|
||||
const [lastSyncAt, setLastSyncAt] = useState<string | null>(null);
|
||||
const [dismissedError, setDismissedError] = useState(false);
|
||||
const { jobs, createJob, refetch } = useJobs({ type: 'gmail-sync' });
|
||||
@@ -88,38 +92,93 @@ export const GoogleAccount = () => {
|
||||
}
|
||||
};
|
||||
|
||||
const handleSaveAppPassword = async () => {
|
||||
if (!appPassword.trim()) return;
|
||||
setSavingPassword(true);
|
||||
try {
|
||||
await client.put('/integrations/google/app-password', { appPassword: appPassword.trim() });
|
||||
setStatus({ ...status, hasAppPassword: true });
|
||||
setAppPassword('');
|
||||
setShowPasswordInput(false);
|
||||
toast.success('App password saved');
|
||||
} catch {
|
||||
toast.error('Failed to save app password');
|
||||
} finally {
|
||||
setSavingPassword(false);
|
||||
}
|
||||
};
|
||||
|
||||
if (isLoading) return null;
|
||||
|
||||
if (!status.configured) {
|
||||
return (
|
||||
<div className="grid gap-4">
|
||||
<p className="text-sm text-duck-dark/60 dark:text-foreground/60">
|
||||
Google integration has not been configured yet. Ask your administrator to set up Google OAuth credentials in
|
||||
the Enterprise settings.
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
const currentStep = activeJob?.steps[activeJob.currentStep];
|
||||
const progress = currentStep?.progress;
|
||||
|
||||
if (status.connected) {
|
||||
const currentStep = activeJob?.steps[activeJob.currentStep];
|
||||
const progress = currentStep?.progress;
|
||||
|
||||
return (
|
||||
return (
|
||||
<div className="grid gap-6">
|
||||
{/* Gmail Sync — independent of OAuth */}
|
||||
<div className="grid gap-4">
|
||||
<div className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
|
||||
<div className="h-2.5 w-2.5 rounded-full bg-green-500 shrink-0" />
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Connected</p>
|
||||
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 truncate">{status.email}</p>
|
||||
</div>
|
||||
{status.picture && (
|
||||
<img src={status.picture} alt="" className="h-9 w-9 rounded-full shrink-0" referrerPolicy="no-referrer" />
|
||||
<div>
|
||||
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Gmail Sync</p>
|
||||
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 mt-1">
|
||||
Import and sync your Gmail emails with your Officer inbox. This uses a Google App Password for a direct IMAP
|
||||
connection — the only thing it can do is download your emails. It cannot send, delete, or modify anything in
|
||||
your account.
|
||||
</p>
|
||||
</div>
|
||||
<div className="rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4 grid gap-2">
|
||||
<p className="text-sm font-medium text-duck-dark dark:text-foreground">App Password</p>
|
||||
{status.hasAppPassword && !showPasswordInput ? (
|
||||
<div className="flex items-center gap-2">
|
||||
<span className="text-xs text-green-600 dark:text-green-400">Configured</span>
|
||||
<button
|
||||
onClick={() => setShowPasswordInput(true)}
|
||||
className="text-xs text-duck-dark/50 dark:text-foreground/50 underline hover:opacity-70 cursor-pointer"
|
||||
>
|
||||
Replace
|
||||
</button>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<div className="text-xs text-duck-dark/50 dark:text-foreground/50 grid gap-1.5">
|
||||
<p>To create an App Password:</p>
|
||||
<ol className="list-decimal ml-4 grid gap-0.5">
|
||||
<li>
|
||||
Go to{' '}
|
||||
<a
|
||||
href="https://myaccount.google.com/apppasswords"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="underline hover:opacity-70"
|
||||
>
|
||||
Google App Passwords
|
||||
</a>
|
||||
</li>
|
||||
<li>You may need to enable 2-Step Verification first</li>
|
||||
<li>Enter a name (e.g. "Officer") and click Create</li>
|
||||
<li>Copy the 16-character password and paste it below</li>
|
||||
</ol>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<input
|
||||
type="password"
|
||||
value={appPassword}
|
||||
onChange={(ev) => setAppPassword(ev.target.value)}
|
||||
placeholder="xxxx xxxx xxxx xxxx"
|
||||
className="flex-1 h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
||||
/>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
disabled={!appPassword.trim() || savingPassword}
|
||||
onClick={handleSaveAppPassword}
|
||||
className="h-9 cursor-pointer"
|
||||
>
|
||||
{savingPassword ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Save'}
|
||||
</Button>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<p className="text-xs text-duck-dark/40 dark:text-foreground/40">
|
||||
Officer has access to your Google Calendar, Gmail, and other enabled services.
|
||||
</p>
|
||||
{activeJob && (
|
||||
<div className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-3">
|
||||
<Loader2 className="h-4 w-4 animate-spin text-duck-dark/60 dark:text-foreground/60 shrink-0" />
|
||||
@@ -163,37 +222,52 @@ export const GoogleAccount = () => {
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
disabled={!!activeJob}
|
||||
disabled={!!activeJob || !status.hasAppPassword}
|
||||
onClick={() => handleSync()}
|
||||
className="w-full h-11 cursor-pointer gap-2"
|
||||
>
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
Sync Gmail Inbox
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
onClick={handleDisconnect}
|
||||
className="w-full h-11 cursor-pointer"
|
||||
>
|
||||
Disconnect
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="grid gap-4">
|
||||
<p className="text-sm text-duck-dark/60 dark:text-foreground/60">
|
||||
Connect your Google account to give Officer access to your Calendar, Gmail, and other Google services.
|
||||
</p>
|
||||
<Button
|
||||
type="button"
|
||||
onClick={handleConnect}
|
||||
className="w-full h-11 bg-duck-yellow hover:bg-duck-yellow/90 text-duck-teal font-bold transition-all duration-200 hover:scale-105 cursor-pointer"
|
||||
>
|
||||
Connect Google Account
|
||||
</Button>
|
||||
{/* OAuth — for Calendar and other Google services */}
|
||||
{status.configured && (
|
||||
<div className="grid gap-4 border-t border-duck-dark/10 dark:border-foreground/10 pt-6">
|
||||
<div>
|
||||
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Google Account</p>
|
||||
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 mt-1">
|
||||
Connect your Google account for Calendar, Contacts, and other Google services. This is separate from Gmail
|
||||
sync above.
|
||||
</p>
|
||||
</div>
|
||||
{status.connected ? (
|
||||
<>
|
||||
<div className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
|
||||
<div className="h-2.5 w-2.5 rounded-full bg-green-500 shrink-0" />
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Connected</p>
|
||||
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 truncate">{status.email}</p>
|
||||
</div>
|
||||
{status.picture && (
|
||||
<img src={status.picture} alt="" className="h-9 w-9 rounded-full shrink-0" referrerPolicy="no-referrer" />
|
||||
)}
|
||||
</div>
|
||||
<Button type="button" variant="outline" onClick={handleDisconnect} className="w-full h-11 cursor-pointer">
|
||||
Disconnect
|
||||
</Button>
|
||||
</>
|
||||
) : (
|
||||
<Button
|
||||
type="button"
|
||||
onClick={handleConnect}
|
||||
className="w-full h-11 bg-duck-yellow hover:bg-duck-yellow/90 text-duck-teal font-bold transition-all duration-200 hover:scale-105 cursor-pointer"
|
||||
>
|
||||
Connect Google Account
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -126,9 +126,28 @@ integrationsRouter.get('/google/status', async (ctx) => {
|
||||
connected: !!connConfig?.accessToken,
|
||||
email: connConfig?.email ?? null,
|
||||
picture: connConfig?.picture ?? null,
|
||||
hasAppPassword: !!connConfig?.imapAppPassword,
|
||||
});
|
||||
});
|
||||
|
||||
integrationsRouter.put('/google/app-password', async (ctx) => {
|
||||
const user = ctx.get('user');
|
||||
const body = ctx.get('body') as { appPassword?: string };
|
||||
if (!body.appPassword) throw BAD_REQUEST('Missing appPassword');
|
||||
|
||||
const connection = await getUserIntegration(user.id, 'google');
|
||||
const connConfig = (connection?.config as Record<string, unknown>) ?? {};
|
||||
|
||||
await upsertUserIntegration({
|
||||
userId: user.id,
|
||||
provider: 'google',
|
||||
serverIntegrationId: connection?.serverIntegrationId ?? undefined,
|
||||
config: { ...connConfig, imapAppPassword: body.appPassword },
|
||||
});
|
||||
|
||||
return ctx.json({ ok: true });
|
||||
});
|
||||
|
||||
integrationsRouter.delete('/google/connection', async (ctx) => {
|
||||
const user = ctx.get('user');
|
||||
const connection = await getUserIntegration(user.id, 'google');
|
||||
|
||||
@@ -93,6 +93,8 @@ export const getAttachmentsDir = (email: string, provider: 'claude' | 'opencode'
|
||||
|
||||
export const getUserEmailDir = (email: string) => join(DATA_PATH, email, 'Gmail', 'emails');
|
||||
|
||||
export const getMaildirPath = (email: string) => join(DATA_PATH, email, 'Gmail', 'Maildir');
|
||||
|
||||
/** Derive a valid Linux username from a display username or email. */
|
||||
export const toShellUsername = (username: string, email: string): string => {
|
||||
const raw = username || email.split('@')[0]!;
|
||||
|
||||
@@ -81,11 +81,16 @@ function kickLane(lane: string) {
|
||||
processNextInLane(lane);
|
||||
}
|
||||
|
||||
function scheduleRetry(lane: string, delayMs: number) {
|
||||
setTimeout(() => kickLane(lane), delayMs);
|
||||
}
|
||||
|
||||
async function processNextInLane(lane: string) {
|
||||
try {
|
||||
const jobs = await listAllJobs();
|
||||
const now = Date.now();
|
||||
const next = jobs
|
||||
.filter((j) => j.lane === lane && j.status === 'queued')
|
||||
.filter((j) => j.lane === lane && j.status === 'queued' && (!j.retryAt || j.retryAt <= now))
|
||||
.sort((a, b) => a.createdAt - b.createdAt)[0];
|
||||
|
||||
if (!next) {
|
||||
@@ -98,7 +103,7 @@ async function processNextInLane(lane: string) {
|
||||
console.error(`[queue] Lane ${lane} processing error:`, err);
|
||||
} finally {
|
||||
const jobs = await listAllJobs();
|
||||
const hasMore = jobs.some((j) => j.lane === lane && j.status === 'queued');
|
||||
const hasMore = jobs.some((j) => j.lane === lane && j.status === 'queued' && (!j.retryAt || j.retryAt <= Date.now()));
|
||||
if (hasMore) {
|
||||
processNextInLane(lane);
|
||||
} else {
|
||||
@@ -119,8 +124,10 @@ async function runJob(job: Job) {
|
||||
|
||||
job.status = 'running';
|
||||
job.startedAt = Date.now();
|
||||
job.retryAt = undefined;
|
||||
await writeJob(job);
|
||||
console.log(`[queue] Running job ${job.id} (${job.type})`);
|
||||
const isRetry = (job.retries ?? 0) > 0;
|
||||
console.log(`[queue] ${isRetry ? 'Resuming' : 'Running'} job ${job.id} (${job.type})${isRetry ? ` retry ${job.retries}` : ''}`);
|
||||
|
||||
const sharedMeta: Record<string, unknown> = { ...(job.meta ?? {}) };
|
||||
|
||||
@@ -134,6 +141,9 @@ async function runJob(job: Job) {
|
||||
const handlerStep = handler.steps[i]!;
|
||||
const step = fresh.steps[i]!;
|
||||
|
||||
// Skip already-completed steps on retry
|
||||
if (step.status === 'completed') continue;
|
||||
|
||||
fresh.currentStep = i;
|
||||
step.status = 'running';
|
||||
step.startedAt = Date.now();
|
||||
@@ -171,9 +181,32 @@ async function runJob(job: Job) {
|
||||
step.status = 'failed';
|
||||
step.error = errorMessage;
|
||||
step.completedAt = Date.now();
|
||||
|
||||
// Check if handler supports retry
|
||||
const retries = (fresh.retries ?? 0) + 1;
|
||||
if (handler.retry && retries <= handler.retry.maxRetries) {
|
||||
// Schedule retry: reset failed step to pending, re-queue
|
||||
step.status = 'pending';
|
||||
step.error = undefined;
|
||||
step.startedAt = undefined;
|
||||
step.completedAt = undefined;
|
||||
step.progress = undefined;
|
||||
fresh.status = 'queued';
|
||||
fresh.error = undefined;
|
||||
fresh.completedAt = undefined;
|
||||
fresh.startedAt = undefined;
|
||||
fresh.retries = retries;
|
||||
fresh.retryAt = Date.now() + handler.retry.delayMs;
|
||||
await writeJob(fresh);
|
||||
console.log(`[queue] Job ${fresh.id} will retry (${retries}/${handler.retry.maxRetries}) in ${handler.retry.delayMs / 1000}s — failed at step "${step.name}": ${errorMessage}`);
|
||||
scheduleRetry(fresh.lane, handler.retry.delayMs);
|
||||
return;
|
||||
}
|
||||
|
||||
fresh.status = 'failed';
|
||||
fresh.error = `Step "${step.name}" failed: ${errorMessage}`;
|
||||
fresh.completedAt = Date.now();
|
||||
fresh.meta = { ...fresh.meta, ...sharedMeta };
|
||||
await writeJob(fresh);
|
||||
console.error(`[queue] Job ${fresh.id} failed at step "${step.name}":`, errorMessage);
|
||||
if (fresh.notify !== false) await notifyFailure(fresh);
|
||||
|
||||
@@ -1,270 +1,311 @@
|
||||
import type { Database } from 'bun:sqlite';
|
||||
import { ImapFlow } from 'imapflow';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { join } from 'node:path';
|
||||
import { readdir, readFile, writeFile, unlink, mkdir, stat } from 'node:fs/promises';
|
||||
import type { JobHandler } from '../types';
|
||||
import { registerHandler } from '../handler-registry';
|
||||
import { openEmailDb, upsertFromRawEml, getSyncMeta, setSyncMeta } from '../../api/email/email-db';
|
||||
import { getServerIntegration, getUserByEmail, getUserIntegration } from 'officerdb';
|
||||
import { openEmailDb, upsertFromRawEml, setSyncMeta } from '../../api/email/email-db';
|
||||
import { getUserByEmail, getUserIntegration } from 'officerdb';
|
||||
import { getMaildirPath } from '@@/data-path';
|
||||
|
||||
type GoogleCredentials = {
|
||||
accessToken: string;
|
||||
refreshToken: string;
|
||||
expiresAt: number;
|
||||
clientId: string;
|
||||
clientSecret: string;
|
||||
};
|
||||
// ── Credentials ──
|
||||
|
||||
async function loadCredentials(email: string): Promise<GoogleCredentials> {
|
||||
const serverGoogle = await getServerIntegration('google');
|
||||
const serverConfig = serverGoogle?.config as Record<string, unknown> | undefined;
|
||||
if (!serverConfig?.clientId || !serverConfig?.clientSecret) {
|
||||
throw new Error('Google OAuth not configured — ask your admin to set up credentials');
|
||||
}
|
||||
type ImapCredentials = { email: string; appPassword: string };
|
||||
|
||||
const dbUser = await getUserByEmail(email);
|
||||
async function loadImapCredentials(userEmail: string): Promise<ImapCredentials> {
|
||||
const dbUser = await getUserByEmail(userEmail);
|
||||
if (!dbUser) throw new Error('User not found');
|
||||
|
||||
const userGoogle = await getUserIntegration(dbUser.id, 'google');
|
||||
const userConfig = userGoogle?.config as Record<string, unknown> | undefined;
|
||||
if (!userConfig?.accessToken) {
|
||||
throw new Error('Google account not connected — connect in Settings → Integrations');
|
||||
const config = userGoogle?.config as Record<string, unknown> | undefined;
|
||||
if (!config?.imapAppPassword) {
|
||||
throw new Error('Gmail App Password not configured — set it in Settings → Integrations');
|
||||
}
|
||||
|
||||
return {
|
||||
accessToken: userConfig.accessToken as string,
|
||||
refreshToken: (userConfig.refreshToken as string) ?? '',
|
||||
expiresAt: (userConfig.expiresAt as number) ?? 0,
|
||||
clientId: serverConfig.clientId as string,
|
||||
clientSecret: serverConfig.clientSecret as string,
|
||||
};
|
||||
const gmailEmail = (config.email as string) ?? userEmail;
|
||||
return { email: gmailEmail, appPassword: config.imapAppPassword as string };
|
||||
}
|
||||
|
||||
let cachedAccessToken: string | null = null;
|
||||
let cachedExpiresAt = 0;
|
||||
// ── mbsync config ──
|
||||
|
||||
async function getValidAccessToken(creds: GoogleCredentials): Promise<string> {
|
||||
if (cachedAccessToken && cachedExpiresAt > Date.now() + 5 * 60 * 1000) {
|
||||
return cachedAccessToken;
|
||||
}
|
||||
function buildMbsyncConfig(email: string, appPassword: string, maildirPath: string): string {
|
||||
return `IMAPAccount gmail
|
||||
Host imap.gmail.com
|
||||
Port 993
|
||||
User ${email}
|
||||
Pass "${appPassword}"
|
||||
SSLType IMAPS
|
||||
AuthMechs LOGIN
|
||||
|
||||
if (creds.expiresAt > Date.now() + 5 * 60 * 1000) {
|
||||
cachedAccessToken = creds.accessToken;
|
||||
cachedExpiresAt = creds.expiresAt;
|
||||
return creds.accessToken;
|
||||
}
|
||||
IMAPStore gmail-remote
|
||||
Account gmail
|
||||
|
||||
if (!creds.refreshToken) throw new Error('Token expired and no refresh token available');
|
||||
MaildirStore gmail-local
|
||||
Path ${maildirPath}/
|
||||
Inbox ${maildirPath}/INBOX
|
||||
SubFolders Verbatim
|
||||
|
||||
const res = await fetch('https://oauth2.googleapis.com/token', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||
body: new URLSearchParams({
|
||||
client_id: creds.clientId,
|
||||
client_secret: creds.clientSecret,
|
||||
refresh_token: creds.refreshToken,
|
||||
grant_type: 'refresh_token',
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const error = await res.text().catch(() => '');
|
||||
throw new Error(`Token refresh failed (${res.status}): ${error}`);
|
||||
}
|
||||
|
||||
const data = (await res.json()) as { access_token: string; expires_in?: number };
|
||||
cachedAccessToken = data.access_token;
|
||||
cachedExpiresAt = Date.now() + (data.expires_in ?? 3600) * 1000;
|
||||
return data.access_token;
|
||||
Channel gmail
|
||||
Far :gmail-remote:
|
||||
Near :gmail-local:
|
||||
Patterns * ![Gmail]/Trash ![Gmail]/Spam
|
||||
Create Near
|
||||
Expunge None
|
||||
SyncState *
|
||||
`;
|
||||
}
|
||||
|
||||
// ── IMAP label mapping ──
|
||||
// ── Folder → label mapping ──
|
||||
|
||||
const SYSTEM_LABEL_MAP: Record<string, string> = {
|
||||
'\\Inbox': 'inbox',
|
||||
'\\Sent': 'sent',
|
||||
'\\Trash': 'trash',
|
||||
'\\Spam': 'spam',
|
||||
'\\Draft': 'draft',
|
||||
'\\Starred': 'starred',
|
||||
'\\Important': 'important',
|
||||
const FOLDER_LABEL_MAP: Record<string, string> = {
|
||||
INBOX: 'inbox',
|
||||
'[Gmail]/Sent Mail': 'sent',
|
||||
'[Gmail]/Drafts': 'draft',
|
||||
'[Gmail]/Starred': 'starred',
|
||||
'[Gmail]/Important': 'important',
|
||||
};
|
||||
|
||||
function mapImapLabels(labels: Set<string> | undefined): string[] {
|
||||
if (!labels) return [];
|
||||
const mapped: string[] = [];
|
||||
for (const label of labels) {
|
||||
const system = SYSTEM_LABEL_MAP[label];
|
||||
if (system) {
|
||||
mapped.push(system);
|
||||
} else {
|
||||
mapped.push(label.toLowerCase());
|
||||
}
|
||||
}
|
||||
return mapped;
|
||||
const SKIP_FOLDERS = new Set(['[Gmail]/All Mail', '[Gmail]/Trash', '[Gmail]/Spam']);
|
||||
|
||||
function folderToLabel(folder: string): string | null {
|
||||
if (SKIP_FOLDERS.has(folder)) return null;
|
||||
if (FOLDER_LABEL_MAP[folder]) return FOLDER_LABEL_MAP[folder]!;
|
||||
// Custom labels / other folders: lowercase the folder name
|
||||
return folder.replace(/^\[Gmail\]\//, '').toLowerCase();
|
||||
}
|
||||
|
||||
// ── IMAP sync ──
|
||||
// ── Stable ID from Message-Id header ──
|
||||
|
||||
type ImapSyncResult = { saved: number; skipped: number; errors: number };
|
||||
function messageIdToStableId(raw: string): string | null {
|
||||
const match = raw.match(/^Message-Id:\s*<?([^>\s]+)>?/im);
|
||||
if (!match?.[1]) return null;
|
||||
return createHash('sha1').update(match[1]).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
// Mailboxes to sync: All Mail has everything except Trash and Spam
|
||||
const SYNC_SPECIAL_USE = ['\\All', '\\Trash', '\\Junk'];
|
||||
// ── Maildir import ──
|
||||
|
||||
async function syncViaImap(
|
||||
accessToken: string,
|
||||
type ImportResult = { saved: number; skipped: number; errors: number };
|
||||
|
||||
async function importMaildir(
|
||||
maildirPath: string,
|
||||
emailAccount: string,
|
||||
db: Database,
|
||||
since: Date | null,
|
||||
onProgress?: (saved: number, skipped: number) => void,
|
||||
): Promise<ImapSyncResult> {
|
||||
const client = new ImapFlow({
|
||||
host: 'imap.gmail.com',
|
||||
port: 993,
|
||||
secure: true,
|
||||
auth: { user: emailAccount, accessToken },
|
||||
logger: false,
|
||||
});
|
||||
|
||||
try {
|
||||
await client.connect();
|
||||
} catch (err) {
|
||||
console.error('[gmail-sync] IMAP connect failed:', err);
|
||||
throw err;
|
||||
}
|
||||
|
||||
): Promise<ImportResult> {
|
||||
let saved = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
// Load existing IDs for dedup (once, shared across mailboxes)
|
||||
// Load existing IDs for fast dedup
|
||||
const existingIds = new Set<string>();
|
||||
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
||||
for (const row of rows) existingIds.add(row.id);
|
||||
|
||||
// First pass: collect all message files with their folders to build label map
|
||||
const messageIdLabels = new Map<string, Set<string>>();
|
||||
const messageFiles = new Map<string, string>(); // id → first file path
|
||||
|
||||
let folders: string[];
|
||||
try {
|
||||
// Find mailboxes by specialUse flag (locale-independent)
|
||||
const allMailboxes = await client.list();
|
||||
const toSync: Array<{ path: string; specialUse: string }> = [];
|
||||
for (const mailbox of allMailboxes) {
|
||||
if (mailbox.specialUse && SYNC_SPECIAL_USE.includes(mailbox.specialUse)) {
|
||||
toSync.push({ path: mailbox.path, specialUse: mailbox.specialUse });
|
||||
}
|
||||
}
|
||||
folders = await readdir(maildirPath);
|
||||
} catch {
|
||||
console.log('[gmail-sync] No Maildir folders found');
|
||||
return { saved, skipped, errors };
|
||||
}
|
||||
|
||||
if (toSync.length === 0) {
|
||||
console.error('[gmail-sync] No mailboxes found to sync');
|
||||
return { saved, skipped, errors };
|
||||
}
|
||||
for (const folder of folders) {
|
||||
const label = folderToLabel(folder);
|
||||
if (label === null) continue;
|
||||
|
||||
for (const mailbox of toSync) {
|
||||
console.log(`[gmail-sync] Opening ${mailbox.path} (${mailbox.specialUse})...`);
|
||||
const lock = await client.getMailboxLock(mailbox.path);
|
||||
for (const subdir of ['cur', 'new']) {
|
||||
const dirPath = join(maildirPath, folder, subdir);
|
||||
let files: string[];
|
||||
try {
|
||||
const searchCriteria = since ? { since } : { all: true };
|
||||
const uids = await client.search(searchCriteria, { uid: true });
|
||||
files = await readdir(dirPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!uids || uids.length === 0) {
|
||||
console.log(`[gmail-sync] ${mailbox.path}: no messages`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`[gmail-sync] ${mailbox.path}: ${uids.length} messages`);
|
||||
|
||||
const uidRange = uids.join(',');
|
||||
const messages = client.fetch(uidRange, {
|
||||
source: true,
|
||||
labels: true,
|
||||
}, { uid: true });
|
||||
|
||||
for await (const msg of messages) {
|
||||
try {
|
||||
if (!msg.emailId || !msg.source) continue;
|
||||
|
||||
const gmailId = BigInt(msg.emailId).toString(16);
|
||||
|
||||
if (existingIds.has(gmailId)) {
|
||||
skipped++;
|
||||
if ((saved + skipped) % 500 === 0) onProgress?.(saved, skipped);
|
||||
continue;
|
||||
}
|
||||
|
||||
const rawEmail = msg.source.toString('utf-8');
|
||||
const labels = mapImapLabels(msg.labels);
|
||||
|
||||
upsertFromRawEml({ db, id: gmailId, raw: rawEmail, integration: 'gmail', emailAccount, labels });
|
||||
existingIds.add(gmailId);
|
||||
saved++;
|
||||
|
||||
if ((saved + skipped) % 100 === 0) {
|
||||
console.log(`[gmail-sync] Progress: saved ${saved}, skipped ${skipped}, errors ${errors}`);
|
||||
onProgress?.(saved, skipped);
|
||||
}
|
||||
} catch {
|
||||
for (const file of files) {
|
||||
const filePath = join(dirPath, file);
|
||||
try {
|
||||
const raw = await readFile(filePath, 'utf-8');
|
||||
const id = messageIdToStableId(raw);
|
||||
if (!id) {
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Track labels
|
||||
const labels = messageIdLabels.get(id) ?? new Set<string>();
|
||||
labels.add(label);
|
||||
messageIdLabels.set(id, labels);
|
||||
|
||||
// Keep first file path for importing
|
||||
if (!messageFiles.has(id)) {
|
||||
messageFiles.set(id, filePath);
|
||||
}
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await client.logout();
|
||||
}
|
||||
|
||||
// Second pass: import messages that aren't already in DB
|
||||
for (const [id, filePath] of messageFiles) {
|
||||
if (existingIds.has(id)) {
|
||||
skipped++;
|
||||
if ((saved + skipped) % 500 === 0) onProgress?.(saved, skipped);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const raw = await readFile(filePath, 'utf-8');
|
||||
const labels = Array.from(messageIdLabels.get(id) ?? []);
|
||||
|
||||
upsertFromRawEml({ db, id, raw, integration: 'gmail', emailAccount, labels });
|
||||
existingIds.add(id);
|
||||
saved++;
|
||||
|
||||
if ((saved + skipped) % 100 === 0) {
|
||||
console.log(`[gmail-sync] Import progress: saved ${saved}, skipped ${skipped}, errors ${errors}`);
|
||||
onProgress?.(saved, skipped);
|
||||
}
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
return { saved, skipped, errors };
|
||||
}
|
||||
|
||||
// ── Maildir stats ──
|
||||
|
||||
async function countMaildirFiles(maildirPath: string): Promise<number> {
|
||||
let count = 0;
|
||||
let folders: string[];
|
||||
try {
|
||||
folders = await readdir(maildirPath);
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
for (const folder of folders) {
|
||||
for (const subdir of ['cur', 'new']) {
|
||||
try {
|
||||
const files = await readdir(join(maildirPath, folder, subdir));
|
||||
count += files.length;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// ── Handler ──
|
||||
|
||||
const gmailSyncHandler: JobHandler = {
|
||||
type: 'gmail-sync',
|
||||
retry: { delayMs: 15 * 60 * 1000, maxRetries: 10 },
|
||||
steps: [
|
||||
{
|
||||
name: 'Verify connection',
|
||||
name: 'Verify credentials',
|
||||
run: async (ctx) => {
|
||||
const creds = await loadCredentials(ctx.job.userId);
|
||||
const token = await getValidAccessToken(creds);
|
||||
ctx.meta.accessToken = token;
|
||||
const creds = await loadImapCredentials(ctx.job.userId);
|
||||
ctx.meta.email = creds.email;
|
||||
ctx.meta.appPassword = creds.appPassword;
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Sync emails',
|
||||
name: 'Sync via mbsync',
|
||||
run: async (ctx) => {
|
||||
const token = ctx.meta.accessToken as string;
|
||||
const year = ctx.meta.year as number | undefined;
|
||||
const email = ctx.meta.email as string;
|
||||
const appPassword = ctx.meta.appPassword as string;
|
||||
const maildirPath = getMaildirPath(ctx.job.userId);
|
||||
|
||||
// Ensure Maildir root exists
|
||||
await mkdir(maildirPath, { recursive: true });
|
||||
|
||||
// Write temp config
|
||||
const configPath = join(maildirPath, '.mbsyncrc');
|
||||
const config = buildMbsyncConfig(email, appPassword, maildirPath);
|
||||
await writeFile(configPath, config, { mode: 0o600 });
|
||||
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Running mbsync...' });
|
||||
|
||||
try {
|
||||
const proc = Bun.spawn(['mbsync', '-c', configPath, '-a', '-V'], {
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
});
|
||||
|
||||
// Stream stderr for live progress
|
||||
let lastLine = '';
|
||||
let stderrBuf = '';
|
||||
const reader = proc.stderr.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
const readLoop = (async () => {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
const chunk = decoder.decode(value, { stream: true });
|
||||
stderrBuf += chunk;
|
||||
const lines = chunk.split('\n');
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) continue;
|
||||
lastLine = trimmed;
|
||||
console.log(`[gmail-sync] mbsync: ${trimmed}`);
|
||||
}
|
||||
if (lastLine) {
|
||||
ctx.updateProgress({ current: 0, total: 0, label: `mbsync: ${lastLine.slice(0, 80)}` });
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
const exitCode = await proc.exited;
|
||||
await readLoop;
|
||||
|
||||
if (exitCode !== 0) {
|
||||
console.error(`[gmail-sync] mbsync failed`);
|
||||
const isOverquota = stderrBuf.includes('OVERQUOTA');
|
||||
const isAuthFail = stderrBuf.includes('AUTHENTICATIONFAILED') || stderrBuf.includes('Invalid credentials');
|
||||
if (isOverquota || isAuthFail) {
|
||||
const emailCount = await countMaildirFiles(maildirPath);
|
||||
ctx.meta.gmailSyncRecoverable = true;
|
||||
ctx.meta.gmailSyncEmailCount = emailCount;
|
||||
ctx.meta.gmailSyncIsAuthFail = isAuthFail;
|
||||
}
|
||||
throw new Error(`mbsync exited with code ${exitCode}: ${stderrBuf.slice(-500)}`);
|
||||
}
|
||||
|
||||
console.log('[gmail-sync] mbsync completed successfully');
|
||||
} finally {
|
||||
// Always clean up config (contains password)
|
||||
await unlink(configPath).catch(() => {});
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Import to database',
|
||||
run: async (ctx) => {
|
||||
const emailAccount = ctx.meta.email as string;
|
||||
const maildirPath = getMaildirPath(ctx.job.userId);
|
||||
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Importing emails...' });
|
||||
|
||||
const db = openEmailDb(ctx.job.userId);
|
||||
try {
|
||||
// Bootstrap sync_meta from existing emails if DB was imported without metadata
|
||||
if (!getSyncMeta(db, 'last_sync_date')) {
|
||||
const newest = db.query('SELECT date FROM emails ORDER BY date DESC LIMIT 1').get() as { date: string } | null;
|
||||
if (newest?.date) {
|
||||
console.log(`[gmail-sync] Bootstrapping last_sync_date from existing DB: ${newest.date}`);
|
||||
setSyncMeta(db, 'last_sync_date', newest.date);
|
||||
}
|
||||
}
|
||||
|
||||
// Determine since date
|
||||
let since: Date | null = null;
|
||||
if (year) {
|
||||
since = new Date(year, 0, 1);
|
||||
} else {
|
||||
const lastSyncDate = getSyncMeta(db, 'last_sync_date');
|
||||
if (lastSyncDate) {
|
||||
since = new Date(lastSyncDate);
|
||||
console.log(`[gmail-sync] Syncing since ${since.toISOString()}`);
|
||||
}
|
||||
}
|
||||
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Connecting via IMAP...' });
|
||||
|
||||
const result = await syncViaImap(token, ctx.job.userId, db, since, (saved, skipped) => {
|
||||
const label = year
|
||||
? `${year} — Saved ${saved.toLocaleString()}, skipped ${skipped.toLocaleString()}`
|
||||
: `Saved ${saved.toLocaleString()}, skipped ${skipped.toLocaleString()}`;
|
||||
ctx.updateProgress({ current: saved + skipped, total: 0, label });
|
||||
const result = await importMaildir(maildirPath, emailAccount, db, (saved, skipped) => {
|
||||
ctx.updateProgress({
|
||||
current: saved + skipped,
|
||||
total: 0,
|
||||
label: `Saved ${saved.toLocaleString()}, skipped ${skipped.toLocaleString()}`,
|
||||
});
|
||||
});
|
||||
|
||||
console.log(`[gmail-sync] Done: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`);
|
||||
console.log(`[gmail-sync] Import done: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`);
|
||||
|
||||
setSyncMeta(db, 'last_sync_date', new Date().toISOString().split('T')[0]!);
|
||||
setSyncMeta(db, 'last_sync_at', new Date().toISOString());
|
||||
|
||||
@@ -1,17 +1,64 @@
|
||||
import Layout from 'emailer/emails/layouts/MainLayout.jsx';
|
||||
import { Container, Text } from '@react-email/components';
|
||||
import { Container, Text, Link } from '@react-email/components';
|
||||
|
||||
type JobEmailData = {
|
||||
job: { type: string; error?: string };
|
||||
job: {
|
||||
type: string;
|
||||
error?: string;
|
||||
meta?: {
|
||||
gmailSyncRecoverable?: boolean;
|
||||
gmailSyncEmailCount?: number;
|
||||
gmailSyncIsAuthFail?: boolean;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
const GmailSyncRecovery = ({ job }: JobEmailData) => {
|
||||
const emailCount = job.meta?.gmailSyncEmailCount ?? 0;
|
||||
const isAuthFail = job.meta?.gmailSyncIsAuthFail;
|
||||
|
||||
return (
|
||||
<>
|
||||
<Text className="text-base font-semibold text-green-700">Don't worry — your progress is saved.</Text>
|
||||
{emailCount > 0 && (
|
||||
<Text className="text-sm text-gray-700">
|
||||
Your mailbox already has {emailCount.toLocaleString()} emails downloaded. The sync will resume from where it
|
||||
stopped — it won't re-download emails you already have.
|
||||
</Text>
|
||||
)}
|
||||
<Text className="text-sm text-gray-700">
|
||||
{isAuthFail
|
||||
? 'Google temporarily blocked sign-in after hitting bandwidth limits. This usually clears up within a few hours.'
|
||||
: 'Gmail throttled the connection after downloading too much data at once. This is normal for large mailboxes.'}
|
||||
</Text>
|
||||
<Text className="text-sm font-semibold text-gray-800">To resume syncing:</Text>
|
||||
<Text className="text-sm text-gray-700" style={{ marginTop: 0 }}>
|
||||
1. Wait a few hours for Google to lift the block{'\n'}
|
||||
2. Generate a new App Password at{' '}
|
||||
<Link href="https://myaccount.google.com/apppasswords">myaccount.google.com/apppasswords</Link> (Google revokes
|
||||
them after repeated auth failures){'\n'}
|
||||
3. Save the new password in Settings → Integrations → Google Account{'\n'}
|
||||
4. Click "Sync Gmail Inbox" to resume — only new emails will be downloaded
|
||||
</Text>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
const Email = ({ job }: JobEmailData) => {
|
||||
const isRecoverableGmail = job?.type === 'gmail-sync' && job.meta?.gmailSyncRecoverable;
|
||||
|
||||
return (
|
||||
<Layout>
|
||||
<Container>
|
||||
<Text className="pt-4 text-2xl">Job Failed</Text>
|
||||
<Text>Your {job?.type ?? 'unknown'} job has failed.</Text>
|
||||
{job?.error ? <Text className="text-sm text-gray-600">{job.error}</Text> : null}
|
||||
<Text className="pt-4 text-2xl">{isRecoverableGmail ? 'Gmail Sync Paused' : 'Job Failed'}</Text>
|
||||
{isRecoverableGmail ? (
|
||||
<GmailSyncRecovery job={job} />
|
||||
) : (
|
||||
<>
|
||||
<Text>Your {job?.type ?? 'unknown'} job has failed.</Text>
|
||||
{job?.error ? <Text className="text-sm text-gray-600">{job.error}</Text> : null}
|
||||
</>
|
||||
)}
|
||||
</Container>
|
||||
</Layout>
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user