Merge remote-tracking branch 'origin/email-imap'

This commit is contained in:
2026-03-06 07:27:45 +00:00
22 changed files with 2101 additions and 410 deletions
+2 -2
View File
@@ -25,9 +25,9 @@ try {
const db = openEmailDb(email); const db = openEmailDb(email);
console.log(`Importing from ${maildirPath}...`); console.log(`Importing from ${maildirPath}...`);
const result = await importMaildir(maildirPath, email, db, (saved, skipped) => { const result = await importMaildir({ maildirPath, emailAccount: email, db, onProgress: (saved, skipped) => {
process.stdout.write(`\r saved ${saved}, skipped ${skipped}`); process.stdout.write(`\r saved ${saved}, skipped ${skipped}`);
}); }});
console.log(`\nDone: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`); console.log(`\nDone: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`);
db.close(); db.close();
+88
View File
@@ -0,0 +1,88 @@
import { ImapFlow } from 'imapflow';
import { getUserByEmail, getUserIntegration, getServerIntegration } from 'officerdb';
import { openEmailDb, setSyncMeta } from '../src/servers/api/email/email-db';
const userEmail = process.argv[2];
if (!userEmail) {
console.error('Usage: bun run scripts/seed-imap-uids.ts <email>');
process.exit(1);
}
// ── Load credentials ──
const dbUser = await getUserByEmail(userEmail);
if (!dbUser) throw new Error('User not found');
const userGoogle = await getUserIntegration(dbUser.id, 'google');
const config = userGoogle?.config as Record<string, unknown> | undefined;
if (!config?.accessToken) throw new Error('No OAuth tokens found');
// Refresh token if needed
let accessToken = config.accessToken as string;
const expiresAt = config.expiresAt as number | undefined;
if (!expiresAt || expiresAt < Date.now() + 60_000) {
console.log('Refreshing expired token...');
const serverGoogle = await getServerIntegration('google');
const serverConfig = serverGoogle?.config as Record<string, unknown>;
const res = await fetch('https://oauth2.googleapis.com/token', {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams({
client_id: serverConfig.clientId as string,
client_secret: serverConfig.clientSecret as string,
refresh_token: config.refreshToken as string,
grant_type: 'refresh_token',
}),
});
if (!res.ok) throw new Error(`Token refresh failed: ${await res.text()}`);
const data = (await res.json()) as { access_token: string };
accessToken = data.access_token;
}
// ── Connect IMAP ──
const client = new ImapFlow({
host: 'imap.gmail.com',
port: 993,
secure: true,
auth: { user: config.email as string, accessToken },
logger: false,
});
await client.connect();
console.log('Connected to IMAP');
const GMAIL_PREFIX_RE = /^\[(?:Gmail|Google Mail)\]\//;
const SKIP_SUFFIXES = new Set(['All Mail', 'Trash', 'Spam', 'Bin']);
const folders = await client.list();
const db = openEmailDb(userEmail);
let seeded = 0;
for (const folder of folders) {
const suffix = folder.path.replace(GMAIL_PREFIX_RE, '');
const isGmailFolder = suffix !== folder.path;
if (isGmailFolder && SKIP_SUFFIXES.has(suffix)) continue;
if (folder.specialUse && ['\\Trash', '\\Junk', '\\All'].includes(folder.specialUse)) continue;
try {
const status = await client.status(folder.path, { uidNext: true, uidValidity: true });
const lastUid = (status.uidNext ?? 1) - 1;
const uidValidity = String(status.uidValidity);
setSyncMeta(db, `imap_lastuid:${folder.path}`, String(lastUid));
setSyncMeta(db, `imap_uidvalidity:${folder.path}`, uidValidity);
console.log(` ${folder.path}: lastUid=${lastUid}, uidValidity=${uidValidity}`);
seeded++;
} catch (err) {
console.log(` ${folder.path}: skipped (${err instanceof Error ? err.message : err})`);
}
}
db.close();
await client.logout();
console.log(`\nSeeded ${seeded} folders. Next sync will only fetch new messages.`);
process.exit(0);
@@ -6,14 +6,13 @@ import { toast } from 'sonner';
import { Button } from '@/components/ui/button'; import { Button } from '@/components/ui/button';
import { useClient } from 'hooks/useClient'; import { useClient } from 'hooks/useClient';
import { useGlobal } from 'hooks/useGlobal'; import { useGlobal } from 'hooks/useGlobal';
import { useJobs } from 'hooks/useJobs';
import type { EmailSummary } from 'types'; import type { EmailSummary } from 'types';
type GoogleStatus = { type EmailAccountRow = {
configured: boolean; id: number;
connected: boolean; provider: string;
email: string | null; email: string;
picture: string | null; status: string;
}; };
const LIMIT = 50; const LIMIT = 50;
@@ -42,13 +41,13 @@ export const EmailList = () => {
const [selectedId, setSelectedId] = useGlobal<string | null>('EMAIL_SELECTED', null); const [selectedId, setSelectedId] = useGlobal<string | null>('EMAIL_SELECTED', null);
const [folder, setFolder] = useGlobal<string>('EMAIL_FOLDER', 'inbox'); const [folder, setFolder] = useGlobal<string>('EMAIL_FOLDER', 'inbox');
const [page, setPage] = useState(1); const [page, setPage] = useState(1);
const { jobs, createJob } = useJobs({ type: 'gmail-sync' }); const { data: emailAccounts = [], refetch: refetchAccounts } = useQuery({
const isSyncing = jobs.some((j) => j.status === 'queued' || j.status === 'running'); queryKey: ['email-accounts'],
queryFn: () => client.get<EmailAccountRow[]>('/email/accounts'),
const { data: googleStatus } = useQuery({
queryKey: ['google-status'],
queryFn: () => client.get<GoogleStatus>('/integrations/google/status'),
}); });
const syncableAccount = emailAccounts.find((a) => a.status === 'connected' || a.status === 'synced');
const isSyncing = emailAccounts.some((a) => a.status === 'syncing' || a.status === 'queued');
const hasAccounts = emailAccounts.length > 0;
const { data, isLoading } = useQuery({ const { data, isLoading } = useQuery({
queryKey: ['email-messages', page, folder], queryKey: ['email-messages', page, folder],
@@ -75,17 +74,23 @@ export const EmailList = () => {
}; };
const handleSync = async () => { const handleSync = async () => {
if (!syncableAccount) return;
try { try {
await createJob({ lane: 'google-api', type: 'gmail-sync', notify: false }); await client.post(`/email/accounts/${syncableAccount.id}/sync`, {});
toast.success('Gmail sync started'); toast.success('Sync started');
} catch { refetchAccounts();
toast.error('Failed to start sync'); } catch (err) {
toast.error(err instanceof Error ? err.message : 'Failed to start sync');
} }
}; };
// Refresh email list when a sync job completes // Poll accounts while syncing, refresh email list when done
const prevSyncing = useRef(false); const prevSyncing = useRef(false);
useEffect(() => { useEffect(() => {
if (isSyncing) {
const interval = setInterval(refetchAccounts, 5000);
return () => clearInterval(interval);
}
if (prevSyncing.current && !isSyncing) { if (prevSyncing.current && !isSyncing) {
queryClient.invalidateQueries({ queryKey: ['email-messages'] }); queryClient.invalidateQueries({ queryKey: ['email-messages'] });
} }
@@ -134,16 +139,14 @@ export const EmailList = () => {
return ( return (
<div className="flex h-full flex-col items-center justify-center gap-3 text-sm opacity-50"> <div className="flex h-full flex-col items-center justify-center gap-3 text-sm opacity-50">
<Mail className="h-8 w-8" /> <Mail className="h-8 w-8" />
{googleStatus && !googleStatus.configured ? ( {!hasAccounts ? (
<span>Google integration not configured. Contact your administrator.</span>
) : googleStatus && !googleStatus.connected ? (
<div className="flex flex-col items-center gap-2"> <div className="flex flex-col items-center gap-2">
<span>Connect your Google account to sync emails</span> <span>Add an email account to get started</span>
<Button variant="outline" size="sm" asChild> <Button variant="outline" size="sm" asChild>
<Link to="/settings/integrations">Connect Google</Link> <Link to="/settings/integrations">Add Account</Link>
</Button> </Button>
</div> </div>
) : ( ) : syncableAccount ? (
<div className="flex flex-col items-center gap-2"> <div className="flex flex-col items-center gap-2">
<span>No emails synced yet</span> <span>No emails synced yet</span>
<Button variant="outline" size="sm" onClick={handleSync} disabled={isSyncing}> <Button variant="outline" size="sm" onClick={handleSync} disabled={isSyncing}>
@@ -151,6 +154,13 @@ export const EmailList = () => {
Sync Now Sync Now
</Button> </Button>
</div> </div>
) : isSyncing ? (
<div className="flex flex-col items-center gap-2">
<Loader2 className="h-5 w-5 animate-spin" />
<span>Syncing emails...</span>
</div>
) : (
<span>No emails synced yet</span>
)} )}
</div> </div>
); );
@@ -179,18 +189,17 @@ export const EmailList = () => {
})} })}
</div> </div>
<span className="text-xs opacity-50">{total}</span> <span className="text-xs opacity-50">{total}</span>
<button {isSyncing ? (
onClick={handleSync} <Loader2 className="h-3.5 w-3.5 animate-spin shrink-0 opacity-50" title="Syncing..." />
disabled={isSyncing} ) : syncableAccount ? (
className="p-0.5 rounded hover:bg-black/10 transition-colors cursor-pointer disabled:cursor-default disabled:opacity-50 shrink-0" <button
title={isSyncing ? 'Syncing...' : 'Sync emails'} onClick={handleSync}
> className="p-0.5 rounded hover:bg-black/10 transition-colors cursor-pointer shrink-0"
{isSyncing ? ( title="Sync emails"
<Loader2 className="h-3.5 w-3.5 animate-spin" /> >
) : (
<RefreshCw className="h-3.5 w-3.5" /> <RefreshCw className="h-3.5 w-3.5" />
)} </button>
</button> ) : null}
{totalPages > 1 && ( {totalPages > 1 && (
<div className="ml-auto flex items-center gap-2"> <div className="ml-auto flex items-center gap-2">
<button <button
@@ -0,0 +1,508 @@
import { useState, useEffect } from 'react';
import { toast } from 'sonner';
import { Plus, Trash2, Loader2, Mail, Server, KeyRound, RefreshCw } from 'lucide-react';
import { Button } from '@/components/ui/button';
import { useClient } from 'hooks/useClient';
import { useAuth } from 'hooks/useAuth';
import { useJobs } from 'hooks/useJobs';
type EmailAccountRow = {
id: number;
provider: string;
email: string;
displayName: string | null;
enabled: boolean;
status: string;
createdAt: string;
};
type ProviderChoice = 'gmail-oauth' | 'gmail-password' | 'imap';
type FormState = {
email: string;
displayName: string;
password: string;
imapHost: string;
imapPort: string;
imapSecure: boolean;
};
const INITIAL_FORM: FormState = {
email: '',
displayName: '',
password: '',
imapHost: '',
imapPort: '993',
imapSecure: true,
};
export const EmailAccounts = () => {
const client = useClient();
const { user } = useAuth();
const { jobs } = useJobs({ type: 'email-sync' });
const [accounts, setAccounts] = useState<EmailAccountRow[]>([]);
const [loading, setLoading] = useState(true);
const [showPicker, setShowPicker] = useState(false);
const [activeForm, setActiveForm] = useState<ProviderChoice | null>(null);
const [form, setForm] = useState<FormState>(INITIAL_FORM);
const [saving, setSaving] = useState(false);
const [deleting, setDeleting] = useState<number | null>(null);
const [syncing, setSyncing] = useState<number | null>(null);
const fetchAccounts = () => {
client
.get<EmailAccountRow[]>('/email/accounts')
.then(setAccounts)
.catch(() => {})
.finally(() => setLoading(false));
};
useEffect(() => {
fetchAccounts();
}, []);
// Poll accounts while any are queued/syncing to pick up status changes
const hasSyncingAccount = accounts.some((a) => a.status === 'syncing' || a.status === 'queued');
useEffect(() => {
if (!hasSyncingAccount) return;
const interval = setInterval(fetchAccounts, 5000);
return () => clearInterval(interval);
}, [hasSyncingAccount]);
const resetForm = () => {
setActiveForm(null);
setShowPicker(false);
setForm(INITIAL_FORM);
};
const handlePickProvider = (choice: ProviderChoice) => {
setShowPicker(false);
setActiveForm(choice);
if (choice === 'gmail-oauth' || choice === 'gmail-password') {
setForm({ ...INITIAL_FORM, imapHost: 'imap.gmail.com', imapPort: '993', imapSecure: true });
} else {
setForm(INITIAL_FORM);
}
};
const handleGmailOAuth = async () => {
setSaving(true);
try {
const status = await client.get<{ connected: boolean; email: string | null }>('/integrations/google/status');
if (!status.connected) {
const params = new URLSearchParams({ token: client.token ?? '', origin: window.location.origin });
window.location.href = `/api/integrations/google/authorize?${params.toString()}`;
return;
}
const result = await client.post<{ id: number; provider: string; email: string }>('/email/accounts', {
provider: 'gmail',
email: status.email,
displayName: form.displayName || undefined,
imapHost: 'imap.gmail.com',
imapPort: 993,
imapSecure: true,
authType: 'oauth',
credentials: { userIntegrationId: true },
});
toast.success(`Added ${result.email}`);
resetForm();
fetchAccounts();
} catch (err) {
toast.error(err instanceof Error ? err.message : 'Failed to add Gmail account');
} finally {
setSaving(false);
}
};
const handlePasswordSubmit = async () => {
if (!form.email.trim() || !form.password.trim()) return;
setSaving(true);
try {
const provider = form.imapHost === 'imap.gmail.com' ? 'gmail' : 'imap';
const result = await client.post<{ id: number; provider: string; email: string }>('/email/accounts', {
provider,
email: form.email.trim(),
displayName: form.displayName.trim() || undefined,
imapHost: form.imapHost,
imapPort: Number(form.imapPort),
imapSecure: form.imapSecure,
authType: 'password',
credentials: { password: form.password },
});
toast.success(`Added ${result.email}`);
resetForm();
fetchAccounts();
} catch (err) {
toast.error(err instanceof Error ? err.message : 'Failed to add account');
} finally {
setSaving(false);
}
};
const handleDelete = async (id: number) => {
setDeleting(id);
try {
await client.delete(`/email/accounts/${id}`);
setAccounts((prev) => prev.filter((a) => a.id !== id));
toast.success('Account removed');
} catch {
toast.error('Failed to remove account');
} finally {
setDeleting(null);
}
};
const handleSync = async (id: number) => {
setSyncing(id);
try {
await client.post(`/email/accounts/${id}/sync`, {});
toast.success('Sync started');
// Update local state immediately
setAccounts((prev) => prev.map((a) => (a.id === id ? { ...a, status: 'queued' } : a)));
} catch (err) {
toast.error(err instanceof Error ? err.message : 'Failed to start sync');
} finally {
setSyncing(null);
}
};
if (loading) return null;
return (
<div className="grid gap-4">
<div>
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Email Accounts</p>
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 mt-1">
Connect email accounts for syncing your inbox. Supports Gmail and any IMAP provider.
</p>
</div>
{/* Account list */}
{accounts.length > 0 && (
<div className="grid gap-2">
{accounts.map((account) => {
const accountJob = jobs.find(
(j) =>
(j.status === 'queued' || j.status === 'running') &&
(j.meta as Record<string, unknown> | undefined)?.emailAccountId === account.id,
);
const progress = accountJob?.steps[accountJob.currentStep]?.progress;
return (
<div
key={account.id}
className="rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-3"
>
<div className="flex items-center gap-3">
<ProviderIcon provider={account.provider} />
<div className="min-w-0 flex-1">
<p className="text-sm font-medium text-duck-dark dark:text-foreground truncate">
{account.displayName ?? account.email}
</p>
{account.displayName && (
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 truncate">{account.email}</p>
)}
</div>
<StatusBadge status={account.status} />
{account.status === 'queued' && (
<span className="text-xs text-duck-dark/50 dark:text-foreground/50">Waiting...</span>
)}
{account.status === 'connected' && (
<Button
type="button"
variant="outline"
size="sm"
onClick={() => handleSync(account.id)}
disabled={syncing === account.id}
className="h-7 text-xs cursor-pointer gap-1.5"
>
{syncing === account.id ? (
<Loader2 className="h-3 w-3 animate-spin" />
) : (
<RefreshCw className="h-3 w-3" />
)}
Initial Sync
</Button>
)}
<button
onClick={() => handleDelete(account.id)}
disabled={deleting === account.id || account.status === 'syncing' || account.status === 'queued'}
className="text-duck-dark/40 dark:text-foreground/40 hover:text-red-500 transition-colors cursor-pointer disabled:opacity-30 disabled:cursor-not-allowed"
>
{deleting === account.id ? (
<Loader2 className="h-4 w-4 animate-spin" />
) : (
<Trash2 className="h-4 w-4" />
)}
</button>
</div>
{/* Sync progress */}
{account.status === 'syncing' && progress && (
<div className="mt-2 flex items-center gap-2">
<Loader2 className="h-3 w-3 animate-spin text-blue-500 shrink-0" />
<p className="text-xs text-duck-dark/60 dark:text-foreground/60 truncate">
{progress.label ?? 'Syncing...'}
</p>
</div>
)}
{account.status === 'syncing' && !progress && (
<div className="mt-2 flex items-center gap-2">
<Loader2 className="h-3 w-3 animate-spin text-blue-500 shrink-0" />
<p className="text-xs text-duck-dark/60 dark:text-foreground/60">Syncing...</p>
</div>
)}
</div>
);
})}
</div>
)}
{/* Provider picker */}
{showPicker && !activeForm && (
<div className="grid gap-2">
<button
onClick={() => handlePickProvider('gmail-oauth')}
className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-3 hover:bg-duck-dark/5 dark:hover:bg-foreground/5 transition-colors cursor-pointer text-left"
>
<Mail className="h-4 w-4 text-duck-dark/60 dark:text-foreground/60" />
<div>
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Gmail (OAuth)</p>
<p className="text-xs text-duck-dark/50 dark:text-foreground/50">Uses your existing Google connection</p>
</div>
</button>
<button
onClick={() => handlePickProvider('gmail-password')}
className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-3 hover:bg-duck-dark/5 dark:hover:bg-foreground/5 transition-colors cursor-pointer text-left"
>
<KeyRound className="h-4 w-4 text-duck-dark/60 dark:text-foreground/60" />
<div>
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Gmail (App Password)</p>
<p className="text-xs text-duck-dark/50 dark:text-foreground/50">
Direct IMAP with a Google App Password
</p>
</div>
</button>
<button
onClick={() => handlePickProvider('imap')}
className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-3 hover:bg-duck-dark/5 dark:hover:bg-foreground/5 transition-colors cursor-pointer text-left"
>
<Server className="h-4 w-4 text-duck-dark/60 dark:text-foreground/60" />
<div>
<p className="text-sm font-medium text-duck-dark dark:text-foreground">IMAP Account</p>
<p className="text-xs text-duck-dark/50 dark:text-foreground/50">Any email provider with IMAP access</p>
</div>
</button>
<Button type="button" variant="ghost" onClick={resetForm} className="h-9 cursor-pointer">
Cancel
</Button>
</div>
)}
{/* Gmail OAuth form */}
{activeForm === 'gmail-oauth' && (
<div className="grid gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Add Gmail (OAuth)</p>
<input
type="text"
value={form.displayName}
onChange={(ev) => setForm({ ...form, displayName: ev.target.value })}
placeholder="Display name (optional)"
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
/>
<div className="flex gap-2">
<Button type="button" variant="ghost" onClick={resetForm} className="h-9 cursor-pointer">
Cancel
</Button>
<Button type="button" onClick={handleGmailOAuth} disabled={saving} className="h-9 flex-1 cursor-pointer">
{saving ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Connect & Add'}
</Button>
</div>
</div>
)}
{/* Gmail App Password form */}
{activeForm === 'gmail-password' && (
<div className="grid gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Add Gmail (App Password)</p>
<div className="text-xs text-duck-dark/50 dark:text-foreground/50 grid gap-1.5">
<p>To create an App Password:</p>
<ol className="list-decimal ml-4 grid gap-0.5">
<li>
Go to{' '}
<a
href="https://myaccount.google.com/apppasswords"
target="_blank"
rel="noopener noreferrer"
className="underline hover:opacity-70"
>
Google App Passwords
</a>
</li>
<li>You may need to enable 2-Step Verification first</li>
<li>Enter a name (e.g. "Officer") and click Create</li>
<li>Copy the 16-character password and paste it below</li>
</ol>
</div>
<input
type="email"
value={form.email}
onChange={(ev) => setForm({ ...form, email: ev.target.value })}
placeholder="your@gmail.com"
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
/>
<input
type="text"
value={form.displayName}
onChange={(ev) => setForm({ ...form, displayName: ev.target.value })}
placeholder="Display name (optional)"
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
/>
<input
type="password"
value={form.password}
onChange={(ev) => setForm({ ...form, password: ev.target.value })}
placeholder="xxxx xxxx xxxx xxxx"
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
/>
<div className="flex gap-2">
<Button type="button" variant="ghost" onClick={resetForm} className="h-9 cursor-pointer">
Cancel
</Button>
<Button
type="button"
onClick={handlePasswordSubmit}
disabled={!form.email.trim() || !form.password.trim() || saving}
className="h-9 flex-1 cursor-pointer"
>
{saving ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Test & Save'}
</Button>
</div>
</div>
)}
{/* Generic IMAP form */}
{activeForm === 'imap' && (
<div className="grid gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Add IMAP Account</p>
<input
type="email"
value={form.email}
onChange={(ev) => setForm({ ...form, email: ev.target.value })}
placeholder="you@example.com"
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
/>
<input
type="text"
value={form.displayName}
onChange={(ev) => setForm({ ...form, displayName: ev.target.value })}
placeholder="Display name (optional)"
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
/>
<div className="grid grid-cols-[1fr_auto_auto] gap-2">
<input
type="text"
value={form.imapHost}
onChange={(ev) => setForm({ ...form, imapHost: ev.target.value })}
placeholder="imap.example.com"
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
/>
<input
type="number"
value={form.imapPort}
onChange={(ev) => setForm({ ...form, imapPort: ev.target.value })}
placeholder="993"
className="h-9 w-20 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
/>
<label className="flex items-center gap-1.5 text-xs text-duck-dark/60 dark:text-foreground/60 cursor-pointer">
<input
type="checkbox"
checked={form.imapSecure}
onChange={(ev) => setForm({ ...form, imapSecure: ev.target.checked })}
className="cursor-pointer"
/>
SSL
</label>
</div>
<input
type="password"
value={form.password}
onChange={(ev) => setForm({ ...form, password: ev.target.value })}
placeholder="Password"
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
/>
<div className="flex gap-2">
<Button type="button" variant="ghost" onClick={resetForm} className="h-9 cursor-pointer">
Cancel
</Button>
<Button
type="button"
onClick={handlePasswordSubmit}
disabled={!form.email.trim() || !form.password.trim() || !form.imapHost.trim() || saving}
className="h-9 flex-1 cursor-pointer"
>
{saving ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Test & Save'}
</Button>
</div>
</div>
)}
{/* Add account button */}
{!showPicker && !activeForm && (
<Button
type="button"
variant="outline"
onClick={() => setShowPicker(true)}
className="w-full h-11 cursor-pointer gap-2"
>
<Plus className="h-4 w-4" />
Add Account
</Button>
)}
</div>
);
};
const StatusBadge = ({ status }: { status: string }) => {
switch (status) {
case 'connected':
return (
<span className="text-[10px] font-medium px-1.5 py-0.5 rounded-full bg-yellow-500/10 text-yellow-600 dark:text-yellow-400">
Connected
</span>
);
case 'queued':
return (
<span className="text-[10px] font-medium px-1.5 py-0.5 rounded-full bg-orange-500/10 text-orange-600 dark:text-orange-400">
Queued
</span>
);
case 'syncing':
return (
<span className="text-[10px] font-medium px-1.5 py-0.5 rounded-full bg-blue-500/10 text-blue-600 dark:text-blue-400">
Syncing
</span>
);
case 'synced':
return (
<span className="text-[10px] font-medium px-1.5 py-0.5 rounded-full bg-green-500/10 text-green-600 dark:text-green-400">
Synced
</span>
);
default:
return null;
}
};
const ProviderIcon = ({ provider }: { provider: string }) => {
switch (provider) {
case 'gmail':
return <Mail className="h-4 w-4 text-red-500 shrink-0" />;
case 'outlook':
return <Mail className="h-4 w-4 text-blue-500 shrink-0" />;
default:
return <Server className="h-4 w-4 text-duck-dark/60 dark:text-foreground/60 shrink-0" />;
}
};
@@ -1,64 +1,32 @@
import { useState, useEffect } from 'react'; import { useState, useEffect } from 'react';
import { toast } from 'sonner'; import { toast } from 'sonner';
import { RefreshCw, Loader2, CheckCircle2, XCircle } from 'lucide-react';
import { Button } from '@/components/ui/button'; import { Button } from '@/components/ui/button';
import { useClient } from 'hooks/useClient'; import { useClient } from 'hooks/useClient';
import { useAuth } from 'hooks/useAuth';
import { useQueryClient } from '@tanstack/react-query';
import { useJobs } from 'hooks/useJobs';
type GoogleStatus = { type GoogleStatus = {
connected: boolean; connected: boolean;
email: string | null; email: string | null;
picture: string | null; picture: string | null;
configured: boolean; configured: boolean;
hasAppPassword: boolean;
};
const formatTime = (ts: number | string) => {
const d = new Date(ts);
return d.toLocaleString(undefined, { dateStyle: 'medium', timeStyle: 'short' });
}; };
export const GoogleAccount = () => { export const GoogleAccount = () => {
const client = useClient(); const client = useClient();
const { user } = useAuth();
const queryClient = useQueryClient();
const [isLoading, setIsLoading] = useState(true); const [isLoading, setIsLoading] = useState(true);
const [status, setStatus] = useState<GoogleStatus>({ const [status, setStatus] = useState<GoogleStatus>({
connected: false, connected: false,
email: null, email: null,
picture: null, picture: null,
configured: false, configured: false,
hasAppPassword: false,
}); });
const [gmailEmail, setGmailEmail] = useState('');
const [appPassword, setAppPassword] = useState('');
const [showPasswordInput, setShowPasswordInput] = useState(false);
const [savingPassword, setSavingPassword] = useState(false);
const [lastSyncAt, setLastSyncAt] = useState<string | null>(null);
const [dismissedError, setDismissedError] = useState(false);
const { jobs, createJob, refetch } = useJobs({ type: 'gmail-sync' });
const activeJob = jobs.find((j) => j.status === 'queued' || j.status === 'running');
const lastJob = jobs[0];
const fetchStatus = () => {
client
.get<GoogleStatus>('/integrations/google/status')
.then((s) => {
setStatus(s);
setGmailEmail(s.email ?? user?.email ?? '');
})
.catch(() => {})
.finally(() => setIsLoading(false));
client
.get<{ lastSyncAt: string | null }>('/email/sync-status')
.then((res) => setLastSyncAt(res.lastSyncAt))
.catch(() => {});
};
useEffect(() => { useEffect(() => {
fetchStatus(); client
.get<GoogleStatus>('/integrations/google/status')
.then(setStatus)
.catch(() => {})
.finally(() => setIsLoading(false));
const params = new URLSearchParams(window.location.search); const params = new URLSearchParams(window.location.search);
const result = params.get('google'); const result = params.get('google');
if (result === 'success') { if (result === 'success') {
@@ -71,27 +39,6 @@ export const GoogleAccount = () => {
} }
}, []); }, []);
// Refresh sync status and dock when a job finishes
useEffect(() => {
if (!activeJob && lastJob?.status === 'completed') {
client
.get<{ lastSyncAt: string | null }>('/email/sync-status')
.then((res) => setLastSyncAt(res.lastSyncAt))
.catch(() => {});
queryClient.invalidateQueries({ queryKey: ['DOCK'] });
}
}, [activeJob, lastJob?.status]);
const handleSync = async (year?: number) => {
try {
await createJob({ lane: 'google-api', type: 'gmail-sync', meta: year ? { year } : undefined });
await refetch();
toast.success("Gmail sync started — you'll receive an email when it's done");
} catch {
toast.error('Failed to start Gmail sync');
}
};
const handleConnect = () => { const handleConnect = () => {
const params = new URLSearchParams({ const params = new URLSearchParams({
token: client.token ?? '', token: client.token ?? '',
@@ -110,201 +57,41 @@ export const GoogleAccount = () => {
} }
}; };
const handleSaveAppPassword = async () => {
if (!appPassword.trim() || !gmailEmail.trim()) return;
setSavingPassword(true);
try {
await client.put('/integrations/google/app-password', {
appPassword: appPassword.trim(),
email: gmailEmail.trim(),
});
setStatus({ ...status, hasAppPassword: true, email: gmailEmail.trim() });
setAppPassword('');
setShowPasswordInput(false);
toast.success('App password saved');
} catch {
toast.error('Failed to save app password');
} finally {
setSavingPassword(false);
}
};
if (isLoading) return null; if (isLoading) return null;
if (!status.configured) return null;
const currentStep = activeJob?.steps[activeJob.currentStep];
const progress = currentStep?.progress;
return ( return (
<div className="grid gap-6"> <div className="grid gap-4">
{/* Gmail Sync — independent of OAuth */} <div>
<div className="grid gap-4"> <p className="text-sm font-medium text-duck-dark dark:text-foreground">Google Account</p>
<div> <p className="text-xs text-duck-dark/50 dark:text-foreground/50 mt-1">
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Gmail Sync</p> Connect your Google account for Calendar, Contacts, and other Google services.
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 mt-1"> </p>
Import and sync your Gmail emails with your Officer inbox. This uses a Google App Password for a direct IMAP </div>
connection the only thing it can do is download your emails. It cannot send, delete, or modify anything in {status.connected ? (
your account. <>
</p> <div className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
</div> <div className="h-2.5 w-2.5 rounded-full bg-green-500 shrink-0" />
<div className="rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4 grid gap-2">
<p className="text-sm font-medium text-duck-dark dark:text-foreground">App Password</p>
{status.hasAppPassword && !showPasswordInput ? (
<div className="flex items-center gap-2">
<span className="text-xs text-green-600 dark:text-green-400">Configured</span>
{status.email && (
<span className="text-xs text-duck-dark/50 dark:text-foreground/50">({status.email})</span>
)}
<button
onClick={() => setShowPasswordInput(true)}
className="text-xs text-duck-dark/50 dark:text-foreground/50 underline hover:opacity-70 cursor-pointer"
>
Replace
</button>
</div>
) : (
<>
<div className="text-xs text-duck-dark/50 dark:text-foreground/50 grid gap-1.5">
<p>To create an App Password:</p>
<ol className="list-decimal ml-4 grid gap-0.5">
<li>
Go to{' '}
<a
href="https://myaccount.google.com/apppasswords"
target="_blank"
rel="noopener noreferrer"
className="underline hover:opacity-70"
>
Google App Passwords
</a>
</li>
<li>You may need to enable 2-Step Verification first</li>
<li>Enter a name (e.g. "Officer") and click Create</li>
<li>Copy the 16-character password and paste it below</li>
</ol>
</div>
<div className="grid gap-2">
<input
type="email"
value={gmailEmail}
onChange={(ev) => setGmailEmail(ev.target.value)}
placeholder="your@gmail.com"
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
/>
<div className="flex gap-2">
<input
type="password"
value={appPassword}
onChange={(ev) => setAppPassword(ev.target.value)}
placeholder="xxxx xxxx xxxx xxxx"
className="flex-1 h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
/>
<Button
type="button"
variant="outline"
disabled={!appPassword.trim() || !gmailEmail.trim() || savingPassword}
onClick={handleSaveAppPassword}
className="h-9 cursor-pointer"
>
{savingPassword ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Save'}
</Button>
</div>
</div>
</>
)}
</div>
{activeJob && (
<div className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-3">
<Loader2 className="h-4 w-4 animate-spin text-duck-dark/60 dark:text-foreground/60 shrink-0" />
<div className="min-w-0 flex-1"> <div className="min-w-0 flex-1">
<p className="text-sm font-medium text-duck-dark dark:text-foreground"> <p className="text-sm font-medium text-duck-dark dark:text-foreground">Connected</p>
{activeJob.status === 'queued' ? 'Queued' : 'Syncing'} <p className="text-xs text-duck-dark/50 dark:text-foreground/50 truncate">{status.email}</p>
{progress?.label ? `${progress.label}` : ''}
</p>
{progress && progress.total > 0 && (
<div className="mt-1.5 h-1.5 rounded-full bg-duck-dark/10 dark:bg-foreground/10 overflow-hidden">
<div
className="h-full rounded-full bg-duck-yellow transition-all duration-300"
style={{ width: `${Math.round((progress.current / progress.total) * 100)}%` }}
/>
</div>
)}
{progress && progress.total === 0 && progress.current > 0 && (
<p className="mt-1 text-xs text-duck-dark/50 dark:text-foreground/50">
{progress.current.toLocaleString()} emails processed
</p>
)}
</div> </div>
{status.picture && (
<img src={status.picture} alt="" className="h-9 w-9 rounded-full shrink-0" referrerPolicy="no-referrer" />
)}
</div> </div>
)} <Button type="button" variant="outline" onClick={handleDisconnect} className="w-full h-11 cursor-pointer">
{!activeJob && lastJob?.status === 'failed' && !dismissedError && ( Disconnect
<button </Button>
onClick={() => setDismissedError(true)} </>
className="flex items-center gap-2 text-xs text-red-500 hover:opacity-70 transition-opacity cursor-pointer text-left" ) : (
title="Click to dismiss"
>
<XCircle className="h-3.5 w-3.5 shrink-0" />
Last sync failed{lastJob.error ? `: ${lastJob.error}` : ''}
</button>
)}
{!activeJob && lastSyncAt && (
<div className="flex items-center gap-2 text-xs text-duck-dark/50 dark:text-foreground/50">
<CheckCircle2 className="h-3.5 w-3.5 text-green-500 shrink-0" />
Last sync completed {formatTime(lastSyncAt)}
</div>
)}
<Button <Button
type="button" type="button"
variant="outline" onClick={handleConnect}
disabled={!!activeJob || !status.hasAppPassword} className="w-full h-11 bg-duck-yellow hover:bg-duck-yellow/90 text-duck-teal font-bold transition-all duration-200 hover:scale-105 cursor-pointer"
onClick={() => handleSync()}
className="w-full h-11 cursor-pointer gap-2"
> >
<RefreshCw className="h-4 w-4" /> Connect Google Account
Sync Gmail Inbox
</Button> </Button>
</div>
{/* OAuth — for Calendar and other Google services */}
{status.configured && (
<div className="grid gap-4 border-t border-duck-dark/10 dark:border-foreground/10 pt-6">
<div>
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Google Account</p>
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 mt-1">
Connect your Google account for Calendar, Contacts, and other Google services. This is separate from Gmail
sync above.
</p>
</div>
{status.connected ? (
<>
<div className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
<div className="h-2.5 w-2.5 rounded-full bg-green-500 shrink-0" />
<div className="min-w-0 flex-1">
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Connected</p>
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 truncate">{status.email}</p>
</div>
{status.picture && (
<img
src={status.picture}
alt=""
className="h-9 w-9 rounded-full shrink-0"
referrerPolicy="no-referrer"
/>
)}
</div>
<Button type="button" variant="outline" onClick={handleDisconnect} className="w-full h-11 cursor-pointer">
Disconnect
</Button>
</>
) : (
<Button
type="button"
onClick={handleConnect}
className="w-full h-11 bg-duck-yellow hover:bg-duck-yellow/90 text-duck-teal font-bold transition-all duration-200 hover:scale-105 cursor-pointer"
>
Connect Google Account
</Button>
)}
</div>
)} )}
</div> </div>
); );
@@ -1,5 +1,5 @@
import { useMemo } from 'react'; import { useMemo } from 'react';
import { Puzzle, KeyRound, UserCircle, MessageCircle, Globe, Wrench } from 'lucide-react'; import { Puzzle, KeyRound, UserCircle, MessageCircle, Globe, Wrench, Mail } from 'lucide-react';
import type { LayoutNode, PanelComponents } from 'officerdev'; import type { LayoutNode, PanelComponents } from 'officerdev';
import { WorkspaceLayout } from 'officerdev'; import { WorkspaceLayout } from 'officerdev';
import { useAuth } from 'hooks/useAuth'; import { useAuth } from 'hooks/useAuth';
@@ -17,6 +17,7 @@ import { WhatsAppBotConfig } from './WhatsAppBotConfig';
import { WhatsAppAccount } from './WhatsAppAccount'; import { WhatsAppAccount } from './WhatsAppAccount';
import { BrowserRelay } from './BrowserRelay'; import { BrowserRelay } from './BrowserRelay';
import { ApifyConfig } from './ApifyConfig'; import { ApifyConfig } from './ApifyConfig';
import { EmailAccounts } from './EmailAccounts';
const GLOBAL_KEY = 'INTEGRATIONS_SETTINGS_SELECTED'; const GLOBAL_KEY = 'INTEGRATIONS_SETTINGS_SELECTED';
const TAB_KEY = 'INTEGRATIONS_SETTINGS_TAB'; const TAB_KEY = 'INTEGRATIONS_SETTINGS_TAB';
@@ -60,6 +61,13 @@ const enterpriseSections: SettingsSection[] = [
]; ];
const personalSections: SettingsSection[] = [ const personalSections: SettingsSection[] = [
{
key: 'email-accounts',
icon: Mail,
title: 'Email',
description: 'Connect email accounts for inbox sync',
content: <EmailAccounts />,
},
{ {
key: 'google-account', key: 'google-account',
icon: UserCircle, icon: UserCircle,
+18 -1
View File
@@ -20,7 +20,14 @@ export {
export { readServerSettings, writeServerSettings, readConfigValue, writeConfigValue } from './queries/server-config'; export { readServerSettings, writeServerSettings, readConfigValue, writeConfigValue } from './queries/server-config';
export { getUserSettings, setUserSettings, getUserState, patchUserState, getDockPaths, setDockPaths } from './queries/user-data'; export {
getUserSettings,
setUserSettings,
getUserState,
patchUserState,
getDockPaths,
setDockPaths,
} from './queries/user-data';
export { export {
getServerIntegrations, getServerIntegrations,
@@ -35,5 +42,15 @@ export {
findUserByIntegrationConfig, findUserByIntegrationConfig,
} from './queries/integrations'; } from './queries/integrations';
export {
getEmailAccounts,
getEmailAccount,
createEmailAccount,
deleteEmailAccount,
updateEmailAccountStatus,
updateEmailAccountSyncMeta,
getAllSyncedAccounts,
} from './queries/email-accounts';
export { db } from './db'; export { db } from './db';
export * as schema from './schema'; export * as schema from './schema';
@@ -0,0 +1,41 @@
import { eq, and } from 'drizzle-orm';
import { db } from '../db';
import { emailAccounts } from '../schema';
import type { EmailAccountInsert, EmailAccountSelect } from '../types';
export async function getEmailAccounts(userId: number): Promise<EmailAccountSelect[]> {
return db.select().from(emailAccounts).where(eq(emailAccounts.userId, userId));
}
export async function getEmailAccount(id: number): Promise<EmailAccountSelect | undefined> {
const [row] = await db.select().from(emailAccounts).where(eq(emailAccounts.id, id));
return row;
}
export async function createEmailAccount(params: EmailAccountInsert): Promise<EmailAccountSelect> {
const [row] = await db.insert(emailAccounts).values(params).returning();
return row!;
}
export async function deleteEmailAccount(id: number, userId: number): Promise<boolean> {
const [row] = await db
.delete(emailAccounts)
.where(and(eq(emailAccounts.id, id), eq(emailAccounts.userId, userId)))
.returning({ id: emailAccounts.id });
return !!row;
}
export async function updateEmailAccountStatus(id: number, status: string): Promise<void> {
await db.update(emailAccounts).set({ status }).where(eq(emailAccounts.id, id));
}
export async function updateEmailAccountSyncMeta(id: number, syncMeta: Record<string, unknown>): Promise<void> {
await db.update(emailAccounts).set({ syncMeta }).where(eq(emailAccounts.id, id));
}
export async function getAllSyncedAccounts(): Promise<EmailAccountSelect[]> {
return db
.select()
.from(emailAccounts)
.where(and(eq(emailAccounts.status, 'synced'), eq(emailAccounts.enabled, true)));
}
@@ -0,0 +1,26 @@
import { pgTable, serial, integer, text, boolean, timestamp, jsonb, unique } from 'drizzle-orm/pg-core';
import { users } from './auth';
export const emailAccounts = pgTable(
'email_accounts',
{
id: serial('id').primaryKey(),
userId: integer('user_id')
.notNull()
.references(() => users.id, { onDelete: 'cascade' }),
provider: text('provider').notNull(),
email: text('email').notNull(),
displayName: text('display_name'),
imapHost: text('imap_host').notNull(),
imapPort: integer('imap_port').notNull(),
imapSecure: boolean('imap_secure').notNull().default(true),
authType: text('auth_type').notNull(),
credentials: jsonb('credentials').notNull().default({}),
enabled: boolean('enabled').notNull().default(true),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
status: text('status').notNull().default('connected'),
syncMeta: jsonb('sync_meta').notNull().default({}),
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
},
(table) => [unique('uq_email_accounts_user_email').on(table.userId, table.email)],
);
@@ -5,3 +5,4 @@ export * from './dashboards';
export * from './agent-items'; export * from './agent-items';
export * from './operations'; export * from './operations';
export * from './server'; export * from './server';
export * from './email';
+5
View File
@@ -100,6 +100,11 @@ export type QueueJobInsert = typeof Schema.queueJobs.$inferInsert;
export type TerminalContainerSelect = typeof Schema.terminalContainers.$inferSelect; export type TerminalContainerSelect = typeof Schema.terminalContainers.$inferSelect;
export type TerminalContainerInsert = typeof Schema.terminalContainers.$inferInsert; export type TerminalContainerInsert = typeof Schema.terminalContainers.$inferInsert;
// ── Email ──
export type EmailAccountSelect = typeof Schema.emailAccounts.$inferSelect;
export type EmailAccountInsert = typeof Schema.emailAccounts.$inferInsert;
// ── Server ── // ── Server ──
export type ServerConfigSelect = typeof Schema.serverConfig.$inferSelect; export type ServerConfigSelect = typeof Schema.serverConfig.$inferSelect;
+222
View File
@@ -0,0 +1,222 @@
import { createRouter } from '../../create-router';
import { BAD_REQUEST, NOT_FOUND } from '../../custom-errors';
import {
getEmailAccounts,
getEmailAccount,
createEmailAccount,
deleteEmailAccount,
getUserIntegration,
updateEmailAccountStatus,
} from 'officerdb';
import { validateImapConnection } from './imap-validate';
import * as sidecar from '../../sidecar-client';
type CreateAccountBody = {
provider: string;
email: string;
displayName?: string;
imapHost: string;
imapPort: number;
imapSecure: boolean;
authType: string;
credentials: Record<string, unknown>;
};
type ValidateBody = {
imapHost: string;
imapPort: number;
imapSecure: boolean;
authType: string;
email: string;
credentials: Record<string, unknown>;
};
export const accountsRouter = createRouter();
accountsRouter.get('/', async (ctx) => {
const user = ctx.get('user');
const accounts = await getEmailAccounts(user.id);
// Check for stale syncing/queued accounts with no active job
const staleIds: number[] = [];
const hasActiveAccounts = accounts.some((a) => a.status === 'syncing' || a.status === 'queued');
let activeJobAccountIds = new Set<number>();
if (hasActiveAccounts) {
try {
const jobs = await sidecar.listJobs();
activeJobAccountIds = new Set(
jobs
.filter((j) => j.type === 'email-sync' && (j.status === 'queued' || j.status === 'running'))
.map((j) => (j.meta as Record<string, unknown> | undefined)?.emailAccountId as number)
.filter(Boolean),
);
} catch {
// Sidecar unavailable — all syncing/queued accounts are stale
}
for (const a of accounts) {
if ((a.status === 'syncing' || a.status === 'queued') && !activeJobAccountIds.has(a.id)) {
staleIds.push(a.id);
}
}
// Reset stale accounts in background
if (staleIds.length > 0) {
for (const id of staleIds) {
updateEmailAccountStatus(id, 'connected').catch(() => {});
}
}
}
return ctx.json(
accounts.map((a) => ({
id: a.id,
provider: a.provider,
email: a.email,
displayName: a.displayName,
enabled: a.enabled,
status: staleIds.includes(a.id) ? 'connected' : a.status,
createdAt: a.createdAt,
})),
);
});
accountsRouter.post('/', async (ctx) => {
const user = ctx.get('user');
const body = ctx.get('body') as CreateAccountBody;
if (!body.provider || !body.email || !body.imapHost || !body.imapPort || !body.authType) {
throw BAD_REQUEST('Missing required fields');
}
const authResult = await resolveAuth(user.id, body.authType, body.email, body.credentials);
if (!authResult.ok) throw BAD_REQUEST(authResult.error);
const validation = await validateImapConnection({
host: body.imapHost,
port: body.imapPort,
secure: body.imapSecure,
user: body.email,
...authResult.auth,
});
if (!validation.ok) throw BAD_REQUEST(`IMAP connection failed: ${validation.error}`);
const account = await createEmailAccount({
userId: user.id,
provider: body.provider,
email: body.email,
displayName: body.displayName,
imapHost: body.imapHost,
imapPort: body.imapPort,
imapSecure: body.imapSecure,
authType: body.authType,
credentials: body.credentials,
});
return ctx.json({ id: account.id, provider: account.provider, email: account.email }, 201);
});
accountsRouter.delete('/:id', async (ctx) => {
const user = ctx.get('user');
const id = Number(ctx.req.param('id'));
const deleted = await deleteEmailAccount(id, user.id);
if (!deleted) throw NOT_FOUND('Account not found');
return ctx.json({ ok: true });
});
accountsRouter.post('/:id/sync', async (ctx) => {
const user = ctx.get('user');
const id = Number(ctx.req.param('id'));
const account = await getEmailAccount(id);
if (!account || account.userId !== user.id) throw NOT_FOUND('Account not found');
if (account.status === 'queued') throw BAD_REQUEST('Sync is already queued');
if (account.status === 'syncing') throw BAD_REQUEST('Account is already syncing');
// Resolve auth before enqueueing
const authResult = await resolveAuth(user.id, account.authType, account.email, account.credentials as Record<string, unknown>);
if (!authResult.ok) throw BAD_REQUEST(authResult.error);
// Set status immediately so the UI reflects the queued state
await updateEmailAccountStatus(id, 'queued');
const job = await sidecar.enqueueJob({
lane: 'email',
type: 'email-sync',
userId: user.email,
meta: {
emailAccountId: id,
userEmail: user.email,
account: {
id: account.id,
userId: account.userId,
email: account.email,
imapHost: account.imapHost,
imapPort: account.imapPort,
imapSecure: account.imapSecure,
provider: account.provider,
authType: account.authType,
credentials: account.credentials,
},
imapAuth: { user: account.email, ...authResult.auth },
},
});
return ctx.json({ ok: true, jobId: job.id }, 201);
});
accountsRouter.post('/validate', async (ctx) => {
const user = ctx.get('user');
const body = ctx.get('body') as ValidateBody;
if (!body.imapHost || !body.imapPort || !body.authType || !body.email) {
throw BAD_REQUEST('Missing required fields');
}
const authResult = await resolveAuth(user.id, body.authType, body.email, body.credentials);
if (!authResult.ok) return ctx.json({ ok: false, error: authResult.error });
const result = await validateImapConnection({
host: body.imapHost,
port: body.imapPort,
secure: body.imapSecure,
user: body.email,
...authResult.auth,
});
return ctx.json(result);
});
type AuthResult = { ok: true; auth: { pass?: string; accessToken?: string } } | { ok: false; error: string };
async function resolveAuth(
userId: number,
authType: string,
email: string,
credentials: Record<string, unknown>,
): Promise<AuthResult> {
if (authType === 'oauth') {
const integrationId = credentials.userIntegrationId as number | undefined;
if (!integrationId) return { ok: false, error: 'Missing userIntegrationId for OAuth' };
const integration = await getUserIntegration(userId, 'google');
if (!integration) return { ok: false, error: 'Google integration not found' };
const config = integration.config as Record<string, unknown>;
const accessToken = config.accessToken as string | undefined;
if (!accessToken) return { ok: false, error: 'No access token available — reconnect Google account' };
return { ok: true, auth: { accessToken } };
}
if (authType === 'password') {
const pass = credentials.password as string | undefined;
if (!pass) return { ok: false, error: 'Missing password' };
return { ok: true, auth: { pass } };
}
return { ok: false, error: `Unknown auth type: ${authType}` };
}
+30 -13
View File
@@ -4,9 +4,12 @@ import type { EmailMessage } from 'types';
import { createRouter } from '../../create-router'; import { createRouter } from '../../create-router';
import { DATA_PATH } from '@@/data-path'; import { DATA_PATH } from '@@/data-path';
import { openEmailDb, rowToSummary, getSyncMeta } from './email-db'; import { openEmailDb, rowToSummary, getSyncMeta } from './email-db';
import { accountsRouter } from './accounts';
export const emailRouter = createRouter(); export const emailRouter = createRouter();
emailRouter.route('/accounts', accountsRouter);
emailRouter.get('/messages', async (ctx) => { emailRouter.get('/messages', async (ctx) => {
const email = ctx.get('user').email; const email = ctx.get('user').email;
const page = Number(ctx.req.query('page') ?? '1'); const page = Number(ctx.req.query('page') ?? '1');
@@ -18,7 +21,9 @@ emailRouter.get('/messages', async (ctx) => {
const db = openEmailDb(email); const db = openEmailDb(email);
try { try {
const rows = db.query(`SELECT * FROM emails WHERE ${folderWhere} ORDER BY date DESC LIMIT ? OFFSET ?`).all(limit, offset) as Record<string, unknown>[]; const rows = db
.query(`SELECT * FROM emails WHERE ${folderWhere} ORDER BY date DESC LIMIT ? OFFSET ?`)
.all(limit, offset) as Record<string, unknown>[];
const countRow = db.query(`SELECT COUNT(*) as total FROM emails WHERE ${folderWhere}`).get() as { total: number }; const countRow = db.query(`SELECT COUNT(*) as total FROM emails WHERE ${folderWhere}`).get() as { total: number };
const messages = rows.map(rowToSummary); const messages = rows.map(rowToSummary);
return ctx.json({ messages, total: countRow.total }); return ctx.json({ messages, total: countRow.total });
@@ -36,11 +41,11 @@ emailRouter.get('/messages/:id', async (ctx) => {
const row = db.query('SELECT * FROM emails WHERE id = ? AND deleted = 0').get(id) as Record<string, unknown> | null; const row = db.query('SELECT * FROM emails WHERE id = ? AND deleted = 0').get(id) as Record<string, unknown> | null;
if (!row) return ctx.text('Not found', 404); if (!row) return ctx.text('Not found', 404);
const attachmentRows = db.query('SELECT * FROM attachments WHERE email_id = ? ORDER BY idx').all(id) as Array<Record<string, unknown>>; const attachmentRows = db.query('SELECT * FROM attachments WHERE email_id = ? ORDER BY idx').all(id) as Array<
Record<string, unknown>
>;
const from = row.from_name const from = row.from_name ? `${row.from_name} <${row.from_address}>` : (row.from_address as string);
? `${row.from_name} <${row.from_address}>`
: (row.from_address as string);
const message: EmailMessage = { const message: EmailMessage = {
id: row.id as string, id: row.id as string,
@@ -75,7 +80,10 @@ emailRouter.post('/messages/:id/attachments/:index/extract', async (ctx) => {
const db = openEmailDb(email); const db = openEmailDb(email);
try { try {
const row = db.query('SELECT filename, content FROM attachments WHERE email_id = ? AND idx = ?').get(id, index) as { filename: string; content: string | null } | null; const row = db.query('SELECT filename, content FROM attachments WHERE email_id = ? AND idx = ?').get(id, index) as {
filename: string;
content: string | null;
} | null;
if (!row || !row.content) return ctx.text('Attachment not found', 404); if (!row || !row.content) return ctx.text('Attachment not found', 404);
const fileName = row.filename ?? 'unknown'; const fileName = row.filename ?? 'unknown';
@@ -144,9 +152,18 @@ emailRouter.get('/stats', async (ctx) => {
const db = openEmailDb(email); const db = openEmailDb(email);
try { try {
const total = (db.query(`SELECT COUNT(*) as count FROM emails WHERE ${folderWhere}`).get() as { count: number }).count; const total = (db.query(`SELECT COUNT(*) as count FROM emails WHERE ${folderWhere}`).get() as { count: number })
const byDomain = db.query(`SELECT from_domain, COUNT(*) as count FROM emails WHERE ${folderWhere} GROUP BY from_domain ORDER BY count DESC LIMIT 20`).all() as Array<{ from_domain: string; count: number }>; .count;
const bySender = db.query(`SELECT from_address, from_name, COUNT(*) as count FROM emails WHERE ${folderWhere} GROUP BY from_address ORDER BY count DESC LIMIT 20`).all() as Array<{ from_address: string; from_name: string; count: number }>; const byDomain = db
.query(
`SELECT from_domain, COUNT(*) as count FROM emails WHERE ${folderWhere} GROUP BY from_domain ORDER BY count DESC LIMIT 20`,
)
.all() as Array<{ from_domain: string; count: number }>;
const bySender = db
.query(
`SELECT from_address, from_name, COUNT(*) as count FROM emails WHERE ${folderWhere} GROUP BY from_address ORDER BY count DESC LIMIT 20`,
)
.all() as Array<{ from_address: string; from_name: string; count: number }>;
return ctx.json({ total, byDomain, bySender }); return ctx.json({ total, byDomain, bySender });
} finally { } finally {
@@ -159,7 +176,9 @@ emailRouter.get('/labels', async (ctx) => {
const db = openEmailDb(email); const db = openEmailDb(email);
try { try {
const rows = db.query('SELECT labels FROM emails WHERE deleted = 0 AND labels IS NOT NULL').all() as Array<{ labels: string }>; const rows = db.query('SELECT labels FROM emails WHERE deleted = 0 AND labels IS NOT NULL').all() as Array<{
labels: string;
}>;
const counts = new Map<string, number>(); const counts = new Map<string, number>();
for (const row of rows) { for (const row of rows) {
@@ -169,9 +188,7 @@ emailRouter.get('/labels', async (ctx) => {
} }
} }
const labels = [...counts.entries()] const labels = [...counts.entries()].map(([label, count]) => ({ label, count })).sort((a, b) => b.count - a.count);
.map(([label, count]) => ({ label, count }))
.sort((a, b) => b.count - a.count);
return ctx.json({ labels }); return ctx.json({ labels });
} finally { } finally {
+54
View File
@@ -0,0 +1,54 @@
type ValidateImapParams = {
host: string;
port: number;
secure: boolean;
user: string;
pass?: string;
accessToken?: string;
};
type ValidateImapResult = { ok: true; folderCount: number } | { ok: false; error: string };
export async function validateImapConnection(params: ValidateImapParams): Promise<ValidateImapResult> {
const { ImapFlow } = await import('imapflow');
const auth: { user: string; pass?: string; accessToken?: string } = { user: params.user };
if (params.accessToken) {
auth.accessToken = params.accessToken;
} else if (params.pass) {
auth.pass = params.pass;
} else {
return { ok: false, error: 'No authentication credentials provided' };
}
const client = new ImapFlow({
host: params.host,
port: params.port,
secure: params.secure,
auth,
logger: false,
greetingTimeout: 60_000,
socketTimeout: 60_000,
});
try {
const result = await Promise.race([
(async () => {
await client.connect();
const folders = await client.list();
await client.logout();
return { ok: true as const, folderCount: folders.length };
})(),
new Promise<ValidateImapResult>((_, reject) =>
setTimeout(() => reject(new Error('Connection timed out')), 90_000),
),
]);
return result;
} catch (err) {
try {
client.close();
} catch {}
const message = err instanceof Error ? err.message : 'Connection failed';
return { ok: false, error: message };
}
}
@@ -0,0 +1,34 @@
import { getServerIntegration } from 'officerdb';
import { PermanentError } from '../../queue/types';
type TokenRefreshResult = { accessToken: string; expiresAt: number };
export async function refreshGoogleAccessToken(refreshToken: string): Promise<TokenRefreshResult> {
const serverGoogle = await getServerIntegration('google');
const serverConfig = serverGoogle?.config as Record<string, unknown> | undefined;
if (!serverConfig?.clientId || !serverConfig?.clientSecret) {
throw new PermanentError('Google OAuth not configured on server');
}
const response = await fetch('https://oauth2.googleapis.com/token', {
method: 'POST',
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams({
client_id: serverConfig.clientId as string,
client_secret: serverConfig.clientSecret as string,
refresh_token: refreshToken,
grant_type: 'refresh_token',
}),
});
if (!response.ok) {
const text = await response.text();
throw new Error(`Google token refresh failed: ${text}`);
}
const data = (await response.json()) as { access_token: string; expires_in: number };
return {
accessToken: data.access_token,
expiresAt: Date.now() + data.expires_in * 1000,
};
}
@@ -259,11 +259,16 @@ export const googleCallbackHandler = async (ctx: any) => {
const serverIntegration = await getServerIntegration('google'); const serverIntegration = await getServerIntegration('google');
// Merge with existing config to preserve fields like imapAppPassword
const existingIntegration = await getUserIntegration(dbUser.id, 'google');
const existingConfig = (existingIntegration?.config as Record<string, unknown>) ?? {};
await upsertUserIntegration({ await upsertUserIntegration({
userId: dbUser.id, userId: dbUser.id,
provider: 'google', provider: 'google',
serverIntegrationId: serverIntegration?.id, serverIntegrationId: serverIntegration?.id,
config: { config: {
...existingConfig,
accessToken: tokens.access_token, accessToken: tokens.access_token,
refreshToken: tokens.refresh_token, refreshToken: tokens.refresh_token,
expiresAt: Date.now() + tokens.expires_in * 1000, expiresAt: Date.now() + tokens.expires_in * 1000,
+381
View File
@@ -0,0 +1,381 @@
import { createHash } from 'node:crypto';
import type { JobHandler } from '../types';
import { PermanentError } from '../types';
import { registerHandler } from '../handler-registry';
import { openEmailDb, upsertFromRawEml } from '../../api/email/email-db';
import { refreshGoogleAccessToken } from '../../api/integrations/google-auth';
import {
getEmailAccount,
getUserIntegration,
upsertUserIntegration,
getServerIntegration,
updateEmailAccountStatus,
updateEmailAccountSyncMeta,
getDockPaths,
setDockPaths,
} from 'officerdb';
// ── Types for job meta (passed by the API server at enqueue time) ──
type EmailSyncMeta = {
emailAccountId: number;
userEmail: string;
account: {
id: number;
userId: number;
email: string;
imapHost: string;
imapPort: number;
imapSecure: boolean;
provider: string;
authType: string;
credentials: Record<string, unknown>;
};
imapAuth: { user: string; pass?: string; accessToken?: string };
saved?: number;
};
// ── Stable ID from Message-Id header ──
function messageIdToStableId(raw: string): string | null {
const match = raw.match(/^Message-Id:\s*<?([^>\s]+)>?/im);
if (!match?.[1]) return null;
return createHash('sha1').update(match[1]).digest('hex').slice(0, 16);
}
// ── IMAP folder → label mapping ──
const SPECIAL_USE_LABEL_MAP: Record<string, string> = {
'\\Inbox': 'inbox',
'\\Sent': 'sent',
'\\Drafts': 'draft',
'\\Flagged': 'starred',
'\\Trash': 'trash',
'\\Junk': 'spam',
'\\All': 'archive',
};
const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
type FolderInfo = { specialUse?: string; path: string; flags: Set<string>; status?: { uidNext?: number; uidValidity?: number } };
function shouldSkipFolder(folder: FolderInfo): boolean {
if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
return false;
}
function folderToLabel(folder: FolderInfo): string {
if (folder.specialUse && SPECIAL_USE_LABEL_MAP[folder.specialUse]) {
return SPECIAL_USE_LABEL_MAP[folder.specialUse]!;
}
if (folder.path === 'INBOX') return 'inbox';
return folder.path.toLowerCase();
}
// ── OAuth token refresh ──
async function resolveImapAuth(meta: EmailSyncMeta): Promise<{ user: string; pass?: string; accessToken?: string }> {
if (meta.account.authType !== 'oauth') return meta.imapAuth;
// Refresh OAuth token if expired
const userGoogle = await getUserIntegration(meta.account.userId, 'google');
const config = userGoogle?.config as Record<string, unknown> | undefined;
if (!config?.refreshToken) {
throw new PermanentError('Google OAuth not configured — reconnect your Google account');
}
const expiresAt = config.expiresAt as number | undefined;
const tokenExpired = !expiresAt || expiresAt < Date.now() + 60_000;
if (!tokenExpired && config.accessToken) {
return { user: meta.account.email, accessToken: config.accessToken as string };
}
console.log('[email-sync] Refreshing OAuth token');
const refreshed = await refreshGoogleAccessToken(config.refreshToken as string);
const serverGoogle = await getServerIntegration('google');
await upsertUserIntegration({
userId: meta.account.userId,
provider: 'google',
serverIntegrationId: serverGoogle?.id,
config: { ...config, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
});
return { user: meta.account.email, accessToken: refreshed.accessToken };
}
// ── Handler ──
const emailSyncHandler: JobHandler = {
type: 'email-sync',
retry: { delayMs: 15 * 60 * 1000, maxRetries: 5 },
steps: [
{
name: 'Sync emails',
run: async (ctx) => {
const { ImapFlow } = await import('imapflow');
const meta = ctx.meta as unknown as EmailSyncMeta;
const { account, userEmail } = meta;
// Resolve auth (refreshes OAuth token if needed)
const imapAuth = await resolveImapAuth(meta);
// Load syncMeta from DB (always fresh, not from job meta)
const freshAccount = await getEmailAccount(account.id);
if (!freshAccount) throw new PermanentError(`Email account ${account.id} not found`);
const syncMeta = (freshAccount.syncMeta ?? {}) as Record<string, unknown>;
const isIncremental = !!syncMeta.last_sync_at;
await updateEmailAccountStatus(account.id, 'syncing');
console.log(`[email-sync] ${isIncremental ? 'Incremental' : 'Initial'} sync for ${account.email}`);
const MAX_RECONNECTS = 10;
const RECONNECT_DELAY_MS = 5_000;
let reconnects = 0;
let saved = 0;
let skipped = 0;
let errors = 0;
let allDone = false;
const db = openEmailDb(userEmail);
// Load existing IDs for dedup (once, shared across reconnections)
const existingIds = new Set<string>();
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
for (const row of rows) existingIds.add(row.id);
try {
while (!allDone && reconnects <= MAX_RECONNECTS) {
if (reconnects > 0) {
console.log(`[email-sync] Reconnecting (${reconnects}/${MAX_RECONNECTS}) after ${RECONNECT_DELAY_MS / 1000}s...`);
await ctx.updateProgress({ current: saved + skipped, total: 0, label: `Reconnecting (${reconnects}/${MAX_RECONNECTS})...` });
await new Promise((r) => setTimeout(r, RECONNECT_DELAY_MS));
// Re-read syncMeta from DB to get latest saved UIDs
const updated = await getEmailAccount(account.id);
if (updated?.syncMeta) {
Object.assign(syncMeta, updated.syncMeta as Record<string, unknown>);
}
} else {
await ctx.updateProgress({ current: 0, total: 0, label: 'Connecting...' });
}
const client = new ImapFlow({
host: account.imapHost,
port: account.imapPort,
secure: account.imapSecure,
auth: imapAuth,
logger: false,
socketTimeout: 30 * 60 * 1000,
});
const connState = { error: null as Error | null };
client.on('error', (err: Error) => {
console.log(`[email-sync] IMAP connection error: ${err.message}`);
connState.error = err;
});
try {
await client.connect();
console.log(`[email-sync] IMAP connected${reconnects > 0 ? ` (reconnect ${reconnects})` : ''}`);
const folders = await client.list({ statusQuery: { uidNext: true, uidValidity: true } }) as FolderInfo[];
// Filter to folders that still need syncing
const foldersToSync: Array<{ folder: FolderInfo; lastUid: number }> = [];
for (const folder of folders) {
if (shouldSkipFolder(folder)) continue;
const uidValidityKey = `imap_uidvalidity:${folder.path}`;
const lastUidKey = `imap_lastuid:${folder.path}`;
const storedUidValidity = syncMeta[uidValidityKey] as string | undefined;
const storedLastUid = syncMeta[lastUidKey] as string | undefined;
const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
const uidNext = folder.status?.uidNext ?? 0;
let lastUid = storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
if (storedUidValidity && uidValidity && storedUidValidity !== uidValidity) {
console.log(`[email-sync] UIDVALIDITY changed for ${folder.path} — will re-scan`);
lastUid = 0;
}
if (lastUid > 0 && uidNext <= lastUid + 1) continue;
foldersToSync.push({ folder, lastUid });
}
if (foldersToSync.length === 0) {
console.log('[email-sync] All folders synced');
allDone = true;
await client.logout().catch(() => {});
break;
}
console.log(`[email-sync] ${foldersToSync.length} folder(s) to sync`);
let connectionLost = false;
for (let fi = 0; fi < foldersToSync.length; fi++) {
const { folder, lastUid } = foldersToSync[fi]!;
if (connState.error) { connectionLost = true; break; }
console.log(`[email-sync] Syncing folder ${fi + 1}/${foldersToSync.length}: ${folder.path}`);
let lock;
try {
lock = await client.getMailboxLock(folder.path);
} catch (err) {
const errMsg = err instanceof Error ? err.message : String(err);
if (errMsg.includes('not available') || errMsg.includes('timeout') || errMsg.includes('closed')) {
console.log(`[email-sync] Connection lost at folder ${folder.path}`);
connectionLost = true;
break;
}
console.log(`[email-sync] Skipping folder ${folder.path}: ${errMsg}`);
continue;
}
try {
const mailbox = client.mailbox;
if (!mailbox) continue;
const uidValidity = String(mailbox.uidValidity);
const effectiveLastUid = syncMeta[`imap_uidvalidity:${folder.path}`] === uidValidity ? lastUid : 0;
let maxUid = effectiveLastUid;
const label = folderToLabel(folder);
const range = effectiveLastUid > 0 ? `${effectiveLastUid + 1}:*` : '1:*';
try {
const fetchOpts: Record<string, unknown> = { source: true, uid: true };
if (account.provider === 'gmail') fetchOpts.labels = true;
for await (const msg of client.fetch(range, fetchOpts, { uid: true })) {
if (msg.uid <= effectiveLastUid) continue;
maxUid = Math.max(maxUid, msg.uid);
if (!msg.source) { errors++; continue; }
const raw = msg.source.toString('utf-8');
const id = messageIdToStableId(raw);
if (!id) { errors++; continue; }
if (existingIds.has(id)) { skipped++; continue; }
const labels = [label];
try {
upsertFromRawEml({ db, id, raw, integration: account.provider, emailAccount: account.email, labels });
existingIds.add(id);
saved++;
} catch {
errors++;
}
if ((saved + skipped) % 50 === 0) {
const progressLabel = `${folder.path}${saved.toLocaleString()} saved, ${skipped.toLocaleString()} skipped`;
console.log(`[email-sync] ${progressLabel}`);
await ctx.updateProgress({ current: saved + skipped, total: 0, label: progressLabel });
if (maxUid > effectiveLastUid) {
syncMeta[`imap_lastuid:${folder.path}`] = String(maxUid);
await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
}
}
}
} catch (fetchErr) {
const errMsg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
if (errMsg.includes('Nothing to fetch')) {
// No messages in range — normal
} else if (errMsg.includes('not available') || errMsg.includes('timeout') || errMsg.includes('closed')) {
console.log(`[email-sync] Connection lost during fetch in ${folder.path}: ${errMsg}`);
connectionLost = true;
} else {
console.log(`[email-sync] Fetch error in ${folder.path}: ${errMsg}`);
errors++;
}
}
syncMeta[`imap_uidvalidity:${folder.path}`] = uidValidity;
if (maxUid > effectiveLastUid) {
syncMeta[`imap_lastuid:${folder.path}`] = String(maxUid);
}
console.log(`[email-sync] ${folder.path} done: ${saved} saved, ${skipped} skipped, ${errors} errors`);
await ctx.updateProgress({ current: saved + skipped, total: 0, label: `${folder.path}: ${saved} saved` });
await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
} finally {
lock.release();
}
if (connectionLost) break;
}
await client.logout().catch(() => {});
if (connectionLost) {
console.log(`[email-sync] Connection lost after saving ${saved} emails — will reconnect`);
reconnects++;
continue;
}
allDone = true;
} catch (err) {
await client.logout().catch(() => {});
if (reconnects < MAX_RECONNECTS) {
console.log(`[email-sync] Error: ${err instanceof Error ? err.message : String(err)} — will reconnect`);
reconnects++;
continue;
}
throw err;
}
}
if (!allDone) {
throw new Error(`IMAP sync incomplete after ${MAX_RECONNECTS} reconnection attempts`);
}
// Mark sync complete
syncMeta.last_sync_at = new Date().toISOString();
await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
} finally {
db.close();
}
console.log(`[email-sync] Done: saved ${saved}, skipped ${skipped}, errors ${errors}, reconnects ${reconnects}`);
ctx.meta.saved = saved;
await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${saved} new emails` });
},
},
{
name: 'Finalize',
run: async (ctx) => {
const meta = ctx.meta as unknown as EmailSyncMeta;
const saved = meta.saved ?? 0;
await updateEmailAccountStatus(meta.account.id, 'synced');
if (saved > 0) {
try {
const paths = await getDockPaths(meta.account.userId);
if (!paths) {
const defaults = ['/', '/files', '/automation', '/projects', '/dashboards', '/chat'];
await setDockPaths(meta.account.userId, [...defaults, '/email']);
} else if (!paths.includes('/email')) {
await setDockPaths(meta.account.userId, [...paths, '/email']);
}
} catch {
// Non-fatal
}
}
console.log(`[email-sync] ${meta.account.email} status set to synced`);
},
},
],
};
registerHandler(emailSyncHandler);
+488 -112
View File
@@ -4,28 +4,59 @@ import { join } from 'node:path';
import { readdir, readFile, writeFile, unlink, mkdir, stat } from 'node:fs/promises'; import { readdir, readFile, writeFile, unlink, mkdir, stat } from 'node:fs/promises';
import { type JobHandler, PermanentError } from '../types'; import { type JobHandler, PermanentError } from '../types';
import { registerHandler } from '../handler-registry'; import { registerHandler } from '../handler-registry';
import { openEmailDb, upsertFromRawEml, setSyncMeta } from '../../api/email/email-db'; import { openEmailDb, upsertFromRawEml, setSyncMeta, getSyncMeta } from '../../api/email/email-db';
import { getUserByEmail, getUserIntegration, getDockPaths, setDockPaths } from 'officerdb'; import {
getUserByEmail,
getUserIntegration,
upsertUserIntegration,
getServerIntegration,
getDockPaths,
setDockPaths,
} from 'officerdb';
import { getMaildirPath } from '@@/data-path'; import { getMaildirPath } from '@@/data-path';
import { refreshGoogleAccessToken } from '@@/api/integrations/google-auth';
// ── Credentials ── // ── Credentials ──
type ImapCredentials = { email: string; appPassword: string }; type GmailCredentials = {
email: string;
userId: number;
appPassword?: string;
accessToken?: string;
refreshToken?: string;
expiresAt?: number;
};
async function loadImapCredentials(userEmail: string): Promise<ImapCredentials> { async function loadGmailCredentials(userEmail: string): Promise<GmailCredentials> {
const dbUser = await getUserByEmail(userEmail); const dbUser = await getUserByEmail(userEmail);
if (!dbUser) throw new PermanentError('User not found'); if (!dbUser) throw new PermanentError('User not found');
const userGoogle = await getUserIntegration(dbUser.id, 'google'); const userGoogle = await getUserIntegration(dbUser.id, 'google');
const config = userGoogle?.config as Record<string, unknown> | undefined; const config = userGoogle?.config as Record<string, unknown> | undefined;
const gmailEmail = (config?.email as string) ?? userEmail;
// Prefer OAuth tokens, fall back to app password
if (config?.accessToken && config?.refreshToken) {
return {
email: gmailEmail,
userId: dbUser.id,
accessToken: config.accessToken as string,
refreshToken: config.refreshToken as string,
expiresAt: config.expiresAt as number | undefined,
appPassword: config.imapAppPassword as string | undefined,
};
}
if (!config?.imapAppPassword) { if (!config?.imapAppPassword) {
throw new PermanentError('Gmail App Password not configured — set it in Settings → Integrations'); throw new PermanentError('Gmail App Password not configured — set it in Settings → Integrations');
} }
const gmailEmail = (config.email as string) ?? userEmail; return { email: gmailEmail, userId: dbUser.id, appPassword: config.imapAppPassword as string };
return { email: gmailEmail, appPassword: config.imapAppPassword as string };
} }
// refreshGoogleAccessToken is imported from @@/api/integrations/google-auth
// ── mbsync config ── // ── mbsync config ──
function buildMbsyncConfig(email: string, appPassword: string, maildirPath: string): string { function buildMbsyncConfig(email: string, appPassword: string, maildirPath: string): string {
@@ -48,7 +79,7 @@ SubFolders Verbatim
Channel gmail Channel gmail
Far :gmail-remote: Far :gmail-remote:
Near :gmail-local: Near :gmail-local:
Patterns * ![Gmail]/Trash ![Gmail]/Spam ![Gmail]/Bin ![Google Mail]/Trash ![Google Mail]/Spam ![Google Mail]/Bin Patterns * ![Gmail]/Trash ![Gmail]/Spam ![Gmail]/Bin !"[Gmail]/All Mail" ![Google Mail]/Trash ![Google Mail]/Spam ![Google Mail]/Bin !"[Google Mail]/All Mail"
Create Near Create Near
Expunge None Expunge None
SyncState * SyncState *
@@ -99,21 +130,40 @@ function messageIdToStableId(raw: string): string | null {
type ImportResult = { saved: number; skipped: number; errors: number }; type ImportResult = { saved: number; skipped: number; errors: number };
export async function importMaildir( type ImportMaildirParams = {
maildirPath: string, maildirPath: string;
emailAccount: string, emailAccount: string;
db: Database, db: Database;
onProgress?: (saved: number, skipped: number) => void, lastSyncAt?: string | null;
): Promise<ImportResult> { onProgress?: (saved: number, skipped: number) => void;
};
export async function importMaildir({
maildirPath,
emailAccount,
db,
lastSyncAt,
onProgress,
}: ImportMaildirParams): Promise<ImportResult> {
let saved = 0; let saved = 0;
let skipped = 0; let skipped = 0;
let errors = 0; let errors = 0;
// For incremental syncs, skip files older than last sync (with 60s buffer for clock skew)
const mtimeCutoff = lastSyncAt ? new Date(lastSyncAt).getTime() - 60_000 : 0;
const isIncremental = mtimeCutoff > 0;
// Load existing IDs for fast dedup // Load existing IDs for fast dedup
const existingIds = new Set<string>(); const existingIds = new Set<string>();
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>; const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
for (const row of rows) existingIds.add(row.id); for (const row of rows) existingIds.add(row.id);
if (isIncremental) {
console.log(
`[gmail-sync] Incremental import — only reading files newer than ${new Date(mtimeCutoff).toISOString()}`,
);
}
// First pass: collect all message files with their folders to build label map // First pass: collect all message files with their folders to build label map
const messageIdLabels = new Map<string, Set<string>>(); const messageIdLabels = new Map<string, Set<string>>();
const messageFiles = new Map<string, string>(); // id → first file path const messageFiles = new Map<string, string>(); // id → first file path
@@ -162,6 +212,12 @@ export async function importMaildir(
for (const file of files) { for (const file of files) {
const filePath = join(dirPath, file); const filePath = join(dirPath, file);
try { try {
// Skip files older than last sync for incremental imports
if (isIncremental) {
const fileStat = await stat(filePath);
if (fileStat.mtimeMs < mtimeCutoff) continue;
}
const raw = await readFile(filePath, 'utf-8'); const raw = await readFile(filePath, 'utf-8');
const id = messageIdToStableId(raw); const id = messageIdToStableId(raw);
if (!id) { if (!id) {
@@ -238,6 +294,220 @@ async function countMaildirFiles(maildirPath: string): Promise<number> {
return count; return count;
} }
// ── Gmail IMAP label → our label mapping ──
const GMAIL_LABEL_MAP: Record<string, string> = {
'\\Inbox': 'inbox',
'\\Sent': 'sent',
'\\Drafts': 'draft',
'\\Starred': 'starred',
'\\Important': 'important',
'\\All': 'archive',
'\\Trash': 'trash',
'\\Junk': 'spam',
};
function gmailLabelsToLabels(gmailLabels: Set<string>): string[] {
const labels: string[] = [];
for (const gl of gmailLabels) {
const mapped = GMAIL_LABEL_MAP[gl];
if (mapped) {
labels.push(mapped);
} else if (!gl.startsWith('\\')) {
// Custom label — lowercase it
labels.push(gl.toLowerCase());
}
}
// If only "archive" and no specific folder, keep it; otherwise drop "archive"
if (labels.length > 1 && labels.includes('archive')) {
return labels.filter((l) => l !== 'archive');
}
return labels;
}
// ── IMAP special-use folders to skip ──
const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
function shouldSkipFolder(folder: { specialUse?: string; path: string; flags: Set<string> }): boolean {
if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
const norm = normalizeGmailFolder(folder.path);
return norm === 'All Mail' || norm === 'Trash' || norm === 'Spam' || norm === 'Bin';
}
// ── Incremental IMAP sync ──
type IncrementalSyncParams = {
creds: GmailCredentials;
db: Database;
onProgress?: (fetched: number, folder: string) => void;
};
type IncrementalSyncResult = { saved: number; skipped: number; errors: number };
async function incrementalImapSync({ creds, db, onProgress }: IncrementalSyncParams): Promise<IncrementalSyncResult> {
const { ImapFlow } = await import('imapflow');
// Determine auth method: prefer OAuth, fall back to app password
const auth: { user: string; pass?: string; accessToken?: string } = { user: creds.email };
if (creds.accessToken) {
auth.accessToken = creds.accessToken;
} else if (creds.appPassword) {
auth.pass = creds.appPassword;
} else {
throw new PermanentError('No authentication method available for IMAP');
}
const client = new ImapFlow({
host: 'imap.gmail.com',
port: 993,
secure: true,
auth,
logger: false,
});
let saved = 0;
let skipped = 0;
let errors = 0;
try {
await client.connect();
console.log('[gmail-sync] Incremental IMAP connected');
// Get all folders with status in a single LIST command
const folders = await client.list({ statusQuery: { uidNext: true, uidValidity: true } });
// Load existing IDs for dedup
const existingIds = new Set<string>();
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
for (const row of rows) existingIds.add(row.id);
// Filter to only folders with new messages
const foldersToSync: typeof folders = [];
for (const folder of folders) {
if (shouldSkipFolder(folder)) continue;
const folderPath = folder.path;
const uidValidityKey = `imap_uidvalidity:${folderPath}`;
const lastUidKey = `imap_lastuid:${folderPath}`;
const storedUidValidity = getSyncMeta(db, uidValidityKey);
const storedLastUid = getSyncMeta(db, lastUidKey);
const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
const uidNext = folder.status?.uidNext ?? 0;
const lastUid =
storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
if (uidValidity) setSyncMeta(db, uidValidityKey, uidValidity);
if (lastUid > 0 && uidNext <= lastUid + 1) continue; // no new messages
if (storedUidValidity && uidValidity && storedUidValidity !== uidValidity) {
console.log(`[gmail-sync] UIDVALIDITY changed for ${folderPath} — will re-scan`);
}
foldersToSync.push(folder);
}
console.log(`[gmail-sync] ${foldersToSync.length} folder(s) with new messages`);
for (const folder of foldersToSync) {
const folderPath = folder.path;
const uidValidityKey = `imap_uidvalidity:${folderPath}`;
const lastUidKey = `imap_lastuid:${folderPath}`;
const storedUidValidity = getSyncMeta(db, uidValidityKey);
const storedLastUid = getSyncMeta(db, lastUidKey);
// Open folder and fetch new messages
let lock;
try {
lock = await client.getMailboxLock(folderPath);
} catch (err) {
console.log(`[gmail-sync] Skipping folder ${folderPath}: ${err instanceof Error ? err.message : err}`);
continue;
}
try {
const mailbox = client.mailbox;
if (!mailbox) continue;
const uidValidity = String(mailbox.uidValidity);
let lastUid = 0;
if (storedUidValidity === uidValidity && storedLastUid) {
lastUid = parseInt(storedLastUid, 10);
}
const range = lastUid > 0 ? `${lastUid + 1}:*` : '1:*';
let maxUid = lastUid;
let folderFetched = 0;
try {
for await (const msg of client.fetch(range, { source: true, labels: true, uid: true }, { uid: true })) {
if (msg.uid <= lastUid) continue;
maxUid = Math.max(maxUid, msg.uid);
folderFetched++;
if (!msg.source) {
errors++;
continue;
}
const raw = msg.source.toString('utf-8');
const id = messageIdToStableId(raw);
if (!id) {
errors++;
continue;
}
if (existingIds.has(id)) {
skipped++;
continue;
}
// Gmail X-GM-EXT-1 labels don't include folder membership (e.g. \Inbox),
// so always use the folder we're fetching from as the base label
const folderLabel = folderToLabel(folderPath);
const gmailLabels = msg.labels ? gmailLabelsToLabels(msg.labels) : [];
const labels = folderLabel ? [...new Set([folderLabel, ...gmailLabels])] : gmailLabels;
try {
upsertFromRawEml({ db, id, raw, integration: 'gmail', emailAccount: creds.email, labels });
existingIds.add(id);
saved++;
} catch {
errors++;
}
}
} catch (fetchErr) {
const msg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
if (!msg.includes('Nothing to fetch')) {
console.log(`[gmail-sync] Fetch error in ${folderPath}: ${msg}`);
errors++;
}
}
if (maxUid > lastUid) {
setSyncMeta(db, lastUidKey, String(maxUid));
}
setSyncMeta(db, uidValidityKey, uidValidity);
if (folderFetched > 0) {
console.log(`[gmail-sync] ${folderPath}: fetched ${folderFetched}, saved ${saved}, skipped ${skipped}`);
onProgress?.(saved + skipped, folderPath);
}
} finally {
lock.release();
}
}
} finally {
await client.logout().catch(() => {});
}
return { saved, skipped, errors };
}
// ── Handler ── // ── Handler ──
const gmailSyncHandler: JobHandler = { const gmailSyncHandler: JobHandler = {
@@ -247,119 +517,219 @@ const gmailSyncHandler: JobHandler = {
{ {
name: 'Verify credentials', name: 'Verify credentials',
run: async (ctx) => { run: async (ctx) => {
const creds = await loadImapCredentials(ctx.job.userId); const creds = await loadGmailCredentials(ctx.job.userId);
// Determine sync mode: incremental if we have a previous sync
const db = openEmailDb(ctx.job.userId);
try {
const lastSyncAt = getSyncMeta(db, 'last_sync_at');
ctx.meta.isIncremental = !!lastSyncAt;
} finally {
db.close();
}
// For incremental sync with OAuth, refresh token if expired
if (ctx.meta.isIncremental && creds.accessToken && creds.refreshToken) {
const tokenExpired = !creds.expiresAt || creds.expiresAt < Date.now() + 60_000;
if (tokenExpired) {
console.log('[gmail-sync] Refreshing OAuth token for incremental sync');
try {
const refreshed = await refreshGoogleAccessToken(creds.refreshToken);
creds.accessToken = refreshed.accessToken;
creds.expiresAt = refreshed.expiresAt;
// Persist refreshed token
const userGoogle = await getUserIntegration(creds.userId, 'google');
const existingConfig = (userGoogle?.config as Record<string, unknown>) ?? {};
const serverGoogle = await getServerIntegration('google');
await upsertUserIntegration({
userId: creds.userId,
provider: 'google',
serverIntegrationId: serverGoogle?.id,
config: { ...existingConfig, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
});
} catch (err) {
console.log(`[gmail-sync] OAuth refresh failed, will fall back to app password: ${err}`);
// Clear OAuth so we fall back to app password
creds.accessToken = undefined;
creds.refreshToken = undefined;
}
}
}
ctx.meta.creds = creds;
ctx.meta.email = creds.email; ctx.meta.email = creds.email;
ctx.meta.appPassword = creds.appPassword; ctx.meta.appPassword = creds.appPassword;
if (ctx.meta.isIncremental) {
console.log(`[gmail-sync] Incremental sync mode (${creds.accessToken ? 'OAuth' : 'App Password'})`);
} else {
console.log('[gmail-sync] Full sync mode (mbsync)');
if (!creds.appPassword) {
throw new PermanentError('Gmail App Password required for initial sync');
}
}
}, },
}, },
{ {
name: 'Sync via mbsync', name: 'Sync emails',
run: async (ctx) => { run: async (ctx) => {
const email = ctx.meta.email as string; if (ctx.meta.isIncremental) {
const appPassword = ctx.meta.appPassword as string; // ── Incremental: direct IMAP via imapflow ──
const maildirPath = getMaildirPath(ctx.job.userId); const creds = ctx.meta.creds as GmailCredentials;
await ctx.updateProgress({ current: 0, total: 0, label: 'Connecting to Gmail...' });
// Ensure Maildir root exists const db = openEmailDb(ctx.job.userId);
await mkdir(maildirPath, { recursive: true });
// Write temp config
const configPath = join(maildirPath, '.mbsyncrc');
const config = buildMbsyncConfig(email, appPassword, maildirPath);
await writeFile(configPath, config, { mode: 0o600 });
await ctx.updateProgress({ current: 0, total: 0, label: 'Running mbsync...' });
try {
let proc: ReturnType<typeof Bun.spawn>;
try { try {
proc = Bun.spawn(['mbsync', '-c', configPath, '-a', '-V'], { const result = await incrementalImapSync({
stdout: 'pipe', creds,
stderr: 'pipe', db,
onProgress: (fetched, folder) => {
ctx.updateProgress({ current: fetched, total: 0, label: `Syncing ${folder}...` });
},
}); });
} catch (spawnErr) {
const msg = spawnErr instanceof Error ? spawnErr.message : String(spawnErr);
throw new PermanentError(`Failed to start mbsync: ${msg}`);
}
// Periodically count downloaded emails and update progress console.log(
let emailCount = 0; `[gmail-sync] Incremental sync done: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`,
let counting = true; );
const countLoop = (async () => {
while (counting) { setSyncMeta(db, 'last_sync_date', new Date().toISOString().split('T')[0]!);
await new Promise((r) => setTimeout(r, 3000)); setSyncMeta(db, 'last_sync_at', new Date().toISOString());
if (!counting) break;
ctx.meta.syncResult = result;
await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${result.saved} new emails` });
} finally {
db.close();
}
} else {
// ── Full sync: mbsync ──
const email = ctx.meta.email as string;
const appPassword = ctx.meta.appPassword as string;
const maildirPath = getMaildirPath(ctx.job.userId);
await mkdir(maildirPath, { recursive: true });
const configPath = join(maildirPath, '.mbsyncrc');
const config = buildMbsyncConfig(email, appPassword, maildirPath);
await writeFile(configPath, config, { mode: 0o600 });
await ctx.updateProgress({ current: 0, total: 0, label: 'Running mbsync...' });
try {
let proc: ReturnType<typeof Bun.spawn>;
try {
proc = Bun.spawn(['mbsync', '-c', configPath, '-a'], {
stdout: 'pipe',
stderr: 'pipe',
});
} catch (spawnErr) {
const msg = spawnErr instanceof Error ? spawnErr.message : String(spawnErr);
throw new PermanentError(`Failed to start mbsync: ${msg}`);
}
let emailCount = 0;
let counting = true;
const countLoop = (async () => {
while (counting) {
await new Promise((r) => setTimeout(r, 3000));
if (!counting) break;
emailCount = await countMaildirFiles(maildirPath);
ctx.updateProgress({
current: emailCount,
total: 0,
label: `Downloading — ${emailCount.toLocaleString()} emails`,
});
}
})();
let stderrBuf = '';
const reader = (proc.stderr as ReadableStream<Uint8Array>).getReader();
const decoder = new TextDecoder();
const readLoop = (async () => {
while (true) {
const { done, value } = await reader.read();
if (done) break;
const chunk = decoder.decode(value, { stream: true });
stderrBuf += chunk;
const lines = chunk.split('\n');
for (const line of lines) {
const trimmed = line.trim();
if (trimmed) console.log(`[gmail-sync] mbsync: ${trimmed}`);
}
}
})();
const exitCode = await proc.exited;
counting = false;
await readLoop;
await countLoop;
if (exitCode !== 0) {
const isOverquota = stderrBuf.includes('OVERQUOTA');
const isAuthFail =
stderrBuf.includes('AUTHENTICATIONFAILED') || stderrBuf.includes('Invalid credentials');
if (isOverquota) {
const emailCount = await countMaildirFiles(maildirPath);
console.log(`[gmail-sync] Gmail OVERQUOTA — proceeding to import ${emailCount} downloaded emails`);
ctx.meta.gmailSyncPartial = true;
ctx.meta.gmailSyncEmailCount = emailCount;
} else {
console.error(`[gmail-sync] mbsync failed`);
if (isAuthFail) {
const emailCount = await countMaildirFiles(maildirPath);
ctx.meta.gmailSyncRecoverable = true;
ctx.meta.gmailSyncEmailCount = emailCount;
ctx.meta.gmailSyncIsAuthFail = true;
throw new PermanentError(`Authentication failed — check your App Password`);
}
throw new Error(`mbsync exited with code ${exitCode}: ${stderrBuf.slice(-500)}`);
}
} else {
emailCount = await countMaildirFiles(maildirPath); emailCount = await countMaildirFiles(maildirPath);
ctx.updateProgress({ console.log(`[gmail-sync] mbsync completed successfully — ${emailCount} emails`);
await ctx.updateProgress({
current: emailCount, current: emailCount,
total: 0, total: emailCount,
label: `Downloading${emailCount.toLocaleString()} emails`, label: `Download complete${emailCount.toLocaleString()} emails`,
}); });
} }
})(); } finally {
await unlink(configPath).catch(() => {});
// Stream stderr for logging
let stderrBuf = '';
const reader = (proc.stderr as ReadableStream<Uint8Array>).getReader();
const decoder = new TextDecoder();
const readLoop = (async () => {
while (true) {
const { done, value } = await reader.read();
if (done) break;
const chunk = decoder.decode(value, { stream: true });
stderrBuf += chunk;
const lines = chunk.split('\n');
for (const line of lines) {
const trimmed = line.trim();
if (trimmed) console.log(`[gmail-sync] mbsync: ${trimmed}`);
}
}
})();
const exitCode = await proc.exited;
counting = false;
await readLoop;
await countLoop;
if (exitCode !== 0) {
const isOverquota = stderrBuf.includes('OVERQUOTA');
const isAuthFail = stderrBuf.includes('AUTHENTICATIONFAILED') || stderrBuf.includes('Invalid credentials');
if (isOverquota) {
// Gmail throttled us — don't retry, just import what we have
const emailCount = await countMaildirFiles(maildirPath);
console.log(`[gmail-sync] Gmail OVERQUOTA — proceeding to import ${emailCount} downloaded emails`);
ctx.meta.gmailSyncPartial = true;
ctx.meta.gmailSyncEmailCount = emailCount;
// Fall through to import step
} else {
console.error(`[gmail-sync] mbsync failed`);
if (isAuthFail) {
const emailCount = await countMaildirFiles(maildirPath);
ctx.meta.gmailSyncRecoverable = true;
ctx.meta.gmailSyncEmailCount = emailCount;
ctx.meta.gmailSyncIsAuthFail = true;
throw new PermanentError(`Authentication failed — check your App Password`);
}
throw new Error(`mbsync exited with code ${exitCode}: ${stderrBuf.slice(-500)}`);
}
} else {
emailCount = await countMaildirFiles(maildirPath);
console.log(`[gmail-sync] mbsync completed successfully — ${emailCount} emails`);
await ctx.updateProgress({
current: emailCount,
total: emailCount,
label: `Download complete — ${emailCount.toLocaleString()} emails`,
});
} }
} finally {
// Always clean up config (contains password)
await unlink(configPath).catch(() => {});
} }
}, },
}, },
{ {
name: 'Import to database', name: 'Import to database',
run: async (ctx) => { run: async (ctx) => {
// Incremental sync already imported in the previous step
if (ctx.meta.isIncremental) {
const result = ctx.meta.syncResult as IncrementalSyncResult | undefined;
// Auto-add /email to dock
if (result && result.saved > 0) {
try {
const dbUser = await getUserByEmail(ctx.job.userId);
if (dbUser) {
const paths = await getDockPaths(dbUser.id);
if (!paths) {
const defaults = ['/', '/files', '/automation', '/projects', '/dashboards', '/chat'];
await setDockPaths(dbUser.id, [...defaults, '/email']);
} else if (!paths.includes('/email')) {
await setDockPaths(dbUser.id, [...paths, '/email']);
}
}
} catch {
// Non-fatal
}
}
await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${result?.saved ?? 0} new emails` });
return;
}
// Full sync: import from Maildir
const emailAccount = ctx.meta.email as string; const emailAccount = ctx.meta.email as string;
const maildirPath = getMaildirPath(ctx.job.userId); const maildirPath = getMaildirPath(ctx.job.userId);
@@ -367,12 +737,19 @@ const gmailSyncHandler: JobHandler = {
const db = openEmailDb(ctx.job.userId); const db = openEmailDb(ctx.job.userId);
try { try {
const result = await importMaildir(maildirPath, emailAccount, db, (saved, skipped) => { const lastSyncAt = getSyncMeta(db, 'last_sync_at');
ctx.updateProgress({ const result = await importMaildir({
current: saved + skipped, maildirPath,
total: 0, emailAccount,
label: `Saved ${saved.toLocaleString()}, skipped ${skipped.toLocaleString()}`, db,
}); lastSyncAt,
onProgress: (saved, skipped) => {
ctx.updateProgress({
current: saved + skipped,
total: 0,
label: `Saved ${saved.toLocaleString()}, skipped ${skipped.toLocaleString()}`,
});
},
}); });
console.log( console.log(
@@ -389,7 +766,6 @@ const gmailSyncHandler: JobHandler = {
if (dbUser) { if (dbUser) {
const paths = await getDockPaths(dbUser.id); const paths = await getDockPaths(dbUser.id);
if (!paths) { if (!paths) {
// User hasn't customized dock — initialize with defaults + /email
const defaults = ['/', '/files', '/automation', '/projects', '/dashboards', '/chat']; const defaults = ['/', '/files', '/automation', '/projects', '/dashboards', '/chat'];
await setDockPaths(dbUser.id, [...defaults, '/email']); await setDockPaths(dbUser.id, [...defaults, '/email']);
} else if (!paths.includes('/email')) { } else if (!paths.includes('/email')) {
+1
View File
@@ -1 +1,2 @@
import './gmail-sync'; import './gmail-sync';
import './email-sync';
+87
View File
@@ -0,0 +1,87 @@
import { getAllSyncedAccounts, getUserById, getUserIntegration } from 'officerdb';
import * as queueRunner from './queue-runner';
const INTERVAL_MS = 10 * 60 * 1000; // 10 minutes
let timer: ReturnType<typeof setInterval> | null = null;
async function tick() {
try {
const accounts = await getAllSyncedAccounts();
if (accounts.length === 0) return;
const allJobs = await queueRunner.listAllJobs();
const activeEmailSyncIds = new Set(
allJobs
.filter((j) => j.type === 'email-sync' && (j.status === 'queued' || j.status === 'running'))
.map((j) => (j.meta as Record<string, unknown> | undefined)?.emailAccountId),
);
for (const account of accounts) {
if (activeEmailSyncIds.has(account.id)) continue;
const user = await getUserById(account.userId);
if (!user) continue;
// Resolve IMAP auth
const imapAuth: Record<string, unknown> = { user: account.email };
if (account.authType === 'oauth') {
const integration = await getUserIntegration(account.userId, 'google');
const config = integration?.config as Record<string, unknown> | undefined;
const accessToken = config?.accessToken as string | undefined;
if (!accessToken) {
console.log(`[email-cron] Skipping ${account.email}: no OAuth access token`);
continue;
}
imapAuth.accessToken = accessToken;
} else {
const creds = account.credentials as Record<string, unknown>;
imapAuth.pass = creds.password;
}
try {
await queueRunner.enqueue({
lane: 'email',
type: 'email-sync',
userId: user.email,
meta: {
emailAccountId: account.id,
userEmail: user.email,
account: {
id: account.id,
userId: account.userId,
email: account.email,
imapHost: account.imapHost,
imapPort: account.imapPort,
imapSecure: account.imapSecure,
provider: account.provider,
authType: account.authType,
credentials: account.credentials,
},
imapAuth,
},
});
console.log(`[email-cron] Enqueued incremental sync for ${account.email}`);
} catch (err) {
console.error(`[email-cron] Failed to enqueue sync for ${account.email}:`, err instanceof Error ? err.message : err);
}
}
} catch (err) {
console.error('[email-cron] Error:', err instanceof Error ? err.message : err);
}
}
export function initEmailCron() {
if (timer) return;
console.log(`[email-cron] Starting email sync cron (every ${INTERVAL_MS / 60_000} min)`);
timer = setInterval(tick, INTERVAL_MS);
// Run first tick after a short delay to let the queue initialize
setTimeout(tick, 30_000);
}
export function stopEmailCron() {
if (timer) {
clearInterval(timer);
timer = null;
}
}
+5
View File
@@ -5,6 +5,7 @@ import { startAnthropicProxy, getProxySecret, ensureProxySecret } from './proxy'
import * as claudeManager from './claude-manager'; import * as claudeManager from './claude-manager';
import * as piManager from './pi-manager'; import * as piManager from './pi-manager';
import * as queueRunner from './queue-runner'; import * as queueRunner from './queue-runner';
import { initEmailCron, stopEmailCron } from './email-cron';
const PORT = Number(process.env.SIDECAR_PORT ?? '5100'); const PORT = Number(process.env.SIDECAR_PORT ?? '5100');
const startedAt = Date.now(); const startedAt = Date.now();
@@ -31,6 +32,9 @@ queueRunner.initQueue().catch((err) => {
console.error('[sidecar] failed to initialize queue:', err); console.error('[sidecar] failed to initialize queue:', err);
}); });
// Start email sync cron
// initEmailCron(); // TODO: re-enable after initial sync testing
// ── WebSocket connections ── // ── WebSocket connections ──
const clients = new Set<ServerWebSocket<unknown>>(); const clients = new Set<ServerWebSocket<unknown>>();
@@ -251,6 +255,7 @@ console.log(`[sidecar] listening on 127.0.0.1:${PORT}`);
async function shutdown(signal: string) { async function shutdown(signal: string) {
console.log(`[sidecar] ${signal} received, saving state...`); console.log(`[sidecar] ${signal} received, saving state...`);
stopEmailCron();
await flushAndSave(); await flushAndSave();
releaseLock(); releaseLock();
process.exit(0); process.exit(0);
+22 -3
View File
@@ -5,6 +5,17 @@ import { getHandler } from '../queue/handler-registry';
// Import handlers to register them // Import handlers to register them
import '../queue/handlers'; import '../queue/handlers';
function formatDuration(ms: number): string {
const s = Math.floor(ms / 1000);
if (s < 60) return `${s}s`;
const m = Math.floor(s / 60);
const rem = s % 60;
if (m < 60) return rem > 0 ? `${m}m${rem}s` : `${m}m`;
const h = Math.floor(m / 60);
const remM = m % 60;
return remM > 0 ? `${h}h${remM}m` : `${h}h`;
}
const activeLanes = new Map<string, boolean>(); const activeLanes = new Map<string, boolean>();
const PROGRESS_THROTTLE_MS = 1000; const PROGRESS_THROTTLE_MS = 1000;
@@ -73,6 +84,12 @@ async function resumeInterruptedJobs() {
console.log(`[sidecar:queue] reset interrupted job ${job.id} back to queued`); console.log(`[sidecar:queue] reset interrupted job ${job.id} back to queued`);
lanesToKick.add(job.lane); lanesToKick.add(job.lane);
} else if (job.status === 'queued') { } else if (job.status === 'queued') {
// Clear retry delay on restart — no reason to wait after a sidecar restart
if (job.retryAt) {
job.retryAt = undefined;
await writeJob(job);
console.log(`[sidecar:queue] cleared retry delay for job ${job.id}`);
}
lanesToKick.add(job.lane); lanesToKick.add(job.lane);
} }
} }
@@ -136,8 +153,9 @@ async function runJob(job: Job) {
job.retryAt = undefined; job.retryAt = undefined;
await writeJob(job); await writeJob(job);
const isRetry = (job.retries ?? 0) > 0; const isRetry = (job.retries ?? 0) > 0;
const startTime = Date.now();
console.log( console.log(
`[sidecar:queue] ${isRetry ? 'resuming' : 'running'} job ${job.id} (${job.type})${isRetry ? ` retry ${job.retries}` : ''}`, `[sidecar:queue] ${isRetry ? 'resuming' : 'running'} job ${job.id} (${job.type})${isRetry ? ` retry ${job.retries}` : ''}`,
); );
const sharedMeta: Record<string, unknown> = { ...(job.meta ?? {}) }; const sharedMeta: Record<string, unknown> = { ...(job.meta ?? {}) };
@@ -187,6 +205,7 @@ async function runJob(job: Job) {
await writeJob(fresh); await writeJob(fresh);
} catch (err) { } catch (err) {
const errorMessage = err instanceof Error ? err.message : String(err); const errorMessage = err instanceof Error ? err.message : String(err);
console.error(`[sidecar:queue] step "${step.name}" failed: ${errorMessage}`);
step.status = 'failed'; step.status = 'failed';
step.error = errorMessage; step.error = errorMessage;
step.completedAt = Date.now(); step.completedAt = Date.now();
@@ -218,7 +237,7 @@ async function runJob(job: Job) {
fresh.completedAt = Date.now(); fresh.completedAt = Date.now();
fresh.meta = { ...fresh.meta, ...sharedMeta }; fresh.meta = { ...fresh.meta, ...sharedMeta };
await writeJob(fresh); await writeJob(fresh);
console.error(`[sidecar:queue] job ${fresh.id} failed at step "${step.name}":`, errorMessage); console.error(`[sidecar:queue] job ${fresh.id} failed at step "${step.name}" in ${formatDuration(Date.now() - startTime)}:`, errorMessage);
await notifyFailure(fresh); await notifyFailure(fresh);
return; return;
} }
@@ -230,7 +249,7 @@ async function runJob(job: Job) {
final.completedAt = Date.now(); final.completedAt = Date.now();
final.meta = { ...final.meta, ...sharedMeta }; final.meta = { ...final.meta, ...sharedMeta };
await writeJob(final); await writeJob(final);
console.log(`[sidecar:queue] job ${final.id} completed`); console.log(`[sidecar:queue] job ${final.id} completed in ${formatDuration(Date.now() - startTime)}`);
await notifyCompletion(final); await notifyCompletion(final);
} }
} }