Merge remote-tracking branch 'origin/email-imap'
This commit is contained in:
@@ -25,9 +25,9 @@ try {
|
|||||||
const db = openEmailDb(email);
|
const db = openEmailDb(email);
|
||||||
|
|
||||||
console.log(`Importing from ${maildirPath}...`);
|
console.log(`Importing from ${maildirPath}...`);
|
||||||
const result = await importMaildir(maildirPath, email, db, (saved, skipped) => {
|
const result = await importMaildir({ maildirPath, emailAccount: email, db, onProgress: (saved, skipped) => {
|
||||||
process.stdout.write(`\r saved ${saved}, skipped ${skipped}`);
|
process.stdout.write(`\r saved ${saved}, skipped ${skipped}`);
|
||||||
});
|
}});
|
||||||
|
|
||||||
console.log(`\nDone: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`);
|
console.log(`\nDone: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`);
|
||||||
db.close();
|
db.close();
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
import { ImapFlow } from 'imapflow';
|
||||||
|
import { getUserByEmail, getUserIntegration, getServerIntegration } from 'officerdb';
|
||||||
|
import { openEmailDb, setSyncMeta } from '../src/servers/api/email/email-db';
|
||||||
|
|
||||||
|
const userEmail = process.argv[2];
|
||||||
|
if (!userEmail) {
|
||||||
|
console.error('Usage: bun run scripts/seed-imap-uids.ts <email>');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Load credentials ──
|
||||||
|
|
||||||
|
const dbUser = await getUserByEmail(userEmail);
|
||||||
|
if (!dbUser) throw new Error('User not found');
|
||||||
|
|
||||||
|
const userGoogle = await getUserIntegration(dbUser.id, 'google');
|
||||||
|
const config = userGoogle?.config as Record<string, unknown> | undefined;
|
||||||
|
if (!config?.accessToken) throw new Error('No OAuth tokens found');
|
||||||
|
|
||||||
|
// Refresh token if needed
|
||||||
|
let accessToken = config.accessToken as string;
|
||||||
|
const expiresAt = config.expiresAt as number | undefined;
|
||||||
|
if (!expiresAt || expiresAt < Date.now() + 60_000) {
|
||||||
|
console.log('Refreshing expired token...');
|
||||||
|
const serverGoogle = await getServerIntegration('google');
|
||||||
|
const serverConfig = serverGoogle?.config as Record<string, unknown>;
|
||||||
|
const res = await fetch('https://oauth2.googleapis.com/token', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||||
|
body: new URLSearchParams({
|
||||||
|
client_id: serverConfig.clientId as string,
|
||||||
|
client_secret: serverConfig.clientSecret as string,
|
||||||
|
refresh_token: config.refreshToken as string,
|
||||||
|
grant_type: 'refresh_token',
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
if (!res.ok) throw new Error(`Token refresh failed: ${await res.text()}`);
|
||||||
|
const data = (await res.json()) as { access_token: string };
|
||||||
|
accessToken = data.access_token;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Connect IMAP ──
|
||||||
|
|
||||||
|
const client = new ImapFlow({
|
||||||
|
host: 'imap.gmail.com',
|
||||||
|
port: 993,
|
||||||
|
secure: true,
|
||||||
|
auth: { user: config.email as string, accessToken },
|
||||||
|
logger: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
await client.connect();
|
||||||
|
console.log('Connected to IMAP');
|
||||||
|
|
||||||
|
const GMAIL_PREFIX_RE = /^\[(?:Gmail|Google Mail)\]\//;
|
||||||
|
const SKIP_SUFFIXES = new Set(['All Mail', 'Trash', 'Spam', 'Bin']);
|
||||||
|
|
||||||
|
const folders = await client.list();
|
||||||
|
const db = openEmailDb(userEmail);
|
||||||
|
|
||||||
|
let seeded = 0;
|
||||||
|
|
||||||
|
for (const folder of folders) {
|
||||||
|
const suffix = folder.path.replace(GMAIL_PREFIX_RE, '');
|
||||||
|
const isGmailFolder = suffix !== folder.path;
|
||||||
|
if (isGmailFolder && SKIP_SUFFIXES.has(suffix)) continue;
|
||||||
|
if (folder.specialUse && ['\\Trash', '\\Junk', '\\All'].includes(folder.specialUse)) continue;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const status = await client.status(folder.path, { uidNext: true, uidValidity: true });
|
||||||
|
const lastUid = (status.uidNext ?? 1) - 1;
|
||||||
|
const uidValidity = String(status.uidValidity);
|
||||||
|
|
||||||
|
setSyncMeta(db, `imap_lastuid:${folder.path}`, String(lastUid));
|
||||||
|
setSyncMeta(db, `imap_uidvalidity:${folder.path}`, uidValidity);
|
||||||
|
|
||||||
|
console.log(` ${folder.path}: lastUid=${lastUid}, uidValidity=${uidValidity}`);
|
||||||
|
seeded++;
|
||||||
|
} catch (err) {
|
||||||
|
console.log(` ${folder.path}: skipped (${err instanceof Error ? err.message : err})`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
db.close();
|
||||||
|
await client.logout();
|
||||||
|
|
||||||
|
console.log(`\nSeeded ${seeded} folders. Next sync will only fetch new messages.`);
|
||||||
|
process.exit(0);
|
||||||
@@ -6,14 +6,13 @@ import { toast } from 'sonner';
|
|||||||
import { Button } from '@/components/ui/button';
|
import { Button } from '@/components/ui/button';
|
||||||
import { useClient } from 'hooks/useClient';
|
import { useClient } from 'hooks/useClient';
|
||||||
import { useGlobal } from 'hooks/useGlobal';
|
import { useGlobal } from 'hooks/useGlobal';
|
||||||
import { useJobs } from 'hooks/useJobs';
|
|
||||||
import type { EmailSummary } from 'types';
|
import type { EmailSummary } from 'types';
|
||||||
|
|
||||||
type GoogleStatus = {
|
type EmailAccountRow = {
|
||||||
configured: boolean;
|
id: number;
|
||||||
connected: boolean;
|
provider: string;
|
||||||
email: string | null;
|
email: string;
|
||||||
picture: string | null;
|
status: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
const LIMIT = 50;
|
const LIMIT = 50;
|
||||||
@@ -42,13 +41,13 @@ export const EmailList = () => {
|
|||||||
const [selectedId, setSelectedId] = useGlobal<string | null>('EMAIL_SELECTED', null);
|
const [selectedId, setSelectedId] = useGlobal<string | null>('EMAIL_SELECTED', null);
|
||||||
const [folder, setFolder] = useGlobal<string>('EMAIL_FOLDER', 'inbox');
|
const [folder, setFolder] = useGlobal<string>('EMAIL_FOLDER', 'inbox');
|
||||||
const [page, setPage] = useState(1);
|
const [page, setPage] = useState(1);
|
||||||
const { jobs, createJob } = useJobs({ type: 'gmail-sync' });
|
const { data: emailAccounts = [], refetch: refetchAccounts } = useQuery({
|
||||||
const isSyncing = jobs.some((j) => j.status === 'queued' || j.status === 'running');
|
queryKey: ['email-accounts'],
|
||||||
|
queryFn: () => client.get<EmailAccountRow[]>('/email/accounts'),
|
||||||
const { data: googleStatus } = useQuery({
|
|
||||||
queryKey: ['google-status'],
|
|
||||||
queryFn: () => client.get<GoogleStatus>('/integrations/google/status'),
|
|
||||||
});
|
});
|
||||||
|
const syncableAccount = emailAccounts.find((a) => a.status === 'connected' || a.status === 'synced');
|
||||||
|
const isSyncing = emailAccounts.some((a) => a.status === 'syncing' || a.status === 'queued');
|
||||||
|
const hasAccounts = emailAccounts.length > 0;
|
||||||
|
|
||||||
const { data, isLoading } = useQuery({
|
const { data, isLoading } = useQuery({
|
||||||
queryKey: ['email-messages', page, folder],
|
queryKey: ['email-messages', page, folder],
|
||||||
@@ -75,17 +74,23 @@ export const EmailList = () => {
|
|||||||
};
|
};
|
||||||
|
|
||||||
const handleSync = async () => {
|
const handleSync = async () => {
|
||||||
|
if (!syncableAccount) return;
|
||||||
try {
|
try {
|
||||||
await createJob({ lane: 'google-api', type: 'gmail-sync', notify: false });
|
await client.post(`/email/accounts/${syncableAccount.id}/sync`, {});
|
||||||
toast.success('Gmail sync started');
|
toast.success('Sync started');
|
||||||
} catch {
|
refetchAccounts();
|
||||||
toast.error('Failed to start sync');
|
} catch (err) {
|
||||||
|
toast.error(err instanceof Error ? err.message : 'Failed to start sync');
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Refresh email list when a sync job completes
|
// Poll accounts while syncing, refresh email list when done
|
||||||
const prevSyncing = useRef(false);
|
const prevSyncing = useRef(false);
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
if (isSyncing) {
|
||||||
|
const interval = setInterval(refetchAccounts, 5000);
|
||||||
|
return () => clearInterval(interval);
|
||||||
|
}
|
||||||
if (prevSyncing.current && !isSyncing) {
|
if (prevSyncing.current && !isSyncing) {
|
||||||
queryClient.invalidateQueries({ queryKey: ['email-messages'] });
|
queryClient.invalidateQueries({ queryKey: ['email-messages'] });
|
||||||
}
|
}
|
||||||
@@ -134,16 +139,14 @@ export const EmailList = () => {
|
|||||||
return (
|
return (
|
||||||
<div className="flex h-full flex-col items-center justify-center gap-3 text-sm opacity-50">
|
<div className="flex h-full flex-col items-center justify-center gap-3 text-sm opacity-50">
|
||||||
<Mail className="h-8 w-8" />
|
<Mail className="h-8 w-8" />
|
||||||
{googleStatus && !googleStatus.configured ? (
|
{!hasAccounts ? (
|
||||||
<span>Google integration not configured. Contact your administrator.</span>
|
|
||||||
) : googleStatus && !googleStatus.connected ? (
|
|
||||||
<div className="flex flex-col items-center gap-2">
|
<div className="flex flex-col items-center gap-2">
|
||||||
<span>Connect your Google account to sync emails</span>
|
<span>Add an email account to get started</span>
|
||||||
<Button variant="outline" size="sm" asChild>
|
<Button variant="outline" size="sm" asChild>
|
||||||
<Link to="/settings/integrations">Connect Google</Link>
|
<Link to="/settings/integrations">Add Account</Link>
|
||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : syncableAccount ? (
|
||||||
<div className="flex flex-col items-center gap-2">
|
<div className="flex flex-col items-center gap-2">
|
||||||
<span>No emails synced yet</span>
|
<span>No emails synced yet</span>
|
||||||
<Button variant="outline" size="sm" onClick={handleSync} disabled={isSyncing}>
|
<Button variant="outline" size="sm" onClick={handleSync} disabled={isSyncing}>
|
||||||
@@ -151,6 +154,13 @@ export const EmailList = () => {
|
|||||||
Sync Now
|
Sync Now
|
||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
|
) : isSyncing ? (
|
||||||
|
<div className="flex flex-col items-center gap-2">
|
||||||
|
<Loader2 className="h-5 w-5 animate-spin" />
|
||||||
|
<span>Syncing emails...</span>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<span>No emails synced yet</span>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
@@ -179,18 +189,17 @@ export const EmailList = () => {
|
|||||||
})}
|
})}
|
||||||
</div>
|
</div>
|
||||||
<span className="text-xs opacity-50">{total}</span>
|
<span className="text-xs opacity-50">{total}</span>
|
||||||
<button
|
{isSyncing ? (
|
||||||
onClick={handleSync}
|
<Loader2 className="h-3.5 w-3.5 animate-spin shrink-0 opacity-50" title="Syncing..." />
|
||||||
disabled={isSyncing}
|
) : syncableAccount ? (
|
||||||
className="p-0.5 rounded hover:bg-black/10 transition-colors cursor-pointer disabled:cursor-default disabled:opacity-50 shrink-0"
|
<button
|
||||||
title={isSyncing ? 'Syncing...' : 'Sync emails'}
|
onClick={handleSync}
|
||||||
>
|
className="p-0.5 rounded hover:bg-black/10 transition-colors cursor-pointer shrink-0"
|
||||||
{isSyncing ? (
|
title="Sync emails"
|
||||||
<Loader2 className="h-3.5 w-3.5 animate-spin" />
|
>
|
||||||
) : (
|
|
||||||
<RefreshCw className="h-3.5 w-3.5" />
|
<RefreshCw className="h-3.5 w-3.5" />
|
||||||
)}
|
</button>
|
||||||
</button>
|
) : null}
|
||||||
{totalPages > 1 && (
|
{totalPages > 1 && (
|
||||||
<div className="ml-auto flex items-center gap-2">
|
<div className="ml-auto flex items-center gap-2">
|
||||||
<button
|
<button
|
||||||
|
|||||||
+508
@@ -0,0 +1,508 @@
|
|||||||
|
import { useState, useEffect } from 'react';
|
||||||
|
import { toast } from 'sonner';
|
||||||
|
import { Plus, Trash2, Loader2, Mail, Server, KeyRound, RefreshCw } from 'lucide-react';
|
||||||
|
import { Button } from '@/components/ui/button';
|
||||||
|
import { useClient } from 'hooks/useClient';
|
||||||
|
import { useAuth } from 'hooks/useAuth';
|
||||||
|
import { useJobs } from 'hooks/useJobs';
|
||||||
|
|
||||||
|
type EmailAccountRow = {
|
||||||
|
id: number;
|
||||||
|
provider: string;
|
||||||
|
email: string;
|
||||||
|
displayName: string | null;
|
||||||
|
enabled: boolean;
|
||||||
|
status: string;
|
||||||
|
createdAt: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ProviderChoice = 'gmail-oauth' | 'gmail-password' | 'imap';
|
||||||
|
|
||||||
|
type FormState = {
|
||||||
|
email: string;
|
||||||
|
displayName: string;
|
||||||
|
password: string;
|
||||||
|
imapHost: string;
|
||||||
|
imapPort: string;
|
||||||
|
imapSecure: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
const INITIAL_FORM: FormState = {
|
||||||
|
email: '',
|
||||||
|
displayName: '',
|
||||||
|
password: '',
|
||||||
|
imapHost: '',
|
||||||
|
imapPort: '993',
|
||||||
|
imapSecure: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
export const EmailAccounts = () => {
|
||||||
|
const client = useClient();
|
||||||
|
const { user } = useAuth();
|
||||||
|
const { jobs } = useJobs({ type: 'email-sync' });
|
||||||
|
const [accounts, setAccounts] = useState<EmailAccountRow[]>([]);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [showPicker, setShowPicker] = useState(false);
|
||||||
|
const [activeForm, setActiveForm] = useState<ProviderChoice | null>(null);
|
||||||
|
const [form, setForm] = useState<FormState>(INITIAL_FORM);
|
||||||
|
const [saving, setSaving] = useState(false);
|
||||||
|
const [deleting, setDeleting] = useState<number | null>(null);
|
||||||
|
const [syncing, setSyncing] = useState<number | null>(null);
|
||||||
|
|
||||||
|
const fetchAccounts = () => {
|
||||||
|
client
|
||||||
|
.get<EmailAccountRow[]>('/email/accounts')
|
||||||
|
.then(setAccounts)
|
||||||
|
.catch(() => {})
|
||||||
|
.finally(() => setLoading(false));
|
||||||
|
};
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
fetchAccounts();
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Poll accounts while any are queued/syncing to pick up status changes
|
||||||
|
const hasSyncingAccount = accounts.some((a) => a.status === 'syncing' || a.status === 'queued');
|
||||||
|
useEffect(() => {
|
||||||
|
if (!hasSyncingAccount) return;
|
||||||
|
const interval = setInterval(fetchAccounts, 5000);
|
||||||
|
return () => clearInterval(interval);
|
||||||
|
}, [hasSyncingAccount]);
|
||||||
|
|
||||||
|
const resetForm = () => {
|
||||||
|
setActiveForm(null);
|
||||||
|
setShowPicker(false);
|
||||||
|
setForm(INITIAL_FORM);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handlePickProvider = (choice: ProviderChoice) => {
|
||||||
|
setShowPicker(false);
|
||||||
|
setActiveForm(choice);
|
||||||
|
if (choice === 'gmail-oauth' || choice === 'gmail-password') {
|
||||||
|
setForm({ ...INITIAL_FORM, imapHost: 'imap.gmail.com', imapPort: '993', imapSecure: true });
|
||||||
|
} else {
|
||||||
|
setForm(INITIAL_FORM);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleGmailOAuth = async () => {
|
||||||
|
setSaving(true);
|
||||||
|
try {
|
||||||
|
const status = await client.get<{ connected: boolean; email: string | null }>('/integrations/google/status');
|
||||||
|
if (!status.connected) {
|
||||||
|
const params = new URLSearchParams({ token: client.token ?? '', origin: window.location.origin });
|
||||||
|
window.location.href = `/api/integrations/google/authorize?${params.toString()}`;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await client.post<{ id: number; provider: string; email: string }>('/email/accounts', {
|
||||||
|
provider: 'gmail',
|
||||||
|
email: status.email,
|
||||||
|
displayName: form.displayName || undefined,
|
||||||
|
imapHost: 'imap.gmail.com',
|
||||||
|
imapPort: 993,
|
||||||
|
imapSecure: true,
|
||||||
|
authType: 'oauth',
|
||||||
|
credentials: { userIntegrationId: true },
|
||||||
|
});
|
||||||
|
|
||||||
|
toast.success(`Added ${result.email}`);
|
||||||
|
resetForm();
|
||||||
|
fetchAccounts();
|
||||||
|
} catch (err) {
|
||||||
|
toast.error(err instanceof Error ? err.message : 'Failed to add Gmail account');
|
||||||
|
} finally {
|
||||||
|
setSaving(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handlePasswordSubmit = async () => {
|
||||||
|
if (!form.email.trim() || !form.password.trim()) return;
|
||||||
|
setSaving(true);
|
||||||
|
try {
|
||||||
|
const provider = form.imapHost === 'imap.gmail.com' ? 'gmail' : 'imap';
|
||||||
|
const result = await client.post<{ id: number; provider: string; email: string }>('/email/accounts', {
|
||||||
|
provider,
|
||||||
|
email: form.email.trim(),
|
||||||
|
displayName: form.displayName.trim() || undefined,
|
||||||
|
imapHost: form.imapHost,
|
||||||
|
imapPort: Number(form.imapPort),
|
||||||
|
imapSecure: form.imapSecure,
|
||||||
|
authType: 'password',
|
||||||
|
credentials: { password: form.password },
|
||||||
|
});
|
||||||
|
|
||||||
|
toast.success(`Added ${result.email}`);
|
||||||
|
resetForm();
|
||||||
|
fetchAccounts();
|
||||||
|
} catch (err) {
|
||||||
|
toast.error(err instanceof Error ? err.message : 'Failed to add account');
|
||||||
|
} finally {
|
||||||
|
setSaving(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleDelete = async (id: number) => {
|
||||||
|
setDeleting(id);
|
||||||
|
try {
|
||||||
|
await client.delete(`/email/accounts/${id}`);
|
||||||
|
setAccounts((prev) => prev.filter((a) => a.id !== id));
|
||||||
|
toast.success('Account removed');
|
||||||
|
} catch {
|
||||||
|
toast.error('Failed to remove account');
|
||||||
|
} finally {
|
||||||
|
setDeleting(null);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleSync = async (id: number) => {
|
||||||
|
setSyncing(id);
|
||||||
|
try {
|
||||||
|
await client.post(`/email/accounts/${id}/sync`, {});
|
||||||
|
toast.success('Sync started');
|
||||||
|
// Update local state immediately
|
||||||
|
setAccounts((prev) => prev.map((a) => (a.id === id ? { ...a, status: 'queued' } : a)));
|
||||||
|
} catch (err) {
|
||||||
|
toast.error(err instanceof Error ? err.message : 'Failed to start sync');
|
||||||
|
} finally {
|
||||||
|
setSyncing(null);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (loading) return null;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="grid gap-4">
|
||||||
|
<div>
|
||||||
|
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Email Accounts</p>
|
||||||
|
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 mt-1">
|
||||||
|
Connect email accounts for syncing your inbox. Supports Gmail and any IMAP provider.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* Account list */}
|
||||||
|
{accounts.length > 0 && (
|
||||||
|
<div className="grid gap-2">
|
||||||
|
{accounts.map((account) => {
|
||||||
|
const accountJob = jobs.find(
|
||||||
|
(j) =>
|
||||||
|
(j.status === 'queued' || j.status === 'running') &&
|
||||||
|
(j.meta as Record<string, unknown> | undefined)?.emailAccountId === account.id,
|
||||||
|
);
|
||||||
|
const progress = accountJob?.steps[accountJob.currentStep]?.progress;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={account.id}
|
||||||
|
className="rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-3"
|
||||||
|
>
|
||||||
|
<div className="flex items-center gap-3">
|
||||||
|
<ProviderIcon provider={account.provider} />
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<p className="text-sm font-medium text-duck-dark dark:text-foreground truncate">
|
||||||
|
{account.displayName ?? account.email}
|
||||||
|
</p>
|
||||||
|
{account.displayName && (
|
||||||
|
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 truncate">{account.email}</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<StatusBadge status={account.status} />
|
||||||
|
{account.status === 'queued' && (
|
||||||
|
<span className="text-xs text-duck-dark/50 dark:text-foreground/50">Waiting...</span>
|
||||||
|
)}
|
||||||
|
{account.status === 'connected' && (
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="outline"
|
||||||
|
size="sm"
|
||||||
|
onClick={() => handleSync(account.id)}
|
||||||
|
disabled={syncing === account.id}
|
||||||
|
className="h-7 text-xs cursor-pointer gap-1.5"
|
||||||
|
>
|
||||||
|
{syncing === account.id ? (
|
||||||
|
<Loader2 className="h-3 w-3 animate-spin" />
|
||||||
|
) : (
|
||||||
|
<RefreshCw className="h-3 w-3" />
|
||||||
|
)}
|
||||||
|
Initial Sync
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
<button
|
||||||
|
onClick={() => handleDelete(account.id)}
|
||||||
|
disabled={deleting === account.id || account.status === 'syncing' || account.status === 'queued'}
|
||||||
|
className="text-duck-dark/40 dark:text-foreground/40 hover:text-red-500 transition-colors cursor-pointer disabled:opacity-30 disabled:cursor-not-allowed"
|
||||||
|
>
|
||||||
|
{deleting === account.id ? (
|
||||||
|
<Loader2 className="h-4 w-4 animate-spin" />
|
||||||
|
) : (
|
||||||
|
<Trash2 className="h-4 w-4" />
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{/* Sync progress */}
|
||||||
|
{account.status === 'syncing' && progress && (
|
||||||
|
<div className="mt-2 flex items-center gap-2">
|
||||||
|
<Loader2 className="h-3 w-3 animate-spin text-blue-500 shrink-0" />
|
||||||
|
<p className="text-xs text-duck-dark/60 dark:text-foreground/60 truncate">
|
||||||
|
{progress.label ?? 'Syncing...'}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{account.status === 'syncing' && !progress && (
|
||||||
|
<div className="mt-2 flex items-center gap-2">
|
||||||
|
<Loader2 className="h-3 w-3 animate-spin text-blue-500 shrink-0" />
|
||||||
|
<p className="text-xs text-duck-dark/60 dark:text-foreground/60">Syncing...</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Provider picker */}
|
||||||
|
{showPicker && !activeForm && (
|
||||||
|
<div className="grid gap-2">
|
||||||
|
<button
|
||||||
|
onClick={() => handlePickProvider('gmail-oauth')}
|
||||||
|
className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-3 hover:bg-duck-dark/5 dark:hover:bg-foreground/5 transition-colors cursor-pointer text-left"
|
||||||
|
>
|
||||||
|
<Mail className="h-4 w-4 text-duck-dark/60 dark:text-foreground/60" />
|
||||||
|
<div>
|
||||||
|
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Gmail (OAuth)</p>
|
||||||
|
<p className="text-xs text-duck-dark/50 dark:text-foreground/50">Uses your existing Google connection</p>
|
||||||
|
</div>
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={() => handlePickProvider('gmail-password')}
|
||||||
|
className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-3 hover:bg-duck-dark/5 dark:hover:bg-foreground/5 transition-colors cursor-pointer text-left"
|
||||||
|
>
|
||||||
|
<KeyRound className="h-4 w-4 text-duck-dark/60 dark:text-foreground/60" />
|
||||||
|
<div>
|
||||||
|
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Gmail (App Password)</p>
|
||||||
|
<p className="text-xs text-duck-dark/50 dark:text-foreground/50">
|
||||||
|
Direct IMAP with a Google App Password
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
onClick={() => handlePickProvider('imap')}
|
||||||
|
className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-3 hover:bg-duck-dark/5 dark:hover:bg-foreground/5 transition-colors cursor-pointer text-left"
|
||||||
|
>
|
||||||
|
<Server className="h-4 w-4 text-duck-dark/60 dark:text-foreground/60" />
|
||||||
|
<div>
|
||||||
|
<p className="text-sm font-medium text-duck-dark dark:text-foreground">IMAP Account</p>
|
||||||
|
<p className="text-xs text-duck-dark/50 dark:text-foreground/50">Any email provider with IMAP access</p>
|
||||||
|
</div>
|
||||||
|
</button>
|
||||||
|
<Button type="button" variant="ghost" onClick={resetForm} className="h-9 cursor-pointer">
|
||||||
|
Cancel
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Gmail OAuth form */}
|
||||||
|
{activeForm === 'gmail-oauth' && (
|
||||||
|
<div className="grid gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
|
||||||
|
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Add Gmail (OAuth)</p>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={form.displayName}
|
||||||
|
onChange={(ev) => setForm({ ...form, displayName: ev.target.value })}
|
||||||
|
placeholder="Display name (optional)"
|
||||||
|
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
||||||
|
/>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<Button type="button" variant="ghost" onClick={resetForm} className="h-9 cursor-pointer">
|
||||||
|
Cancel
|
||||||
|
</Button>
|
||||||
|
<Button type="button" onClick={handleGmailOAuth} disabled={saving} className="h-9 flex-1 cursor-pointer">
|
||||||
|
{saving ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Connect & Add'}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Gmail App Password form */}
|
||||||
|
{activeForm === 'gmail-password' && (
|
||||||
|
<div className="grid gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
|
||||||
|
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Add Gmail (App Password)</p>
|
||||||
|
<div className="text-xs text-duck-dark/50 dark:text-foreground/50 grid gap-1.5">
|
||||||
|
<p>To create an App Password:</p>
|
||||||
|
<ol className="list-decimal ml-4 grid gap-0.5">
|
||||||
|
<li>
|
||||||
|
Go to{' '}
|
||||||
|
<a
|
||||||
|
href="https://myaccount.google.com/apppasswords"
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
className="underline hover:opacity-70"
|
||||||
|
>
|
||||||
|
Google App Passwords
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
<li>You may need to enable 2-Step Verification first</li>
|
||||||
|
<li>Enter a name (e.g. "Officer") and click Create</li>
|
||||||
|
<li>Copy the 16-character password and paste it below</li>
|
||||||
|
</ol>
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
type="email"
|
||||||
|
value={form.email}
|
||||||
|
onChange={(ev) => setForm({ ...form, email: ev.target.value })}
|
||||||
|
placeholder="your@gmail.com"
|
||||||
|
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={form.displayName}
|
||||||
|
onChange={(ev) => setForm({ ...form, displayName: ev.target.value })}
|
||||||
|
placeholder="Display name (optional)"
|
||||||
|
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
type="password"
|
||||||
|
value={form.password}
|
||||||
|
onChange={(ev) => setForm({ ...form, password: ev.target.value })}
|
||||||
|
placeholder="xxxx xxxx xxxx xxxx"
|
||||||
|
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
||||||
|
/>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<Button type="button" variant="ghost" onClick={resetForm} className="h-9 cursor-pointer">
|
||||||
|
Cancel
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
onClick={handlePasswordSubmit}
|
||||||
|
disabled={!form.email.trim() || !form.password.trim() || saving}
|
||||||
|
className="h-9 flex-1 cursor-pointer"
|
||||||
|
>
|
||||||
|
{saving ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Test & Save'}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Generic IMAP form */}
|
||||||
|
{activeForm === 'imap' && (
|
||||||
|
<div className="grid gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
|
||||||
|
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Add IMAP Account</p>
|
||||||
|
<input
|
||||||
|
type="email"
|
||||||
|
value={form.email}
|
||||||
|
onChange={(ev) => setForm({ ...form, email: ev.target.value })}
|
||||||
|
placeholder="you@example.com"
|
||||||
|
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={form.displayName}
|
||||||
|
onChange={(ev) => setForm({ ...form, displayName: ev.target.value })}
|
||||||
|
placeholder="Display name (optional)"
|
||||||
|
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
||||||
|
/>
|
||||||
|
<div className="grid grid-cols-[1fr_auto_auto] gap-2">
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
value={form.imapHost}
|
||||||
|
onChange={(ev) => setForm({ ...form, imapHost: ev.target.value })}
|
||||||
|
placeholder="imap.example.com"
|
||||||
|
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
value={form.imapPort}
|
||||||
|
onChange={(ev) => setForm({ ...form, imapPort: ev.target.value })}
|
||||||
|
placeholder="993"
|
||||||
|
className="h-9 w-20 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
||||||
|
/>
|
||||||
|
<label className="flex items-center gap-1.5 text-xs text-duck-dark/60 dark:text-foreground/60 cursor-pointer">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={form.imapSecure}
|
||||||
|
onChange={(ev) => setForm({ ...form, imapSecure: ev.target.checked })}
|
||||||
|
className="cursor-pointer"
|
||||||
|
/>
|
||||||
|
SSL
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
type="password"
|
||||||
|
value={form.password}
|
||||||
|
onChange={(ev) => setForm({ ...form, password: ev.target.value })}
|
||||||
|
placeholder="Password"
|
||||||
|
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
||||||
|
/>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<Button type="button" variant="ghost" onClick={resetForm} className="h-9 cursor-pointer">
|
||||||
|
Cancel
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
onClick={handlePasswordSubmit}
|
||||||
|
disabled={!form.email.trim() || !form.password.trim() || !form.imapHost.trim() || saving}
|
||||||
|
className="h-9 flex-1 cursor-pointer"
|
||||||
|
>
|
||||||
|
{saving ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Test & Save'}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{/* Add account button */}
|
||||||
|
{!showPicker && !activeForm && (
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="outline"
|
||||||
|
onClick={() => setShowPicker(true)}
|
||||||
|
className="w-full h-11 cursor-pointer gap-2"
|
||||||
|
>
|
||||||
|
<Plus className="h-4 w-4" />
|
||||||
|
Add Account
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const StatusBadge = ({ status }: { status: string }) => {
|
||||||
|
switch (status) {
|
||||||
|
case 'connected':
|
||||||
|
return (
|
||||||
|
<span className="text-[10px] font-medium px-1.5 py-0.5 rounded-full bg-yellow-500/10 text-yellow-600 dark:text-yellow-400">
|
||||||
|
Connected
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
case 'queued':
|
||||||
|
return (
|
||||||
|
<span className="text-[10px] font-medium px-1.5 py-0.5 rounded-full bg-orange-500/10 text-orange-600 dark:text-orange-400">
|
||||||
|
Queued
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
case 'syncing':
|
||||||
|
return (
|
||||||
|
<span className="text-[10px] font-medium px-1.5 py-0.5 rounded-full bg-blue-500/10 text-blue-600 dark:text-blue-400">
|
||||||
|
Syncing
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
case 'synced':
|
||||||
|
return (
|
||||||
|
<span className="text-[10px] font-medium px-1.5 py-0.5 rounded-full bg-green-500/10 text-green-600 dark:text-green-400">
|
||||||
|
Synced
|
||||||
|
</span>
|
||||||
|
);
|
||||||
|
default:
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const ProviderIcon = ({ provider }: { provider: string }) => {
|
||||||
|
switch (provider) {
|
||||||
|
case 'gmail':
|
||||||
|
return <Mail className="h-4 w-4 text-red-500 shrink-0" />;
|
||||||
|
case 'outlook':
|
||||||
|
return <Mail className="h-4 w-4 text-blue-500 shrink-0" />;
|
||||||
|
default:
|
||||||
|
return <Server className="h-4 w-4 text-duck-dark/60 dark:text-foreground/60 shrink-0" />;
|
||||||
|
}
|
||||||
|
};
|
||||||
+31
-244
@@ -1,64 +1,32 @@
|
|||||||
import { useState, useEffect } from 'react';
|
import { useState, useEffect } from 'react';
|
||||||
import { toast } from 'sonner';
|
import { toast } from 'sonner';
|
||||||
import { RefreshCw, Loader2, CheckCircle2, XCircle } from 'lucide-react';
|
|
||||||
import { Button } from '@/components/ui/button';
|
import { Button } from '@/components/ui/button';
|
||||||
import { useClient } from 'hooks/useClient';
|
import { useClient } from 'hooks/useClient';
|
||||||
import { useAuth } from 'hooks/useAuth';
|
|
||||||
import { useQueryClient } from '@tanstack/react-query';
|
|
||||||
import { useJobs } from 'hooks/useJobs';
|
|
||||||
|
|
||||||
type GoogleStatus = {
|
type GoogleStatus = {
|
||||||
connected: boolean;
|
connected: boolean;
|
||||||
email: string | null;
|
email: string | null;
|
||||||
picture: string | null;
|
picture: string | null;
|
||||||
configured: boolean;
|
configured: boolean;
|
||||||
hasAppPassword: boolean;
|
|
||||||
};
|
|
||||||
|
|
||||||
const formatTime = (ts: number | string) => {
|
|
||||||
const d = new Date(ts);
|
|
||||||
return d.toLocaleString(undefined, { dateStyle: 'medium', timeStyle: 'short' });
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export const GoogleAccount = () => {
|
export const GoogleAccount = () => {
|
||||||
const client = useClient();
|
const client = useClient();
|
||||||
const { user } = useAuth();
|
|
||||||
const queryClient = useQueryClient();
|
|
||||||
const [isLoading, setIsLoading] = useState(true);
|
const [isLoading, setIsLoading] = useState(true);
|
||||||
const [status, setStatus] = useState<GoogleStatus>({
|
const [status, setStatus] = useState<GoogleStatus>({
|
||||||
connected: false,
|
connected: false,
|
||||||
email: null,
|
email: null,
|
||||||
picture: null,
|
picture: null,
|
||||||
configured: false,
|
configured: false,
|
||||||
hasAppPassword: false,
|
|
||||||
});
|
});
|
||||||
const [gmailEmail, setGmailEmail] = useState('');
|
|
||||||
const [appPassword, setAppPassword] = useState('');
|
|
||||||
const [showPasswordInput, setShowPasswordInput] = useState(false);
|
|
||||||
const [savingPassword, setSavingPassword] = useState(false);
|
|
||||||
const [lastSyncAt, setLastSyncAt] = useState<string | null>(null);
|
|
||||||
const [dismissedError, setDismissedError] = useState(false);
|
|
||||||
const { jobs, createJob, refetch } = useJobs({ type: 'gmail-sync' });
|
|
||||||
const activeJob = jobs.find((j) => j.status === 'queued' || j.status === 'running');
|
|
||||||
const lastJob = jobs[0];
|
|
||||||
|
|
||||||
const fetchStatus = () => {
|
|
||||||
client
|
|
||||||
.get<GoogleStatus>('/integrations/google/status')
|
|
||||||
.then((s) => {
|
|
||||||
setStatus(s);
|
|
||||||
setGmailEmail(s.email ?? user?.email ?? '');
|
|
||||||
})
|
|
||||||
.catch(() => {})
|
|
||||||
.finally(() => setIsLoading(false));
|
|
||||||
client
|
|
||||||
.get<{ lastSyncAt: string | null }>('/email/sync-status')
|
|
||||||
.then((res) => setLastSyncAt(res.lastSyncAt))
|
|
||||||
.catch(() => {});
|
|
||||||
};
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
fetchStatus();
|
client
|
||||||
|
.get<GoogleStatus>('/integrations/google/status')
|
||||||
|
.then(setStatus)
|
||||||
|
.catch(() => {})
|
||||||
|
.finally(() => setIsLoading(false));
|
||||||
|
|
||||||
const params = new URLSearchParams(window.location.search);
|
const params = new URLSearchParams(window.location.search);
|
||||||
const result = params.get('google');
|
const result = params.get('google');
|
||||||
if (result === 'success') {
|
if (result === 'success') {
|
||||||
@@ -71,27 +39,6 @@ export const GoogleAccount = () => {
|
|||||||
}
|
}
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
// Refresh sync status and dock when a job finishes
|
|
||||||
useEffect(() => {
|
|
||||||
if (!activeJob && lastJob?.status === 'completed') {
|
|
||||||
client
|
|
||||||
.get<{ lastSyncAt: string | null }>('/email/sync-status')
|
|
||||||
.then((res) => setLastSyncAt(res.lastSyncAt))
|
|
||||||
.catch(() => {});
|
|
||||||
queryClient.invalidateQueries({ queryKey: ['DOCK'] });
|
|
||||||
}
|
|
||||||
}, [activeJob, lastJob?.status]);
|
|
||||||
|
|
||||||
const handleSync = async (year?: number) => {
|
|
||||||
try {
|
|
||||||
await createJob({ lane: 'google-api', type: 'gmail-sync', meta: year ? { year } : undefined });
|
|
||||||
await refetch();
|
|
||||||
toast.success("Gmail sync started — you'll receive an email when it's done");
|
|
||||||
} catch {
|
|
||||||
toast.error('Failed to start Gmail sync');
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleConnect = () => {
|
const handleConnect = () => {
|
||||||
const params = new URLSearchParams({
|
const params = new URLSearchParams({
|
||||||
token: client.token ?? '',
|
token: client.token ?? '',
|
||||||
@@ -110,201 +57,41 @@ export const GoogleAccount = () => {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleSaveAppPassword = async () => {
|
|
||||||
if (!appPassword.trim() || !gmailEmail.trim()) return;
|
|
||||||
setSavingPassword(true);
|
|
||||||
try {
|
|
||||||
await client.put('/integrations/google/app-password', {
|
|
||||||
appPassword: appPassword.trim(),
|
|
||||||
email: gmailEmail.trim(),
|
|
||||||
});
|
|
||||||
setStatus({ ...status, hasAppPassword: true, email: gmailEmail.trim() });
|
|
||||||
setAppPassword('');
|
|
||||||
setShowPasswordInput(false);
|
|
||||||
toast.success('App password saved');
|
|
||||||
} catch {
|
|
||||||
toast.error('Failed to save app password');
|
|
||||||
} finally {
|
|
||||||
setSavingPassword(false);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
if (isLoading) return null;
|
if (isLoading) return null;
|
||||||
|
if (!status.configured) return null;
|
||||||
const currentStep = activeJob?.steps[activeJob.currentStep];
|
|
||||||
const progress = currentStep?.progress;
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="grid gap-6">
|
<div className="grid gap-4">
|
||||||
{/* Gmail Sync — independent of OAuth */}
|
<div>
|
||||||
<div className="grid gap-4">
|
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Google Account</p>
|
||||||
<div>
|
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 mt-1">
|
||||||
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Gmail Sync</p>
|
Connect your Google account for Calendar, Contacts, and other Google services.
|
||||||
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 mt-1">
|
</p>
|
||||||
Import and sync your Gmail emails with your Officer inbox. This uses a Google App Password for a direct IMAP
|
</div>
|
||||||
connection — the only thing it can do is download your emails. It cannot send, delete, or modify anything in
|
{status.connected ? (
|
||||||
your account.
|
<>
|
||||||
</p>
|
<div className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
|
||||||
</div>
|
<div className="h-2.5 w-2.5 rounded-full bg-green-500 shrink-0" />
|
||||||
<div className="rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4 grid gap-2">
|
|
||||||
<p className="text-sm font-medium text-duck-dark dark:text-foreground">App Password</p>
|
|
||||||
{status.hasAppPassword && !showPasswordInput ? (
|
|
||||||
<div className="flex items-center gap-2">
|
|
||||||
<span className="text-xs text-green-600 dark:text-green-400">Configured</span>
|
|
||||||
{status.email && (
|
|
||||||
<span className="text-xs text-duck-dark/50 dark:text-foreground/50">({status.email})</span>
|
|
||||||
)}
|
|
||||||
<button
|
|
||||||
onClick={() => setShowPasswordInput(true)}
|
|
||||||
className="text-xs text-duck-dark/50 dark:text-foreground/50 underline hover:opacity-70 cursor-pointer"
|
|
||||||
>
|
|
||||||
Replace
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
) : (
|
|
||||||
<>
|
|
||||||
<div className="text-xs text-duck-dark/50 dark:text-foreground/50 grid gap-1.5">
|
|
||||||
<p>To create an App Password:</p>
|
|
||||||
<ol className="list-decimal ml-4 grid gap-0.5">
|
|
||||||
<li>
|
|
||||||
Go to{' '}
|
|
||||||
<a
|
|
||||||
href="https://myaccount.google.com/apppasswords"
|
|
||||||
target="_blank"
|
|
||||||
rel="noopener noreferrer"
|
|
||||||
className="underline hover:opacity-70"
|
|
||||||
>
|
|
||||||
Google App Passwords
|
|
||||||
</a>
|
|
||||||
</li>
|
|
||||||
<li>You may need to enable 2-Step Verification first</li>
|
|
||||||
<li>Enter a name (e.g. "Officer") and click Create</li>
|
|
||||||
<li>Copy the 16-character password and paste it below</li>
|
|
||||||
</ol>
|
|
||||||
</div>
|
|
||||||
<div className="grid gap-2">
|
|
||||||
<input
|
|
||||||
type="email"
|
|
||||||
value={gmailEmail}
|
|
||||||
onChange={(ev) => setGmailEmail(ev.target.value)}
|
|
||||||
placeholder="your@gmail.com"
|
|
||||||
className="h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
|
||||||
/>
|
|
||||||
<div className="flex gap-2">
|
|
||||||
<input
|
|
||||||
type="password"
|
|
||||||
value={appPassword}
|
|
||||||
onChange={(ev) => setAppPassword(ev.target.value)}
|
|
||||||
placeholder="xxxx xxxx xxxx xxxx"
|
|
||||||
className="flex-1 h-9 rounded-md border border-duck-dark/10 dark:border-foreground/10 bg-transparent px-3 text-sm"
|
|
||||||
/>
|
|
||||||
<Button
|
|
||||||
type="button"
|
|
||||||
variant="outline"
|
|
||||||
disabled={!appPassword.trim() || !gmailEmail.trim() || savingPassword}
|
|
||||||
onClick={handleSaveAppPassword}
|
|
||||||
className="h-9 cursor-pointer"
|
|
||||||
>
|
|
||||||
{savingPassword ? <Loader2 className="h-4 w-4 animate-spin" /> : 'Save'}
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
{activeJob && (
|
|
||||||
<div className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-3">
|
|
||||||
<Loader2 className="h-4 w-4 animate-spin text-duck-dark/60 dark:text-foreground/60 shrink-0" />
|
|
||||||
<div className="min-w-0 flex-1">
|
<div className="min-w-0 flex-1">
|
||||||
<p className="text-sm font-medium text-duck-dark dark:text-foreground">
|
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Connected</p>
|
||||||
{activeJob.status === 'queued' ? 'Queued' : 'Syncing'}
|
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 truncate">{status.email}</p>
|
||||||
{progress?.label ? ` — ${progress.label}` : ''}
|
|
||||||
</p>
|
|
||||||
{progress && progress.total > 0 && (
|
|
||||||
<div className="mt-1.5 h-1.5 rounded-full bg-duck-dark/10 dark:bg-foreground/10 overflow-hidden">
|
|
||||||
<div
|
|
||||||
className="h-full rounded-full bg-duck-yellow transition-all duration-300"
|
|
||||||
style={{ width: `${Math.round((progress.current / progress.total) * 100)}%` }}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
{progress && progress.total === 0 && progress.current > 0 && (
|
|
||||||
<p className="mt-1 text-xs text-duck-dark/50 dark:text-foreground/50">
|
|
||||||
{progress.current.toLocaleString()} emails processed
|
|
||||||
</p>
|
|
||||||
)}
|
|
||||||
</div>
|
</div>
|
||||||
|
{status.picture && (
|
||||||
|
<img src={status.picture} alt="" className="h-9 w-9 rounded-full shrink-0" referrerPolicy="no-referrer" />
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
<Button type="button" variant="outline" onClick={handleDisconnect} className="w-full h-11 cursor-pointer">
|
||||||
{!activeJob && lastJob?.status === 'failed' && !dismissedError && (
|
Disconnect
|
||||||
<button
|
</Button>
|
||||||
onClick={() => setDismissedError(true)}
|
</>
|
||||||
className="flex items-center gap-2 text-xs text-red-500 hover:opacity-70 transition-opacity cursor-pointer text-left"
|
) : (
|
||||||
title="Click to dismiss"
|
|
||||||
>
|
|
||||||
<XCircle className="h-3.5 w-3.5 shrink-0" />
|
|
||||||
Last sync failed{lastJob.error ? `: ${lastJob.error}` : ''}
|
|
||||||
</button>
|
|
||||||
)}
|
|
||||||
{!activeJob && lastSyncAt && (
|
|
||||||
<div className="flex items-center gap-2 text-xs text-duck-dark/50 dark:text-foreground/50">
|
|
||||||
<CheckCircle2 className="h-3.5 w-3.5 text-green-500 shrink-0" />
|
|
||||||
Last sync completed {formatTime(lastSyncAt)}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
<Button
|
<Button
|
||||||
type="button"
|
type="button"
|
||||||
variant="outline"
|
onClick={handleConnect}
|
||||||
disabled={!!activeJob || !status.hasAppPassword}
|
className="w-full h-11 bg-duck-yellow hover:bg-duck-yellow/90 text-duck-teal font-bold transition-all duration-200 hover:scale-105 cursor-pointer"
|
||||||
onClick={() => handleSync()}
|
|
||||||
className="w-full h-11 cursor-pointer gap-2"
|
|
||||||
>
|
>
|
||||||
<RefreshCw className="h-4 w-4" />
|
Connect Google Account
|
||||||
Sync Gmail Inbox
|
|
||||||
</Button>
|
</Button>
|
||||||
</div>
|
|
||||||
|
|
||||||
{/* OAuth — for Calendar and other Google services */}
|
|
||||||
{status.configured && (
|
|
||||||
<div className="grid gap-4 border-t border-duck-dark/10 dark:border-foreground/10 pt-6">
|
|
||||||
<div>
|
|
||||||
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Google Account</p>
|
|
||||||
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 mt-1">
|
|
||||||
Connect your Google account for Calendar, Contacts, and other Google services. This is separate from Gmail
|
|
||||||
sync above.
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
{status.connected ? (
|
|
||||||
<>
|
|
||||||
<div className="flex items-center gap-3 rounded-lg border border-duck-dark/10 dark:border-foreground/10 p-4">
|
|
||||||
<div className="h-2.5 w-2.5 rounded-full bg-green-500 shrink-0" />
|
|
||||||
<div className="min-w-0 flex-1">
|
|
||||||
<p className="text-sm font-medium text-duck-dark dark:text-foreground">Connected</p>
|
|
||||||
<p className="text-xs text-duck-dark/50 dark:text-foreground/50 truncate">{status.email}</p>
|
|
||||||
</div>
|
|
||||||
{status.picture && (
|
|
||||||
<img
|
|
||||||
src={status.picture}
|
|
||||||
alt=""
|
|
||||||
className="h-9 w-9 rounded-full shrink-0"
|
|
||||||
referrerPolicy="no-referrer"
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
<Button type="button" variant="outline" onClick={handleDisconnect} className="w-full h-11 cursor-pointer">
|
|
||||||
Disconnect
|
|
||||||
</Button>
|
|
||||||
</>
|
|
||||||
) : (
|
|
||||||
<Button
|
|
||||||
type="button"
|
|
||||||
onClick={handleConnect}
|
|
||||||
className="w-full h-11 bg-duck-yellow hover:bg-duck-yellow/90 text-duck-teal font-bold transition-all duration-200 hover:scale-105 cursor-pointer"
|
|
||||||
>
|
|
||||||
Connect Google Account
|
|
||||||
</Button>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { useMemo } from 'react';
|
import { useMemo } from 'react';
|
||||||
import { Puzzle, KeyRound, UserCircle, MessageCircle, Globe, Wrench } from 'lucide-react';
|
import { Puzzle, KeyRound, UserCircle, MessageCircle, Globe, Wrench, Mail } from 'lucide-react';
|
||||||
import type { LayoutNode, PanelComponents } from 'officerdev';
|
import type { LayoutNode, PanelComponents } from 'officerdev';
|
||||||
import { WorkspaceLayout } from 'officerdev';
|
import { WorkspaceLayout } from 'officerdev';
|
||||||
import { useAuth } from 'hooks/useAuth';
|
import { useAuth } from 'hooks/useAuth';
|
||||||
@@ -17,6 +17,7 @@ import { WhatsAppBotConfig } from './WhatsAppBotConfig';
|
|||||||
import { WhatsAppAccount } from './WhatsAppAccount';
|
import { WhatsAppAccount } from './WhatsAppAccount';
|
||||||
import { BrowserRelay } from './BrowserRelay';
|
import { BrowserRelay } from './BrowserRelay';
|
||||||
import { ApifyConfig } from './ApifyConfig';
|
import { ApifyConfig } from './ApifyConfig';
|
||||||
|
import { EmailAccounts } from './EmailAccounts';
|
||||||
|
|
||||||
const GLOBAL_KEY = 'INTEGRATIONS_SETTINGS_SELECTED';
|
const GLOBAL_KEY = 'INTEGRATIONS_SETTINGS_SELECTED';
|
||||||
const TAB_KEY = 'INTEGRATIONS_SETTINGS_TAB';
|
const TAB_KEY = 'INTEGRATIONS_SETTINGS_TAB';
|
||||||
@@ -60,6 +61,13 @@ const enterpriseSections: SettingsSection[] = [
|
|||||||
];
|
];
|
||||||
|
|
||||||
const personalSections: SettingsSection[] = [
|
const personalSections: SettingsSection[] = [
|
||||||
|
{
|
||||||
|
key: 'email-accounts',
|
||||||
|
icon: Mail,
|
||||||
|
title: 'Email',
|
||||||
|
description: 'Connect email accounts for inbox sync',
|
||||||
|
content: <EmailAccounts />,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
key: 'google-account',
|
key: 'google-account',
|
||||||
icon: UserCircle,
|
icon: UserCircle,
|
||||||
|
|||||||
@@ -20,7 +20,14 @@ export {
|
|||||||
|
|
||||||
export { readServerSettings, writeServerSettings, readConfigValue, writeConfigValue } from './queries/server-config';
|
export { readServerSettings, writeServerSettings, readConfigValue, writeConfigValue } from './queries/server-config';
|
||||||
|
|
||||||
export { getUserSettings, setUserSettings, getUserState, patchUserState, getDockPaths, setDockPaths } from './queries/user-data';
|
export {
|
||||||
|
getUserSettings,
|
||||||
|
setUserSettings,
|
||||||
|
getUserState,
|
||||||
|
patchUserState,
|
||||||
|
getDockPaths,
|
||||||
|
setDockPaths,
|
||||||
|
} from './queries/user-data';
|
||||||
|
|
||||||
export {
|
export {
|
||||||
getServerIntegrations,
|
getServerIntegrations,
|
||||||
@@ -35,5 +42,15 @@ export {
|
|||||||
findUserByIntegrationConfig,
|
findUserByIntegrationConfig,
|
||||||
} from './queries/integrations';
|
} from './queries/integrations';
|
||||||
|
|
||||||
|
export {
|
||||||
|
getEmailAccounts,
|
||||||
|
getEmailAccount,
|
||||||
|
createEmailAccount,
|
||||||
|
deleteEmailAccount,
|
||||||
|
updateEmailAccountStatus,
|
||||||
|
updateEmailAccountSyncMeta,
|
||||||
|
getAllSyncedAccounts,
|
||||||
|
} from './queries/email-accounts';
|
||||||
|
|
||||||
export { db } from './db';
|
export { db } from './db';
|
||||||
export * as schema from './schema';
|
export * as schema from './schema';
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { eq, and } from 'drizzle-orm';
|
||||||
|
import { db } from '../db';
|
||||||
|
import { emailAccounts } from '../schema';
|
||||||
|
import type { EmailAccountInsert, EmailAccountSelect } from '../types';
|
||||||
|
|
||||||
|
export async function getEmailAccounts(userId: number): Promise<EmailAccountSelect[]> {
|
||||||
|
return db.select().from(emailAccounts).where(eq(emailAccounts.userId, userId));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getEmailAccount(id: number): Promise<EmailAccountSelect | undefined> {
|
||||||
|
const [row] = await db.select().from(emailAccounts).where(eq(emailAccounts.id, id));
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function createEmailAccount(params: EmailAccountInsert): Promise<EmailAccountSelect> {
|
||||||
|
const [row] = await db.insert(emailAccounts).values(params).returning();
|
||||||
|
return row!;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteEmailAccount(id: number, userId: number): Promise<boolean> {
|
||||||
|
const [row] = await db
|
||||||
|
.delete(emailAccounts)
|
||||||
|
.where(and(eq(emailAccounts.id, id), eq(emailAccounts.userId, userId)))
|
||||||
|
.returning({ id: emailAccounts.id });
|
||||||
|
return !!row;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateEmailAccountStatus(id: number, status: string): Promise<void> {
|
||||||
|
await db.update(emailAccounts).set({ status }).where(eq(emailAccounts.id, id));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function updateEmailAccountSyncMeta(id: number, syncMeta: Record<string, unknown>): Promise<void> {
|
||||||
|
await db.update(emailAccounts).set({ syncMeta }).where(eq(emailAccounts.id, id));
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getAllSyncedAccounts(): Promise<EmailAccountSelect[]> {
|
||||||
|
return db
|
||||||
|
.select()
|
||||||
|
.from(emailAccounts)
|
||||||
|
.where(and(eq(emailAccounts.status, 'synced'), eq(emailAccounts.enabled, true)));
|
||||||
|
}
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
import { pgTable, serial, integer, text, boolean, timestamp, jsonb, unique } from 'drizzle-orm/pg-core';
|
||||||
|
import { users } from './auth';
|
||||||
|
|
||||||
|
export const emailAccounts = pgTable(
|
||||||
|
'email_accounts',
|
||||||
|
{
|
||||||
|
id: serial('id').primaryKey(),
|
||||||
|
userId: integer('user_id')
|
||||||
|
.notNull()
|
||||||
|
.references(() => users.id, { onDelete: 'cascade' }),
|
||||||
|
provider: text('provider').notNull(),
|
||||||
|
email: text('email').notNull(),
|
||||||
|
displayName: text('display_name'),
|
||||||
|
imapHost: text('imap_host').notNull(),
|
||||||
|
imapPort: integer('imap_port').notNull(),
|
||||||
|
imapSecure: boolean('imap_secure').notNull().default(true),
|
||||||
|
authType: text('auth_type').notNull(),
|
||||||
|
credentials: jsonb('credentials').notNull().default({}),
|
||||||
|
enabled: boolean('enabled').notNull().default(true),
|
||||||
|
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
|
||||||
|
status: text('status').notNull().default('connected'),
|
||||||
|
syncMeta: jsonb('sync_meta').notNull().default({}),
|
||||||
|
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
|
||||||
|
},
|
||||||
|
(table) => [unique('uq_email_accounts_user_email').on(table.userId, table.email)],
|
||||||
|
);
|
||||||
@@ -5,3 +5,4 @@ export * from './dashboards';
|
|||||||
export * from './agent-items';
|
export * from './agent-items';
|
||||||
export * from './operations';
|
export * from './operations';
|
||||||
export * from './server';
|
export * from './server';
|
||||||
|
export * from './email';
|
||||||
|
|||||||
@@ -100,6 +100,11 @@ export type QueueJobInsert = typeof Schema.queueJobs.$inferInsert;
|
|||||||
export type TerminalContainerSelect = typeof Schema.terminalContainers.$inferSelect;
|
export type TerminalContainerSelect = typeof Schema.terminalContainers.$inferSelect;
|
||||||
export type TerminalContainerInsert = typeof Schema.terminalContainers.$inferInsert;
|
export type TerminalContainerInsert = typeof Schema.terminalContainers.$inferInsert;
|
||||||
|
|
||||||
|
// ── Email ──
|
||||||
|
|
||||||
|
export type EmailAccountSelect = typeof Schema.emailAccounts.$inferSelect;
|
||||||
|
export type EmailAccountInsert = typeof Schema.emailAccounts.$inferInsert;
|
||||||
|
|
||||||
// ── Server ──
|
// ── Server ──
|
||||||
|
|
||||||
export type ServerConfigSelect = typeof Schema.serverConfig.$inferSelect;
|
export type ServerConfigSelect = typeof Schema.serverConfig.$inferSelect;
|
||||||
|
|||||||
@@ -0,0 +1,222 @@
|
|||||||
|
import { createRouter } from '../../create-router';
|
||||||
|
import { BAD_REQUEST, NOT_FOUND } from '../../custom-errors';
|
||||||
|
import {
|
||||||
|
getEmailAccounts,
|
||||||
|
getEmailAccount,
|
||||||
|
createEmailAccount,
|
||||||
|
deleteEmailAccount,
|
||||||
|
getUserIntegration,
|
||||||
|
updateEmailAccountStatus,
|
||||||
|
} from 'officerdb';
|
||||||
|
import { validateImapConnection } from './imap-validate';
|
||||||
|
import * as sidecar from '../../sidecar-client';
|
||||||
|
|
||||||
|
type CreateAccountBody = {
|
||||||
|
provider: string;
|
||||||
|
email: string;
|
||||||
|
displayName?: string;
|
||||||
|
imapHost: string;
|
||||||
|
imapPort: number;
|
||||||
|
imapSecure: boolean;
|
||||||
|
authType: string;
|
||||||
|
credentials: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ValidateBody = {
|
||||||
|
imapHost: string;
|
||||||
|
imapPort: number;
|
||||||
|
imapSecure: boolean;
|
||||||
|
authType: string;
|
||||||
|
email: string;
|
||||||
|
credentials: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const accountsRouter = createRouter();
|
||||||
|
|
||||||
|
accountsRouter.get('/', async (ctx) => {
|
||||||
|
const user = ctx.get('user');
|
||||||
|
const accounts = await getEmailAccounts(user.id);
|
||||||
|
|
||||||
|
// Check for stale syncing/queued accounts with no active job
|
||||||
|
const staleIds: number[] = [];
|
||||||
|
const hasActiveAccounts = accounts.some((a) => a.status === 'syncing' || a.status === 'queued');
|
||||||
|
let activeJobAccountIds = new Set<number>();
|
||||||
|
|
||||||
|
if (hasActiveAccounts) {
|
||||||
|
try {
|
||||||
|
const jobs = await sidecar.listJobs();
|
||||||
|
activeJobAccountIds = new Set(
|
||||||
|
jobs
|
||||||
|
.filter((j) => j.type === 'email-sync' && (j.status === 'queued' || j.status === 'running'))
|
||||||
|
.map((j) => (j.meta as Record<string, unknown> | undefined)?.emailAccountId as number)
|
||||||
|
.filter(Boolean),
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// Sidecar unavailable — all syncing/queued accounts are stale
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const a of accounts) {
|
||||||
|
if ((a.status === 'syncing' || a.status === 'queued') && !activeJobAccountIds.has(a.id)) {
|
||||||
|
staleIds.push(a.id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset stale accounts in background
|
||||||
|
if (staleIds.length > 0) {
|
||||||
|
for (const id of staleIds) {
|
||||||
|
updateEmailAccountStatus(id, 'connected').catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ctx.json(
|
||||||
|
accounts.map((a) => ({
|
||||||
|
id: a.id,
|
||||||
|
provider: a.provider,
|
||||||
|
email: a.email,
|
||||||
|
displayName: a.displayName,
|
||||||
|
enabled: a.enabled,
|
||||||
|
status: staleIds.includes(a.id) ? 'connected' : a.status,
|
||||||
|
createdAt: a.createdAt,
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
accountsRouter.post('/', async (ctx) => {
|
||||||
|
const user = ctx.get('user');
|
||||||
|
const body = ctx.get('body') as CreateAccountBody;
|
||||||
|
|
||||||
|
if (!body.provider || !body.email || !body.imapHost || !body.imapPort || !body.authType) {
|
||||||
|
throw BAD_REQUEST('Missing required fields');
|
||||||
|
}
|
||||||
|
|
||||||
|
const authResult = await resolveAuth(user.id, body.authType, body.email, body.credentials);
|
||||||
|
if (!authResult.ok) throw BAD_REQUEST(authResult.error);
|
||||||
|
|
||||||
|
const validation = await validateImapConnection({
|
||||||
|
host: body.imapHost,
|
||||||
|
port: body.imapPort,
|
||||||
|
secure: body.imapSecure,
|
||||||
|
user: body.email,
|
||||||
|
...authResult.auth,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!validation.ok) throw BAD_REQUEST(`IMAP connection failed: ${validation.error}`);
|
||||||
|
|
||||||
|
const account = await createEmailAccount({
|
||||||
|
userId: user.id,
|
||||||
|
provider: body.provider,
|
||||||
|
email: body.email,
|
||||||
|
displayName: body.displayName,
|
||||||
|
imapHost: body.imapHost,
|
||||||
|
imapPort: body.imapPort,
|
||||||
|
imapSecure: body.imapSecure,
|
||||||
|
authType: body.authType,
|
||||||
|
credentials: body.credentials,
|
||||||
|
});
|
||||||
|
|
||||||
|
return ctx.json({ id: account.id, provider: account.provider, email: account.email }, 201);
|
||||||
|
});
|
||||||
|
|
||||||
|
accountsRouter.delete('/:id', async (ctx) => {
|
||||||
|
const user = ctx.get('user');
|
||||||
|
const id = Number(ctx.req.param('id'));
|
||||||
|
const deleted = await deleteEmailAccount(id, user.id);
|
||||||
|
if (!deleted) throw NOT_FOUND('Account not found');
|
||||||
|
return ctx.json({ ok: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
accountsRouter.post('/:id/sync', async (ctx) => {
|
||||||
|
const user = ctx.get('user');
|
||||||
|
const id = Number(ctx.req.param('id'));
|
||||||
|
|
||||||
|
const account = await getEmailAccount(id);
|
||||||
|
if (!account || account.userId !== user.id) throw NOT_FOUND('Account not found');
|
||||||
|
|
||||||
|
if (account.status === 'queued') throw BAD_REQUEST('Sync is already queued');
|
||||||
|
if (account.status === 'syncing') throw BAD_REQUEST('Account is already syncing');
|
||||||
|
|
||||||
|
// Resolve auth before enqueueing
|
||||||
|
const authResult = await resolveAuth(user.id, account.authType, account.email, account.credentials as Record<string, unknown>);
|
||||||
|
if (!authResult.ok) throw BAD_REQUEST(authResult.error);
|
||||||
|
|
||||||
|
// Set status immediately so the UI reflects the queued state
|
||||||
|
await updateEmailAccountStatus(id, 'queued');
|
||||||
|
|
||||||
|
const job = await sidecar.enqueueJob({
|
||||||
|
lane: 'email',
|
||||||
|
type: 'email-sync',
|
||||||
|
userId: user.email,
|
||||||
|
meta: {
|
||||||
|
emailAccountId: id,
|
||||||
|
userEmail: user.email,
|
||||||
|
account: {
|
||||||
|
id: account.id,
|
||||||
|
userId: account.userId,
|
||||||
|
email: account.email,
|
||||||
|
imapHost: account.imapHost,
|
||||||
|
imapPort: account.imapPort,
|
||||||
|
imapSecure: account.imapSecure,
|
||||||
|
provider: account.provider,
|
||||||
|
authType: account.authType,
|
||||||
|
credentials: account.credentials,
|
||||||
|
},
|
||||||
|
imapAuth: { user: account.email, ...authResult.auth },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return ctx.json({ ok: true, jobId: job.id }, 201);
|
||||||
|
});
|
||||||
|
|
||||||
|
accountsRouter.post('/validate', async (ctx) => {
|
||||||
|
const user = ctx.get('user');
|
||||||
|
const body = ctx.get('body') as ValidateBody;
|
||||||
|
|
||||||
|
if (!body.imapHost || !body.imapPort || !body.authType || !body.email) {
|
||||||
|
throw BAD_REQUEST('Missing required fields');
|
||||||
|
}
|
||||||
|
|
||||||
|
const authResult = await resolveAuth(user.id, body.authType, body.email, body.credentials);
|
||||||
|
if (!authResult.ok) return ctx.json({ ok: false, error: authResult.error });
|
||||||
|
|
||||||
|
const result = await validateImapConnection({
|
||||||
|
host: body.imapHost,
|
||||||
|
port: body.imapPort,
|
||||||
|
secure: body.imapSecure,
|
||||||
|
user: body.email,
|
||||||
|
...authResult.auth,
|
||||||
|
});
|
||||||
|
|
||||||
|
return ctx.json(result);
|
||||||
|
});
|
||||||
|
|
||||||
|
type AuthResult = { ok: true; auth: { pass?: string; accessToken?: string } } | { ok: false; error: string };
|
||||||
|
|
||||||
|
async function resolveAuth(
|
||||||
|
userId: number,
|
||||||
|
authType: string,
|
||||||
|
email: string,
|
||||||
|
credentials: Record<string, unknown>,
|
||||||
|
): Promise<AuthResult> {
|
||||||
|
if (authType === 'oauth') {
|
||||||
|
const integrationId = credentials.userIntegrationId as number | undefined;
|
||||||
|
if (!integrationId) return { ok: false, error: 'Missing userIntegrationId for OAuth' };
|
||||||
|
|
||||||
|
const integration = await getUserIntegration(userId, 'google');
|
||||||
|
if (!integration) return { ok: false, error: 'Google integration not found' };
|
||||||
|
|
||||||
|
const config = integration.config as Record<string, unknown>;
|
||||||
|
const accessToken = config.accessToken as string | undefined;
|
||||||
|
if (!accessToken) return { ok: false, error: 'No access token available — reconnect Google account' };
|
||||||
|
|
||||||
|
return { ok: true, auth: { accessToken } };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (authType === 'password') {
|
||||||
|
const pass = credentials.password as string | undefined;
|
||||||
|
if (!pass) return { ok: false, error: 'Missing password' };
|
||||||
|
return { ok: true, auth: { pass } };
|
||||||
|
}
|
||||||
|
|
||||||
|
return { ok: false, error: `Unknown auth type: ${authType}` };
|
||||||
|
}
|
||||||
@@ -4,9 +4,12 @@ import type { EmailMessage } from 'types';
|
|||||||
import { createRouter } from '../../create-router';
|
import { createRouter } from '../../create-router';
|
||||||
import { DATA_PATH } from '@@/data-path';
|
import { DATA_PATH } from '@@/data-path';
|
||||||
import { openEmailDb, rowToSummary, getSyncMeta } from './email-db';
|
import { openEmailDb, rowToSummary, getSyncMeta } from './email-db';
|
||||||
|
import { accountsRouter } from './accounts';
|
||||||
|
|
||||||
export const emailRouter = createRouter();
|
export const emailRouter = createRouter();
|
||||||
|
|
||||||
|
emailRouter.route('/accounts', accountsRouter);
|
||||||
|
|
||||||
emailRouter.get('/messages', async (ctx) => {
|
emailRouter.get('/messages', async (ctx) => {
|
||||||
const email = ctx.get('user').email;
|
const email = ctx.get('user').email;
|
||||||
const page = Number(ctx.req.query('page') ?? '1');
|
const page = Number(ctx.req.query('page') ?? '1');
|
||||||
@@ -18,7 +21,9 @@ emailRouter.get('/messages', async (ctx) => {
|
|||||||
|
|
||||||
const db = openEmailDb(email);
|
const db = openEmailDb(email);
|
||||||
try {
|
try {
|
||||||
const rows = db.query(`SELECT * FROM emails WHERE ${folderWhere} ORDER BY date DESC LIMIT ? OFFSET ?`).all(limit, offset) as Record<string, unknown>[];
|
const rows = db
|
||||||
|
.query(`SELECT * FROM emails WHERE ${folderWhere} ORDER BY date DESC LIMIT ? OFFSET ?`)
|
||||||
|
.all(limit, offset) as Record<string, unknown>[];
|
||||||
const countRow = db.query(`SELECT COUNT(*) as total FROM emails WHERE ${folderWhere}`).get() as { total: number };
|
const countRow = db.query(`SELECT COUNT(*) as total FROM emails WHERE ${folderWhere}`).get() as { total: number };
|
||||||
const messages = rows.map(rowToSummary);
|
const messages = rows.map(rowToSummary);
|
||||||
return ctx.json({ messages, total: countRow.total });
|
return ctx.json({ messages, total: countRow.total });
|
||||||
@@ -36,11 +41,11 @@ emailRouter.get('/messages/:id', async (ctx) => {
|
|||||||
const row = db.query('SELECT * FROM emails WHERE id = ? AND deleted = 0').get(id) as Record<string, unknown> | null;
|
const row = db.query('SELECT * FROM emails WHERE id = ? AND deleted = 0').get(id) as Record<string, unknown> | null;
|
||||||
if (!row) return ctx.text('Not found', 404);
|
if (!row) return ctx.text('Not found', 404);
|
||||||
|
|
||||||
const attachmentRows = db.query('SELECT * FROM attachments WHERE email_id = ? ORDER BY idx').all(id) as Array<Record<string, unknown>>;
|
const attachmentRows = db.query('SELECT * FROM attachments WHERE email_id = ? ORDER BY idx').all(id) as Array<
|
||||||
|
Record<string, unknown>
|
||||||
|
>;
|
||||||
|
|
||||||
const from = row.from_name
|
const from = row.from_name ? `${row.from_name} <${row.from_address}>` : (row.from_address as string);
|
||||||
? `${row.from_name} <${row.from_address}>`
|
|
||||||
: (row.from_address as string);
|
|
||||||
|
|
||||||
const message: EmailMessage = {
|
const message: EmailMessage = {
|
||||||
id: row.id as string,
|
id: row.id as string,
|
||||||
@@ -75,7 +80,10 @@ emailRouter.post('/messages/:id/attachments/:index/extract', async (ctx) => {
|
|||||||
|
|
||||||
const db = openEmailDb(email);
|
const db = openEmailDb(email);
|
||||||
try {
|
try {
|
||||||
const row = db.query('SELECT filename, content FROM attachments WHERE email_id = ? AND idx = ?').get(id, index) as { filename: string; content: string | null } | null;
|
const row = db.query('SELECT filename, content FROM attachments WHERE email_id = ? AND idx = ?').get(id, index) as {
|
||||||
|
filename: string;
|
||||||
|
content: string | null;
|
||||||
|
} | null;
|
||||||
if (!row || !row.content) return ctx.text('Attachment not found', 404);
|
if (!row || !row.content) return ctx.text('Attachment not found', 404);
|
||||||
|
|
||||||
const fileName = row.filename ?? 'unknown';
|
const fileName = row.filename ?? 'unknown';
|
||||||
@@ -144,9 +152,18 @@ emailRouter.get('/stats', async (ctx) => {
|
|||||||
|
|
||||||
const db = openEmailDb(email);
|
const db = openEmailDb(email);
|
||||||
try {
|
try {
|
||||||
const total = (db.query(`SELECT COUNT(*) as count FROM emails WHERE ${folderWhere}`).get() as { count: number }).count;
|
const total = (db.query(`SELECT COUNT(*) as count FROM emails WHERE ${folderWhere}`).get() as { count: number })
|
||||||
const byDomain = db.query(`SELECT from_domain, COUNT(*) as count FROM emails WHERE ${folderWhere} GROUP BY from_domain ORDER BY count DESC LIMIT 20`).all() as Array<{ from_domain: string; count: number }>;
|
.count;
|
||||||
const bySender = db.query(`SELECT from_address, from_name, COUNT(*) as count FROM emails WHERE ${folderWhere} GROUP BY from_address ORDER BY count DESC LIMIT 20`).all() as Array<{ from_address: string; from_name: string; count: number }>;
|
const byDomain = db
|
||||||
|
.query(
|
||||||
|
`SELECT from_domain, COUNT(*) as count FROM emails WHERE ${folderWhere} GROUP BY from_domain ORDER BY count DESC LIMIT 20`,
|
||||||
|
)
|
||||||
|
.all() as Array<{ from_domain: string; count: number }>;
|
||||||
|
const bySender = db
|
||||||
|
.query(
|
||||||
|
`SELECT from_address, from_name, COUNT(*) as count FROM emails WHERE ${folderWhere} GROUP BY from_address ORDER BY count DESC LIMIT 20`,
|
||||||
|
)
|
||||||
|
.all() as Array<{ from_address: string; from_name: string; count: number }>;
|
||||||
|
|
||||||
return ctx.json({ total, byDomain, bySender });
|
return ctx.json({ total, byDomain, bySender });
|
||||||
} finally {
|
} finally {
|
||||||
@@ -159,7 +176,9 @@ emailRouter.get('/labels', async (ctx) => {
|
|||||||
|
|
||||||
const db = openEmailDb(email);
|
const db = openEmailDb(email);
|
||||||
try {
|
try {
|
||||||
const rows = db.query('SELECT labels FROM emails WHERE deleted = 0 AND labels IS NOT NULL').all() as Array<{ labels: string }>;
|
const rows = db.query('SELECT labels FROM emails WHERE deleted = 0 AND labels IS NOT NULL').all() as Array<{
|
||||||
|
labels: string;
|
||||||
|
}>;
|
||||||
|
|
||||||
const counts = new Map<string, number>();
|
const counts = new Map<string, number>();
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
@@ -169,9 +188,7 @@ emailRouter.get('/labels', async (ctx) => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const labels = [...counts.entries()]
|
const labels = [...counts.entries()].map(([label, count]) => ({ label, count })).sort((a, b) => b.count - a.count);
|
||||||
.map(([label, count]) => ({ label, count }))
|
|
||||||
.sort((a, b) => b.count - a.count);
|
|
||||||
|
|
||||||
return ctx.json({ labels });
|
return ctx.json({ labels });
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
type ValidateImapParams = {
|
||||||
|
host: string;
|
||||||
|
port: number;
|
||||||
|
secure: boolean;
|
||||||
|
user: string;
|
||||||
|
pass?: string;
|
||||||
|
accessToken?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ValidateImapResult = { ok: true; folderCount: number } | { ok: false; error: string };
|
||||||
|
|
||||||
|
export async function validateImapConnection(params: ValidateImapParams): Promise<ValidateImapResult> {
|
||||||
|
const { ImapFlow } = await import('imapflow');
|
||||||
|
|
||||||
|
const auth: { user: string; pass?: string; accessToken?: string } = { user: params.user };
|
||||||
|
if (params.accessToken) {
|
||||||
|
auth.accessToken = params.accessToken;
|
||||||
|
} else if (params.pass) {
|
||||||
|
auth.pass = params.pass;
|
||||||
|
} else {
|
||||||
|
return { ok: false, error: 'No authentication credentials provided' };
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = new ImapFlow({
|
||||||
|
host: params.host,
|
||||||
|
port: params.port,
|
||||||
|
secure: params.secure,
|
||||||
|
auth,
|
||||||
|
logger: false,
|
||||||
|
greetingTimeout: 60_000,
|
||||||
|
socketTimeout: 60_000,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await Promise.race([
|
||||||
|
(async () => {
|
||||||
|
await client.connect();
|
||||||
|
const folders = await client.list();
|
||||||
|
await client.logout();
|
||||||
|
return { ok: true as const, folderCount: folders.length };
|
||||||
|
})(),
|
||||||
|
new Promise<ValidateImapResult>((_, reject) =>
|
||||||
|
setTimeout(() => reject(new Error('Connection timed out')), 90_000),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
return result;
|
||||||
|
} catch (err) {
|
||||||
|
try {
|
||||||
|
client.close();
|
||||||
|
} catch {}
|
||||||
|
const message = err instanceof Error ? err.message : 'Connection failed';
|
||||||
|
return { ok: false, error: message };
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
import { getServerIntegration } from 'officerdb';
|
||||||
|
import { PermanentError } from '../../queue/types';
|
||||||
|
|
||||||
|
type TokenRefreshResult = { accessToken: string; expiresAt: number };
|
||||||
|
|
||||||
|
export async function refreshGoogleAccessToken(refreshToken: string): Promise<TokenRefreshResult> {
|
||||||
|
const serverGoogle = await getServerIntegration('google');
|
||||||
|
const serverConfig = serverGoogle?.config as Record<string, unknown> | undefined;
|
||||||
|
if (!serverConfig?.clientId || !serverConfig?.clientSecret) {
|
||||||
|
throw new PermanentError('Google OAuth not configured on server');
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await fetch('https://oauth2.googleapis.com/token', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||||
|
body: new URLSearchParams({
|
||||||
|
client_id: serverConfig.clientId as string,
|
||||||
|
client_secret: serverConfig.clientSecret as string,
|
||||||
|
refresh_token: refreshToken,
|
||||||
|
grant_type: 'refresh_token',
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const text = await response.text();
|
||||||
|
throw new Error(`Google token refresh failed: ${text}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = (await response.json()) as { access_token: string; expires_in: number };
|
||||||
|
return {
|
||||||
|
accessToken: data.access_token,
|
||||||
|
expiresAt: Date.now() + data.expires_in * 1000,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -259,11 +259,16 @@ export const googleCallbackHandler = async (ctx: any) => {
|
|||||||
|
|
||||||
const serverIntegration = await getServerIntegration('google');
|
const serverIntegration = await getServerIntegration('google');
|
||||||
|
|
||||||
|
// Merge with existing config to preserve fields like imapAppPassword
|
||||||
|
const existingIntegration = await getUserIntegration(dbUser.id, 'google');
|
||||||
|
const existingConfig = (existingIntegration?.config as Record<string, unknown>) ?? {};
|
||||||
|
|
||||||
await upsertUserIntegration({
|
await upsertUserIntegration({
|
||||||
userId: dbUser.id,
|
userId: dbUser.id,
|
||||||
provider: 'google',
|
provider: 'google',
|
||||||
serverIntegrationId: serverIntegration?.id,
|
serverIntegrationId: serverIntegration?.id,
|
||||||
config: {
|
config: {
|
||||||
|
...existingConfig,
|
||||||
accessToken: tokens.access_token,
|
accessToken: tokens.access_token,
|
||||||
refreshToken: tokens.refresh_token,
|
refreshToken: tokens.refresh_token,
|
||||||
expiresAt: Date.now() + tokens.expires_in * 1000,
|
expiresAt: Date.now() + tokens.expires_in * 1000,
|
||||||
|
|||||||
@@ -0,0 +1,381 @@
|
|||||||
|
import { createHash } from 'node:crypto';
|
||||||
|
import type { JobHandler } from '../types';
|
||||||
|
import { PermanentError } from '../types';
|
||||||
|
import { registerHandler } from '../handler-registry';
|
||||||
|
import { openEmailDb, upsertFromRawEml } from '../../api/email/email-db';
|
||||||
|
import { refreshGoogleAccessToken } from '../../api/integrations/google-auth';
|
||||||
|
import {
|
||||||
|
getEmailAccount,
|
||||||
|
getUserIntegration,
|
||||||
|
upsertUserIntegration,
|
||||||
|
getServerIntegration,
|
||||||
|
updateEmailAccountStatus,
|
||||||
|
updateEmailAccountSyncMeta,
|
||||||
|
getDockPaths,
|
||||||
|
setDockPaths,
|
||||||
|
} from 'officerdb';
|
||||||
|
|
||||||
|
// ── Types for job meta (passed by the API server at enqueue time) ──
|
||||||
|
|
||||||
|
type EmailSyncMeta = {
|
||||||
|
emailAccountId: number;
|
||||||
|
userEmail: string;
|
||||||
|
account: {
|
||||||
|
id: number;
|
||||||
|
userId: number;
|
||||||
|
email: string;
|
||||||
|
imapHost: string;
|
||||||
|
imapPort: number;
|
||||||
|
imapSecure: boolean;
|
||||||
|
provider: string;
|
||||||
|
authType: string;
|
||||||
|
credentials: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
imapAuth: { user: string; pass?: string; accessToken?: string };
|
||||||
|
saved?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
// ── Stable ID from Message-Id header ──
|
||||||
|
|
||||||
|
function messageIdToStableId(raw: string): string | null {
|
||||||
|
const match = raw.match(/^Message-Id:\s*<?([^>\s]+)>?/im);
|
||||||
|
if (!match?.[1]) return null;
|
||||||
|
return createHash('sha1').update(match[1]).digest('hex').slice(0, 16);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── IMAP folder → label mapping ──
|
||||||
|
|
||||||
|
const SPECIAL_USE_LABEL_MAP: Record<string, string> = {
|
||||||
|
'\\Inbox': 'inbox',
|
||||||
|
'\\Sent': 'sent',
|
||||||
|
'\\Drafts': 'draft',
|
||||||
|
'\\Flagged': 'starred',
|
||||||
|
'\\Trash': 'trash',
|
||||||
|
'\\Junk': 'spam',
|
||||||
|
'\\All': 'archive',
|
||||||
|
};
|
||||||
|
|
||||||
|
const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
|
||||||
|
|
||||||
|
type FolderInfo = { specialUse?: string; path: string; flags: Set<string>; status?: { uidNext?: number; uidValidity?: number } };
|
||||||
|
|
||||||
|
function shouldSkipFolder(folder: FolderInfo): boolean {
|
||||||
|
if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
|
||||||
|
if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function folderToLabel(folder: FolderInfo): string {
|
||||||
|
if (folder.specialUse && SPECIAL_USE_LABEL_MAP[folder.specialUse]) {
|
||||||
|
return SPECIAL_USE_LABEL_MAP[folder.specialUse]!;
|
||||||
|
}
|
||||||
|
if (folder.path === 'INBOX') return 'inbox';
|
||||||
|
return folder.path.toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── OAuth token refresh ──
|
||||||
|
|
||||||
|
async function resolveImapAuth(meta: EmailSyncMeta): Promise<{ user: string; pass?: string; accessToken?: string }> {
|
||||||
|
if (meta.account.authType !== 'oauth') return meta.imapAuth;
|
||||||
|
|
||||||
|
// Refresh OAuth token if expired
|
||||||
|
const userGoogle = await getUserIntegration(meta.account.userId, 'google');
|
||||||
|
const config = userGoogle?.config as Record<string, unknown> | undefined;
|
||||||
|
if (!config?.refreshToken) {
|
||||||
|
throw new PermanentError('Google OAuth not configured — reconnect your Google account');
|
||||||
|
}
|
||||||
|
|
||||||
|
const expiresAt = config.expiresAt as number | undefined;
|
||||||
|
const tokenExpired = !expiresAt || expiresAt < Date.now() + 60_000;
|
||||||
|
|
||||||
|
if (!tokenExpired && config.accessToken) {
|
||||||
|
return { user: meta.account.email, accessToken: config.accessToken as string };
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log('[email-sync] Refreshing OAuth token');
|
||||||
|
const refreshed = await refreshGoogleAccessToken(config.refreshToken as string);
|
||||||
|
|
||||||
|
const serverGoogle = await getServerIntegration('google');
|
||||||
|
await upsertUserIntegration({
|
||||||
|
userId: meta.account.userId,
|
||||||
|
provider: 'google',
|
||||||
|
serverIntegrationId: serverGoogle?.id,
|
||||||
|
config: { ...config, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
|
||||||
|
});
|
||||||
|
|
||||||
|
return { user: meta.account.email, accessToken: refreshed.accessToken };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Handler ──
|
||||||
|
|
||||||
|
const emailSyncHandler: JobHandler = {
|
||||||
|
type: 'email-sync',
|
||||||
|
retry: { delayMs: 15 * 60 * 1000, maxRetries: 5 },
|
||||||
|
steps: [
|
||||||
|
{
|
||||||
|
name: 'Sync emails',
|
||||||
|
run: async (ctx) => {
|
||||||
|
const { ImapFlow } = await import('imapflow');
|
||||||
|
const meta = ctx.meta as unknown as EmailSyncMeta;
|
||||||
|
const { account, userEmail } = meta;
|
||||||
|
|
||||||
|
// Resolve auth (refreshes OAuth token if needed)
|
||||||
|
const imapAuth = await resolveImapAuth(meta);
|
||||||
|
|
||||||
|
// Load syncMeta from DB (always fresh, not from job meta)
|
||||||
|
const freshAccount = await getEmailAccount(account.id);
|
||||||
|
if (!freshAccount) throw new PermanentError(`Email account ${account.id} not found`);
|
||||||
|
const syncMeta = (freshAccount.syncMeta ?? {}) as Record<string, unknown>;
|
||||||
|
const isIncremental = !!syncMeta.last_sync_at;
|
||||||
|
|
||||||
|
await updateEmailAccountStatus(account.id, 'syncing');
|
||||||
|
console.log(`[email-sync] ${isIncremental ? 'Incremental' : 'Initial'} sync for ${account.email}`);
|
||||||
|
|
||||||
|
const MAX_RECONNECTS = 10;
|
||||||
|
const RECONNECT_DELAY_MS = 5_000;
|
||||||
|
let reconnects = 0;
|
||||||
|
let saved = 0;
|
||||||
|
let skipped = 0;
|
||||||
|
let errors = 0;
|
||||||
|
let allDone = false;
|
||||||
|
|
||||||
|
const db = openEmailDb(userEmail);
|
||||||
|
|
||||||
|
// Load existing IDs for dedup (once, shared across reconnections)
|
||||||
|
const existingIds = new Set<string>();
|
||||||
|
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
||||||
|
for (const row of rows) existingIds.add(row.id);
|
||||||
|
|
||||||
|
try {
|
||||||
|
while (!allDone && reconnects <= MAX_RECONNECTS) {
|
||||||
|
if (reconnects > 0) {
|
||||||
|
console.log(`[email-sync] Reconnecting (${reconnects}/${MAX_RECONNECTS}) after ${RECONNECT_DELAY_MS / 1000}s...`);
|
||||||
|
await ctx.updateProgress({ current: saved + skipped, total: 0, label: `Reconnecting (${reconnects}/${MAX_RECONNECTS})...` });
|
||||||
|
await new Promise((r) => setTimeout(r, RECONNECT_DELAY_MS));
|
||||||
|
|
||||||
|
// Re-read syncMeta from DB to get latest saved UIDs
|
||||||
|
const updated = await getEmailAccount(account.id);
|
||||||
|
if (updated?.syncMeta) {
|
||||||
|
Object.assign(syncMeta, updated.syncMeta as Record<string, unknown>);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
await ctx.updateProgress({ current: 0, total: 0, label: 'Connecting...' });
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = new ImapFlow({
|
||||||
|
host: account.imapHost,
|
||||||
|
port: account.imapPort,
|
||||||
|
secure: account.imapSecure,
|
||||||
|
auth: imapAuth,
|
||||||
|
logger: false,
|
||||||
|
socketTimeout: 30 * 60 * 1000,
|
||||||
|
});
|
||||||
|
|
||||||
|
const connState = { error: null as Error | null };
|
||||||
|
client.on('error', (err: Error) => {
|
||||||
|
console.log(`[email-sync] IMAP connection error: ${err.message}`);
|
||||||
|
connState.error = err;
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
await client.connect();
|
||||||
|
console.log(`[email-sync] IMAP connected${reconnects > 0 ? ` (reconnect ${reconnects})` : ''}`);
|
||||||
|
|
||||||
|
const folders = await client.list({ statusQuery: { uidNext: true, uidValidity: true } }) as FolderInfo[];
|
||||||
|
|
||||||
|
// Filter to folders that still need syncing
|
||||||
|
const foldersToSync: Array<{ folder: FolderInfo; lastUid: number }> = [];
|
||||||
|
for (const folder of folders) {
|
||||||
|
if (shouldSkipFolder(folder)) continue;
|
||||||
|
|
||||||
|
const uidValidityKey = `imap_uidvalidity:${folder.path}`;
|
||||||
|
const lastUidKey = `imap_lastuid:${folder.path}`;
|
||||||
|
const storedUidValidity = syncMeta[uidValidityKey] as string | undefined;
|
||||||
|
const storedLastUid = syncMeta[lastUidKey] as string | undefined;
|
||||||
|
|
||||||
|
const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
|
||||||
|
const uidNext = folder.status?.uidNext ?? 0;
|
||||||
|
let lastUid = storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
|
||||||
|
|
||||||
|
if (storedUidValidity && uidValidity && storedUidValidity !== uidValidity) {
|
||||||
|
console.log(`[email-sync] UIDVALIDITY changed for ${folder.path} — will re-scan`);
|
||||||
|
lastUid = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lastUid > 0 && uidNext <= lastUid + 1) continue;
|
||||||
|
|
||||||
|
foldersToSync.push({ folder, lastUid });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (foldersToSync.length === 0) {
|
||||||
|
console.log('[email-sync] All folders synced');
|
||||||
|
allDone = true;
|
||||||
|
await client.logout().catch(() => {});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[email-sync] ${foldersToSync.length} folder(s) to sync`);
|
||||||
|
|
||||||
|
let connectionLost = false;
|
||||||
|
|
||||||
|
for (let fi = 0; fi < foldersToSync.length; fi++) {
|
||||||
|
const { folder, lastUid } = foldersToSync[fi]!;
|
||||||
|
|
||||||
|
if (connState.error) { connectionLost = true; break; }
|
||||||
|
|
||||||
|
console.log(`[email-sync] Syncing folder ${fi + 1}/${foldersToSync.length}: ${folder.path}`);
|
||||||
|
|
||||||
|
let lock;
|
||||||
|
try {
|
||||||
|
lock = await client.getMailboxLock(folder.path);
|
||||||
|
} catch (err) {
|
||||||
|
const errMsg = err instanceof Error ? err.message : String(err);
|
||||||
|
if (errMsg.includes('not available') || errMsg.includes('timeout') || errMsg.includes('closed')) {
|
||||||
|
console.log(`[email-sync] Connection lost at folder ${folder.path}`);
|
||||||
|
connectionLost = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
console.log(`[email-sync] Skipping folder ${folder.path}: ${errMsg}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const mailbox = client.mailbox;
|
||||||
|
if (!mailbox) continue;
|
||||||
|
|
||||||
|
const uidValidity = String(mailbox.uidValidity);
|
||||||
|
const effectiveLastUid = syncMeta[`imap_uidvalidity:${folder.path}`] === uidValidity ? lastUid : 0;
|
||||||
|
let maxUid = effectiveLastUid;
|
||||||
|
const label = folderToLabel(folder);
|
||||||
|
const range = effectiveLastUid > 0 ? `${effectiveLastUid + 1}:*` : '1:*';
|
||||||
|
|
||||||
|
try {
|
||||||
|
const fetchOpts: Record<string, unknown> = { source: true, uid: true };
|
||||||
|
if (account.provider === 'gmail') fetchOpts.labels = true;
|
||||||
|
|
||||||
|
for await (const msg of client.fetch(range, fetchOpts, { uid: true })) {
|
||||||
|
if (msg.uid <= effectiveLastUid) continue;
|
||||||
|
|
||||||
|
maxUid = Math.max(maxUid, msg.uid);
|
||||||
|
|
||||||
|
if (!msg.source) { errors++; continue; }
|
||||||
|
|
||||||
|
const raw = msg.source.toString('utf-8');
|
||||||
|
const id = messageIdToStableId(raw);
|
||||||
|
if (!id) { errors++; continue; }
|
||||||
|
|
||||||
|
if (existingIds.has(id)) { skipped++; continue; }
|
||||||
|
|
||||||
|
const labels = [label];
|
||||||
|
try {
|
||||||
|
upsertFromRawEml({ db, id, raw, integration: account.provider, emailAccount: account.email, labels });
|
||||||
|
existingIds.add(id);
|
||||||
|
saved++;
|
||||||
|
} catch {
|
||||||
|
errors++;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((saved + skipped) % 50 === 0) {
|
||||||
|
const progressLabel = `${folder.path} — ${saved.toLocaleString()} saved, ${skipped.toLocaleString()} skipped`;
|
||||||
|
console.log(`[email-sync] ${progressLabel}`);
|
||||||
|
await ctx.updateProgress({ current: saved + skipped, total: 0, label: progressLabel });
|
||||||
|
|
||||||
|
if (maxUid > effectiveLastUid) {
|
||||||
|
syncMeta[`imap_lastuid:${folder.path}`] = String(maxUid);
|
||||||
|
await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (fetchErr) {
|
||||||
|
const errMsg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
|
||||||
|
if (errMsg.includes('Nothing to fetch')) {
|
||||||
|
// No messages in range — normal
|
||||||
|
} else if (errMsg.includes('not available') || errMsg.includes('timeout') || errMsg.includes('closed')) {
|
||||||
|
console.log(`[email-sync] Connection lost during fetch in ${folder.path}: ${errMsg}`);
|
||||||
|
connectionLost = true;
|
||||||
|
} else {
|
||||||
|
console.log(`[email-sync] Fetch error in ${folder.path}: ${errMsg}`);
|
||||||
|
errors++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
syncMeta[`imap_uidvalidity:${folder.path}`] = uidValidity;
|
||||||
|
if (maxUid > effectiveLastUid) {
|
||||||
|
syncMeta[`imap_lastuid:${folder.path}`] = String(maxUid);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[email-sync] ${folder.path} done: ${saved} saved, ${skipped} skipped, ${errors} errors`);
|
||||||
|
await ctx.updateProgress({ current: saved + skipped, total: 0, label: `${folder.path}: ${saved} saved` });
|
||||||
|
await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
|
||||||
|
} finally {
|
||||||
|
lock.release();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (connectionLost) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
await client.logout().catch(() => {});
|
||||||
|
|
||||||
|
if (connectionLost) {
|
||||||
|
console.log(`[email-sync] Connection lost after saving ${saved} emails — will reconnect`);
|
||||||
|
reconnects++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
allDone = true;
|
||||||
|
} catch (err) {
|
||||||
|
await client.logout().catch(() => {});
|
||||||
|
if (reconnects < MAX_RECONNECTS) {
|
||||||
|
console.log(`[email-sync] Error: ${err instanceof Error ? err.message : String(err)} — will reconnect`);
|
||||||
|
reconnects++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!allDone) {
|
||||||
|
throw new Error(`IMAP sync incomplete after ${MAX_RECONNECTS} reconnection attempts`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark sync complete
|
||||||
|
syncMeta.last_sync_at = new Date().toISOString();
|
||||||
|
await updateEmailAccountSyncMeta(account.id, syncMeta as Record<string, unknown>);
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[email-sync] Done: saved ${saved}, skipped ${skipped}, errors ${errors}, reconnects ${reconnects}`);
|
||||||
|
ctx.meta.saved = saved;
|
||||||
|
await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${saved} new emails` });
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'Finalize',
|
||||||
|
run: async (ctx) => {
|
||||||
|
const meta = ctx.meta as unknown as EmailSyncMeta;
|
||||||
|
const saved = meta.saved ?? 0;
|
||||||
|
|
||||||
|
await updateEmailAccountStatus(meta.account.id, 'synced');
|
||||||
|
|
||||||
|
if (saved > 0) {
|
||||||
|
try {
|
||||||
|
const paths = await getDockPaths(meta.account.userId);
|
||||||
|
if (!paths) {
|
||||||
|
const defaults = ['/', '/files', '/automation', '/projects', '/dashboards', '/chat'];
|
||||||
|
await setDockPaths(meta.account.userId, [...defaults, '/email']);
|
||||||
|
} else if (!paths.includes('/email')) {
|
||||||
|
await setDockPaths(meta.account.userId, [...paths, '/email']);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Non-fatal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[email-sync] ${meta.account.email} status set to synced`);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
|
||||||
|
registerHandler(emailSyncHandler);
|
||||||
@@ -4,28 +4,59 @@ import { join } from 'node:path';
|
|||||||
import { readdir, readFile, writeFile, unlink, mkdir, stat } from 'node:fs/promises';
|
import { readdir, readFile, writeFile, unlink, mkdir, stat } from 'node:fs/promises';
|
||||||
import { type JobHandler, PermanentError } from '../types';
|
import { type JobHandler, PermanentError } from '../types';
|
||||||
import { registerHandler } from '../handler-registry';
|
import { registerHandler } from '../handler-registry';
|
||||||
import { openEmailDb, upsertFromRawEml, setSyncMeta } from '../../api/email/email-db';
|
import { openEmailDb, upsertFromRawEml, setSyncMeta, getSyncMeta } from '../../api/email/email-db';
|
||||||
import { getUserByEmail, getUserIntegration, getDockPaths, setDockPaths } from 'officerdb';
|
import {
|
||||||
|
getUserByEmail,
|
||||||
|
getUserIntegration,
|
||||||
|
upsertUserIntegration,
|
||||||
|
getServerIntegration,
|
||||||
|
getDockPaths,
|
||||||
|
setDockPaths,
|
||||||
|
} from 'officerdb';
|
||||||
import { getMaildirPath } from '@@/data-path';
|
import { getMaildirPath } from '@@/data-path';
|
||||||
|
import { refreshGoogleAccessToken } from '@@/api/integrations/google-auth';
|
||||||
|
|
||||||
// ── Credentials ──
|
// ── Credentials ──
|
||||||
|
|
||||||
type ImapCredentials = { email: string; appPassword: string };
|
type GmailCredentials = {
|
||||||
|
email: string;
|
||||||
|
userId: number;
|
||||||
|
appPassword?: string;
|
||||||
|
accessToken?: string;
|
||||||
|
refreshToken?: string;
|
||||||
|
expiresAt?: number;
|
||||||
|
};
|
||||||
|
|
||||||
async function loadImapCredentials(userEmail: string): Promise<ImapCredentials> {
|
async function loadGmailCredentials(userEmail: string): Promise<GmailCredentials> {
|
||||||
const dbUser = await getUserByEmail(userEmail);
|
const dbUser = await getUserByEmail(userEmail);
|
||||||
if (!dbUser) throw new PermanentError('User not found');
|
if (!dbUser) throw new PermanentError('User not found');
|
||||||
|
|
||||||
const userGoogle = await getUserIntegration(dbUser.id, 'google');
|
const userGoogle = await getUserIntegration(dbUser.id, 'google');
|
||||||
const config = userGoogle?.config as Record<string, unknown> | undefined;
|
const config = userGoogle?.config as Record<string, unknown> | undefined;
|
||||||
|
|
||||||
|
const gmailEmail = (config?.email as string) ?? userEmail;
|
||||||
|
|
||||||
|
// Prefer OAuth tokens, fall back to app password
|
||||||
|
if (config?.accessToken && config?.refreshToken) {
|
||||||
|
return {
|
||||||
|
email: gmailEmail,
|
||||||
|
userId: dbUser.id,
|
||||||
|
accessToken: config.accessToken as string,
|
||||||
|
refreshToken: config.refreshToken as string,
|
||||||
|
expiresAt: config.expiresAt as number | undefined,
|
||||||
|
appPassword: config.imapAppPassword as string | undefined,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (!config?.imapAppPassword) {
|
if (!config?.imapAppPassword) {
|
||||||
throw new PermanentError('Gmail App Password not configured — set it in Settings → Integrations');
|
throw new PermanentError('Gmail App Password not configured — set it in Settings → Integrations');
|
||||||
}
|
}
|
||||||
|
|
||||||
const gmailEmail = (config.email as string) ?? userEmail;
|
return { email: gmailEmail, userId: dbUser.id, appPassword: config.imapAppPassword as string };
|
||||||
return { email: gmailEmail, appPassword: config.imapAppPassword as string };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refreshGoogleAccessToken is imported from @@/api/integrations/google-auth
|
||||||
|
|
||||||
// ── mbsync config ──
|
// ── mbsync config ──
|
||||||
|
|
||||||
function buildMbsyncConfig(email: string, appPassword: string, maildirPath: string): string {
|
function buildMbsyncConfig(email: string, appPassword: string, maildirPath: string): string {
|
||||||
@@ -48,7 +79,7 @@ SubFolders Verbatim
|
|||||||
Channel gmail
|
Channel gmail
|
||||||
Far :gmail-remote:
|
Far :gmail-remote:
|
||||||
Near :gmail-local:
|
Near :gmail-local:
|
||||||
Patterns * ![Gmail]/Trash ![Gmail]/Spam ![Gmail]/Bin ![Google Mail]/Trash ![Google Mail]/Spam ![Google Mail]/Bin
|
Patterns * ![Gmail]/Trash ![Gmail]/Spam ![Gmail]/Bin !"[Gmail]/All Mail" ![Google Mail]/Trash ![Google Mail]/Spam ![Google Mail]/Bin !"[Google Mail]/All Mail"
|
||||||
Create Near
|
Create Near
|
||||||
Expunge None
|
Expunge None
|
||||||
SyncState *
|
SyncState *
|
||||||
@@ -99,21 +130,40 @@ function messageIdToStableId(raw: string): string | null {
|
|||||||
|
|
||||||
type ImportResult = { saved: number; skipped: number; errors: number };
|
type ImportResult = { saved: number; skipped: number; errors: number };
|
||||||
|
|
||||||
export async function importMaildir(
|
type ImportMaildirParams = {
|
||||||
maildirPath: string,
|
maildirPath: string;
|
||||||
emailAccount: string,
|
emailAccount: string;
|
||||||
db: Database,
|
db: Database;
|
||||||
onProgress?: (saved: number, skipped: number) => void,
|
lastSyncAt?: string | null;
|
||||||
): Promise<ImportResult> {
|
onProgress?: (saved: number, skipped: number) => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
export async function importMaildir({
|
||||||
|
maildirPath,
|
||||||
|
emailAccount,
|
||||||
|
db,
|
||||||
|
lastSyncAt,
|
||||||
|
onProgress,
|
||||||
|
}: ImportMaildirParams): Promise<ImportResult> {
|
||||||
let saved = 0;
|
let saved = 0;
|
||||||
let skipped = 0;
|
let skipped = 0;
|
||||||
let errors = 0;
|
let errors = 0;
|
||||||
|
|
||||||
|
// For incremental syncs, skip files older than last sync (with 60s buffer for clock skew)
|
||||||
|
const mtimeCutoff = lastSyncAt ? new Date(lastSyncAt).getTime() - 60_000 : 0;
|
||||||
|
const isIncremental = mtimeCutoff > 0;
|
||||||
|
|
||||||
// Load existing IDs for fast dedup
|
// Load existing IDs for fast dedup
|
||||||
const existingIds = new Set<string>();
|
const existingIds = new Set<string>();
|
||||||
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
||||||
for (const row of rows) existingIds.add(row.id);
|
for (const row of rows) existingIds.add(row.id);
|
||||||
|
|
||||||
|
if (isIncremental) {
|
||||||
|
console.log(
|
||||||
|
`[gmail-sync] Incremental import — only reading files newer than ${new Date(mtimeCutoff).toISOString()}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// First pass: collect all message files with their folders to build label map
|
// First pass: collect all message files with their folders to build label map
|
||||||
const messageIdLabels = new Map<string, Set<string>>();
|
const messageIdLabels = new Map<string, Set<string>>();
|
||||||
const messageFiles = new Map<string, string>(); // id → first file path
|
const messageFiles = new Map<string, string>(); // id → first file path
|
||||||
@@ -162,6 +212,12 @@ export async function importMaildir(
|
|||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
const filePath = join(dirPath, file);
|
const filePath = join(dirPath, file);
|
||||||
try {
|
try {
|
||||||
|
// Skip files older than last sync for incremental imports
|
||||||
|
if (isIncremental) {
|
||||||
|
const fileStat = await stat(filePath);
|
||||||
|
if (fileStat.mtimeMs < mtimeCutoff) continue;
|
||||||
|
}
|
||||||
|
|
||||||
const raw = await readFile(filePath, 'utf-8');
|
const raw = await readFile(filePath, 'utf-8');
|
||||||
const id = messageIdToStableId(raw);
|
const id = messageIdToStableId(raw);
|
||||||
if (!id) {
|
if (!id) {
|
||||||
@@ -238,6 +294,220 @@ async function countMaildirFiles(maildirPath: string): Promise<number> {
|
|||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Gmail IMAP label → our label mapping ──
|
||||||
|
|
||||||
|
const GMAIL_LABEL_MAP: Record<string, string> = {
|
||||||
|
'\\Inbox': 'inbox',
|
||||||
|
'\\Sent': 'sent',
|
||||||
|
'\\Drafts': 'draft',
|
||||||
|
'\\Starred': 'starred',
|
||||||
|
'\\Important': 'important',
|
||||||
|
'\\All': 'archive',
|
||||||
|
'\\Trash': 'trash',
|
||||||
|
'\\Junk': 'spam',
|
||||||
|
};
|
||||||
|
|
||||||
|
function gmailLabelsToLabels(gmailLabels: Set<string>): string[] {
|
||||||
|
const labels: string[] = [];
|
||||||
|
for (const gl of gmailLabels) {
|
||||||
|
const mapped = GMAIL_LABEL_MAP[gl];
|
||||||
|
if (mapped) {
|
||||||
|
labels.push(mapped);
|
||||||
|
} else if (!gl.startsWith('\\')) {
|
||||||
|
// Custom label — lowercase it
|
||||||
|
labels.push(gl.toLowerCase());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// If only "archive" and no specific folder, keep it; otherwise drop "archive"
|
||||||
|
if (labels.length > 1 && labels.includes('archive')) {
|
||||||
|
return labels.filter((l) => l !== 'archive');
|
||||||
|
}
|
||||||
|
return labels;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── IMAP special-use folders to skip ──
|
||||||
|
|
||||||
|
const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
|
||||||
|
|
||||||
|
function shouldSkipFolder(folder: { specialUse?: string; path: string; flags: Set<string> }): boolean {
|
||||||
|
if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
|
||||||
|
if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
|
||||||
|
const norm = normalizeGmailFolder(folder.path);
|
||||||
|
return norm === 'All Mail' || norm === 'Trash' || norm === 'Spam' || norm === 'Bin';
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Incremental IMAP sync ──
|
||||||
|
|
||||||
|
type IncrementalSyncParams = {
|
||||||
|
creds: GmailCredentials;
|
||||||
|
db: Database;
|
||||||
|
onProgress?: (fetched: number, folder: string) => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
type IncrementalSyncResult = { saved: number; skipped: number; errors: number };
|
||||||
|
|
||||||
|
async function incrementalImapSync({ creds, db, onProgress }: IncrementalSyncParams): Promise<IncrementalSyncResult> {
|
||||||
|
const { ImapFlow } = await import('imapflow');
|
||||||
|
|
||||||
|
// Determine auth method: prefer OAuth, fall back to app password
|
||||||
|
const auth: { user: string; pass?: string; accessToken?: string } = { user: creds.email };
|
||||||
|
if (creds.accessToken) {
|
||||||
|
auth.accessToken = creds.accessToken;
|
||||||
|
} else if (creds.appPassword) {
|
||||||
|
auth.pass = creds.appPassword;
|
||||||
|
} else {
|
||||||
|
throw new PermanentError('No authentication method available for IMAP');
|
||||||
|
}
|
||||||
|
|
||||||
|
const client = new ImapFlow({
|
||||||
|
host: 'imap.gmail.com',
|
||||||
|
port: 993,
|
||||||
|
secure: true,
|
||||||
|
auth,
|
||||||
|
logger: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
let saved = 0;
|
||||||
|
let skipped = 0;
|
||||||
|
let errors = 0;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await client.connect();
|
||||||
|
console.log('[gmail-sync] Incremental IMAP connected');
|
||||||
|
|
||||||
|
// Get all folders with status in a single LIST command
|
||||||
|
const folders = await client.list({ statusQuery: { uidNext: true, uidValidity: true } });
|
||||||
|
|
||||||
|
// Load existing IDs for dedup
|
||||||
|
const existingIds = new Set<string>();
|
||||||
|
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
||||||
|
for (const row of rows) existingIds.add(row.id);
|
||||||
|
|
||||||
|
// Filter to only folders with new messages
|
||||||
|
const foldersToSync: typeof folders = [];
|
||||||
|
for (const folder of folders) {
|
||||||
|
if (shouldSkipFolder(folder)) continue;
|
||||||
|
|
||||||
|
const folderPath = folder.path;
|
||||||
|
const uidValidityKey = `imap_uidvalidity:${folderPath}`;
|
||||||
|
const lastUidKey = `imap_lastuid:${folderPath}`;
|
||||||
|
const storedUidValidity = getSyncMeta(db, uidValidityKey);
|
||||||
|
const storedLastUid = getSyncMeta(db, lastUidKey);
|
||||||
|
|
||||||
|
const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
|
||||||
|
const uidNext = folder.status?.uidNext ?? 0;
|
||||||
|
const lastUid =
|
||||||
|
storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
|
||||||
|
|
||||||
|
if (uidValidity) setSyncMeta(db, uidValidityKey, uidValidity);
|
||||||
|
|
||||||
|
if (lastUid > 0 && uidNext <= lastUid + 1) continue; // no new messages
|
||||||
|
|
||||||
|
if (storedUidValidity && uidValidity && storedUidValidity !== uidValidity) {
|
||||||
|
console.log(`[gmail-sync] UIDVALIDITY changed for ${folderPath} — will re-scan`);
|
||||||
|
}
|
||||||
|
|
||||||
|
foldersToSync.push(folder);
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(`[gmail-sync] ${foldersToSync.length} folder(s) with new messages`);
|
||||||
|
|
||||||
|
for (const folder of foldersToSync) {
|
||||||
|
const folderPath = folder.path;
|
||||||
|
const uidValidityKey = `imap_uidvalidity:${folderPath}`;
|
||||||
|
const lastUidKey = `imap_lastuid:${folderPath}`;
|
||||||
|
const storedUidValidity = getSyncMeta(db, uidValidityKey);
|
||||||
|
const storedLastUid = getSyncMeta(db, lastUidKey);
|
||||||
|
|
||||||
|
// Open folder and fetch new messages
|
||||||
|
let lock;
|
||||||
|
try {
|
||||||
|
lock = await client.getMailboxLock(folderPath);
|
||||||
|
} catch (err) {
|
||||||
|
console.log(`[gmail-sync] Skipping folder ${folderPath}: ${err instanceof Error ? err.message : err}`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const mailbox = client.mailbox;
|
||||||
|
if (!mailbox) continue;
|
||||||
|
|
||||||
|
const uidValidity = String(mailbox.uidValidity);
|
||||||
|
let lastUid = 0;
|
||||||
|
if (storedUidValidity === uidValidity && storedLastUid) {
|
||||||
|
lastUid = parseInt(storedLastUid, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
const range = lastUid > 0 ? `${lastUid + 1}:*` : '1:*';
|
||||||
|
let maxUid = lastUid;
|
||||||
|
let folderFetched = 0;
|
||||||
|
|
||||||
|
try {
|
||||||
|
for await (const msg of client.fetch(range, { source: true, labels: true, uid: true }, { uid: true })) {
|
||||||
|
if (msg.uid <= lastUid) continue;
|
||||||
|
|
||||||
|
maxUid = Math.max(maxUid, msg.uid);
|
||||||
|
folderFetched++;
|
||||||
|
|
||||||
|
if (!msg.source) {
|
||||||
|
errors++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const raw = msg.source.toString('utf-8');
|
||||||
|
const id = messageIdToStableId(raw);
|
||||||
|
if (!id) {
|
||||||
|
errors++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (existingIds.has(id)) {
|
||||||
|
skipped++;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gmail X-GM-EXT-1 labels don't include folder membership (e.g. \Inbox),
|
||||||
|
// so always use the folder we're fetching from as the base label
|
||||||
|
const folderLabel = folderToLabel(folderPath);
|
||||||
|
const gmailLabels = msg.labels ? gmailLabelsToLabels(msg.labels) : [];
|
||||||
|
const labels = folderLabel ? [...new Set([folderLabel, ...gmailLabels])] : gmailLabels;
|
||||||
|
|
||||||
|
try {
|
||||||
|
upsertFromRawEml({ db, id, raw, integration: 'gmail', emailAccount: creds.email, labels });
|
||||||
|
existingIds.add(id);
|
||||||
|
saved++;
|
||||||
|
} catch {
|
||||||
|
errors++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (fetchErr) {
|
||||||
|
const msg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
|
||||||
|
if (!msg.includes('Nothing to fetch')) {
|
||||||
|
console.log(`[gmail-sync] Fetch error in ${folderPath}: ${msg}`);
|
||||||
|
errors++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (maxUid > lastUid) {
|
||||||
|
setSyncMeta(db, lastUidKey, String(maxUid));
|
||||||
|
}
|
||||||
|
setSyncMeta(db, uidValidityKey, uidValidity);
|
||||||
|
|
||||||
|
if (folderFetched > 0) {
|
||||||
|
console.log(`[gmail-sync] ${folderPath}: fetched ${folderFetched}, saved ${saved}, skipped ${skipped}`);
|
||||||
|
onProgress?.(saved + skipped, folderPath);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
lock.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await client.logout().catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
return { saved, skipped, errors };
|
||||||
|
}
|
||||||
|
|
||||||
// ── Handler ──
|
// ── Handler ──
|
||||||
|
|
||||||
const gmailSyncHandler: JobHandler = {
|
const gmailSyncHandler: JobHandler = {
|
||||||
@@ -247,119 +517,219 @@ const gmailSyncHandler: JobHandler = {
|
|||||||
{
|
{
|
||||||
name: 'Verify credentials',
|
name: 'Verify credentials',
|
||||||
run: async (ctx) => {
|
run: async (ctx) => {
|
||||||
const creds = await loadImapCredentials(ctx.job.userId);
|
const creds = await loadGmailCredentials(ctx.job.userId);
|
||||||
|
|
||||||
|
// Determine sync mode: incremental if we have a previous sync
|
||||||
|
const db = openEmailDb(ctx.job.userId);
|
||||||
|
try {
|
||||||
|
const lastSyncAt = getSyncMeta(db, 'last_sync_at');
|
||||||
|
ctx.meta.isIncremental = !!lastSyncAt;
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
// For incremental sync with OAuth, refresh token if expired
|
||||||
|
if (ctx.meta.isIncremental && creds.accessToken && creds.refreshToken) {
|
||||||
|
const tokenExpired = !creds.expiresAt || creds.expiresAt < Date.now() + 60_000;
|
||||||
|
if (tokenExpired) {
|
||||||
|
console.log('[gmail-sync] Refreshing OAuth token for incremental sync');
|
||||||
|
try {
|
||||||
|
const refreshed = await refreshGoogleAccessToken(creds.refreshToken);
|
||||||
|
creds.accessToken = refreshed.accessToken;
|
||||||
|
creds.expiresAt = refreshed.expiresAt;
|
||||||
|
|
||||||
|
// Persist refreshed token
|
||||||
|
const userGoogle = await getUserIntegration(creds.userId, 'google');
|
||||||
|
const existingConfig = (userGoogle?.config as Record<string, unknown>) ?? {};
|
||||||
|
const serverGoogle = await getServerIntegration('google');
|
||||||
|
await upsertUserIntegration({
|
||||||
|
userId: creds.userId,
|
||||||
|
provider: 'google',
|
||||||
|
serverIntegrationId: serverGoogle?.id,
|
||||||
|
config: { ...existingConfig, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
console.log(`[gmail-sync] OAuth refresh failed, will fall back to app password: ${err}`);
|
||||||
|
// Clear OAuth so we fall back to app password
|
||||||
|
creds.accessToken = undefined;
|
||||||
|
creds.refreshToken = undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.meta.creds = creds;
|
||||||
ctx.meta.email = creds.email;
|
ctx.meta.email = creds.email;
|
||||||
ctx.meta.appPassword = creds.appPassword;
|
ctx.meta.appPassword = creds.appPassword;
|
||||||
|
|
||||||
|
if (ctx.meta.isIncremental) {
|
||||||
|
console.log(`[gmail-sync] Incremental sync mode (${creds.accessToken ? 'OAuth' : 'App Password'})`);
|
||||||
|
} else {
|
||||||
|
console.log('[gmail-sync] Full sync mode (mbsync)');
|
||||||
|
if (!creds.appPassword) {
|
||||||
|
throw new PermanentError('Gmail App Password required for initial sync');
|
||||||
|
}
|
||||||
|
}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'Sync via mbsync',
|
name: 'Sync emails',
|
||||||
run: async (ctx) => {
|
run: async (ctx) => {
|
||||||
const email = ctx.meta.email as string;
|
if (ctx.meta.isIncremental) {
|
||||||
const appPassword = ctx.meta.appPassword as string;
|
// ── Incremental: direct IMAP via imapflow ──
|
||||||
const maildirPath = getMaildirPath(ctx.job.userId);
|
const creds = ctx.meta.creds as GmailCredentials;
|
||||||
|
await ctx.updateProgress({ current: 0, total: 0, label: 'Connecting to Gmail...' });
|
||||||
|
|
||||||
// Ensure Maildir root exists
|
const db = openEmailDb(ctx.job.userId);
|
||||||
await mkdir(maildirPath, { recursive: true });
|
|
||||||
|
|
||||||
// Write temp config
|
|
||||||
const configPath = join(maildirPath, '.mbsyncrc');
|
|
||||||
const config = buildMbsyncConfig(email, appPassword, maildirPath);
|
|
||||||
await writeFile(configPath, config, { mode: 0o600 });
|
|
||||||
|
|
||||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Running mbsync...' });
|
|
||||||
|
|
||||||
try {
|
|
||||||
let proc: ReturnType<typeof Bun.spawn>;
|
|
||||||
try {
|
try {
|
||||||
proc = Bun.spawn(['mbsync', '-c', configPath, '-a', '-V'], {
|
const result = await incrementalImapSync({
|
||||||
stdout: 'pipe',
|
creds,
|
||||||
stderr: 'pipe',
|
db,
|
||||||
|
onProgress: (fetched, folder) => {
|
||||||
|
ctx.updateProgress({ current: fetched, total: 0, label: `Syncing ${folder}...` });
|
||||||
|
},
|
||||||
});
|
});
|
||||||
} catch (spawnErr) {
|
|
||||||
const msg = spawnErr instanceof Error ? spawnErr.message : String(spawnErr);
|
|
||||||
throw new PermanentError(`Failed to start mbsync: ${msg}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Periodically count downloaded emails and update progress
|
console.log(
|
||||||
let emailCount = 0;
|
`[gmail-sync] Incremental sync done: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`,
|
||||||
let counting = true;
|
);
|
||||||
const countLoop = (async () => {
|
|
||||||
while (counting) {
|
setSyncMeta(db, 'last_sync_date', new Date().toISOString().split('T')[0]!);
|
||||||
await new Promise((r) => setTimeout(r, 3000));
|
setSyncMeta(db, 'last_sync_at', new Date().toISOString());
|
||||||
if (!counting) break;
|
|
||||||
|
ctx.meta.syncResult = result;
|
||||||
|
await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${result.saved} new emails` });
|
||||||
|
} finally {
|
||||||
|
db.close();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// ── Full sync: mbsync ──
|
||||||
|
const email = ctx.meta.email as string;
|
||||||
|
const appPassword = ctx.meta.appPassword as string;
|
||||||
|
const maildirPath = getMaildirPath(ctx.job.userId);
|
||||||
|
|
||||||
|
await mkdir(maildirPath, { recursive: true });
|
||||||
|
|
||||||
|
const configPath = join(maildirPath, '.mbsyncrc');
|
||||||
|
const config = buildMbsyncConfig(email, appPassword, maildirPath);
|
||||||
|
await writeFile(configPath, config, { mode: 0o600 });
|
||||||
|
|
||||||
|
await ctx.updateProgress({ current: 0, total: 0, label: 'Running mbsync...' });
|
||||||
|
|
||||||
|
try {
|
||||||
|
let proc: ReturnType<typeof Bun.spawn>;
|
||||||
|
try {
|
||||||
|
proc = Bun.spawn(['mbsync', '-c', configPath, '-a'], {
|
||||||
|
stdout: 'pipe',
|
||||||
|
stderr: 'pipe',
|
||||||
|
});
|
||||||
|
} catch (spawnErr) {
|
||||||
|
const msg = spawnErr instanceof Error ? spawnErr.message : String(spawnErr);
|
||||||
|
throw new PermanentError(`Failed to start mbsync: ${msg}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
let emailCount = 0;
|
||||||
|
let counting = true;
|
||||||
|
const countLoop = (async () => {
|
||||||
|
while (counting) {
|
||||||
|
await new Promise((r) => setTimeout(r, 3000));
|
||||||
|
if (!counting) break;
|
||||||
|
emailCount = await countMaildirFiles(maildirPath);
|
||||||
|
ctx.updateProgress({
|
||||||
|
current: emailCount,
|
||||||
|
total: 0,
|
||||||
|
label: `Downloading — ${emailCount.toLocaleString()} emails`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
|
let stderrBuf = '';
|
||||||
|
const reader = (proc.stderr as ReadableStream<Uint8Array>).getReader();
|
||||||
|
const decoder = new TextDecoder();
|
||||||
|
const readLoop = (async () => {
|
||||||
|
while (true) {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
const chunk = decoder.decode(value, { stream: true });
|
||||||
|
stderrBuf += chunk;
|
||||||
|
const lines = chunk.split('\n');
|
||||||
|
for (const line of lines) {
|
||||||
|
const trimmed = line.trim();
|
||||||
|
if (trimmed) console.log(`[gmail-sync] mbsync: ${trimmed}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
|
const exitCode = await proc.exited;
|
||||||
|
counting = false;
|
||||||
|
await readLoop;
|
||||||
|
await countLoop;
|
||||||
|
|
||||||
|
if (exitCode !== 0) {
|
||||||
|
const isOverquota = stderrBuf.includes('OVERQUOTA');
|
||||||
|
const isAuthFail =
|
||||||
|
stderrBuf.includes('AUTHENTICATIONFAILED') || stderrBuf.includes('Invalid credentials');
|
||||||
|
|
||||||
|
if (isOverquota) {
|
||||||
|
const emailCount = await countMaildirFiles(maildirPath);
|
||||||
|
console.log(`[gmail-sync] Gmail OVERQUOTA — proceeding to import ${emailCount} downloaded emails`);
|
||||||
|
ctx.meta.gmailSyncPartial = true;
|
||||||
|
ctx.meta.gmailSyncEmailCount = emailCount;
|
||||||
|
} else {
|
||||||
|
console.error(`[gmail-sync] mbsync failed`);
|
||||||
|
if (isAuthFail) {
|
||||||
|
const emailCount = await countMaildirFiles(maildirPath);
|
||||||
|
ctx.meta.gmailSyncRecoverable = true;
|
||||||
|
ctx.meta.gmailSyncEmailCount = emailCount;
|
||||||
|
ctx.meta.gmailSyncIsAuthFail = true;
|
||||||
|
throw new PermanentError(`Authentication failed — check your App Password`);
|
||||||
|
}
|
||||||
|
throw new Error(`mbsync exited with code ${exitCode}: ${stderrBuf.slice(-500)}`);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
emailCount = await countMaildirFiles(maildirPath);
|
emailCount = await countMaildirFiles(maildirPath);
|
||||||
ctx.updateProgress({
|
console.log(`[gmail-sync] mbsync completed successfully — ${emailCount} emails`);
|
||||||
|
await ctx.updateProgress({
|
||||||
current: emailCount,
|
current: emailCount,
|
||||||
total: 0,
|
total: emailCount,
|
||||||
label: `Downloading — ${emailCount.toLocaleString()} emails`,
|
label: `Download complete — ${emailCount.toLocaleString()} emails`,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
})();
|
} finally {
|
||||||
|
await unlink(configPath).catch(() => {});
|
||||||
// Stream stderr for logging
|
|
||||||
let stderrBuf = '';
|
|
||||||
const reader = (proc.stderr as ReadableStream<Uint8Array>).getReader();
|
|
||||||
const decoder = new TextDecoder();
|
|
||||||
const readLoop = (async () => {
|
|
||||||
while (true) {
|
|
||||||
const { done, value } = await reader.read();
|
|
||||||
if (done) break;
|
|
||||||
const chunk = decoder.decode(value, { stream: true });
|
|
||||||
stderrBuf += chunk;
|
|
||||||
const lines = chunk.split('\n');
|
|
||||||
for (const line of lines) {
|
|
||||||
const trimmed = line.trim();
|
|
||||||
if (trimmed) console.log(`[gmail-sync] mbsync: ${trimmed}`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
|
|
||||||
const exitCode = await proc.exited;
|
|
||||||
counting = false;
|
|
||||||
await readLoop;
|
|
||||||
await countLoop;
|
|
||||||
|
|
||||||
if (exitCode !== 0) {
|
|
||||||
const isOverquota = stderrBuf.includes('OVERQUOTA');
|
|
||||||
const isAuthFail = stderrBuf.includes('AUTHENTICATIONFAILED') || stderrBuf.includes('Invalid credentials');
|
|
||||||
|
|
||||||
if (isOverquota) {
|
|
||||||
// Gmail throttled us — don't retry, just import what we have
|
|
||||||
const emailCount = await countMaildirFiles(maildirPath);
|
|
||||||
console.log(`[gmail-sync] Gmail OVERQUOTA — proceeding to import ${emailCount} downloaded emails`);
|
|
||||||
ctx.meta.gmailSyncPartial = true;
|
|
||||||
ctx.meta.gmailSyncEmailCount = emailCount;
|
|
||||||
// Fall through to import step
|
|
||||||
} else {
|
|
||||||
console.error(`[gmail-sync] mbsync failed`);
|
|
||||||
if (isAuthFail) {
|
|
||||||
const emailCount = await countMaildirFiles(maildirPath);
|
|
||||||
ctx.meta.gmailSyncRecoverable = true;
|
|
||||||
ctx.meta.gmailSyncEmailCount = emailCount;
|
|
||||||
ctx.meta.gmailSyncIsAuthFail = true;
|
|
||||||
throw new PermanentError(`Authentication failed — check your App Password`);
|
|
||||||
}
|
|
||||||
throw new Error(`mbsync exited with code ${exitCode}: ${stderrBuf.slice(-500)}`);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
emailCount = await countMaildirFiles(maildirPath);
|
|
||||||
console.log(`[gmail-sync] mbsync completed successfully — ${emailCount} emails`);
|
|
||||||
await ctx.updateProgress({
|
|
||||||
current: emailCount,
|
|
||||||
total: emailCount,
|
|
||||||
label: `Download complete — ${emailCount.toLocaleString()} emails`,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
} finally {
|
|
||||||
// Always clean up config (contains password)
|
|
||||||
await unlink(configPath).catch(() => {});
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'Import to database',
|
name: 'Import to database',
|
||||||
run: async (ctx) => {
|
run: async (ctx) => {
|
||||||
|
// Incremental sync already imported in the previous step
|
||||||
|
if (ctx.meta.isIncremental) {
|
||||||
|
const result = ctx.meta.syncResult as IncrementalSyncResult | undefined;
|
||||||
|
|
||||||
|
// Auto-add /email to dock
|
||||||
|
if (result && result.saved > 0) {
|
||||||
|
try {
|
||||||
|
const dbUser = await getUserByEmail(ctx.job.userId);
|
||||||
|
if (dbUser) {
|
||||||
|
const paths = await getDockPaths(dbUser.id);
|
||||||
|
if (!paths) {
|
||||||
|
const defaults = ['/', '/files', '/automation', '/projects', '/dashboards', '/chat'];
|
||||||
|
await setDockPaths(dbUser.id, [...defaults, '/email']);
|
||||||
|
} else if (!paths.includes('/email')) {
|
||||||
|
await setDockPaths(dbUser.id, [...paths, '/email']);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Non-fatal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${result?.saved ?? 0} new emails` });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Full sync: import from Maildir
|
||||||
const emailAccount = ctx.meta.email as string;
|
const emailAccount = ctx.meta.email as string;
|
||||||
const maildirPath = getMaildirPath(ctx.job.userId);
|
const maildirPath = getMaildirPath(ctx.job.userId);
|
||||||
|
|
||||||
@@ -367,12 +737,19 @@ const gmailSyncHandler: JobHandler = {
|
|||||||
|
|
||||||
const db = openEmailDb(ctx.job.userId);
|
const db = openEmailDb(ctx.job.userId);
|
||||||
try {
|
try {
|
||||||
const result = await importMaildir(maildirPath, emailAccount, db, (saved, skipped) => {
|
const lastSyncAt = getSyncMeta(db, 'last_sync_at');
|
||||||
ctx.updateProgress({
|
const result = await importMaildir({
|
||||||
current: saved + skipped,
|
maildirPath,
|
||||||
total: 0,
|
emailAccount,
|
||||||
label: `Saved ${saved.toLocaleString()}, skipped ${skipped.toLocaleString()}`,
|
db,
|
||||||
});
|
lastSyncAt,
|
||||||
|
onProgress: (saved, skipped) => {
|
||||||
|
ctx.updateProgress({
|
||||||
|
current: saved + skipped,
|
||||||
|
total: 0,
|
||||||
|
label: `Saved ${saved.toLocaleString()}, skipped ${skipped.toLocaleString()}`,
|
||||||
|
});
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
console.log(
|
console.log(
|
||||||
@@ -389,7 +766,6 @@ const gmailSyncHandler: JobHandler = {
|
|||||||
if (dbUser) {
|
if (dbUser) {
|
||||||
const paths = await getDockPaths(dbUser.id);
|
const paths = await getDockPaths(dbUser.id);
|
||||||
if (!paths) {
|
if (!paths) {
|
||||||
// User hasn't customized dock — initialize with defaults + /email
|
|
||||||
const defaults = ['/', '/files', '/automation', '/projects', '/dashboards', '/chat'];
|
const defaults = ['/', '/files', '/automation', '/projects', '/dashboards', '/chat'];
|
||||||
await setDockPaths(dbUser.id, [...defaults, '/email']);
|
await setDockPaths(dbUser.id, [...defaults, '/email']);
|
||||||
} else if (!paths.includes('/email')) {
|
} else if (!paths.includes('/email')) {
|
||||||
|
|||||||
@@ -1 +1,2 @@
|
|||||||
import './gmail-sync';
|
import './gmail-sync';
|
||||||
|
import './email-sync';
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
import { getAllSyncedAccounts, getUserById, getUserIntegration } from 'officerdb';
|
||||||
|
import * as queueRunner from './queue-runner';
|
||||||
|
|
||||||
|
const INTERVAL_MS = 10 * 60 * 1000; // 10 minutes
|
||||||
|
|
||||||
|
let timer: ReturnType<typeof setInterval> | null = null;
|
||||||
|
|
||||||
|
async function tick() {
|
||||||
|
try {
|
||||||
|
const accounts = await getAllSyncedAccounts();
|
||||||
|
if (accounts.length === 0) return;
|
||||||
|
|
||||||
|
const allJobs = await queueRunner.listAllJobs();
|
||||||
|
const activeEmailSyncIds = new Set(
|
||||||
|
allJobs
|
||||||
|
.filter((j) => j.type === 'email-sync' && (j.status === 'queued' || j.status === 'running'))
|
||||||
|
.map((j) => (j.meta as Record<string, unknown> | undefined)?.emailAccountId),
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const account of accounts) {
|
||||||
|
if (activeEmailSyncIds.has(account.id)) continue;
|
||||||
|
|
||||||
|
const user = await getUserById(account.userId);
|
||||||
|
if (!user) continue;
|
||||||
|
|
||||||
|
// Resolve IMAP auth
|
||||||
|
const imapAuth: Record<string, unknown> = { user: account.email };
|
||||||
|
if (account.authType === 'oauth') {
|
||||||
|
const integration = await getUserIntegration(account.userId, 'google');
|
||||||
|
const config = integration?.config as Record<string, unknown> | undefined;
|
||||||
|
const accessToken = config?.accessToken as string | undefined;
|
||||||
|
if (!accessToken) {
|
||||||
|
console.log(`[email-cron] Skipping ${account.email}: no OAuth access token`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
imapAuth.accessToken = accessToken;
|
||||||
|
} else {
|
||||||
|
const creds = account.credentials as Record<string, unknown>;
|
||||||
|
imapAuth.pass = creds.password;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await queueRunner.enqueue({
|
||||||
|
lane: 'email',
|
||||||
|
type: 'email-sync',
|
||||||
|
userId: user.email,
|
||||||
|
meta: {
|
||||||
|
emailAccountId: account.id,
|
||||||
|
userEmail: user.email,
|
||||||
|
account: {
|
||||||
|
id: account.id,
|
||||||
|
userId: account.userId,
|
||||||
|
email: account.email,
|
||||||
|
imapHost: account.imapHost,
|
||||||
|
imapPort: account.imapPort,
|
||||||
|
imapSecure: account.imapSecure,
|
||||||
|
provider: account.provider,
|
||||||
|
authType: account.authType,
|
||||||
|
credentials: account.credentials,
|
||||||
|
},
|
||||||
|
imapAuth,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
console.log(`[email-cron] Enqueued incremental sync for ${account.email}`);
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`[email-cron] Failed to enqueue sync for ${account.email}:`, err instanceof Error ? err.message : err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error('[email-cron] Error:', err instanceof Error ? err.message : err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function initEmailCron() {
|
||||||
|
if (timer) return;
|
||||||
|
console.log(`[email-cron] Starting email sync cron (every ${INTERVAL_MS / 60_000} min)`);
|
||||||
|
timer = setInterval(tick, INTERVAL_MS);
|
||||||
|
// Run first tick after a short delay to let the queue initialize
|
||||||
|
setTimeout(tick, 30_000);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stopEmailCron() {
|
||||||
|
if (timer) {
|
||||||
|
clearInterval(timer);
|
||||||
|
timer = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import { startAnthropicProxy, getProxySecret, ensureProxySecret } from './proxy'
|
|||||||
import * as claudeManager from './claude-manager';
|
import * as claudeManager from './claude-manager';
|
||||||
import * as piManager from './pi-manager';
|
import * as piManager from './pi-manager';
|
||||||
import * as queueRunner from './queue-runner';
|
import * as queueRunner from './queue-runner';
|
||||||
|
import { initEmailCron, stopEmailCron } from './email-cron';
|
||||||
|
|
||||||
const PORT = Number(process.env.SIDECAR_PORT ?? '5100');
|
const PORT = Number(process.env.SIDECAR_PORT ?? '5100');
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
@@ -31,6 +32,9 @@ queueRunner.initQueue().catch((err) => {
|
|||||||
console.error('[sidecar] failed to initialize queue:', err);
|
console.error('[sidecar] failed to initialize queue:', err);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Start email sync cron
|
||||||
|
// initEmailCron(); // TODO: re-enable after initial sync testing
|
||||||
|
|
||||||
// ── WebSocket connections ──
|
// ── WebSocket connections ──
|
||||||
|
|
||||||
const clients = new Set<ServerWebSocket<unknown>>();
|
const clients = new Set<ServerWebSocket<unknown>>();
|
||||||
@@ -251,6 +255,7 @@ console.log(`[sidecar] listening on 127.0.0.1:${PORT}`);
|
|||||||
|
|
||||||
async function shutdown(signal: string) {
|
async function shutdown(signal: string) {
|
||||||
console.log(`[sidecar] ${signal} received, saving state...`);
|
console.log(`[sidecar] ${signal} received, saving state...`);
|
||||||
|
stopEmailCron();
|
||||||
await flushAndSave();
|
await flushAndSave();
|
||||||
releaseLock();
|
releaseLock();
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
|
|||||||
@@ -5,6 +5,17 @@ import { getHandler } from '../queue/handler-registry';
|
|||||||
// Import handlers to register them
|
// Import handlers to register them
|
||||||
import '../queue/handlers';
|
import '../queue/handlers';
|
||||||
|
|
||||||
|
function formatDuration(ms: number): string {
|
||||||
|
const s = Math.floor(ms / 1000);
|
||||||
|
if (s < 60) return `${s}s`;
|
||||||
|
const m = Math.floor(s / 60);
|
||||||
|
const rem = s % 60;
|
||||||
|
if (m < 60) return rem > 0 ? `${m}m${rem}s` : `${m}m`;
|
||||||
|
const h = Math.floor(m / 60);
|
||||||
|
const remM = m % 60;
|
||||||
|
return remM > 0 ? `${h}h${remM}m` : `${h}h`;
|
||||||
|
}
|
||||||
|
|
||||||
const activeLanes = new Map<string, boolean>();
|
const activeLanes = new Map<string, boolean>();
|
||||||
const PROGRESS_THROTTLE_MS = 1000;
|
const PROGRESS_THROTTLE_MS = 1000;
|
||||||
|
|
||||||
@@ -73,6 +84,12 @@ async function resumeInterruptedJobs() {
|
|||||||
console.log(`[sidecar:queue] reset interrupted job ${job.id} back to queued`);
|
console.log(`[sidecar:queue] reset interrupted job ${job.id} back to queued`);
|
||||||
lanesToKick.add(job.lane);
|
lanesToKick.add(job.lane);
|
||||||
} else if (job.status === 'queued') {
|
} else if (job.status === 'queued') {
|
||||||
|
// Clear retry delay on restart — no reason to wait after a sidecar restart
|
||||||
|
if (job.retryAt) {
|
||||||
|
job.retryAt = undefined;
|
||||||
|
await writeJob(job);
|
||||||
|
console.log(`[sidecar:queue] cleared retry delay for job ${job.id}`);
|
||||||
|
}
|
||||||
lanesToKick.add(job.lane);
|
lanesToKick.add(job.lane);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -136,8 +153,9 @@ async function runJob(job: Job) {
|
|||||||
job.retryAt = undefined;
|
job.retryAt = undefined;
|
||||||
await writeJob(job);
|
await writeJob(job);
|
||||||
const isRetry = (job.retries ?? 0) > 0;
|
const isRetry = (job.retries ?? 0) > 0;
|
||||||
|
const startTime = Date.now();
|
||||||
console.log(
|
console.log(
|
||||||
`[sidecar:queue] ${isRetry ? 'resuming' : 'running'} job ${job.id} (${job.type})${isRetry ? ` retry ${job.retries}` : ''}`,
|
`[sidecar:queue] ▶ ${isRetry ? 'resuming' : 'running'} job ${job.id} (${job.type})${isRetry ? ` retry ${job.retries}` : ''}`,
|
||||||
);
|
);
|
||||||
|
|
||||||
const sharedMeta: Record<string, unknown> = { ...(job.meta ?? {}) };
|
const sharedMeta: Record<string, unknown> = { ...(job.meta ?? {}) };
|
||||||
@@ -187,6 +205,7 @@ async function runJob(job: Job) {
|
|||||||
await writeJob(fresh);
|
await writeJob(fresh);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
const errorMessage = err instanceof Error ? err.message : String(err);
|
const errorMessage = err instanceof Error ? err.message : String(err);
|
||||||
|
console.error(`[sidecar:queue] step "${step.name}" failed: ${errorMessage}`);
|
||||||
step.status = 'failed';
|
step.status = 'failed';
|
||||||
step.error = errorMessage;
|
step.error = errorMessage;
|
||||||
step.completedAt = Date.now();
|
step.completedAt = Date.now();
|
||||||
@@ -218,7 +237,7 @@ async function runJob(job: Job) {
|
|||||||
fresh.completedAt = Date.now();
|
fresh.completedAt = Date.now();
|
||||||
fresh.meta = { ...fresh.meta, ...sharedMeta };
|
fresh.meta = { ...fresh.meta, ...sharedMeta };
|
||||||
await writeJob(fresh);
|
await writeJob(fresh);
|
||||||
console.error(`[sidecar:queue] job ${fresh.id} failed at step "${step.name}":`, errorMessage);
|
console.error(`[sidecar:queue] ✗ job ${fresh.id} failed at step "${step.name}" in ${formatDuration(Date.now() - startTime)}:`, errorMessage);
|
||||||
await notifyFailure(fresh);
|
await notifyFailure(fresh);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -230,7 +249,7 @@ async function runJob(job: Job) {
|
|||||||
final.completedAt = Date.now();
|
final.completedAt = Date.now();
|
||||||
final.meta = { ...final.meta, ...sharedMeta };
|
final.meta = { ...final.meta, ...sharedMeta };
|
||||||
await writeJob(final);
|
await writeJob(final);
|
||||||
console.log(`[sidecar:queue] job ${final.id} completed`);
|
console.log(`[sidecar:queue] ✓ job ${final.id} completed in ${formatDuration(Date.now() - startTime)}`);
|
||||||
await notifyCompletion(final);
|
await notifyCompletion(final);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user