workspaces to dashboards, imap email sync, ffmpeg tool, tts fix, file browser refresh, automation sidebar reorder

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-02 20:00:45 +00:00
co-authored by Claude Opus 4.6
parent bd362dc586
commit 927267e041
98 changed files with 1176 additions and 802 deletions
+138 -320
View File
@@ -1,8 +1,8 @@
import type { Database } from 'bun:sqlite';
import { ImapFlow } from 'imapflow';
import type { JobHandler } from '../types';
import { registerHandler } from '../handler-registry';
import { DATA_PATH } from '../../data-path';
import { openEmailDb, upsertFromRawEml, getSyncMeta, setSyncMeta, updateEmailLabels } from '../../api/email/email-db';
import { openEmailDb, upsertFromRawEml, getSyncMeta, setSyncMeta } from '../../api/email/email-db';
import { getServerIntegration, getUserByEmail, getUserIntegration } from 'officerdb';
type GoogleCredentials = {
@@ -76,267 +76,145 @@ async function getValidAccessToken(creds: GoogleCredentials): Promise<string> {
return data.access_token;
}
const GMAIL_BASE = 'https://gmail.googleapis.com/gmail/v1/users/me';
// ── IMAP label mapping ──
async function gmailGet(token: string, path: string, params?: Record<string, string>): Promise<unknown> {
const url = new URL(`${GMAIL_BASE}${path}`);
if (params) {
for (const [k, v] of Object.entries(params)) {
if (v) url.searchParams.set(k, v);
}
}
const res = await fetch(url.toString(), {
headers: { Authorization: `Bearer ${token}` },
});
if (!res.ok) {
const error = await res.text().catch(() => '');
throw new Error(`Gmail API error (${res.status}): ${error}`);
}
return res.json();
}
// ── Pre-flight message count ──
async function countMessages(token: string, query?: string): Promise<number> {
let count = 0;
let pageToken: string | undefined;
do {
const params: Record<string, string> = { maxResults: '500' };
if (query) params.q = query;
if (pageToken) params.pageToken = pageToken;
const list = (await gmailGet(token, '/messages', params)) as {
messages?: Array<{ id: string }>;
nextPageToken?: string;
};
count += list.messages?.length ?? 0;
if (!list.messages?.length) break;
pageToken = list.nextPageToken;
} while (pageToken);
return count;
}
// ── Full sync: Gmail API → SQLite directly ──
type SyncProgress = { saved: number; skipped: number; errors: number; page: number };
type OnProgress = (progress: SyncProgress) => void;
type SyncResult = {
saved: number;
skipped: number;
errors: number;
labelMap: Map<string, string[]>;
maxHistoryId: string | null;
const SYSTEM_LABEL_MAP: Record<string, string> = {
'\\Inbox': 'inbox',
'\\Sent': 'sent',
'\\Trash': 'trash',
'\\Spam': 'spam',
'\\Draft': 'draft',
'\\Starred': 'starred',
'\\Important': 'important',
};
async function syncInbox(
token: string,
db: Database,
function mapImapLabels(labels: Set<string> | undefined): string[] {
if (!labels) return [];
const mapped: string[] = [];
for (const label of labels) {
const system = SYSTEM_LABEL_MAP[label];
if (system) {
mapped.push(system);
} else {
mapped.push(label.toLowerCase());
}
}
return mapped;
}
// ── IMAP sync ──
type ImapSyncResult = { saved: number; skipped: number; errors: number };
// Mailboxes to sync: All Mail has everything except Trash and Spam
const SYNC_SPECIAL_USE = ['\\All', '\\Trash', '\\Junk'];
async function syncViaImap(
accessToken: string,
emailAccount: string,
query?: string,
onProgress?: OnProgress,
): Promise<SyncResult> {
// Dedup via DB
const existingIds = new Set<string>();
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
for (const row of rows) existingIds.add(row.id);
db: Database,
since: Date | null,
onProgress?: (saved: number, skipped: number) => void,
): Promise<ImapSyncResult> {
const client = new ImapFlow({
host: 'imap.gmail.com',
port: 993,
secure: true,
auth: { user: emailAccount, accessToken },
logger: false,
});
try {
await client.connect();
} catch (err) {
console.error('[gmail-sync] IMAP connect failed:', err);
throw err;
}
let saved = 0;
let skipped = 0;
let errors = 0;
let page = 0;
let pageToken: string | undefined;
const labelMap = new Map<string, string[]>();
let maxHistoryId: bigint | null = null;
do {
const params: Record<string, string> = { maxResults: '100' };
if (query) params.q = query;
if (pageToken) params.pageToken = pageToken;
// Load existing IDs for dedup (once, shared across mailboxes)
const existingIds = new Set<string>();
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
for (const row of rows) existingIds.add(row.id);
const list = (await gmailGet(token, '/messages', params)) as {
messages?: Array<{ id: string }>;
nextPageToken?: string;
};
try {
// Find mailboxes by specialUse flag (locale-independent)
const allMailboxes = await client.list();
const toSync: Array<{ path: string; specialUse: string }> = [];
for (const mailbox of allMailboxes) {
if (mailbox.specialUse && SYNC_SPECIAL_USE.includes(mailbox.specialUse)) {
toSync.push({ path: mailbox.path, specialUse: mailbox.specialUse });
}
}
const messages = list.messages ?? [];
if (messages.length === 0) break;
if (toSync.length === 0) {
console.error('[gmail-sync] No mailboxes found to sync');
return { saved, skipped, errors };
}
for (let i = 0; i < messages.length; i += 5) {
const batch = messages.slice(i, i + 5);
await Promise.all(
batch.map(async ({ id }) => {
if (existingIds.has(id)) {
skipped++;
return;
}
for (const mailbox of toSync) {
console.log(`[gmail-sync] Opening ${mailbox.path} (${mailbox.specialUse})...`);
const lock = await client.getMailboxLock(mailbox.path);
try {
const searchCriteria = since ? { since } : { all: true };
const uids = await client.search(searchCriteria, { uid: true });
if (!uids || uids.length === 0) {
console.log(`[gmail-sync] ${mailbox.path}: no messages`);
continue;
}
console.log(`[gmail-sync] ${mailbox.path}: ${uids.length} messages`);
const uidRange = uids.join(',');
const messages = client.fetch(uidRange, {
source: true,
labels: true,
}, { uid: true });
for await (const msg of messages) {
try {
const msg = (await gmailGet(token, `/messages/${id}`, { format: 'raw' })) as {
id: string;
internalDate?: string;
raw: string;
labelIds?: string[];
historyId?: string;
};
const rawEmail = Buffer.from(msg.raw, 'base64url').toString('utf-8');
upsertFromRawEml({ db, id, raw: rawEmail, integration: 'gmail', emailAccount, labels: msg.labelIds });
existingIds.add(id);
if (!msg.emailId || !msg.source) continue;
const gmailId = BigInt(msg.emailId).toString(16);
if (existingIds.has(gmailId)) {
skipped++;
if ((saved + skipped) % 500 === 0) onProgress?.(saved, skipped);
continue;
}
const rawEmail = msg.source.toString('utf-8');
const labels = mapImapLabels(msg.labels);
upsertFromRawEml({ db, id: gmailId, raw: rawEmail, integration: 'gmail', emailAccount, labels });
existingIds.add(gmailId);
saved++;
if (msg.labelIds) labelMap.set(id, msg.labelIds);
if (msg.historyId) {
const hid = BigInt(msg.historyId);
if (maxHistoryId === null || hid > maxHistoryId) maxHistoryId = hid;
if ((saved + skipped) % 100 === 0) {
console.log(`[gmail-sync] Progress: saved ${saved}, skipped ${skipped}, errors ${errors}`);
onProgress?.(saved, skipped);
}
} catch {
errors++;
}
}),
);
}
page++;
console.log(`[gmail-sync] Page ${page}: saved ${saved}, skipped ${skipped}, errors ${errors}`);
onProgress?.({ saved, skipped, errors, page });
pageToken = list.nextPageToken;
} while (pageToken);
return { saved, skipped, errors, labelMap, maxHistoryId: maxHistoryId !== null ? String(maxHistoryId) : null };
}
// ── Gmail History API (incremental sync) ──
type HistoryMessage = { id: string; labelIds?: string[] };
type HistoryRecord = {
id: string;
messagesAdded?: Array<{ message: HistoryMessage }>;
messagesDeleted?: Array<{ message: HistoryMessage }>;
labelsAdded?: Array<{ message: HistoryMessage; labelIds: string[] }>;
labelsRemoved?: Array<{ message: HistoryMessage; labelIds: string[] }>;
};
type HistoryResponse = {
history?: HistoryRecord[];
nextPageToken?: string;
historyId: string;
};
type IncrementalResult =
| { stale: false; added: number; deleted: number; relabeled: number; maxHistoryId: string }
| { stale: true };
async function syncIncremental(
token: string,
db: Database,
lastHistoryId: string,
emailAccount: string,
): Promise<IncrementalResult> {
let pageToken: string | undefined;
let added = 0;
let deleted = 0;
let relabeled = 0;
let latestHistoryId = lastHistoryId;
do {
const url = new URL(`${GMAIL_BASE}/history`);
url.searchParams.set('startHistoryId', lastHistoryId);
for (const ht of ['messageAdded', 'messageDeleted', 'labelAdded', 'labelRemoved']) {
url.searchParams.append('historyTypes', ht);
}
if (pageToken) url.searchParams.set('pageToken', pageToken);
let data: HistoryResponse;
try {
const res = await fetch(url.toString(), { headers: { Authorization: `Bearer ${token}` } });
if (!res.ok) {
const error = await res.text().catch(() => '');
throw new Error(`Gmail API error (${res.status}): ${error}`);
}
data = (await res.json()) as HistoryResponse;
} catch (err) {
// historyId too old — Gmail returns 404
if (err instanceof Error && err.message.includes('404')) {
return { stale: true };
}
throw err;
}
latestHistoryId = data.historyId;
for (const record of data.history ?? []) {
// New messages
for (const { message } of record.messagesAdded ?? []) {
try {
const msg = (await gmailGet(token, `/messages/${message.id}`, { format: 'raw' })) as {
id: string;
internalDate?: string;
raw: string;
labelIds?: string[];
historyId?: string;
};
const rawEmail = Buffer.from(msg.raw, 'base64url').toString('utf-8');
upsertFromRawEml({ db, id: msg.id, raw: rawEmail, integration: 'gmail', emailAccount, labels: msg.labelIds });
added++;
} catch {
/* skip individual failures */
}
}
// Deleted messages
for (const { message } of record.messagesDeleted ?? []) {
db.run('UPDATE emails SET deleted = 1 WHERE id = ?', [message.id]);
deleted++;
}
// Labels added
for (const { message, labelIds } of record.labelsAdded ?? []) {
const row = db.query('SELECT labels FROM emails WHERE id = ?').get(message.id) as { labels: string | null } | null;
if (row) {
const existing = row.labels ? row.labels.split(',') : [];
const merged = [...new Set([...existing, ...labelIds.map((l) => l.toLowerCase())])];
updateEmailLabels(db, message.id, merged);
relabeled++;
}
}
// Labels removed
for (const { message, labelIds } of record.labelsRemoved ?? []) {
const row = db.query('SELECT labels FROM emails WHERE id = ?').get(message.id) as { labels: string | null } | null;
if (row) {
const removeSet = new Set(labelIds.map((l) => l.toLowerCase()));
const remaining = (row.labels ? row.labels.split(',') : []).filter((l) => !removeSet.has(l));
updateEmailLabels(db, message.id, remaining);
relabeled++;
}
} finally {
lock.release();
}
}
pageToken = data.nextPageToken;
} while (pageToken);
return { stale: false, added, deleted, relabeled, maxHistoryId: latestHistoryId };
}
const MONTH_NAMES = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'];
function buildMonthRanges(year: number): Array<{ label: string; after: string; before: string }> {
const now = new Date();
const currentMonth = now.getFullYear() === year ? now.getMonth() : 11;
const ranges: Array<{ label: string; after: string; before: string }> = [];
for (let m = 0; m <= currentMonth; m++) {
const after = `${year}/${m + 1}/1`;
const before = m < 11 ? `${year}/${m + 2}/1` : `${year + 1}/1/1`;
ranges.push({ label: `${MONTH_NAMES[m]!} ${year}`, after, before });
} finally {
await client.logout();
}
return ranges;
return { saved, skipped, errors };
}
// ── Handler ──
const gmailSyncHandler: JobHandler = {
type: 'gmail-sync',
steps: [
@@ -356,102 +234,42 @@ const gmailSyncHandler: JobHandler = {
const db = openEmailDb(ctx.job.userId);
try {
// Try incremental sync first (only for non-year-scoped syncs)
if (!year) {
const lastHistoryId = getSyncMeta(db, 'last_history_id');
if (lastHistoryId) {
await ctx.updateProgress({ current: 0, total: 0, label: 'Incremental sync' });
console.log(`[gmail-sync] Attempting incremental sync from historyId ${lastHistoryId}`);
const result = await syncIncremental(token, db, lastHistoryId, ctx.job.userId);
if (!result.stale) {
setSyncMeta(db, 'last_history_id', result.maxHistoryId);
setSyncMeta(db, 'last_sync_at', new Date().toISOString());
console.log(`[gmail-sync] Incremental: +${result.added} added, -${result.deleted} deleted, ~${result.relabeled} relabeled`);
await ctx.updateProgress({ current: 1, total: 1, label: 'Done (incremental)' });
return;
}
console.log('[gmail-sync] historyId stale, falling back to full sync');
// Bootstrap sync_meta from existing emails if DB was imported without metadata
if (!getSyncMeta(db, 'last_sync_date')) {
const newest = db.query('SELECT date FROM emails ORDER BY date DESC LIMIT 1').get() as { date: string } | null;
if (newest?.date) {
console.log(`[gmail-sync] Bootstrapping last_sync_date from existing DB: ${newest.date}`);
setSyncMeta(db, 'last_sync_date', newest.date);
}
}
// Full sync
let totalSaved = 0;
let totalSkipped = 0;
let totalErrors = 0;
let allLabelMaps: Map<string, string[]> = new Map();
let maxHistoryId: string | null = null;
// Determine since date
let since: Date | null = null;
if (year) {
// Year-scoped sync: count total emails first, then sync month by month
const yearQuery = `after:${year}/1/1 before:${year + 1}/1/1`;
await ctx.updateProgress({ current: 0, total: 0, label: 'Counting emails...' });
const totalEmails = await countMessages(token, yearQuery);
console.log(`[gmail-sync] Pre-flight: ${totalEmails} emails for ${year}`);
const months = buildMonthRanges(year);
for (let i = 0; i < months.length; i++) {
const month = months[i]!;
await ctx.updateProgress({ current: totalSaved + totalSkipped, total: totalEmails, label: month.label });
const query = `after:${month.after} before:${month.before}`;
const result = await syncInbox(token, db, ctx.job.userId, query, (p) => {
const current = totalSaved + p.saved + p.skipped + p.errors;
const label = `${month.label} — Saved ${(totalSaved + p.saved).toLocaleString()} of ${totalEmails.toLocaleString()}`;
ctx.updateProgress({ current, total: totalEmails, label });
});
totalSaved += result.saved;
totalSkipped += result.skipped;
totalErrors += result.errors;
for (const [id, labels] of result.labelMap) allLabelMaps.set(id, labels);
if (result.maxHistoryId) {
if (!maxHistoryId || BigInt(result.maxHistoryId) > BigInt(maxHistoryId)) {
maxHistoryId = result.maxHistoryId;
}
}
}
await ctx.updateProgress({ current: totalEmails, total: totalEmails, label: 'Done' });
since = new Date(year, 0, 1);
} else {
const lastSyncDate = getSyncMeta(db, 'last_sync_date');
let syncQuery: string | undefined;
if (lastSyncDate) {
const d = new Date(lastSyncDate);
syncQuery = `after:${d.getFullYear()}/${d.getMonth() + 1}/${d.getDate()}`;
console.log(`[gmail-sync] Scoping full sync with query: ${syncQuery}`);
since = new Date(lastSyncDate);
console.log(`[gmail-sync] Syncing since ${since.toISOString()}`);
}
// Count total emails for accurate progress
await ctx.updateProgress({ current: 0, total: 0, label: 'Counting emails...' });
const totalEmails = await countMessages(token, syncQuery);
console.log(`[gmail-sync] Pre-flight: ${totalEmails} emails`);
await ctx.updateProgress({ current: 0, total: totalEmails, label: 'Starting sync' });
const result = await syncInbox(token, db, ctx.job.userId, syncQuery, (p) => {
const current = p.saved + p.skipped + p.errors;
const label = `Saved ${p.saved.toLocaleString()} of ${totalEmails.toLocaleString()}`;
ctx.updateProgress({ current, total: totalEmails, label });
});
totalSaved = result.saved;
totalSkipped = result.skipped;
totalErrors = result.errors;
allLabelMaps = result.labelMap;
maxHistoryId = result.maxHistoryId;
await ctx.updateProgress({ current: totalEmails, total: totalEmails, label: 'Done' });
}
console.log(`[gmail-sync] Full: saved ${totalSaved}, skipped ${totalSkipped}, errors ${totalErrors}`);
await ctx.updateProgress({ current: 0, total: 0, label: 'Connecting via IMAP...' });
// Update labels for emails that were skipped but have new label data
let labelsUpdated = 0;
for (const [id, labels] of allLabelMaps) {
updateEmailLabels(db, id, labels);
labelsUpdated++;
}
if (labelsUpdated > 0) console.log(`[gmail-sync] Updated labels for ${labelsUpdated} emails`);
const result = await syncViaImap(token, ctx.job.userId, db, since, (saved, skipped) => {
const label = year
? `${year} — Saved ${saved.toLocaleString()}, skipped ${skipped.toLocaleString()}`
: `Saved ${saved.toLocaleString()}, skipped ${skipped.toLocaleString()}`;
ctx.updateProgress({ current: saved + skipped, total: 0, label });
});
// Store sync state
if (maxHistoryId) {
setSyncMeta(db, 'last_history_id', maxHistoryId);
}
console.log(`[gmail-sync] Done: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`);
setSyncMeta(db, 'last_sync_date', new Date().toISOString().split('T')[0]!);
setSyncMeta(db, 'last_sync_at', new Date().toISOString());
await ctx.updateProgress({ current: 1, total: 1, label: `Done — ${result.saved} new emails` });
} finally {
db.close();
}