email, extracted from the officer platform
IMAP and Gmail sync into a per-account SQLite store, with the mail read and composed in the browser. moved whole from src/servers/sidecar/email (14 files), src/servers/api/email, officer_db/src/email, and the Screens/Dashboard/Email tree. what changed on the way out: - credentials are encrypted. the column was plaintext jsonb holding an IMAP app password that the SMTP transport read straight out of the row; it is now AES-256-GCM under this plugin's own 'email' secret-store purpose, sealed and opened in queries.ts alone so nothing above it can hold ciphertext by accident. no plaintext fallback on read - a value that does not verify is an error. - the router derives its prefix from mountPrefix() instead of the literal '/api/email', which was only correct for a first-party publisher. - the frontend became panels.ts + layout.ts. the two panel appTypes that were null are now declared, because the platform screen used to override them by panel id and a plugin has no such hook. - account management became a modal. it was a section in the platform's Settings, which a plugin cannot contribute to, and a panel does not work either: PluginScreen renders the workspace locked, so a panel outside the default layout is unreachable. it follows Compose, which had already solved this. - 17 palette classes replaced with semantic tokens. not verified: this has never run. it was switched off in the platform on 2026-08-13 and its table was never created, so nothing has exercised sync, sending or the UI since. the first install is the first test.
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,327 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { getUserIntegration, upsertUserIntegration, getServerIntegration } from 'officerdb';
|
||||
import { getEmailAccount } from '../db/queries';
|
||||
import { refreshGoogleAccessToken } from '@@/api/integrations/google-auth';
|
||||
import { openEmailDb, getSyncMeta, setSyncMeta, upsertFromRawEml } from './store';
|
||||
import { readSyncMeta, writeSyncMeta } from './sync-meta';
|
||||
import { type GmailCredentials, loadGmailCredentials, gmailApiSync } from './gmail-api';
|
||||
|
||||
export type ResyncResult = { saved: number; skipped: number; errors: number };
|
||||
|
||||
// ── Gmail resync (REST API, history-based) ──
|
||||
|
||||
async function refreshCredentials(creds: GmailCredentials): Promise<GmailCredentials> {
|
||||
if (!creds.accessToken || !creds.refreshToken) return creds;
|
||||
|
||||
const tokenExpired = !creds.expiresAt || creds.expiresAt < Date.now() + 60_000;
|
||||
if (!tokenExpired) return creds;
|
||||
|
||||
console.log('[resync] Refreshing OAuth token');
|
||||
const refreshed = await refreshGoogleAccessToken(creds.refreshToken);
|
||||
|
||||
const userGoogle = await getUserIntegration(creds.userId, 'google');
|
||||
const existingConfig = (userGoogle?.config as Record<string, unknown>) ?? {};
|
||||
const serverGoogle = await getServerIntegration('google');
|
||||
await upsertUserIntegration({
|
||||
userId: creds.userId,
|
||||
provider: 'google',
|
||||
serverIntegrationId: serverGoogle?.id,
|
||||
config: { ...existingConfig, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
|
||||
});
|
||||
|
||||
return { ...creds, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt };
|
||||
}
|
||||
|
||||
async function gmailResync(userEmail: string, accountEmail: string): Promise<ResyncResult> {
|
||||
let creds = await loadGmailCredentials(userEmail);
|
||||
if (!creds.accessToken) {
|
||||
throw new Error('OAuth not configured — connect Google in Settings → Integrations for resyncs');
|
||||
}
|
||||
|
||||
creds = await refreshCredentials(creds);
|
||||
|
||||
const db = openEmailDb(userEmail, accountEmail);
|
||||
try {
|
||||
const result = await gmailApiSync({ creds, db });
|
||||
|
||||
setSyncMeta(db, 'last_sync_date', new Date().toISOString().split('T')[0]!);
|
||||
setSyncMeta(db, 'last_sync_at', new Date().toISOString());
|
||||
|
||||
console.log(`[resync] Gmail done: saved ${result.saved}, skipped ${result.skipped}, errors ${result.errors}`);
|
||||
return result;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
}
|
||||
|
||||
// ── Generic IMAP resync ──
|
||||
|
||||
type ImapAccountInfo = {
|
||||
id: number;
|
||||
userId: number;
|
||||
email: string;
|
||||
imapHost: string;
|
||||
imapPort: number;
|
||||
imapSecure: boolean;
|
||||
provider: string;
|
||||
authType: string;
|
||||
credentials: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type FolderInfo = {
|
||||
specialUse?: string;
|
||||
path: string;
|
||||
flags: Set<string>;
|
||||
status?: { uidNext?: number; uidValidity?: number };
|
||||
};
|
||||
|
||||
const SPECIAL_USE_LABEL_MAP: Record<string, string> = {
|
||||
'\\Inbox': 'inbox',
|
||||
'\\Sent': 'sent',
|
||||
'\\Drafts': 'draft',
|
||||
'\\Flagged': 'starred',
|
||||
'\\Trash': 'trash',
|
||||
'\\Junk': 'spam',
|
||||
'\\All': 'archive',
|
||||
};
|
||||
|
||||
const SKIP_SPECIAL_USE = new Set(['\\Trash', '\\Junk', '\\All']);
|
||||
|
||||
function shouldSkipFolder(folder: FolderInfo): boolean {
|
||||
if (folder.flags.has('\\Noselect') || folder.flags.has('\\NonExistent')) return true;
|
||||
if (folder.specialUse && SKIP_SPECIAL_USE.has(folder.specialUse)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
function folderToLabel(folder: FolderInfo): string {
|
||||
if (folder.specialUse && SPECIAL_USE_LABEL_MAP[folder.specialUse]) {
|
||||
return SPECIAL_USE_LABEL_MAP[folder.specialUse]!;
|
||||
}
|
||||
if (folder.path === 'INBOX') return 'inbox';
|
||||
return folder.path.toLowerCase();
|
||||
}
|
||||
|
||||
function messageIdToStableId(raw: string): string | null {
|
||||
const match = raw.match(/^Message-Id:\s*<?([^>\s]+)>?/im);
|
||||
if (!match?.[1]) return null;
|
||||
return createHash('sha1').update(match[1]).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
async function resolveImapAuth(
|
||||
account: ImapAccountInfo,
|
||||
): Promise<{ user: string; pass?: string; accessToken?: string }> {
|
||||
if (account.authType !== 'oauth') {
|
||||
return { user: account.email, pass: account.credentials.password as string };
|
||||
}
|
||||
|
||||
const userGoogle = await getUserIntegration(account.userId, 'google');
|
||||
const config = userGoogle?.config as Record<string, unknown> | undefined;
|
||||
if (!config?.refreshToken) throw new Error('Google OAuth not configured — reconnect your Google account');
|
||||
|
||||
const expiresAt = config.expiresAt as number | undefined;
|
||||
const tokenExpired = !expiresAt || expiresAt < Date.now() + 60_000;
|
||||
|
||||
if (!tokenExpired && config.accessToken) {
|
||||
return { user: account.email, accessToken: config.accessToken as string };
|
||||
}
|
||||
|
||||
const refreshed = await refreshGoogleAccessToken(config.refreshToken as string);
|
||||
const serverGoogle = await getServerIntegration('google');
|
||||
await upsertUserIntegration({
|
||||
userId: account.userId,
|
||||
provider: 'google',
|
||||
serverIntegrationId: serverGoogle?.id,
|
||||
config: { ...config, accessToken: refreshed.accessToken, expiresAt: refreshed.expiresAt },
|
||||
});
|
||||
|
||||
return { user: account.email, accessToken: refreshed.accessToken };
|
||||
}
|
||||
|
||||
async function imapResync(account: ImapAccountInfo, userEmail: string): Promise<ResyncResult> {
|
||||
const { ImapFlow } = await import('imapflow');
|
||||
|
||||
const imapAuth = await resolveImapAuth(account);
|
||||
|
||||
const freshAccount = await getEmailAccount(account.id);
|
||||
if (!freshAccount) throw new Error(`Email account ${account.id} not found`);
|
||||
|
||||
const db = openEmailDb(userEmail, account.email);
|
||||
// The position lives in this file now; the Postgres column is only a one-time backfill source for
|
||||
// accounts that predate the move. See sync-meta.ts.
|
||||
const syncMeta: Record<string, unknown> = readSyncMeta(db, freshAccount.syncMeta as Record<string, unknown>);
|
||||
const existingIds = new Set<string>();
|
||||
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
||||
for (const row of rows) existingIds.add(row.id);
|
||||
|
||||
let saved = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
|
||||
const client = new ImapFlow({
|
||||
host: account.imapHost,
|
||||
port: account.imapPort,
|
||||
secure: account.imapSecure,
|
||||
auth: imapAuth,
|
||||
logger: false,
|
||||
socketTimeout: 5 * 60 * 1000,
|
||||
});
|
||||
|
||||
try {
|
||||
await client.connect();
|
||||
console.log('[resync] IMAP connected');
|
||||
|
||||
const folders = (await client.list({ statusQuery: { uidNext: true, uidValidity: true } })) as FolderInfo[];
|
||||
|
||||
for (const folder of folders) {
|
||||
if (shouldSkipFolder(folder)) continue;
|
||||
|
||||
const uidValidityKey = `imap_uidvalidity:${folder.path}`;
|
||||
const lastUidKey = `imap_lastuid:${folder.path}`;
|
||||
const storedUidValidity = syncMeta[uidValidityKey] as string | undefined;
|
||||
const storedLastUid = syncMeta[lastUidKey] as string | undefined;
|
||||
const uidValidity = folder.status?.uidValidity ? String(folder.status.uidValidity) : null;
|
||||
const uidNext = folder.status?.uidNext ?? 0;
|
||||
const lastUid =
|
||||
storedUidValidity && uidValidity === storedUidValidity && storedLastUid ? parseInt(storedLastUid, 10) : 0;
|
||||
|
||||
if (lastUid > 0 && uidNext <= lastUid + 1) continue;
|
||||
|
||||
let lock;
|
||||
try {
|
||||
lock = await client.getMailboxLock(folder.path);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const mailbox = client.mailbox;
|
||||
if (!mailbox) continue;
|
||||
|
||||
const mbUidValidity = String(mailbox.uidValidity);
|
||||
const effectiveLastUid = storedUidValidity === mbUidValidity ? lastUid : 0;
|
||||
let maxUid = effectiveLastUid;
|
||||
const label = folderToLabel(folder);
|
||||
const range = effectiveLastUid > 0 ? `${effectiveLastUid + 1}:*` : '1:*';
|
||||
|
||||
try {
|
||||
const fetchOpts: Record<string, unknown> = { source: true, uid: true };
|
||||
if (account.provider === 'gmail') fetchOpts.labels = true;
|
||||
|
||||
for await (const msg of client.fetch(range, fetchOpts, { uid: true })) {
|
||||
if (msg.uid <= effectiveLastUid) continue;
|
||||
maxUid = Math.max(maxUid, msg.uid);
|
||||
|
||||
if (!msg.source) {
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const raw = msg.source.toString('utf-8');
|
||||
const id = messageIdToStableId(raw);
|
||||
if (!id) {
|
||||
errors++;
|
||||
continue;
|
||||
}
|
||||
if (existingIds.has(id)) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
upsertFromRawEml({
|
||||
db,
|
||||
id,
|
||||
raw,
|
||||
integration: account.provider,
|
||||
emailAccount: account.email,
|
||||
labels: [label],
|
||||
});
|
||||
existingIds.add(id);
|
||||
saved++;
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
} catch (fetchErr) {
|
||||
const errMsg = fetchErr instanceof Error ? fetchErr.message : String(fetchErr);
|
||||
if (!errMsg.includes('Nothing to fetch')) {
|
||||
console.log(`[resync] Fetch error in ${folder.path}: ${errMsg}`);
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
syncMeta[uidValidityKey] = mbUidValidity;
|
||||
if (maxUid > effectiveLastUid) {
|
||||
syncMeta[lastUidKey] = String(maxUid);
|
||||
}
|
||||
} finally {
|
||||
lock.release();
|
||||
}
|
||||
}
|
||||
|
||||
await client.logout().catch(() => {});
|
||||
} catch (err) {
|
||||
await client.logout().catch(() => {});
|
||||
throw err;
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
syncMeta.last_sync_at = new Date().toISOString();
|
||||
// Reopened deliberately: the sync loop above closes the db in its `finally`, and the position must be
|
||||
// written after the messages it describes are safely stored.
|
||||
const metaDb = openEmailDb(userEmail, account.email);
|
||||
try {
|
||||
writeSyncMeta(metaDb, syncMeta);
|
||||
} finally {
|
||||
metaDb.close();
|
||||
}
|
||||
|
||||
console.log(`[resync] IMAP done: saved ${saved}, skipped ${skipped}, errors ${errors}`);
|
||||
return { saved, skipped, errors };
|
||||
}
|
||||
|
||||
// ── Public API ──
|
||||
|
||||
type ResyncParams = {
|
||||
accountId: number;
|
||||
userEmail: string;
|
||||
userId: number;
|
||||
};
|
||||
|
||||
// Coalesce concurrent resyncs of the same account within this process (IDLE + cron + manual can all
|
||||
// fire) — a caller arriving mid-resync just awaits the one already running.
|
||||
const resyncInFlight = new Map<number, Promise<ResyncResult>>();
|
||||
|
||||
export function performResync(params: ResyncParams): Promise<ResyncResult> {
|
||||
const running = resyncInFlight.get(params.accountId);
|
||||
if (running) return running;
|
||||
const p = doResync(params).finally(() => resyncInFlight.delete(params.accountId));
|
||||
resyncInFlight.set(params.accountId, p);
|
||||
return p;
|
||||
}
|
||||
|
||||
async function doResync({ accountId, userEmail, userId }: ResyncParams): Promise<ResyncResult> {
|
||||
const account = await getEmailAccount(accountId);
|
||||
if (!account || account.userId !== userId) throw new Error('Account not found');
|
||||
|
||||
// Gmail API resync needs OAuth; a gmail account authed with an app password resyncs over IMAP.
|
||||
if (account.provider === 'gmail' && account.authType === 'oauth') {
|
||||
return gmailResync(userEmail, account.email);
|
||||
}
|
||||
|
||||
return imapResync(
|
||||
{
|
||||
id: account.id,
|
||||
userId: account.userId,
|
||||
email: account.email,
|
||||
imapHost: account.imapHost,
|
||||
imapPort: account.imapPort,
|
||||
imapSecure: account.imapSecure,
|
||||
provider: account.provider,
|
||||
authType: account.authType,
|
||||
credentials: account.credentials as Record<string, unknown>,
|
||||
},
|
||||
userEmail,
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user