Email was the one sidecar built inside out. The platform held ~1,800 lines — the per-account
SQLite store, all 14 HTTP routes, account CRUD, resync, IMAP validation — while the 314-line
sidecar was a scheduler that reached BACK into the platform to do anything
(`import { performResync } from '../../api/email/resync'`).
The sidecar now serves its own HTTP listener and announces `email:server`, and
/api/email/* on the platform is createSidecarProxy like every other one: 1,801 lines down
to 22, with no mail knowledge left in it — not a message, not a folder, not a credential.
The routes moved verbatim, Hono and all. http.ts only reconstructs what the platform's
middleware used to provide: `user` on the context, from the X-Officer-User header the proxy
injects (trusted because this server binds loopback), and an error handler that turns
custom-errors into status codes.
The /email/events SSE stream went with them, which removes a whole round trip: the IDLE
watcher used to send `email:new` over the registration socket so the platform could push to
its SSE clients. Those clients are here now, so it calls broadcastEmailNew in-process and
`email:new` is gone from the wire protocol.
DELIBERATELY NOT DONE YET, and left backwards on purpose rather than half-moved:
- The two sync handlers (email-sync 381 lines, gmail-sync 712) still run in the platform's
queue and now import the store from its new home — a platform → sidecar import, which is
the wrong direction and is temporary. Moving them is option (A) from the plan: the sidecar
schedules its own syncs, independent of the platform Jobs list.
- accounts.ts still imports queue/init to enqueue a sync and to report sync status, and
index.ts still carries the queue-over-WS shim that inversion needs.
- The three channel handlers still open the mail store directly rather than asking over HTTP.
Two things worth knowing while testing: a from-scratch sync holds a proxied request open
well past the 60s idle default, hence timeoutSeconds on the proxy; and `gmail-sync` is
hardcoded in all three channel handlers even though the only account is provider=gmail with
auth_type=password, which routes to IMAP — so "sync emails" from a chat channel is
almost certainly already broken, and folds into the next stage.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
747 lines
28 KiB
TypeScript
747 lines
28 KiB
TypeScript
import { Database } from 'bun:sqlite';
|
|
import { createHash } from 'node:crypto';
|
|
import { join, dirname } from 'node:path';
|
|
import { chmodSync, mkdirSync } from 'node:fs';
|
|
import { getEmailDbPath } from '@@/data-path';
|
|
import { getEmailAccounts } from 'officerdb';
|
|
import type { EmailSummary } from 'types';
|
|
|
|
const SCHEMA_TABLES = `
|
|
CREATE TABLE IF NOT EXISTS emails (
|
|
id TEXT PRIMARY KEY,
|
|
integration TEXT NOT NULL DEFAULT 'gmail',
|
|
email_account TEXT NOT NULL DEFAULT '',
|
|
from_name TEXT,
|
|
from_address TEXT,
|
|
from_domain TEXT,
|
|
to_address TEXT,
|
|
cc TEXT,
|
|
subject TEXT,
|
|
date TEXT,
|
|
snippet TEXT,
|
|
html TEXT,
|
|
text_body TEXT,
|
|
attachment_count INTEGER DEFAULT 0,
|
|
read INTEGER DEFAULT 0,
|
|
deleted INTEGER DEFAULT 0,
|
|
labels TEXT,
|
|
thread_id TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS attachments (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
email_id TEXT REFERENCES emails(id) ON DELETE CASCADE,
|
|
idx INTEGER,
|
|
filename TEXT,
|
|
size INTEGER,
|
|
content_type TEXT,
|
|
content TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS sync_meta (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT
|
|
);
|
|
`;
|
|
|
|
const SCHEMA_INDEXES = `
|
|
CREATE INDEX IF NOT EXISTS idx_emails_date ON emails(date);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_from_domain ON emails(from_domain);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_from_address ON emails(from_address);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_integration ON emails(integration);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_email_account ON emails(email_account);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_labels ON emails(labels);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_thread ON emails(thread_id);
|
|
`;
|
|
|
|
/** Convert label IDs to lowercase comma-separated string for storage */
|
|
function labelsToString(labels?: string[]): string | null {
|
|
if (!labels || labels.length === 0) return null;
|
|
return labels.map((l) => l.toLowerCase()).join(',');
|
|
}
|
|
|
|
/** Convert stored comma-separated labels back to array */
|
|
function labelsFromString(value: unknown): string[] | undefined {
|
|
if (typeof value !== 'string' || !value) return undefined;
|
|
return value.split(',');
|
|
}
|
|
|
|
function extractAddress(headerValue: string): { name: string; address: string } {
|
|
const match = headerValue.match(/^"?(.+?)"?\s*<(.+?)>$/);
|
|
if (match) return { name: match[1]!.trim(), address: match[2]!.toLowerCase() };
|
|
const bare = headerValue.trim().toLowerCase();
|
|
return { name: '', address: bare };
|
|
}
|
|
|
|
function extractDomain(address: string): string {
|
|
const at = address.lastIndexOf('@');
|
|
return at >= 0 ? address.slice(at + 1) : '';
|
|
}
|
|
|
|
// ── Conversation threading ──
|
|
// `id` is sha1(Message-Id) (see resync.messageIdToStableId), so hashing a referenced Message-Id the
|
|
// same way yields the *id of that referenced email*. That makes header-based threading trivial:
|
|
// a reply's thread_id is the hash of its root Message-Id, which equals the root email's own id.
|
|
|
|
const hashMsgId = (msgId: string): string => createHash('sha1').update(msgId).digest('hex').slice(0, 16);
|
|
|
|
/** Ordered Message-Ids this email references (References first, root→leaf; else In-Reply-To). */
|
|
function extractReferenceIds(raw: string): string[] {
|
|
const refs = extractFullHeader(raw, 'References') || extractFullHeader(raw, 'In-Reply-To');
|
|
return Array.from(refs.matchAll(/<([^>]+)>/g), (m) => m[1]!.trim()).filter(Boolean);
|
|
}
|
|
|
|
/** Compute a header-based thread_id for a freshly ingested email (falls back to its own id = new thread). */
|
|
function computeThreadId(db: Database, id: string, raw: string): string {
|
|
const refIds = extractReferenceIds(raw);
|
|
if (refIds.length === 0) return id; // no ancestors → this email is a thread root
|
|
const hashed = refIds.map(hashMsgId);
|
|
// Adopt an ancestor's thread if we already have one stored (robust to In-Reply-To-only clients).
|
|
const placeholders = hashed.map(() => '?').join(',');
|
|
const found = db
|
|
.query(`SELECT thread_id FROM emails WHERE id IN (${placeholders}) AND thread_id IS NOT NULL LIMIT 1`)
|
|
.get(...hashed) as { thread_id: string } | null;
|
|
return found?.thread_id ?? hashMsgId(refIds[0]!);
|
|
}
|
|
|
|
const RE_PREFIX = /^\s*((re|fwd?|aw|wg|sv|vs|res|antw)\s*(\[\d+\])?\s*:\s*)+/i;
|
|
|
|
/** Normalize a subject for fallback grouping: strip reply/forward prefixes, fold whitespace, lowercase. */
|
|
function normalizeSubject(subject: string): string {
|
|
return subject.replace(RE_PREFIX, '').replace(/\s+/g, ' ').trim().toLowerCase();
|
|
}
|
|
|
|
const firstAddress = (value: unknown): string => {
|
|
if (typeof value !== 'string') return '';
|
|
const first = value.split(',')[0] ?? '';
|
|
return (first.match(/<([^>]+)>/)?.[1] ?? first).trim().toLowerCase();
|
|
};
|
|
|
|
/**
|
|
* Subject-based thread key for mail synced before header capture (no References available).
|
|
* Groups by normalized subject + the counterpart address, so recurring 1:1 conversations collapse
|
|
* while unrelated same-subject mail from different people stays apart. Trivial subjects stay ungrouped.
|
|
*/
|
|
function fallbackThreadId(row: { id: string; subject: unknown; from_address: unknown; to_address: unknown; email_account: unknown }): string {
|
|
const norm = normalizeSubject(typeof row.subject === 'string' ? row.subject : '');
|
|
if (!norm) return row.id;
|
|
const me = typeof row.email_account === 'string' ? row.email_account.toLowerCase() : '';
|
|
const from = typeof row.from_address === 'string' ? row.from_address.toLowerCase() : '';
|
|
const counterpart = from && from !== me ? from : firstAddress(row.to_address) || from;
|
|
return `s:${norm}|${counterpart}`;
|
|
}
|
|
|
|
export function openEmailDb(ownerEmail: string, accountEmail: string): Database {
|
|
const dbPath = getEmailDbPath(ownerEmail, accountEmail);
|
|
mkdirSync(dirname(dbPath), { recursive: true });
|
|
const db = new Database(dbPath, { create: true });
|
|
// The API and the email sidecar both open this file; wait out a concurrent writer (e.g. a resync
|
|
// or the one-time thread_id backfill) instead of failing immediately with "database is locked".
|
|
db.exec('PRAGMA busy_timeout = 5000');
|
|
db.exec('PRAGMA journal_mode = DELETE');
|
|
db.exec('PRAGMA foreign_keys = ON');
|
|
db.exec(SCHEMA_TABLES);
|
|
migrate(db);
|
|
db.exec(SCHEMA_INDEXES);
|
|
ensureFts(db);
|
|
// Try to chmod, but don't crash if permission denied (e.g., file owned by different user)
|
|
try {
|
|
chmodSync(dbPath, 0o666);
|
|
} catch (err) {
|
|
// File exists with correct permissions, or owned by another user - that's fine
|
|
}
|
|
return db;
|
|
}
|
|
|
|
/**
|
|
* Open the email DB for a user's configured account. Emails are stored per account
|
|
* (email_accounts/<account>/emails.db); for now we use the user's first account. Returns null if the
|
|
* user has no email account configured yet.
|
|
*/
|
|
export async function openUserEmailDb(ownerEmail: string, userId: number): Promise<Database | null> {
|
|
const accounts = await getEmailAccounts(userId);
|
|
const account = accounts.find((a) => a.enabled) ?? accounts[0];
|
|
return account ? openEmailDb(ownerEmail, account.email) : null;
|
|
}
|
|
|
|
function migrate(db: Database): void {
|
|
const cols = db.query('PRAGMA table_info(emails)').all() as Array<{ name: string }>;
|
|
const colNames = new Set(cols.map((c) => c.name));
|
|
|
|
if (!colNames.has('deleted')) {
|
|
db.exec('ALTER TABLE emails ADD COLUMN deleted INTEGER DEFAULT 0');
|
|
}
|
|
if (!colNames.has('integration')) {
|
|
db.exec("ALTER TABLE emails ADD COLUMN integration TEXT NOT NULL DEFAULT 'gmail'");
|
|
}
|
|
if (!colNames.has('email_account')) {
|
|
db.exec("ALTER TABLE emails ADD COLUMN email_account TEXT NOT NULL DEFAULT ''");
|
|
}
|
|
if (!colNames.has('labels')) {
|
|
db.exec('ALTER TABLE emails ADD COLUMN labels TEXT');
|
|
}
|
|
if (!colNames.has('thread_id')) {
|
|
db.exec('ALTER TABLE emails ADD COLUMN thread_id TEXT');
|
|
}
|
|
|
|
// Backfill thread_id for any rows missing it. Header data isn't kept for already-synced mail, so
|
|
// these use the subject-based fallback. New mail gets an exact header-based thread_id at insert.
|
|
backfillThreadIds(db);
|
|
|
|
// Ensure sync_meta table exists (for DBs created before it was added to SCHEMA_TABLES)
|
|
db.exec('CREATE TABLE IF NOT EXISTS sync_meta (key TEXT PRIMARY KEY, value TEXT)');
|
|
|
|
// Add content column to attachments if missing
|
|
const attCols = db.query('PRAGMA table_info(attachments)').all() as Array<{ name: string }>;
|
|
const attColNames = new Set(attCols.map((c) => c.name));
|
|
if (!attColNames.has('content')) {
|
|
db.exec('ALTER TABLE attachments ADD COLUMN content TEXT');
|
|
}
|
|
}
|
|
|
|
function backfillThreadIds(db: Database): void {
|
|
const rows = db
|
|
.query('SELECT id, subject, from_address, to_address, email_account FROM emails WHERE thread_id IS NULL')
|
|
.all() as Array<{ id: string; subject: unknown; from_address: unknown; to_address: unknown; email_account: unknown }>;
|
|
if (rows.length === 0) return;
|
|
|
|
const update = db.prepare('UPDATE emails SET thread_id = ? WHERE id = ?');
|
|
db.exec('BEGIN');
|
|
try {
|
|
for (const row of rows) update.run(fallbackThreadId(row), row.id);
|
|
db.exec('COMMIT');
|
|
} catch (err) {
|
|
db.exec('ROLLBACK');
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ── Full-text search (FTS5) ──
|
|
// A standalone FTS5 index kept in sync with `emails` on every upsert. unicode61 + diacritic folding
|
|
// gives accent-insensitive matching; per-term prefix queries make it feel incremental.
|
|
|
|
function ensureFts(db: Database): void {
|
|
db.exec(
|
|
`CREATE VIRTUAL TABLE IF NOT EXISTS emails_fts USING fts5(
|
|
id UNINDEXED, subject, sender, recipients, snippet, body,
|
|
tokenize = 'unicode61 remove_diacritics 2'
|
|
)`,
|
|
);
|
|
const fts = (db.query('SELECT count(*) AS c FROM emails_fts').get() as { c: number }).c;
|
|
const total = (db.query('SELECT count(*) AS c FROM emails').get() as { c: number }).c;
|
|
if (fts === 0 && total > 0) {
|
|
// Backfill existing mail (first run after this feature ships).
|
|
db.exec(
|
|
`INSERT INTO emails_fts (id, subject, sender, recipients, snippet, body)
|
|
SELECT id, COALESCE(subject, ''),
|
|
TRIM(COALESCE(from_name, '') || ' ' || COALESCE(from_address, '')),
|
|
TRIM(COALESCE(to_address, '') || ' ' || COALESCE(cc, '')),
|
|
COALESCE(snippet, ''), COALESCE(text_body, '')
|
|
FROM emails`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const ftsDeleteStmt = 'DELETE FROM emails_fts WHERE id = ?';
|
|
const ftsInsertStmt = 'INSERT INTO emails_fts (id, subject, sender, recipients, snippet, body) VALUES (?, ?, ?, ?, ?, ?)';
|
|
|
|
function syncFtsRow(db: Database, id: string, subject: string, sender: string, recipients: string, snippet: string, body: string): void {
|
|
db.run(ftsDeleteStmt, [id]);
|
|
db.run(ftsInsertStmt, [id, subject, sender, recipients, snippet, body]);
|
|
}
|
|
|
|
// Gmail-style query parsing. Free text → full-text (prefix-AND). Operators:
|
|
// from:/to:/subject:/body: → FTS5 column filters
|
|
// has:attachment, is:unread/read, label:X, before:/after:YYYY-MM-DD → SQL filters on `emails`
|
|
// Unknown operators fall back to free text. Quoted values (from:"a b") match as an exact phrase.
|
|
const FTS_COLUMNS: Record<string, string> = { from: 'sender', to: 'recipients', subject: 'subject', body: 'body' };
|
|
|
|
const ftsTerm = (value: string, column: string | null, prefix: boolean): string => {
|
|
const esc = value.replace(/"/g, '""');
|
|
return `${column ? column + ':' : ''}"${esc}"${prefix ? '*' : ''}`;
|
|
};
|
|
|
|
const parseSearchDate = (v: string): string | null => {
|
|
const iso = /^\d{4}-\d{2}-\d{2}$/.test(v) ? `${v}T00:00:00.000Z` : v;
|
|
const d = new Date(iso);
|
|
return Number.isNaN(d.getTime()) ? null : d.toISOString();
|
|
};
|
|
|
|
// One AND-group of terms (a single OR branch). `fts` is the FTS5 MATCH expression for this branch;
|
|
// `where`/`params` are its structured SQL filters.
|
|
type Branch = { fts: string; where: string[]; params: string[] };
|
|
|
|
function parseBranch(q: string): Branch {
|
|
const fts: string[] = [];
|
|
const where: string[] = [];
|
|
const params: string[] = [];
|
|
const re = /(\w+):("[^"]*"|\S+)|"([^"]*)"|(\S+)/g;
|
|
let m: RegExpExecArray | null;
|
|
while ((m = re.exec(q)) !== null) {
|
|
if (m[1]) {
|
|
const op = m[1].toLowerCase();
|
|
const raw = m[2]!;
|
|
const quoted = raw.startsWith('"') && raw.endsWith('"');
|
|
const val = quoted ? raw.slice(1, -1) : raw;
|
|
if (!val) continue;
|
|
const lower = val.toLowerCase();
|
|
if (FTS_COLUMNS[op]) {
|
|
fts.push(ftsTerm(val, FTS_COLUMNS[op], !quoted));
|
|
} else if (op === 'label') {
|
|
where.push('e.labels LIKE ?');
|
|
params.push(`%${lower}%`);
|
|
} else if (op === 'has' && (lower === 'attachment' || lower === 'attachments')) {
|
|
where.push('e.attachment_count > 0');
|
|
} else if (op === 'is' && (lower === 'unread' || lower === 'read')) {
|
|
where.push(lower === 'unread' ? 'e.read = 0' : 'e.read = 1');
|
|
} else if (op === 'before' || op === 'older') {
|
|
const d = parseSearchDate(val);
|
|
if (d) {
|
|
where.push('e.date < ?');
|
|
params.push(d);
|
|
}
|
|
} else if (op === 'after' || op === 'newer') {
|
|
const d = parseSearchDate(val);
|
|
if (d) {
|
|
where.push('e.date >= ?');
|
|
params.push(d);
|
|
}
|
|
} else {
|
|
// Unknown operator — treat the whole "op:val" token as free text.
|
|
fts.push(ftsTerm(`${op}:${val}`, null, !quoted));
|
|
}
|
|
} else if (m[3] !== undefined) {
|
|
if (m[3].trim()) fts.push(ftsTerm(m[3], null, false)); // quoted phrase → exact
|
|
} else if (m[4]) {
|
|
fts.push(ftsTerm(m[4], null, true)); // bare word → prefix
|
|
}
|
|
}
|
|
return { fts: fts.join(' '), where, params };
|
|
}
|
|
|
|
export function searchEmails(db: Database, q: string, limit: number, offset: number): { rows: Record<string, unknown>[]; total: number } {
|
|
// Split on top-level uppercase OR into branches (Gmail-style; lowercase "or" stays a search word).
|
|
const branches = q
|
|
.split(/\s+OR\s+/)
|
|
.map(parseBranch)
|
|
.filter((b) => b.fts || b.where.length > 0);
|
|
if (branches.length === 0) return { rows: [], total: 0 };
|
|
|
|
// Each branch becomes one self-contained condition: its FTS terms via an `id IN (FTS subquery)` so
|
|
// full-text and structured filters share a WHERE and branches can be OR'd. All ANDed within a branch.
|
|
const conds: string[] = [];
|
|
const params: string[] = [];
|
|
for (const b of branches) {
|
|
const parts: string[] = [];
|
|
if (b.fts) {
|
|
parts.push('e.id IN (SELECT emails_fts.id FROM emails_fts WHERE emails_fts MATCH ?)');
|
|
params.push(b.fts);
|
|
}
|
|
parts.push(...b.where);
|
|
params.push(...b.params);
|
|
conds.push(`(${parts.join(' AND ')})`);
|
|
}
|
|
|
|
const whereSql = `e.deleted = 0 AND (${conds.join(' OR ')})`;
|
|
const rows = db.query(`SELECT e.* FROM emails e WHERE ${whereSql} ORDER BY e.date DESC LIMIT ? OFFSET ?`).all(...params, limit, offset) as Record<string, unknown>[];
|
|
const total = (db.query(`SELECT count(*) AS c FROM emails e WHERE ${whereSql}`).get(...params) as { c: number }).c;
|
|
return { rows, total };
|
|
}
|
|
|
|
type ParsedEmail = {
|
|
id: string;
|
|
integration: string;
|
|
emailAccount: string;
|
|
fromName: string;
|
|
fromAddress: string;
|
|
to: string;
|
|
cc?: string;
|
|
subject: string;
|
|
date: string;
|
|
snippet: string;
|
|
html?: string;
|
|
text?: string;
|
|
attachments: Array<{ filename: string; size: number; contentType: string; content: string }>;
|
|
labels?: string[];
|
|
threadId?: string;
|
|
};
|
|
|
|
const upsertEmailStmt = `
|
|
INSERT OR REPLACE INTO emails (id, integration, email_account, from_name, from_address, from_domain, to_address, cc, subject, date, snippet, html, text_body, attachment_count, labels, thread_id)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
`;
|
|
|
|
const deleteAttachmentsStmt = 'DELETE FROM attachments WHERE email_id = ?';
|
|
const insertAttachmentStmt = 'INSERT INTO attachments (email_id, idx, filename, size, content_type, content) VALUES (?, ?, ?, ?, ?, ?)';
|
|
|
|
export function upsertEmail(db: Database, email: ParsedEmail): void {
|
|
const domain = extractDomain(email.fromAddress);
|
|
|
|
db.exec('BEGIN');
|
|
try {
|
|
db.run(upsertEmailStmt, [
|
|
email.id,
|
|
email.integration,
|
|
email.emailAccount,
|
|
email.fromName,
|
|
email.fromAddress,
|
|
domain,
|
|
email.to,
|
|
email.cc ?? null,
|
|
email.subject,
|
|
email.date,
|
|
email.snippet,
|
|
email.html ?? null,
|
|
email.text ?? null,
|
|
email.attachments.length,
|
|
labelsToString(email.labels),
|
|
email.threadId ?? email.id,
|
|
]);
|
|
|
|
db.run(deleteAttachmentsStmt, [email.id]);
|
|
for (let i = 0; i < email.attachments.length; i++) {
|
|
const att = email.attachments[i]!;
|
|
db.run(insertAttachmentStmt, [email.id, i, att.filename, att.size, att.contentType, att.content]);
|
|
}
|
|
|
|
syncFtsRow(db, email.id, email.subject ?? '', `${email.fromName ?? ''} ${email.fromAddress ?? ''}`.trim(), `${email.to ?? ''} ${email.cc ?? ''}`.trim(), email.snippet ?? '', email.text ?? '');
|
|
|
|
db.exec('COMMIT');
|
|
} catch (err) {
|
|
db.exec('ROLLBACK');
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
/** Upsert a single email from its raw RFC822 text using fast header parsing. */
|
|
type UpsertFromRawEmlParams = {
|
|
db: Database;
|
|
id: string;
|
|
raw: string;
|
|
integration: string;
|
|
emailAccount: string;
|
|
labels?: string[];
|
|
};
|
|
|
|
export function upsertFromRawEml({ db, id, raw, integration, emailAccount, labels }: UpsertFromRawEmlParams): void {
|
|
const from = extractHeader(raw, 'From');
|
|
const { name, address } = extractAddress(from);
|
|
const to = extractHeader(raw, 'To');
|
|
const cc = extractHeader(raw, 'Cc') || null;
|
|
const subject = extractHeader(raw, 'Subject') || '(no subject)';
|
|
const dateStr = extractHeader(raw, 'Date');
|
|
const date = dateStr ? new Date(dateStr).toISOString() : new Date(0).toISOString();
|
|
const snippet = extractSnippet(raw);
|
|
const attachments = parseAttachments(raw);
|
|
const domain = extractDomain(address);
|
|
|
|
const { html, text } = extractBody(raw);
|
|
const threadId = computeThreadId(db, id, raw);
|
|
|
|
db.run(upsertEmailStmt, [
|
|
id, integration, emailAccount, name, address, domain, to, cc, subject, date, snippet, html, text, attachments.length, labelsToString(labels), threadId,
|
|
]);
|
|
|
|
if (attachments.length > 0) {
|
|
db.run(deleteAttachmentsStmt, [id]);
|
|
for (let i = 0; i < attachments.length; i++) {
|
|
const att = attachments[i]!;
|
|
db.run(insertAttachmentStmt, [id, i, att.filename, att.size, att.contentType, att.content]);
|
|
}
|
|
}
|
|
|
|
syncFtsRow(db, id, subject, `${name} ${address}`.trim(), `${to} ${cc ?? ''}`.trim(), snippet, text ?? '');
|
|
}
|
|
|
|
/** Convert a db row to an EmailSummary for the API */
|
|
export function rowToSummary(row: Record<string, unknown>): EmailSummary {
|
|
const from = row.from_name
|
|
? `${row.from_name} <${row.from_address}>`
|
|
: (row.from_address as string);
|
|
|
|
const labels = labelsFromString(row.labels);
|
|
|
|
return {
|
|
id: row.id as string,
|
|
from,
|
|
to: row.to_address as string,
|
|
subject: row.subject as string,
|
|
date: row.date as string,
|
|
snippet: row.snippet as string,
|
|
...(row.attachment_count ? { attachmentCount: row.attachment_count as number } : {}),
|
|
...(row.read ? { read: true } : {}),
|
|
...(labels ? { labels } : {}),
|
|
...(row.thread_count && (row.thread_count as number) > 1 ? { threadCount: row.thread_count as number } : {}),
|
|
...(row.thread_unread ? { threadUnread: row.thread_unread as number } : {}),
|
|
};
|
|
}
|
|
|
|
// ── Sync meta helpers ──
|
|
|
|
export function getSyncMeta(db: Database, key: string): string | null {
|
|
const row = db.query('SELECT value FROM sync_meta WHERE key = ?').get(key) as { value: string } | null;
|
|
return row?.value ?? null;
|
|
}
|
|
|
|
export function setSyncMeta(db: Database, key: string, value: string): void {
|
|
db.run('INSERT OR REPLACE INTO sync_meta (key, value) VALUES (?, ?)', [key, value]);
|
|
}
|
|
|
|
export function updateEmailLabels(db: Database, id: string, labels: string[]): void {
|
|
db.run('UPDATE emails SET labels = ? WHERE id = ?', [labelsToString(labels), id]);
|
|
}
|
|
|
|
// ── Header parsing helpers (same logic as gmail-sync) ──
|
|
|
|
function decodeMimeWords(text: string): string {
|
|
return text.replace(/=\?([^?]+)\?(B|Q)\?([^?]+)\?=/gi, (_, charset, encoding, encoded) => {
|
|
try {
|
|
const normalizedCs = normalizeCharset(charset.toLowerCase());
|
|
if (encoding.toUpperCase() === 'B') {
|
|
const buf = Buffer.from(encoded, 'base64');
|
|
return new TextDecoder(normalizedCs, { fatal: false }).decode(buf);
|
|
}
|
|
const bytes: number[] = [];
|
|
for (let i = 0; i < encoded.length; i++) {
|
|
if (encoded[i] === '_') {
|
|
bytes.push(0x20);
|
|
} else if (encoded[i] === '=' && i + 2 < encoded.length) {
|
|
bytes.push(parseInt(encoded.slice(i + 1, i + 3), 16));
|
|
i += 2;
|
|
} else {
|
|
bytes.push(encoded.charCodeAt(i));
|
|
}
|
|
}
|
|
return new TextDecoder(normalizedCs, { fatal: false }).decode(Buffer.from(bytes));
|
|
} catch {
|
|
return encoded;
|
|
}
|
|
});
|
|
}
|
|
|
|
function extractHeader(raw: string, name: string): string {
|
|
const match = raw.match(new RegExp(`^${name}:\\s*(.+)$`, 'mi'));
|
|
return match?.[1]?.trim() ? decodeMimeWords(match[1].trim()) : '';
|
|
}
|
|
|
|
/** Extract a header value including folded continuation lines (lines starting with whitespace) */
|
|
function extractFullHeader(raw: string, name: string): string {
|
|
const headerEnd = findHeaderEnd(raw);
|
|
const headerBlock = headerEnd !== -1 ? raw.slice(0, headerEnd) : raw.slice(0, 4096);
|
|
const lines = headerBlock.split(/\r?\n/);
|
|
let result = '';
|
|
let capturing = false;
|
|
for (const line of lines) {
|
|
if (new RegExp(`^${name}:\\s*`, 'i').test(line)) {
|
|
result = line.replace(new RegExp(`^${name}:\\s*`, 'i'), '');
|
|
capturing = true;
|
|
} else if (capturing && /^[\t ]/.test(line)) {
|
|
result += ' ' + line.trim();
|
|
} else if (capturing) {
|
|
break;
|
|
}
|
|
}
|
|
return result.trim();
|
|
}
|
|
|
|
function findHeaderEnd(text: string): number {
|
|
const crlf = text.indexOf('\r\n\r\n');
|
|
const lf = text.indexOf('\n\n');
|
|
if (crlf !== -1) return crlf + 4;
|
|
if (lf !== -1) return lf + 2;
|
|
return -1;
|
|
}
|
|
|
|
function extractSnippet(raw: string): string {
|
|
const idx = findHeaderEnd(raw);
|
|
if (idx === -1) return '';
|
|
|
|
let body = raw.slice(idx);
|
|
|
|
if (body.trimStart().startsWith('--')) {
|
|
const afterBoundary = body.slice(body.indexOf('\n') + 1);
|
|
const partBodyStart = findHeaderEnd(afterBoundary);
|
|
if (partBodyStart !== -1) body = afterBoundary.slice(partBodyStart);
|
|
}
|
|
|
|
const nextBoundary = body.indexOf('\n--');
|
|
if (nextBoundary !== -1) body = body.slice(0, nextBoundary);
|
|
|
|
return body.replace(/\s+/g, ' ').trim().slice(0, 120);
|
|
}
|
|
|
|
function decodeQuotedPrintableBytes(text: string): Buffer {
|
|
const cleaned = text.replace(/=\r?\n/g, '');
|
|
const bytes: number[] = [];
|
|
for (let i = 0; i < cleaned.length; i++) {
|
|
if (cleaned[i] === '=' && i + 2 < cleaned.length) {
|
|
const hex = cleaned.slice(i + 1, i + 3);
|
|
const val = parseInt(hex, 16);
|
|
if (!isNaN(val)) {
|
|
bytes.push(val);
|
|
i += 2;
|
|
continue;
|
|
}
|
|
}
|
|
bytes.push(cleaned.charCodeAt(i));
|
|
}
|
|
return Buffer.from(bytes);
|
|
}
|
|
|
|
function extractCharset(contentType: string): string {
|
|
const match = contentType.match(/charset=["']?([^"';\s]+)/i);
|
|
return match?.[1]?.toLowerCase() ?? 'utf-8';
|
|
}
|
|
|
|
function normalizeCharset(charset: string): string {
|
|
const map: Record<string, string> = {
|
|
'iso-8859-1': 'latin1',
|
|
'iso_8859-1': 'latin1',
|
|
'iso-8859-15': 'latin1',
|
|
'iso_8859-15': 'latin1',
|
|
'windows-1250': 'latin1',
|
|
'windows-1251': 'latin1',
|
|
'windows-1252': 'latin1',
|
|
'windows-1254': 'latin1',
|
|
'us-ascii': 'ascii',
|
|
'ascii': 'ascii',
|
|
};
|
|
return map[charset] ?? charset;
|
|
}
|
|
|
|
function decodePartBody(body: string, encoding: string, charset = 'utf-8'): string {
|
|
const enc = encoding.toLowerCase();
|
|
const normalizedCharset = normalizeCharset(charset);
|
|
if (enc === 'base64') {
|
|
const buf = Buffer.from(body.replace(/\s/g, ''), 'base64');
|
|
return new TextDecoder(normalizedCharset, { fatal: false }).decode(buf);
|
|
}
|
|
if (enc === 'quoted-printable') {
|
|
const buf = decodeQuotedPrintableBytes(body);
|
|
return new TextDecoder(normalizedCharset, { fatal: false }).decode(buf);
|
|
}
|
|
return body;
|
|
}
|
|
|
|
function extractBody(raw: string): { html: string | null; text: string | null } {
|
|
const headerEnd = findHeaderEnd(raw);
|
|
if (headerEnd === -1) return { html: null, text: null };
|
|
|
|
const topCtRaw = extractFullHeader(raw, 'Content-Type');
|
|
const topCt = topCtRaw.toLowerCase();
|
|
const topEncoding = extractFullHeader(raw, 'Content-Transfer-Encoding');
|
|
|
|
// Non-multipart: single body
|
|
if (!topCt.includes('multipart')) {
|
|
const body = raw.slice(headerEnd);
|
|
const charset = extractCharset(topCtRaw);
|
|
const decoded = decodePartBody(body, topEncoding, charset);
|
|
if (topCt.includes('text/html')) return { html: decoded, text: null };
|
|
return { html: null, text: decoded };
|
|
}
|
|
|
|
// Multipart: extract boundary from the raw (case-sensitive) header
|
|
const boundaryMatch = topCtRaw.match(/boundary=["']?([^"';\s]+)/i);
|
|
if (!boundaryMatch) return { html: null, text: null };
|
|
const boundary = boundaryMatch[1]!;
|
|
|
|
let html: string | null = null;
|
|
let text: string | null = null;
|
|
|
|
const parts = raw.slice(headerEnd).split(`--${boundary}`);
|
|
for (const part of parts) {
|
|
if (part.startsWith('--') || !part.trim()) continue;
|
|
const partHeaderEnd = findHeaderEnd(part);
|
|
if (partHeaderEnd === -1) continue;
|
|
|
|
const partCtRaw = extractFullHeader(part, 'Content-Type');
|
|
const partCt = partCtRaw.toLowerCase();
|
|
const partEnc = extractFullHeader(part, 'Content-Transfer-Encoding');
|
|
const partCharset = extractCharset(partCtRaw);
|
|
const partBody = part.slice(partHeaderEnd);
|
|
|
|
// Recurse into nested multipart (e.g. multipart/alternative inside multipart/mixed)
|
|
if (partCt.includes('multipart')) {
|
|
const nested = extractBody(part.trim());
|
|
if (nested.html && !html) html = nested.html;
|
|
if (nested.text && !text) text = nested.text;
|
|
continue;
|
|
}
|
|
|
|
if (partCt.includes('text/html') && !html) {
|
|
html = decodePartBody(partBody, partEnc, partCharset);
|
|
} else if (partCt.includes('text/plain') && !text) {
|
|
text = decodePartBody(partBody, partEnc, partCharset);
|
|
}
|
|
}
|
|
|
|
return { html, text };
|
|
}
|
|
|
|
type AttachmentMeta = { filename: string; size: number; contentType: string; content: string };
|
|
|
|
function parseAttachments(raw: string): AttachmentMeta[] {
|
|
const results: AttachmentMeta[] = [];
|
|
// Match both "attachment" and "inline" dispositions
|
|
const regex = /^Content-Disposition:\s*(?:attachment|inline)[^\n]*/gim;
|
|
let match: RegExpExecArray | null;
|
|
|
|
while ((match = regex.exec(raw)) !== null) {
|
|
const pos = match.index;
|
|
|
|
// Walk backwards to find the start of this MIME part's headers
|
|
const partStart = raw.lastIndexOf('\n--', pos);
|
|
const headerBlock = partStart !== -1 ? raw.slice(partStart, pos + 500) : raw.slice(Math.max(0, pos - 500), pos + 500);
|
|
|
|
// Skip inline parts without a filename (e.g. inline text/plain body parts)
|
|
const hasFilename = /filename/i.test(headerBlock);
|
|
if (!hasFilename) continue;
|
|
|
|
// Extract filename from Content-Disposition or Content-Type
|
|
const fnMatch = headerBlock.match(/filename\*?=(?:"([^"]+)"|([^\s;]+))/i);
|
|
let rawFilename = fnMatch ? (fnMatch[1] ?? fnMatch[2] ?? 'unknown') : 'unknown';
|
|
// RFC 5987: filename*=charset''percent-encoded
|
|
const rfc5987Match = rawFilename.match(/^([^']*)'[^']*'(.+)/);
|
|
if (rfc5987Match) {
|
|
const cs = normalizeCharset(rfc5987Match[1]!.toLowerCase() || 'utf-8');
|
|
const encoded = rfc5987Match[2]!;
|
|
const bytes = encoded.replace(/%([0-9A-Fa-f]{2})/g, (_, h: string) => String.fromCharCode(parseInt(h, 16)));
|
|
rawFilename = new TextDecoder(cs, { fatal: false }).decode(Buffer.from(bytes, 'binary'));
|
|
}
|
|
const filename = decodeMimeWords(rawFilename);
|
|
|
|
// Extract content-type
|
|
const ctMatch = headerBlock.match(/^Content-Type:\s*([^\s;]+)/im);
|
|
const contentType = ctMatch?.[1] ?? 'application/octet-stream';
|
|
|
|
// Extract full body content as base64
|
|
const partHeaderEnd = findHeaderEnd(raw.slice(pos));
|
|
let content = '';
|
|
let size = 0;
|
|
if (partHeaderEnd !== -1) {
|
|
const bodyStart = pos + partHeaderEnd;
|
|
const boundaryEnd = raw.indexOf('\n--', bodyStart);
|
|
const bodyRaw = boundaryEnd !== -1 ? raw.slice(bodyStart, boundaryEnd) : raw.slice(bodyStart);
|
|
|
|
// Detect encoding from part headers
|
|
const encMatch = headerBlock.match(/^Content-Transfer-Encoding:\s*(\S+)/im);
|
|
const encoding = encMatch?.[1]?.toLowerCase() ?? 'base64';
|
|
|
|
if (encoding === 'base64') {
|
|
content = bodyRaw.replace(/\s/g, '');
|
|
} else {
|
|
// For quoted-printable or 7bit/8bit, re-encode to base64
|
|
const buf = encoding === 'quoted-printable'
|
|
? decodeQuotedPrintableBytes(bodyRaw)
|
|
: Buffer.from(bodyRaw);
|
|
content = buf.toString('base64');
|
|
}
|
|
size = Math.floor(content.length * 3 / 4);
|
|
}
|
|
|
|
results.push({ filename, size, contentType, content });
|
|
}
|
|
|
|
return results;
|
|
}
|