IMAP and Gmail sync into a per-account SQLite store, with the mail read and composed in the browser. moved whole from src/servers/sidecar/email (14 files), src/servers/api/email, officer_db/src/email, and the Screens/Dashboard/Email tree. what changed on the way out: - credentials are encrypted. the column was plaintext jsonb holding an IMAP app password that the SMTP transport read straight out of the row; it is now AES-256-GCM under this plugin's own 'email' secret-store purpose, sealed and opened in queries.ts alone so nothing above it can hold ciphertext by accident. no plaintext fallback on read - a value that does not verify is an error. - the router derives its prefix from mountPrefix() instead of the literal '/api/email', which was only correct for a first-party publisher. - the frontend became panels.ts + layout.ts. the two panel appTypes that were null are now declared, because the platform screen used to override them by panel id and a plugin has no such hook. - account management became a modal. it was a section in the platform's Settings, which a plugin cannot contribute to, and a panel does not work either: PluginScreen renders the workspace locked, so a panel outside the default layout is unreachable. it follows Compose, which had already solved this. - 17 palette classes replaced with semantic tokens. not verified: this has never run. it was switched off in the platform on 2026-08-13 and its table was never created, so nothing has exercised sync, sending or the UI since. the first install is the first test.
796 lines
28 KiB
TypeScript
796 lines
28 KiB
TypeScript
import { Database } from 'bun:sqlite';
|
|
import { createHash } from 'node:crypto';
|
|
import { join, dirname } from 'node:path';
|
|
import { chmodSync, mkdirSync } from 'node:fs';
|
|
import { getEmailDbPath } from '@@/data-path';
|
|
import { getEmailAccounts } from '../db/queries';
|
|
import type { EmailSummary } from '../db/types';
|
|
|
|
const SCHEMA_TABLES = `
|
|
CREATE TABLE IF NOT EXISTS emails (
|
|
id TEXT PRIMARY KEY,
|
|
integration TEXT NOT NULL DEFAULT 'gmail',
|
|
email_account TEXT NOT NULL DEFAULT '',
|
|
from_name TEXT,
|
|
from_address TEXT,
|
|
from_domain TEXT,
|
|
to_address TEXT,
|
|
cc TEXT,
|
|
subject TEXT,
|
|
date TEXT,
|
|
snippet TEXT,
|
|
html TEXT,
|
|
text_body TEXT,
|
|
attachment_count INTEGER DEFAULT 0,
|
|
read INTEGER DEFAULT 0,
|
|
deleted INTEGER DEFAULT 0,
|
|
labels TEXT,
|
|
thread_id TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS attachments (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
email_id TEXT REFERENCES emails(id) ON DELETE CASCADE,
|
|
idx INTEGER,
|
|
filename TEXT,
|
|
size INTEGER,
|
|
content_type TEXT,
|
|
content TEXT
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS sync_meta (
|
|
key TEXT PRIMARY KEY,
|
|
value TEXT
|
|
);
|
|
`;
|
|
|
|
const SCHEMA_INDEXES = `
|
|
CREATE INDEX IF NOT EXISTS idx_emails_date ON emails(date);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_from_domain ON emails(from_domain);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_from_address ON emails(from_address);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_integration ON emails(integration);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_email_account ON emails(email_account);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_labels ON emails(labels);
|
|
CREATE INDEX IF NOT EXISTS idx_emails_thread ON emails(thread_id);
|
|
`;
|
|
|
|
/** Convert label IDs to lowercase comma-separated string for storage */
|
|
function labelsToString(labels?: string[]): string | null {
|
|
if (!labels || labels.length === 0) return null;
|
|
return labels.map((l) => l.toLowerCase()).join(',');
|
|
}
|
|
|
|
/** Convert stored comma-separated labels back to array */
|
|
function labelsFromString(value: unknown): string[] | undefined {
|
|
if (typeof value !== 'string' || !value) return undefined;
|
|
return value.split(',');
|
|
}
|
|
|
|
function extractAddress(headerValue: string): { name: string; address: string } {
|
|
const match = headerValue.match(/^"?(.+?)"?\s*<(.+?)>$/);
|
|
if (match) return { name: match[1]!.trim(), address: match[2]!.toLowerCase() };
|
|
const bare = headerValue.trim().toLowerCase();
|
|
return { name: '', address: bare };
|
|
}
|
|
|
|
function extractDomain(address: string): string {
|
|
const at = address.lastIndexOf('@');
|
|
return at >= 0 ? address.slice(at + 1) : '';
|
|
}
|
|
|
|
// ── Conversation threading ──
|
|
// `id` is sha1(Message-Id) (see resync.messageIdToStableId), so hashing a referenced Message-Id the
|
|
// same way yields the *id of that referenced email*. That makes header-based threading trivial:
|
|
// a reply's thread_id is the hash of its root Message-Id, which equals the root email's own id.
|
|
|
|
const hashMsgId = (msgId: string): string => createHash('sha1').update(msgId).digest('hex').slice(0, 16);
|
|
|
|
/** Ordered Message-Ids this email references (References first, root→leaf; else In-Reply-To). */
|
|
function extractReferenceIds(raw: string): string[] {
|
|
const refs = extractFullHeader(raw, 'References') || extractFullHeader(raw, 'In-Reply-To');
|
|
return Array.from(refs.matchAll(/<([^>]+)>/g), (m) => m[1]!.trim()).filter(Boolean);
|
|
}
|
|
|
|
/** Compute a header-based thread_id for a freshly ingested email (falls back to its own id = new thread). */
|
|
function computeThreadId(db: Database, id: string, raw: string): string {
|
|
const refIds = extractReferenceIds(raw);
|
|
if (refIds.length === 0) return id; // no ancestors → this email is a thread root
|
|
const hashed = refIds.map(hashMsgId);
|
|
// Adopt an ancestor's thread if we already have one stored (robust to In-Reply-To-only clients).
|
|
const placeholders = hashed.map(() => '?').join(',');
|
|
const found = db
|
|
.query(`SELECT thread_id FROM emails WHERE id IN (${placeholders}) AND thread_id IS NOT NULL LIMIT 1`)
|
|
.get(...hashed) as { thread_id: string } | null;
|
|
return found?.thread_id ?? hashMsgId(refIds[0]!);
|
|
}
|
|
|
|
const RE_PREFIX = /^\s*((re|fwd?|aw|wg|sv|vs|res|antw)\s*(\[\d+\])?\s*:\s*)+/i;
|
|
|
|
/** Normalize a subject for fallback grouping: strip reply/forward prefixes, fold whitespace, lowercase. */
|
|
function normalizeSubject(subject: string): string {
|
|
return subject.replace(RE_PREFIX, '').replace(/\s+/g, ' ').trim().toLowerCase();
|
|
}
|
|
|
|
const firstAddress = (value: unknown): string => {
|
|
if (typeof value !== 'string') return '';
|
|
const first = value.split(',')[0] ?? '';
|
|
return (first.match(/<([^>]+)>/)?.[1] ?? first).trim().toLowerCase();
|
|
};
|
|
|
|
/**
|
|
* Subject-based thread key for mail synced before header capture (no References available).
|
|
* Groups by normalized subject + the counterpart address, so recurring 1:1 conversations collapse
|
|
* while unrelated same-subject mail from different people stays apart. Trivial subjects stay ungrouped.
|
|
*/
|
|
function fallbackThreadId(row: {
|
|
id: string;
|
|
subject: unknown;
|
|
from_address: unknown;
|
|
to_address: unknown;
|
|
email_account: unknown;
|
|
}): string {
|
|
const norm = normalizeSubject(typeof row.subject === 'string' ? row.subject : '');
|
|
if (!norm) return row.id;
|
|
const me = typeof row.email_account === 'string' ? row.email_account.toLowerCase() : '';
|
|
const from = typeof row.from_address === 'string' ? row.from_address.toLowerCase() : '';
|
|
const counterpart = from && from !== me ? from : firstAddress(row.to_address) || from;
|
|
return `s:${norm}|${counterpart}`;
|
|
}
|
|
|
|
export function openEmailDb(ownerEmail: string, accountEmail: string): Database {
|
|
const dbPath = getEmailDbPath(ownerEmail, accountEmail);
|
|
mkdirSync(dirname(dbPath), { recursive: true });
|
|
const db = new Database(dbPath, { create: true });
|
|
// The API and the email sidecar both open this file; wait out a concurrent writer (e.g. a resync
|
|
// or the one-time thread_id backfill) instead of failing immediately with "database is locked".
|
|
db.exec('PRAGMA busy_timeout = 5000');
|
|
db.exec('PRAGMA journal_mode = DELETE');
|
|
db.exec('PRAGMA foreign_keys = ON');
|
|
db.exec(SCHEMA_TABLES);
|
|
migrate(db);
|
|
db.exec(SCHEMA_INDEXES);
|
|
ensureFts(db);
|
|
// Try to chmod, but don't crash if permission denied (e.g., file owned by different user)
|
|
try {
|
|
chmodSync(dbPath, 0o666);
|
|
} catch (err) {
|
|
// File exists with correct permissions, or owned by another user - that's fine
|
|
}
|
|
return db;
|
|
}
|
|
|
|
/**
|
|
* Open the email DB for a user's configured account. Emails are stored per account
|
|
* (email_accounts/<account>/emails.db); for now we use the user's first account. Returns null if the
|
|
* user has no email account configured yet.
|
|
*/
|
|
export async function openUserEmailDb(ownerEmail: string, userId: number): Promise<Database | null> {
|
|
const accounts = await getEmailAccounts(userId);
|
|
const account = accounts.find((a) => a.enabled) ?? accounts[0];
|
|
return account ? openEmailDb(ownerEmail, account.email) : null;
|
|
}
|
|
|
|
function migrate(db: Database): void {
|
|
const cols = db.query('PRAGMA table_info(emails)').all() as Array<{ name: string }>;
|
|
const colNames = new Set(cols.map((c) => c.name));
|
|
|
|
if (!colNames.has('deleted')) {
|
|
db.exec('ALTER TABLE emails ADD COLUMN deleted INTEGER DEFAULT 0');
|
|
}
|
|
if (!colNames.has('integration')) {
|
|
db.exec("ALTER TABLE emails ADD COLUMN integration TEXT NOT NULL DEFAULT 'gmail'");
|
|
}
|
|
if (!colNames.has('email_account')) {
|
|
db.exec("ALTER TABLE emails ADD COLUMN email_account TEXT NOT NULL DEFAULT ''");
|
|
}
|
|
if (!colNames.has('labels')) {
|
|
db.exec('ALTER TABLE emails ADD COLUMN labels TEXT');
|
|
}
|
|
if (!colNames.has('thread_id')) {
|
|
db.exec('ALTER TABLE emails ADD COLUMN thread_id TEXT');
|
|
}
|
|
|
|
// Backfill thread_id for any rows missing it. Header data isn't kept for already-synced mail, so
|
|
// these use the subject-based fallback. New mail gets an exact header-based thread_id at insert.
|
|
backfillThreadIds(db);
|
|
|
|
// Ensure sync_meta table exists (for DBs created before it was added to SCHEMA_TABLES)
|
|
db.exec('CREATE TABLE IF NOT EXISTS sync_meta (key TEXT PRIMARY KEY, value TEXT)');
|
|
|
|
// Add content column to attachments if missing
|
|
const attCols = db.query('PRAGMA table_info(attachments)').all() as Array<{ name: string }>;
|
|
const attColNames = new Set(attCols.map((c) => c.name));
|
|
if (!attColNames.has('content')) {
|
|
db.exec('ALTER TABLE attachments ADD COLUMN content TEXT');
|
|
}
|
|
}
|
|
|
|
function backfillThreadIds(db: Database): void {
|
|
const rows = db
|
|
.query('SELECT id, subject, from_address, to_address, email_account FROM emails WHERE thread_id IS NULL')
|
|
.all() as Array<{
|
|
id: string;
|
|
subject: unknown;
|
|
from_address: unknown;
|
|
to_address: unknown;
|
|
email_account: unknown;
|
|
}>;
|
|
if (rows.length === 0) return;
|
|
|
|
const update = db.prepare('UPDATE emails SET thread_id = ? WHERE id = ?');
|
|
db.exec('BEGIN');
|
|
try {
|
|
for (const row of rows) update.run(fallbackThreadId(row), row.id);
|
|
db.exec('COMMIT');
|
|
} catch (err) {
|
|
db.exec('ROLLBACK');
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ── Full-text search (FTS5) ──
|
|
// A standalone FTS5 index kept in sync with `emails` on every upsert. unicode61 + diacritic folding
|
|
// gives accent-insensitive matching; per-term prefix queries make it feel incremental.
|
|
|
|
function ensureFts(db: Database): void {
|
|
db.exec(
|
|
`CREATE VIRTUAL TABLE IF NOT EXISTS emails_fts USING fts5(
|
|
id UNINDEXED, subject, sender, recipients, snippet, body,
|
|
tokenize = 'unicode61 remove_diacritics 2'
|
|
)`,
|
|
);
|
|
const fts = (db.query('SELECT count(*) AS c FROM emails_fts').get() as { c: number }).c;
|
|
const total = (db.query('SELECT count(*) AS c FROM emails').get() as { c: number }).c;
|
|
if (fts === 0 && total > 0) {
|
|
// Backfill existing mail (first run after this feature ships).
|
|
db.exec(
|
|
`INSERT INTO emails_fts (id, subject, sender, recipients, snippet, body)
|
|
SELECT id, COALESCE(subject, ''),
|
|
TRIM(COALESCE(from_name, '') || ' ' || COALESCE(from_address, '')),
|
|
TRIM(COALESCE(to_address, '') || ' ' || COALESCE(cc, '')),
|
|
COALESCE(snippet, ''), COALESCE(text_body, '')
|
|
FROM emails`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const ftsDeleteStmt = 'DELETE FROM emails_fts WHERE id = ?';
|
|
const ftsInsertStmt =
|
|
'INSERT INTO emails_fts (id, subject, sender, recipients, snippet, body) VALUES (?, ?, ?, ?, ?, ?)';
|
|
|
|
function syncFtsRow(
|
|
db: Database,
|
|
id: string,
|
|
subject: string,
|
|
sender: string,
|
|
recipients: string,
|
|
snippet: string,
|
|
body: string,
|
|
): void {
|
|
db.run(ftsDeleteStmt, [id]);
|
|
db.run(ftsInsertStmt, [id, subject, sender, recipients, snippet, body]);
|
|
}
|
|
|
|
// Gmail-style query parsing. Free text → full-text (prefix-AND). Operators:
|
|
// from:/to:/subject:/body: → FTS5 column filters
|
|
// has:attachment, is:unread/read, label:X, before:/after:YYYY-MM-DD → SQL filters on `emails`
|
|
// Unknown operators fall back to free text. Quoted values (from:"a b") match as an exact phrase.
|
|
const FTS_COLUMNS: Record<string, string> = { from: 'sender', to: 'recipients', subject: 'subject', body: 'body' };
|
|
|
|
const ftsTerm = (value: string, column: string | null, prefix: boolean): string => {
|
|
const esc = value.replace(/"/g, '""');
|
|
return `${column ? column + ':' : ''}"${esc}"${prefix ? '*' : ''}`;
|
|
};
|
|
|
|
const parseSearchDate = (v: string): string | null => {
|
|
const iso = /^\d{4}-\d{2}-\d{2}$/.test(v) ? `${v}T00:00:00.000Z` : v;
|
|
const d = new Date(iso);
|
|
return Number.isNaN(d.getTime()) ? null : d.toISOString();
|
|
};
|
|
|
|
// One AND-group of terms (a single OR branch). `fts` is the FTS5 MATCH expression for this branch;
|
|
// `where`/`params` are its structured SQL filters.
|
|
type Branch = { fts: string; where: string[]; params: string[] };
|
|
|
|
function parseBranch(q: string): Branch {
|
|
const fts: string[] = [];
|
|
const where: string[] = [];
|
|
const params: string[] = [];
|
|
const re = /(\w+):("[^"]*"|\S+)|"([^"]*)"|(\S+)/g;
|
|
let m: RegExpExecArray | null;
|
|
while ((m = re.exec(q)) !== null) {
|
|
if (m[1]) {
|
|
const op = m[1].toLowerCase();
|
|
const raw = m[2]!;
|
|
const quoted = raw.startsWith('"') && raw.endsWith('"');
|
|
const val = quoted ? raw.slice(1, -1) : raw;
|
|
if (!val) continue;
|
|
const lower = val.toLowerCase();
|
|
if (FTS_COLUMNS[op]) {
|
|
fts.push(ftsTerm(val, FTS_COLUMNS[op], !quoted));
|
|
} else if (op === 'label') {
|
|
where.push('e.labels LIKE ?');
|
|
params.push(`%${lower}%`);
|
|
} else if (op === 'has' && (lower === 'attachment' || lower === 'attachments')) {
|
|
where.push('e.attachment_count > 0');
|
|
} else if (op === 'is' && (lower === 'unread' || lower === 'read')) {
|
|
where.push(lower === 'unread' ? 'e.read = 0' : 'e.read = 1');
|
|
} else if (op === 'before' || op === 'older') {
|
|
const d = parseSearchDate(val);
|
|
if (d) {
|
|
where.push('e.date < ?');
|
|
params.push(d);
|
|
}
|
|
} else if (op === 'after' || op === 'newer') {
|
|
const d = parseSearchDate(val);
|
|
if (d) {
|
|
where.push('e.date >= ?');
|
|
params.push(d);
|
|
}
|
|
} else {
|
|
// Unknown operator — treat the whole "op:val" token as free text.
|
|
fts.push(ftsTerm(`${op}:${val}`, null, !quoted));
|
|
}
|
|
} else if (m[3] !== undefined) {
|
|
if (m[3].trim()) fts.push(ftsTerm(m[3], null, false)); // quoted phrase → exact
|
|
} else if (m[4]) {
|
|
fts.push(ftsTerm(m[4], null, true)); // bare word → prefix
|
|
}
|
|
}
|
|
return { fts: fts.join(' '), where, params };
|
|
}
|
|
|
|
export function searchEmails(
|
|
db: Database,
|
|
q: string,
|
|
limit: number,
|
|
offset: number,
|
|
): { rows: Record<string, unknown>[]; total: number } {
|
|
// Split on top-level uppercase OR into branches (Gmail-style; lowercase "or" stays a search word).
|
|
const branches = q
|
|
.split(/\s+OR\s+/)
|
|
.map(parseBranch)
|
|
.filter((b) => b.fts || b.where.length > 0);
|
|
if (branches.length === 0) return { rows: [], total: 0 };
|
|
|
|
// Each branch becomes one self-contained condition: its FTS terms via an `id IN (FTS subquery)` so
|
|
// full-text and structured filters share a WHERE and branches can be OR'd. All ANDed within a branch.
|
|
const conds: string[] = [];
|
|
const params: string[] = [];
|
|
for (const b of branches) {
|
|
const parts: string[] = [];
|
|
if (b.fts) {
|
|
parts.push('e.id IN (SELECT emails_fts.id FROM emails_fts WHERE emails_fts MATCH ?)');
|
|
params.push(b.fts);
|
|
}
|
|
parts.push(...b.where);
|
|
params.push(...b.params);
|
|
conds.push(`(${parts.join(' AND ')})`);
|
|
}
|
|
|
|
const whereSql = `e.deleted = 0 AND (${conds.join(' OR ')})`;
|
|
const rows = db
|
|
.query(`SELECT e.* FROM emails e WHERE ${whereSql} ORDER BY e.date DESC LIMIT ? OFFSET ?`)
|
|
.all(...params, limit, offset) as Record<string, unknown>[];
|
|
const total = (db.query(`SELECT count(*) AS c FROM emails e WHERE ${whereSql}`).get(...params) as { c: number }).c;
|
|
return { rows, total };
|
|
}
|
|
|
|
type ParsedEmail = {
|
|
id: string;
|
|
integration: string;
|
|
emailAccount: string;
|
|
fromName: string;
|
|
fromAddress: string;
|
|
to: string;
|
|
cc?: string;
|
|
subject: string;
|
|
date: string;
|
|
snippet: string;
|
|
html?: string;
|
|
text?: string;
|
|
attachments: Array<{ filename: string; size: number; contentType: string; content: string }>;
|
|
labels?: string[];
|
|
threadId?: string;
|
|
};
|
|
|
|
const upsertEmailStmt = `
|
|
INSERT OR REPLACE INTO emails (id, integration, email_account, from_name, from_address, from_domain, to_address, cc, subject, date, snippet, html, text_body, attachment_count, labels, thread_id)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
`;
|
|
|
|
const deleteAttachmentsStmt = 'DELETE FROM attachments WHERE email_id = ?';
|
|
const insertAttachmentStmt =
|
|
'INSERT INTO attachments (email_id, idx, filename, size, content_type, content) VALUES (?, ?, ?, ?, ?, ?)';
|
|
|
|
export function upsertEmail(db: Database, email: ParsedEmail): void {
|
|
const domain = extractDomain(email.fromAddress);
|
|
|
|
db.exec('BEGIN');
|
|
try {
|
|
db.run(upsertEmailStmt, [
|
|
email.id,
|
|
email.integration,
|
|
email.emailAccount,
|
|
email.fromName,
|
|
email.fromAddress,
|
|
domain,
|
|
email.to,
|
|
email.cc ?? null,
|
|
email.subject,
|
|
email.date,
|
|
email.snippet,
|
|
email.html ?? null,
|
|
email.text ?? null,
|
|
email.attachments.length,
|
|
labelsToString(email.labels),
|
|
email.threadId ?? email.id,
|
|
]);
|
|
|
|
db.run(deleteAttachmentsStmt, [email.id]);
|
|
for (let i = 0; i < email.attachments.length; i++) {
|
|
const att = email.attachments[i]!;
|
|
db.run(insertAttachmentStmt, [email.id, i, att.filename, att.size, att.contentType, att.content]);
|
|
}
|
|
|
|
syncFtsRow(
|
|
db,
|
|
email.id,
|
|
email.subject ?? '',
|
|
`${email.fromName ?? ''} ${email.fromAddress ?? ''}`.trim(),
|
|
`${email.to ?? ''} ${email.cc ?? ''}`.trim(),
|
|
email.snippet ?? '',
|
|
email.text ?? '',
|
|
);
|
|
|
|
db.exec('COMMIT');
|
|
} catch (err) {
|
|
db.exec('ROLLBACK');
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
/** Upsert a single email from its raw RFC822 text using fast header parsing. */
|
|
type UpsertFromRawEmlParams = {
|
|
db: Database;
|
|
id: string;
|
|
raw: string;
|
|
integration: string;
|
|
emailAccount: string;
|
|
labels?: string[];
|
|
};
|
|
|
|
export function upsertFromRawEml({ db, id, raw, integration, emailAccount, labels }: UpsertFromRawEmlParams): void {
|
|
const from = extractHeader(raw, 'From');
|
|
const { name, address } = extractAddress(from);
|
|
const to = extractHeader(raw, 'To');
|
|
const cc = extractHeader(raw, 'Cc') || null;
|
|
const subject = extractHeader(raw, 'Subject') || '(no subject)';
|
|
const dateStr = extractHeader(raw, 'Date');
|
|
const date = dateStr ? new Date(dateStr).toISOString() : new Date(0).toISOString();
|
|
const snippet = extractSnippet(raw);
|
|
const attachments = parseAttachments(raw);
|
|
const domain = extractDomain(address);
|
|
|
|
const { html, text } = extractBody(raw);
|
|
const threadId = computeThreadId(db, id, raw);
|
|
|
|
db.run(upsertEmailStmt, [
|
|
id,
|
|
integration,
|
|
emailAccount,
|
|
name,
|
|
address,
|
|
domain,
|
|
to,
|
|
cc,
|
|
subject,
|
|
date,
|
|
snippet,
|
|
html,
|
|
text,
|
|
attachments.length,
|
|
labelsToString(labels),
|
|
threadId,
|
|
]);
|
|
|
|
if (attachments.length > 0) {
|
|
db.run(deleteAttachmentsStmt, [id]);
|
|
for (let i = 0; i < attachments.length; i++) {
|
|
const att = attachments[i]!;
|
|
db.run(insertAttachmentStmt, [id, i, att.filename, att.size, att.contentType, att.content]);
|
|
}
|
|
}
|
|
|
|
syncFtsRow(db, id, subject, `${name} ${address}`.trim(), `${to} ${cc ?? ''}`.trim(), snippet, text ?? '');
|
|
}
|
|
|
|
/** Convert a db row to an EmailSummary for the API */
|
|
export function rowToSummary(row: Record<string, unknown>): EmailSummary {
|
|
const from = row.from_name ? `${row.from_name} <${row.from_address}>` : (row.from_address as string);
|
|
|
|
const labels = labelsFromString(row.labels);
|
|
|
|
return {
|
|
id: row.id as string,
|
|
from,
|
|
to: row.to_address as string,
|
|
subject: row.subject as string,
|
|
date: row.date as string,
|
|
snippet: row.snippet as string,
|
|
...(row.attachment_count ? { attachmentCount: row.attachment_count as number } : {}),
|
|
...(row.read ? { read: true } : {}),
|
|
...(labels ? { labels } : {}),
|
|
...(row.thread_count && (row.thread_count as number) > 1 ? { threadCount: row.thread_count as number } : {}),
|
|
...(row.thread_unread ? { threadUnread: row.thread_unread as number } : {}),
|
|
};
|
|
}
|
|
|
|
// ── Sync meta helpers ──
|
|
|
|
export function getSyncMeta(db: Database, key: string): string | null {
|
|
const row = db.query('SELECT value FROM sync_meta WHERE key = ?').get(key) as { value: string } | null;
|
|
return row?.value ?? null;
|
|
}
|
|
|
|
export function setSyncMeta(db: Database, key: string, value: string): void {
|
|
db.run('INSERT OR REPLACE INTO sync_meta (key, value) VALUES (?, ?)', [key, value]);
|
|
}
|
|
|
|
export function updateEmailLabels(db: Database, id: string, labels: string[]): void {
|
|
db.run('UPDATE emails SET labels = ? WHERE id = ?', [labelsToString(labels), id]);
|
|
}
|
|
|
|
// ── Header parsing helpers (same logic as gmail-sync) ──
|
|
|
|
function decodeMimeWords(text: string): string {
|
|
return text.replace(/=\?([^?]+)\?(B|Q)\?([^?]+)\?=/gi, (_, charset, encoding, encoded) => {
|
|
try {
|
|
const normalizedCs = normalizeCharset(charset.toLowerCase());
|
|
if (encoding.toUpperCase() === 'B') {
|
|
const buf = Buffer.from(encoded, 'base64');
|
|
return new TextDecoder(normalizedCs, { fatal: false }).decode(buf);
|
|
}
|
|
const bytes: number[] = [];
|
|
for (let i = 0; i < encoded.length; i++) {
|
|
if (encoded[i] === '_') {
|
|
bytes.push(0x20);
|
|
} else if (encoded[i] === '=' && i + 2 < encoded.length) {
|
|
bytes.push(parseInt(encoded.slice(i + 1, i + 3), 16));
|
|
i += 2;
|
|
} else {
|
|
bytes.push(encoded.charCodeAt(i));
|
|
}
|
|
}
|
|
return new TextDecoder(normalizedCs, { fatal: false }).decode(Buffer.from(bytes));
|
|
} catch {
|
|
return encoded;
|
|
}
|
|
});
|
|
}
|
|
|
|
function extractHeader(raw: string, name: string): string {
|
|
const match = raw.match(new RegExp(`^${name}:\\s*(.+)$`, 'mi'));
|
|
return match?.[1]?.trim() ? decodeMimeWords(match[1].trim()) : '';
|
|
}
|
|
|
|
/** Extract a header value including folded continuation lines (lines starting with whitespace) */
|
|
function extractFullHeader(raw: string, name: string): string {
|
|
const headerEnd = findHeaderEnd(raw);
|
|
const headerBlock = headerEnd !== -1 ? raw.slice(0, headerEnd) : raw.slice(0, 4096);
|
|
const lines = headerBlock.split(/\r?\n/);
|
|
let result = '';
|
|
let capturing = false;
|
|
for (const line of lines) {
|
|
if (new RegExp(`^${name}:\\s*`, 'i').test(line)) {
|
|
result = line.replace(new RegExp(`^${name}:\\s*`, 'i'), '');
|
|
capturing = true;
|
|
} else if (capturing && /^[\t ]/.test(line)) {
|
|
result += ' ' + line.trim();
|
|
} else if (capturing) {
|
|
break;
|
|
}
|
|
}
|
|
return result.trim();
|
|
}
|
|
|
|
function findHeaderEnd(text: string): number {
|
|
const crlf = text.indexOf('\r\n\r\n');
|
|
const lf = text.indexOf('\n\n');
|
|
if (crlf !== -1) return crlf + 4;
|
|
if (lf !== -1) return lf + 2;
|
|
return -1;
|
|
}
|
|
|
|
function extractSnippet(raw: string): string {
|
|
const idx = findHeaderEnd(raw);
|
|
if (idx === -1) return '';
|
|
|
|
let body = raw.slice(idx);
|
|
|
|
if (body.trimStart().startsWith('--')) {
|
|
const afterBoundary = body.slice(body.indexOf('\n') + 1);
|
|
const partBodyStart = findHeaderEnd(afterBoundary);
|
|
if (partBodyStart !== -1) body = afterBoundary.slice(partBodyStart);
|
|
}
|
|
|
|
const nextBoundary = body.indexOf('\n--');
|
|
if (nextBoundary !== -1) body = body.slice(0, nextBoundary);
|
|
|
|
return body.replace(/\s+/g, ' ').trim().slice(0, 120);
|
|
}
|
|
|
|
function decodeQuotedPrintableBytes(text: string): Buffer {
|
|
const cleaned = text.replace(/=\r?\n/g, '');
|
|
const bytes: number[] = [];
|
|
for (let i = 0; i < cleaned.length; i++) {
|
|
if (cleaned[i] === '=' && i + 2 < cleaned.length) {
|
|
const hex = cleaned.slice(i + 1, i + 3);
|
|
const val = parseInt(hex, 16);
|
|
if (!isNaN(val)) {
|
|
bytes.push(val);
|
|
i += 2;
|
|
continue;
|
|
}
|
|
}
|
|
bytes.push(cleaned.charCodeAt(i));
|
|
}
|
|
return Buffer.from(bytes);
|
|
}
|
|
|
|
function extractCharset(contentType: string): string {
|
|
const match = contentType.match(/charset=["']?([^"';\s]+)/i);
|
|
return match?.[1]?.toLowerCase() ?? 'utf-8';
|
|
}
|
|
|
|
function normalizeCharset(charset: string): string {
|
|
const map: Record<string, string> = {
|
|
'iso-8859-1': 'latin1',
|
|
'iso_8859-1': 'latin1',
|
|
'iso-8859-15': 'latin1',
|
|
'iso_8859-15': 'latin1',
|
|
'windows-1250': 'latin1',
|
|
'windows-1251': 'latin1',
|
|
'windows-1252': 'latin1',
|
|
'windows-1254': 'latin1',
|
|
'us-ascii': 'ascii',
|
|
ascii: 'ascii',
|
|
};
|
|
return map[charset] ?? charset;
|
|
}
|
|
|
|
function decodePartBody(body: string, encoding: string, charset = 'utf-8'): string {
|
|
const enc = encoding.toLowerCase();
|
|
const normalizedCharset = normalizeCharset(charset);
|
|
if (enc === 'base64') {
|
|
const buf = Buffer.from(body.replace(/\s/g, ''), 'base64');
|
|
return new TextDecoder(normalizedCharset, { fatal: false }).decode(buf);
|
|
}
|
|
if (enc === 'quoted-printable') {
|
|
const buf = decodeQuotedPrintableBytes(body);
|
|
return new TextDecoder(normalizedCharset, { fatal: false }).decode(buf);
|
|
}
|
|
return body;
|
|
}
|
|
|
|
function extractBody(raw: string): { html: string | null; text: string | null } {
|
|
const headerEnd = findHeaderEnd(raw);
|
|
if (headerEnd === -1) return { html: null, text: null };
|
|
|
|
const topCtRaw = extractFullHeader(raw, 'Content-Type');
|
|
const topCt = topCtRaw.toLowerCase();
|
|
const topEncoding = extractFullHeader(raw, 'Content-Transfer-Encoding');
|
|
|
|
// Non-multipart: single body
|
|
if (!topCt.includes('multipart')) {
|
|
const body = raw.slice(headerEnd);
|
|
const charset = extractCharset(topCtRaw);
|
|
const decoded = decodePartBody(body, topEncoding, charset);
|
|
if (topCt.includes('text/html')) return { html: decoded, text: null };
|
|
return { html: null, text: decoded };
|
|
}
|
|
|
|
// Multipart: extract boundary from the raw (case-sensitive) header
|
|
const boundaryMatch = topCtRaw.match(/boundary=["']?([^"';\s]+)/i);
|
|
if (!boundaryMatch) return { html: null, text: null };
|
|
const boundary = boundaryMatch[1]!;
|
|
|
|
let html: string | null = null;
|
|
let text: string | null = null;
|
|
|
|
const parts = raw.slice(headerEnd).split(`--${boundary}`);
|
|
for (const part of parts) {
|
|
if (part.startsWith('--') || !part.trim()) continue;
|
|
const partHeaderEnd = findHeaderEnd(part);
|
|
if (partHeaderEnd === -1) continue;
|
|
|
|
const partCtRaw = extractFullHeader(part, 'Content-Type');
|
|
const partCt = partCtRaw.toLowerCase();
|
|
const partEnc = extractFullHeader(part, 'Content-Transfer-Encoding');
|
|
const partCharset = extractCharset(partCtRaw);
|
|
const partBody = part.slice(partHeaderEnd);
|
|
|
|
// Recurse into nested multipart (e.g. multipart/alternative inside multipart/mixed)
|
|
if (partCt.includes('multipart')) {
|
|
const nested = extractBody(part.trim());
|
|
if (nested.html && !html) html = nested.html;
|
|
if (nested.text && !text) text = nested.text;
|
|
continue;
|
|
}
|
|
|
|
if (partCt.includes('text/html') && !html) {
|
|
html = decodePartBody(partBody, partEnc, partCharset);
|
|
} else if (partCt.includes('text/plain') && !text) {
|
|
text = decodePartBody(partBody, partEnc, partCharset);
|
|
}
|
|
}
|
|
|
|
return { html, text };
|
|
}
|
|
|
|
type AttachmentMeta = { filename: string; size: number; contentType: string; content: string };
|
|
|
|
function parseAttachments(raw: string): AttachmentMeta[] {
|
|
const results: AttachmentMeta[] = [];
|
|
// Match both "attachment" and "inline" dispositions
|
|
const regex = /^Content-Disposition:\s*(?:attachment|inline)[^\n]*/gim;
|
|
let match: RegExpExecArray | null;
|
|
|
|
while ((match = regex.exec(raw)) !== null) {
|
|
const pos = match.index;
|
|
|
|
// Walk backwards to find the start of this MIME part's headers
|
|
const partStart = raw.lastIndexOf('\n--', pos);
|
|
const headerBlock =
|
|
partStart !== -1 ? raw.slice(partStart, pos + 500) : raw.slice(Math.max(0, pos - 500), pos + 500);
|
|
|
|
// Skip inline parts without a filename (e.g. inline text/plain body parts)
|
|
const hasFilename = /filename/i.test(headerBlock);
|
|
if (!hasFilename) continue;
|
|
|
|
// Extract filename from Content-Disposition or Content-Type
|
|
const fnMatch = headerBlock.match(/filename\*?=(?:"([^"]+)"|([^\s;]+))/i);
|
|
let rawFilename = fnMatch ? (fnMatch[1] ?? fnMatch[2] ?? 'unknown') : 'unknown';
|
|
// RFC 5987: filename*=charset''percent-encoded
|
|
const rfc5987Match = rawFilename.match(/^([^']*)'[^']*'(.+)/);
|
|
if (rfc5987Match) {
|
|
const cs = normalizeCharset(rfc5987Match[1]!.toLowerCase() || 'utf-8');
|
|
const encoded = rfc5987Match[2]!;
|
|
const bytes = encoded.replace(/%([0-9A-Fa-f]{2})/g, (_, h: string) => String.fromCharCode(parseInt(h, 16)));
|
|
rawFilename = new TextDecoder(cs, { fatal: false }).decode(Buffer.from(bytes, 'binary'));
|
|
}
|
|
const filename = decodeMimeWords(rawFilename);
|
|
|
|
// Extract content-type
|
|
const ctMatch = headerBlock.match(/^Content-Type:\s*([^\s;]+)/im);
|
|
const contentType = ctMatch?.[1] ?? 'application/octet-stream';
|
|
|
|
// Extract full body content as base64
|
|
const partHeaderEnd = findHeaderEnd(raw.slice(pos));
|
|
let content = '';
|
|
let size = 0;
|
|
if (partHeaderEnd !== -1) {
|
|
const bodyStart = pos + partHeaderEnd;
|
|
const boundaryEnd = raw.indexOf('\n--', bodyStart);
|
|
const bodyRaw = boundaryEnd !== -1 ? raw.slice(bodyStart, boundaryEnd) : raw.slice(bodyStart);
|
|
|
|
// Detect encoding from part headers
|
|
const encMatch = headerBlock.match(/^Content-Transfer-Encoding:\s*(\S+)/im);
|
|
const encoding = encMatch?.[1]?.toLowerCase() ?? 'base64';
|
|
|
|
if (encoding === 'base64') {
|
|
content = bodyRaw.replace(/\s/g, '');
|
|
} else {
|
|
// For quoted-printable or 7bit/8bit, re-encode to base64
|
|
const buf = encoding === 'quoted-printable' ? decodeQuotedPrintableBytes(bodyRaw) : Buffer.from(bodyRaw);
|
|
content = buf.toString('base64');
|
|
}
|
|
size = Math.floor((content.length * 3) / 4);
|
|
}
|
|
|
|
results.push({ filename, size, contentType, content });
|
|
}
|
|
|
|
return results;
|
|
}
|