Files
platform/src/servers/sidecar/email/store.ts
T
pastilhasandClaude Opus 5 af56eb36ff email: move the mail store and every route into the sidecar
Email was the one sidecar built inside out. The platform held ~1,800 lines — the per-account
SQLite store, all 14 HTTP routes, account CRUD, resync, IMAP validation — while the 314-line
sidecar was a scheduler that reached BACK into the platform to do anything
(`import { performResync } from '../../api/email/resync'`).

The sidecar now serves its own HTTP listener and announces `email:server`, and
/api/email/* on the platform is createSidecarProxy like every other one: 1,801 lines down
to 22, with no mail knowledge left in it — not a message, not a folder, not a credential.

The routes moved verbatim, Hono and all. http.ts only reconstructs what the platform's
middleware used to provide: `user` on the context, from the X-Officer-User header the proxy
injects (trusted because this server binds loopback), and an error handler that turns
custom-errors into status codes.

The /email/events SSE stream went with them, which removes a whole round trip: the IDLE
watcher used to send `email:new` over the registration socket so the platform could push to
its SSE clients. Those clients are here now, so it calls broadcastEmailNew in-process and
`email:new` is gone from the wire protocol.

DELIBERATELY NOT DONE YET, and left backwards on purpose rather than half-moved:

- The two sync handlers (email-sync 381 lines, gmail-sync 712) still run in the platform's
  queue and now import the store from its new home — a platform → sidecar import, which is
  the wrong direction and is temporary. Moving them is option (A) from the plan: the sidecar
  schedules its own syncs, independent of the platform Jobs list.
- accounts.ts still imports queue/init to enqueue a sync and to report sync status, and
  index.ts still carries the queue-over-WS shim that inversion needs.
- The three channel handlers still open the mail store directly rather than asking over HTTP.

Two things worth knowing while testing: a from-scratch sync holds a proxied request open
well past the 60s idle default, hence timeoutSeconds on the proxy; and `gmail-sync` is
hardcoded in all three channel handlers even though the only account is provider=gmail with
auth_type=password, which routes to IMAP — so "sync emails" from a chat channel is
almost certainly already broken, and folds into the next stage.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-31 12:10:35 +00:00

747 lines
28 KiB
TypeScript

import { Database } from 'bun:sqlite';
import { createHash } from 'node:crypto';
import { join, dirname } from 'node:path';
import { chmodSync, mkdirSync } from 'node:fs';
import { getEmailDbPath } from '@@/data-path';
import { getEmailAccounts } from 'officerdb';
import type { EmailSummary } from 'types';
const SCHEMA_TABLES = `
CREATE TABLE IF NOT EXISTS emails (
id TEXT PRIMARY KEY,
integration TEXT NOT NULL DEFAULT 'gmail',
email_account TEXT NOT NULL DEFAULT '',
from_name TEXT,
from_address TEXT,
from_domain TEXT,
to_address TEXT,
cc TEXT,
subject TEXT,
date TEXT,
snippet TEXT,
html TEXT,
text_body TEXT,
attachment_count INTEGER DEFAULT 0,
read INTEGER DEFAULT 0,
deleted INTEGER DEFAULT 0,
labels TEXT,
thread_id TEXT
);
CREATE TABLE IF NOT EXISTS attachments (
id INTEGER PRIMARY KEY AUTOINCREMENT,
email_id TEXT REFERENCES emails(id) ON DELETE CASCADE,
idx INTEGER,
filename TEXT,
size INTEGER,
content_type TEXT,
content TEXT
);
CREATE TABLE IF NOT EXISTS sync_meta (
key TEXT PRIMARY KEY,
value TEXT
);
`;
const SCHEMA_INDEXES = `
CREATE INDEX IF NOT EXISTS idx_emails_date ON emails(date);
CREATE INDEX IF NOT EXISTS idx_emails_from_domain ON emails(from_domain);
CREATE INDEX IF NOT EXISTS idx_emails_from_address ON emails(from_address);
CREATE INDEX IF NOT EXISTS idx_emails_integration ON emails(integration);
CREATE INDEX IF NOT EXISTS idx_emails_email_account ON emails(email_account);
CREATE INDEX IF NOT EXISTS idx_emails_labels ON emails(labels);
CREATE INDEX IF NOT EXISTS idx_emails_thread ON emails(thread_id);
`;
/** Convert label IDs to lowercase comma-separated string for storage */
function labelsToString(labels?: string[]): string | null {
if (!labels || labels.length === 0) return null;
return labels.map((l) => l.toLowerCase()).join(',');
}
/** Convert stored comma-separated labels back to array */
function labelsFromString(value: unknown): string[] | undefined {
if (typeof value !== 'string' || !value) return undefined;
return value.split(',');
}
function extractAddress(headerValue: string): { name: string; address: string } {
const match = headerValue.match(/^"?(.+?)"?\s*<(.+?)>$/);
if (match) return { name: match[1]!.trim(), address: match[2]!.toLowerCase() };
const bare = headerValue.trim().toLowerCase();
return { name: '', address: bare };
}
function extractDomain(address: string): string {
const at = address.lastIndexOf('@');
return at >= 0 ? address.slice(at + 1) : '';
}
// ── Conversation threading ──
// `id` is sha1(Message-Id) (see resync.messageIdToStableId), so hashing a referenced Message-Id the
// same way yields the *id of that referenced email*. That makes header-based threading trivial:
// a reply's thread_id is the hash of its root Message-Id, which equals the root email's own id.
const hashMsgId = (msgId: string): string => createHash('sha1').update(msgId).digest('hex').slice(0, 16);
/** Ordered Message-Ids this email references (References first, root→leaf; else In-Reply-To). */
function extractReferenceIds(raw: string): string[] {
const refs = extractFullHeader(raw, 'References') || extractFullHeader(raw, 'In-Reply-To');
return Array.from(refs.matchAll(/<([^>]+)>/g), (m) => m[1]!.trim()).filter(Boolean);
}
/** Compute a header-based thread_id for a freshly ingested email (falls back to its own id = new thread). */
function computeThreadId(db: Database, id: string, raw: string): string {
const refIds = extractReferenceIds(raw);
if (refIds.length === 0) return id; // no ancestors → this email is a thread root
const hashed = refIds.map(hashMsgId);
// Adopt an ancestor's thread if we already have one stored (robust to In-Reply-To-only clients).
const placeholders = hashed.map(() => '?').join(',');
const found = db
.query(`SELECT thread_id FROM emails WHERE id IN (${placeholders}) AND thread_id IS NOT NULL LIMIT 1`)
.get(...hashed) as { thread_id: string } | null;
return found?.thread_id ?? hashMsgId(refIds[0]!);
}
const RE_PREFIX = /^\s*((re|fwd?|aw|wg|sv|vs|res|antw)\s*(\[\d+\])?\s*:\s*)+/i;
/** Normalize a subject for fallback grouping: strip reply/forward prefixes, fold whitespace, lowercase. */
function normalizeSubject(subject: string): string {
return subject.replace(RE_PREFIX, '').replace(/\s+/g, ' ').trim().toLowerCase();
}
const firstAddress = (value: unknown): string => {
if (typeof value !== 'string') return '';
const first = value.split(',')[0] ?? '';
return (first.match(/<([^>]+)>/)?.[1] ?? first).trim().toLowerCase();
};
/**
* Subject-based thread key for mail synced before header capture (no References available).
* Groups by normalized subject + the counterpart address, so recurring 1:1 conversations collapse
* while unrelated same-subject mail from different people stays apart. Trivial subjects stay ungrouped.
*/
function fallbackThreadId(row: { id: string; subject: unknown; from_address: unknown; to_address: unknown; email_account: unknown }): string {
const norm = normalizeSubject(typeof row.subject === 'string' ? row.subject : '');
if (!norm) return row.id;
const me = typeof row.email_account === 'string' ? row.email_account.toLowerCase() : '';
const from = typeof row.from_address === 'string' ? row.from_address.toLowerCase() : '';
const counterpart = from && from !== me ? from : firstAddress(row.to_address) || from;
return `s:${norm}|${counterpart}`;
}
export function openEmailDb(ownerEmail: string, accountEmail: string): Database {
const dbPath = getEmailDbPath(ownerEmail, accountEmail);
mkdirSync(dirname(dbPath), { recursive: true });
const db = new Database(dbPath, { create: true });
// The API and the email sidecar both open this file; wait out a concurrent writer (e.g. a resync
// or the one-time thread_id backfill) instead of failing immediately with "database is locked".
db.exec('PRAGMA busy_timeout = 5000');
db.exec('PRAGMA journal_mode = DELETE');
db.exec('PRAGMA foreign_keys = ON');
db.exec(SCHEMA_TABLES);
migrate(db);
db.exec(SCHEMA_INDEXES);
ensureFts(db);
// Try to chmod, but don't crash if permission denied (e.g., file owned by different user)
try {
chmodSync(dbPath, 0o666);
} catch (err) {
// File exists with correct permissions, or owned by another user - that's fine
}
return db;
}
/**
* Open the email DB for a user's configured account. Emails are stored per account
* (email_accounts/<account>/emails.db); for now we use the user's first account. Returns null if the
* user has no email account configured yet.
*/
export async function openUserEmailDb(ownerEmail: string, userId: number): Promise<Database | null> {
const accounts = await getEmailAccounts(userId);
const account = accounts.find((a) => a.enabled) ?? accounts[0];
return account ? openEmailDb(ownerEmail, account.email) : null;
}
function migrate(db: Database): void {
const cols = db.query('PRAGMA table_info(emails)').all() as Array<{ name: string }>;
const colNames = new Set(cols.map((c) => c.name));
if (!colNames.has('deleted')) {
db.exec('ALTER TABLE emails ADD COLUMN deleted INTEGER DEFAULT 0');
}
if (!colNames.has('integration')) {
db.exec("ALTER TABLE emails ADD COLUMN integration TEXT NOT NULL DEFAULT 'gmail'");
}
if (!colNames.has('email_account')) {
db.exec("ALTER TABLE emails ADD COLUMN email_account TEXT NOT NULL DEFAULT ''");
}
if (!colNames.has('labels')) {
db.exec('ALTER TABLE emails ADD COLUMN labels TEXT');
}
if (!colNames.has('thread_id')) {
db.exec('ALTER TABLE emails ADD COLUMN thread_id TEXT');
}
// Backfill thread_id for any rows missing it. Header data isn't kept for already-synced mail, so
// these use the subject-based fallback. New mail gets an exact header-based thread_id at insert.
backfillThreadIds(db);
// Ensure sync_meta table exists (for DBs created before it was added to SCHEMA_TABLES)
db.exec('CREATE TABLE IF NOT EXISTS sync_meta (key TEXT PRIMARY KEY, value TEXT)');
// Add content column to attachments if missing
const attCols = db.query('PRAGMA table_info(attachments)').all() as Array<{ name: string }>;
const attColNames = new Set(attCols.map((c) => c.name));
if (!attColNames.has('content')) {
db.exec('ALTER TABLE attachments ADD COLUMN content TEXT');
}
}
function backfillThreadIds(db: Database): void {
const rows = db
.query('SELECT id, subject, from_address, to_address, email_account FROM emails WHERE thread_id IS NULL')
.all() as Array<{ id: string; subject: unknown; from_address: unknown; to_address: unknown; email_account: unknown }>;
if (rows.length === 0) return;
const update = db.prepare('UPDATE emails SET thread_id = ? WHERE id = ?');
db.exec('BEGIN');
try {
for (const row of rows) update.run(fallbackThreadId(row), row.id);
db.exec('COMMIT');
} catch (err) {
db.exec('ROLLBACK');
throw err;
}
}
// ── Full-text search (FTS5) ──
// A standalone FTS5 index kept in sync with `emails` on every upsert. unicode61 + diacritic folding
// gives accent-insensitive matching; per-term prefix queries make it feel incremental.
function ensureFts(db: Database): void {
db.exec(
`CREATE VIRTUAL TABLE IF NOT EXISTS emails_fts USING fts5(
id UNINDEXED, subject, sender, recipients, snippet, body,
tokenize = 'unicode61 remove_diacritics 2'
)`,
);
const fts = (db.query('SELECT count(*) AS c FROM emails_fts').get() as { c: number }).c;
const total = (db.query('SELECT count(*) AS c FROM emails').get() as { c: number }).c;
if (fts === 0 && total > 0) {
// Backfill existing mail (first run after this feature ships).
db.exec(
`INSERT INTO emails_fts (id, subject, sender, recipients, snippet, body)
SELECT id, COALESCE(subject, ''),
TRIM(COALESCE(from_name, '') || ' ' || COALESCE(from_address, '')),
TRIM(COALESCE(to_address, '') || ' ' || COALESCE(cc, '')),
COALESCE(snippet, ''), COALESCE(text_body, '')
FROM emails`,
);
}
}
const ftsDeleteStmt = 'DELETE FROM emails_fts WHERE id = ?';
const ftsInsertStmt = 'INSERT INTO emails_fts (id, subject, sender, recipients, snippet, body) VALUES (?, ?, ?, ?, ?, ?)';
function syncFtsRow(db: Database, id: string, subject: string, sender: string, recipients: string, snippet: string, body: string): void {
db.run(ftsDeleteStmt, [id]);
db.run(ftsInsertStmt, [id, subject, sender, recipients, snippet, body]);
}
// Gmail-style query parsing. Free text → full-text (prefix-AND). Operators:
// from:/to:/subject:/body: → FTS5 column filters
// has:attachment, is:unread/read, label:X, before:/after:YYYY-MM-DD → SQL filters on `emails`
// Unknown operators fall back to free text. Quoted values (from:"a b") match as an exact phrase.
const FTS_COLUMNS: Record<string, string> = { from: 'sender', to: 'recipients', subject: 'subject', body: 'body' };
const ftsTerm = (value: string, column: string | null, prefix: boolean): string => {
const esc = value.replace(/"/g, '""');
return `${column ? column + ':' : ''}"${esc}"${prefix ? '*' : ''}`;
};
const parseSearchDate = (v: string): string | null => {
const iso = /^\d{4}-\d{2}-\d{2}$/.test(v) ? `${v}T00:00:00.000Z` : v;
const d = new Date(iso);
return Number.isNaN(d.getTime()) ? null : d.toISOString();
};
// One AND-group of terms (a single OR branch). `fts` is the FTS5 MATCH expression for this branch;
// `where`/`params` are its structured SQL filters.
type Branch = { fts: string; where: string[]; params: string[] };
function parseBranch(q: string): Branch {
const fts: string[] = [];
const where: string[] = [];
const params: string[] = [];
const re = /(\w+):("[^"]*"|\S+)|"([^"]*)"|(\S+)/g;
let m: RegExpExecArray | null;
while ((m = re.exec(q)) !== null) {
if (m[1]) {
const op = m[1].toLowerCase();
const raw = m[2]!;
const quoted = raw.startsWith('"') && raw.endsWith('"');
const val = quoted ? raw.slice(1, -1) : raw;
if (!val) continue;
const lower = val.toLowerCase();
if (FTS_COLUMNS[op]) {
fts.push(ftsTerm(val, FTS_COLUMNS[op], !quoted));
} else if (op === 'label') {
where.push('e.labels LIKE ?');
params.push(`%${lower}%`);
} else if (op === 'has' && (lower === 'attachment' || lower === 'attachments')) {
where.push('e.attachment_count > 0');
} else if (op === 'is' && (lower === 'unread' || lower === 'read')) {
where.push(lower === 'unread' ? 'e.read = 0' : 'e.read = 1');
} else if (op === 'before' || op === 'older') {
const d = parseSearchDate(val);
if (d) {
where.push('e.date < ?');
params.push(d);
}
} else if (op === 'after' || op === 'newer') {
const d = parseSearchDate(val);
if (d) {
where.push('e.date >= ?');
params.push(d);
}
} else {
// Unknown operator — treat the whole "op:val" token as free text.
fts.push(ftsTerm(`${op}:${val}`, null, !quoted));
}
} else if (m[3] !== undefined) {
if (m[3].trim()) fts.push(ftsTerm(m[3], null, false)); // quoted phrase → exact
} else if (m[4]) {
fts.push(ftsTerm(m[4], null, true)); // bare word → prefix
}
}
return { fts: fts.join(' '), where, params };
}
export function searchEmails(db: Database, q: string, limit: number, offset: number): { rows: Record<string, unknown>[]; total: number } {
// Split on top-level uppercase OR into branches (Gmail-style; lowercase "or" stays a search word).
const branches = q
.split(/\s+OR\s+/)
.map(parseBranch)
.filter((b) => b.fts || b.where.length > 0);
if (branches.length === 0) return { rows: [], total: 0 };
// Each branch becomes one self-contained condition: its FTS terms via an `id IN (FTS subquery)` so
// full-text and structured filters share a WHERE and branches can be OR'd. All ANDed within a branch.
const conds: string[] = [];
const params: string[] = [];
for (const b of branches) {
const parts: string[] = [];
if (b.fts) {
parts.push('e.id IN (SELECT emails_fts.id FROM emails_fts WHERE emails_fts MATCH ?)');
params.push(b.fts);
}
parts.push(...b.where);
params.push(...b.params);
conds.push(`(${parts.join(' AND ')})`);
}
const whereSql = `e.deleted = 0 AND (${conds.join(' OR ')})`;
const rows = db.query(`SELECT e.* FROM emails e WHERE ${whereSql} ORDER BY e.date DESC LIMIT ? OFFSET ?`).all(...params, limit, offset) as Record<string, unknown>[];
const total = (db.query(`SELECT count(*) AS c FROM emails e WHERE ${whereSql}`).get(...params) as { c: number }).c;
return { rows, total };
}
type ParsedEmail = {
id: string;
integration: string;
emailAccount: string;
fromName: string;
fromAddress: string;
to: string;
cc?: string;
subject: string;
date: string;
snippet: string;
html?: string;
text?: string;
attachments: Array<{ filename: string; size: number; contentType: string; content: string }>;
labels?: string[];
threadId?: string;
};
const upsertEmailStmt = `
INSERT OR REPLACE INTO emails (id, integration, email_account, from_name, from_address, from_domain, to_address, cc, subject, date, snippet, html, text_body, attachment_count, labels, thread_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`;
const deleteAttachmentsStmt = 'DELETE FROM attachments WHERE email_id = ?';
const insertAttachmentStmt = 'INSERT INTO attachments (email_id, idx, filename, size, content_type, content) VALUES (?, ?, ?, ?, ?, ?)';
export function upsertEmail(db: Database, email: ParsedEmail): void {
const domain = extractDomain(email.fromAddress);
db.exec('BEGIN');
try {
db.run(upsertEmailStmt, [
email.id,
email.integration,
email.emailAccount,
email.fromName,
email.fromAddress,
domain,
email.to,
email.cc ?? null,
email.subject,
email.date,
email.snippet,
email.html ?? null,
email.text ?? null,
email.attachments.length,
labelsToString(email.labels),
email.threadId ?? email.id,
]);
db.run(deleteAttachmentsStmt, [email.id]);
for (let i = 0; i < email.attachments.length; i++) {
const att = email.attachments[i]!;
db.run(insertAttachmentStmt, [email.id, i, att.filename, att.size, att.contentType, att.content]);
}
syncFtsRow(db, email.id, email.subject ?? '', `${email.fromName ?? ''} ${email.fromAddress ?? ''}`.trim(), `${email.to ?? ''} ${email.cc ?? ''}`.trim(), email.snippet ?? '', email.text ?? '');
db.exec('COMMIT');
} catch (err) {
db.exec('ROLLBACK');
throw err;
}
}
/** Upsert a single email from its raw RFC822 text using fast header parsing. */
type UpsertFromRawEmlParams = {
db: Database;
id: string;
raw: string;
integration: string;
emailAccount: string;
labels?: string[];
};
export function upsertFromRawEml({ db, id, raw, integration, emailAccount, labels }: UpsertFromRawEmlParams): void {
const from = extractHeader(raw, 'From');
const { name, address } = extractAddress(from);
const to = extractHeader(raw, 'To');
const cc = extractHeader(raw, 'Cc') || null;
const subject = extractHeader(raw, 'Subject') || '(no subject)';
const dateStr = extractHeader(raw, 'Date');
const date = dateStr ? new Date(dateStr).toISOString() : new Date(0).toISOString();
const snippet = extractSnippet(raw);
const attachments = parseAttachments(raw);
const domain = extractDomain(address);
const { html, text } = extractBody(raw);
const threadId = computeThreadId(db, id, raw);
db.run(upsertEmailStmt, [
id, integration, emailAccount, name, address, domain, to, cc, subject, date, snippet, html, text, attachments.length, labelsToString(labels), threadId,
]);
if (attachments.length > 0) {
db.run(deleteAttachmentsStmt, [id]);
for (let i = 0; i < attachments.length; i++) {
const att = attachments[i]!;
db.run(insertAttachmentStmt, [id, i, att.filename, att.size, att.contentType, att.content]);
}
}
syncFtsRow(db, id, subject, `${name} ${address}`.trim(), `${to} ${cc ?? ''}`.trim(), snippet, text ?? '');
}
/** Convert a db row to an EmailSummary for the API */
export function rowToSummary(row: Record<string, unknown>): EmailSummary {
const from = row.from_name
? `${row.from_name} <${row.from_address}>`
: (row.from_address as string);
const labels = labelsFromString(row.labels);
return {
id: row.id as string,
from,
to: row.to_address as string,
subject: row.subject as string,
date: row.date as string,
snippet: row.snippet as string,
...(row.attachment_count ? { attachmentCount: row.attachment_count as number } : {}),
...(row.read ? { read: true } : {}),
...(labels ? { labels } : {}),
...(row.thread_count && (row.thread_count as number) > 1 ? { threadCount: row.thread_count as number } : {}),
...(row.thread_unread ? { threadUnread: row.thread_unread as number } : {}),
};
}
// ── Sync meta helpers ──
export function getSyncMeta(db: Database, key: string): string | null {
const row = db.query('SELECT value FROM sync_meta WHERE key = ?').get(key) as { value: string } | null;
return row?.value ?? null;
}
export function setSyncMeta(db: Database, key: string, value: string): void {
db.run('INSERT OR REPLACE INTO sync_meta (key, value) VALUES (?, ?)', [key, value]);
}
export function updateEmailLabels(db: Database, id: string, labels: string[]): void {
db.run('UPDATE emails SET labels = ? WHERE id = ?', [labelsToString(labels), id]);
}
// ── Header parsing helpers (same logic as gmail-sync) ──
function decodeMimeWords(text: string): string {
return text.replace(/=\?([^?]+)\?(B|Q)\?([^?]+)\?=/gi, (_, charset, encoding, encoded) => {
try {
const normalizedCs = normalizeCharset(charset.toLowerCase());
if (encoding.toUpperCase() === 'B') {
const buf = Buffer.from(encoded, 'base64');
return new TextDecoder(normalizedCs, { fatal: false }).decode(buf);
}
const bytes: number[] = [];
for (let i = 0; i < encoded.length; i++) {
if (encoded[i] === '_') {
bytes.push(0x20);
} else if (encoded[i] === '=' && i + 2 < encoded.length) {
bytes.push(parseInt(encoded.slice(i + 1, i + 3), 16));
i += 2;
} else {
bytes.push(encoded.charCodeAt(i));
}
}
return new TextDecoder(normalizedCs, { fatal: false }).decode(Buffer.from(bytes));
} catch {
return encoded;
}
});
}
function extractHeader(raw: string, name: string): string {
const match = raw.match(new RegExp(`^${name}:\\s*(.+)$`, 'mi'));
return match?.[1]?.trim() ? decodeMimeWords(match[1].trim()) : '';
}
/** Extract a header value including folded continuation lines (lines starting with whitespace) */
function extractFullHeader(raw: string, name: string): string {
const headerEnd = findHeaderEnd(raw);
const headerBlock = headerEnd !== -1 ? raw.slice(0, headerEnd) : raw.slice(0, 4096);
const lines = headerBlock.split(/\r?\n/);
let result = '';
let capturing = false;
for (const line of lines) {
if (new RegExp(`^${name}:\\s*`, 'i').test(line)) {
result = line.replace(new RegExp(`^${name}:\\s*`, 'i'), '');
capturing = true;
} else if (capturing && /^[\t ]/.test(line)) {
result += ' ' + line.trim();
} else if (capturing) {
break;
}
}
return result.trim();
}
function findHeaderEnd(text: string): number {
const crlf = text.indexOf('\r\n\r\n');
const lf = text.indexOf('\n\n');
if (crlf !== -1) return crlf + 4;
if (lf !== -1) return lf + 2;
return -1;
}
function extractSnippet(raw: string): string {
const idx = findHeaderEnd(raw);
if (idx === -1) return '';
let body = raw.slice(idx);
if (body.trimStart().startsWith('--')) {
const afterBoundary = body.slice(body.indexOf('\n') + 1);
const partBodyStart = findHeaderEnd(afterBoundary);
if (partBodyStart !== -1) body = afterBoundary.slice(partBodyStart);
}
const nextBoundary = body.indexOf('\n--');
if (nextBoundary !== -1) body = body.slice(0, nextBoundary);
return body.replace(/\s+/g, ' ').trim().slice(0, 120);
}
function decodeQuotedPrintableBytes(text: string): Buffer {
const cleaned = text.replace(/=\r?\n/g, '');
const bytes: number[] = [];
for (let i = 0; i < cleaned.length; i++) {
if (cleaned[i] === '=' && i + 2 < cleaned.length) {
const hex = cleaned.slice(i + 1, i + 3);
const val = parseInt(hex, 16);
if (!isNaN(val)) {
bytes.push(val);
i += 2;
continue;
}
}
bytes.push(cleaned.charCodeAt(i));
}
return Buffer.from(bytes);
}
function extractCharset(contentType: string): string {
const match = contentType.match(/charset=["']?([^"';\s]+)/i);
return match?.[1]?.toLowerCase() ?? 'utf-8';
}
function normalizeCharset(charset: string): string {
const map: Record<string, string> = {
'iso-8859-1': 'latin1',
'iso_8859-1': 'latin1',
'iso-8859-15': 'latin1',
'iso_8859-15': 'latin1',
'windows-1250': 'latin1',
'windows-1251': 'latin1',
'windows-1252': 'latin1',
'windows-1254': 'latin1',
'us-ascii': 'ascii',
'ascii': 'ascii',
};
return map[charset] ?? charset;
}
function decodePartBody(body: string, encoding: string, charset = 'utf-8'): string {
const enc = encoding.toLowerCase();
const normalizedCharset = normalizeCharset(charset);
if (enc === 'base64') {
const buf = Buffer.from(body.replace(/\s/g, ''), 'base64');
return new TextDecoder(normalizedCharset, { fatal: false }).decode(buf);
}
if (enc === 'quoted-printable') {
const buf = decodeQuotedPrintableBytes(body);
return new TextDecoder(normalizedCharset, { fatal: false }).decode(buf);
}
return body;
}
function extractBody(raw: string): { html: string | null; text: string | null } {
const headerEnd = findHeaderEnd(raw);
if (headerEnd === -1) return { html: null, text: null };
const topCtRaw = extractFullHeader(raw, 'Content-Type');
const topCt = topCtRaw.toLowerCase();
const topEncoding = extractFullHeader(raw, 'Content-Transfer-Encoding');
// Non-multipart: single body
if (!topCt.includes('multipart')) {
const body = raw.slice(headerEnd);
const charset = extractCharset(topCtRaw);
const decoded = decodePartBody(body, topEncoding, charset);
if (topCt.includes('text/html')) return { html: decoded, text: null };
return { html: null, text: decoded };
}
// Multipart: extract boundary from the raw (case-sensitive) header
const boundaryMatch = topCtRaw.match(/boundary=["']?([^"';\s]+)/i);
if (!boundaryMatch) return { html: null, text: null };
const boundary = boundaryMatch[1]!;
let html: string | null = null;
let text: string | null = null;
const parts = raw.slice(headerEnd).split(`--${boundary}`);
for (const part of parts) {
if (part.startsWith('--') || !part.trim()) continue;
const partHeaderEnd = findHeaderEnd(part);
if (partHeaderEnd === -1) continue;
const partCtRaw = extractFullHeader(part, 'Content-Type');
const partCt = partCtRaw.toLowerCase();
const partEnc = extractFullHeader(part, 'Content-Transfer-Encoding');
const partCharset = extractCharset(partCtRaw);
const partBody = part.slice(partHeaderEnd);
// Recurse into nested multipart (e.g. multipart/alternative inside multipart/mixed)
if (partCt.includes('multipart')) {
const nested = extractBody(part.trim());
if (nested.html && !html) html = nested.html;
if (nested.text && !text) text = nested.text;
continue;
}
if (partCt.includes('text/html') && !html) {
html = decodePartBody(partBody, partEnc, partCharset);
} else if (partCt.includes('text/plain') && !text) {
text = decodePartBody(partBody, partEnc, partCharset);
}
}
return { html, text };
}
type AttachmentMeta = { filename: string; size: number; contentType: string; content: string };
function parseAttachments(raw: string): AttachmentMeta[] {
const results: AttachmentMeta[] = [];
// Match both "attachment" and "inline" dispositions
const regex = /^Content-Disposition:\s*(?:attachment|inline)[^\n]*/gim;
let match: RegExpExecArray | null;
while ((match = regex.exec(raw)) !== null) {
const pos = match.index;
// Walk backwards to find the start of this MIME part's headers
const partStart = raw.lastIndexOf('\n--', pos);
const headerBlock = partStart !== -1 ? raw.slice(partStart, pos + 500) : raw.slice(Math.max(0, pos - 500), pos + 500);
// Skip inline parts without a filename (e.g. inline text/plain body parts)
const hasFilename = /filename/i.test(headerBlock);
if (!hasFilename) continue;
// Extract filename from Content-Disposition or Content-Type
const fnMatch = headerBlock.match(/filename\*?=(?:"([^"]+)"|([^\s;]+))/i);
let rawFilename = fnMatch ? (fnMatch[1] ?? fnMatch[2] ?? 'unknown') : 'unknown';
// RFC 5987: filename*=charset''percent-encoded
const rfc5987Match = rawFilename.match(/^([^']*)'[^']*'(.+)/);
if (rfc5987Match) {
const cs = normalizeCharset(rfc5987Match[1]!.toLowerCase() || 'utf-8');
const encoded = rfc5987Match[2]!;
const bytes = encoded.replace(/%([0-9A-Fa-f]{2})/g, (_, h: string) => String.fromCharCode(parseInt(h, 16)));
rawFilename = new TextDecoder(cs, { fatal: false }).decode(Buffer.from(bytes, 'binary'));
}
const filename = decodeMimeWords(rawFilename);
// Extract content-type
const ctMatch = headerBlock.match(/^Content-Type:\s*([^\s;]+)/im);
const contentType = ctMatch?.[1] ?? 'application/octet-stream';
// Extract full body content as base64
const partHeaderEnd = findHeaderEnd(raw.slice(pos));
let content = '';
let size = 0;
if (partHeaderEnd !== -1) {
const bodyStart = pos + partHeaderEnd;
const boundaryEnd = raw.indexOf('\n--', bodyStart);
const bodyRaw = boundaryEnd !== -1 ? raw.slice(bodyStart, boundaryEnd) : raw.slice(bodyStart);
// Detect encoding from part headers
const encMatch = headerBlock.match(/^Content-Transfer-Encoding:\s*(\S+)/im);
const encoding = encMatch?.[1]?.toLowerCase() ?? 'base64';
if (encoding === 'base64') {
content = bodyRaw.replace(/\s/g, '');
} else {
// For quoted-printable or 7bit/8bit, re-encode to base64
const buf = encoding === 'quoted-printable'
? decodeQuotedPrintableBytes(bodyRaw)
: Buffer.from(bodyRaw);
content = buf.toString('base64');
}
size = Math.floor(content.length * 3 / 4);
}
results.push({ filename, size, contentType, content });
}
return results;
}