email syncyng
This commit is contained in:
@@ -0,0 +1,444 @@
|
||||
import { Database } from 'bun:sqlite';
|
||||
import { join } from 'node:path';
|
||||
import { chmodSync } from 'node:fs';
|
||||
import { DATA_PATH } from '@@/data-path';
|
||||
import type { EmailSummary } from 'types';
|
||||
|
||||
const SCHEMA_TABLES = `
|
||||
CREATE TABLE IF NOT EXISTS emails (
|
||||
id TEXT PRIMARY KEY,
|
||||
integration TEXT NOT NULL DEFAULT 'gmail',
|
||||
email_account TEXT NOT NULL DEFAULT '',
|
||||
from_name TEXT,
|
||||
from_address TEXT,
|
||||
from_domain TEXT,
|
||||
to_address TEXT,
|
||||
cc TEXT,
|
||||
subject TEXT,
|
||||
date TEXT,
|
||||
snippet TEXT,
|
||||
html TEXT,
|
||||
text_body TEXT,
|
||||
attachment_count INTEGER DEFAULT 0,
|
||||
read INTEGER DEFAULT 0,
|
||||
deleted INTEGER DEFAULT 0,
|
||||
labels TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
email_id TEXT REFERENCES emails(id) ON DELETE CASCADE,
|
||||
idx INTEGER,
|
||||
filename TEXT,
|
||||
size INTEGER,
|
||||
content_type TEXT,
|
||||
content TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sync_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
`;
|
||||
|
||||
const SCHEMA_INDEXES = `
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_date ON emails(date);
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_from_domain ON emails(from_domain);
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_from_address ON emails(from_address);
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_integration ON emails(integration);
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_email_account ON emails(email_account);
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_labels ON emails(labels);
|
||||
`;
|
||||
|
||||
/** Convert label IDs to lowercase comma-separated string for storage */
|
||||
function labelsToString(labels?: string[]): string | null {
|
||||
if (!labels || labels.length === 0) return null;
|
||||
return labels.map((l) => l.toLowerCase()).join(',');
|
||||
}
|
||||
|
||||
/** Convert stored comma-separated labels back to array */
|
||||
function labelsFromString(value: unknown): string[] | undefined {
|
||||
if (typeof value !== 'string' || !value) return undefined;
|
||||
return value.split(',');
|
||||
}
|
||||
|
||||
function extractAddress(headerValue: string): { name: string; address: string } {
|
||||
const match = headerValue.match(/^"?(.+?)"?\s*<(.+?)>$/);
|
||||
if (match) return { name: match[1]!.trim(), address: match[2]!.toLowerCase() };
|
||||
const bare = headerValue.trim().toLowerCase();
|
||||
return { name: '', address: bare };
|
||||
}
|
||||
|
||||
function extractDomain(address: string): string {
|
||||
const at = address.lastIndexOf('@');
|
||||
return at >= 0 ? address.slice(at + 1) : '';
|
||||
}
|
||||
|
||||
export function openEmailDb(email: string): Database {
|
||||
const dbPath = join(DATA_PATH, email, 'emails.db');
|
||||
const db = new Database(dbPath, { create: true });
|
||||
db.exec('PRAGMA journal_mode = DELETE');
|
||||
db.exec('PRAGMA foreign_keys = ON');
|
||||
db.exec(SCHEMA_TABLES);
|
||||
migrate(db);
|
||||
db.exec(SCHEMA_INDEXES);
|
||||
chmodSync(dbPath, 0o666);
|
||||
return db;
|
||||
}
|
||||
|
||||
function migrate(db: Database): void {
|
||||
const cols = db.query('PRAGMA table_info(emails)').all() as Array<{ name: string }>;
|
||||
const colNames = new Set(cols.map((c) => c.name));
|
||||
|
||||
if (!colNames.has('deleted')) {
|
||||
db.exec('ALTER TABLE emails ADD COLUMN deleted INTEGER DEFAULT 0');
|
||||
}
|
||||
if (!colNames.has('integration')) {
|
||||
db.exec("ALTER TABLE emails ADD COLUMN integration TEXT NOT NULL DEFAULT 'gmail'");
|
||||
}
|
||||
if (!colNames.has('email_account')) {
|
||||
db.exec("ALTER TABLE emails ADD COLUMN email_account TEXT NOT NULL DEFAULT ''");
|
||||
}
|
||||
if (!colNames.has('labels')) {
|
||||
db.exec('ALTER TABLE emails ADD COLUMN labels TEXT');
|
||||
}
|
||||
|
||||
// Ensure sync_meta table exists (for DBs created before it was added to SCHEMA_TABLES)
|
||||
db.exec('CREATE TABLE IF NOT EXISTS sync_meta (key TEXT PRIMARY KEY, value TEXT)');
|
||||
|
||||
// Add content column to attachments if missing
|
||||
const attCols = db.query('PRAGMA table_info(attachments)').all() as Array<{ name: string }>;
|
||||
const attColNames = new Set(attCols.map((c) => c.name));
|
||||
if (!attColNames.has('content')) {
|
||||
db.exec('ALTER TABLE attachments ADD COLUMN content TEXT');
|
||||
}
|
||||
}
|
||||
|
||||
type ParsedEmail = {
|
||||
id: string;
|
||||
integration: string;
|
||||
emailAccount: string;
|
||||
fromName: string;
|
||||
fromAddress: string;
|
||||
to: string;
|
||||
cc?: string;
|
||||
subject: string;
|
||||
date: string;
|
||||
snippet: string;
|
||||
html?: string;
|
||||
text?: string;
|
||||
attachments: Array<{ filename: string; size: number; contentType: string; content: string }>;
|
||||
labels?: string[];
|
||||
};
|
||||
|
||||
const upsertEmailStmt = `
|
||||
INSERT OR REPLACE INTO emails (id, integration, email_account, from_name, from_address, from_domain, to_address, cc, subject, date, snippet, html, text_body, attachment_count, labels)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`;
|
||||
|
||||
const deleteAttachmentsStmt = 'DELETE FROM attachments WHERE email_id = ?';
|
||||
const insertAttachmentStmt = 'INSERT INTO attachments (email_id, idx, filename, size, content_type, content) VALUES (?, ?, ?, ?, ?, ?)';
|
||||
|
||||
export function upsertEmail(db: Database, email: ParsedEmail): void {
|
||||
const domain = extractDomain(email.fromAddress);
|
||||
|
||||
db.exec('BEGIN');
|
||||
try {
|
||||
db.run(upsertEmailStmt, [
|
||||
email.id,
|
||||
email.integration,
|
||||
email.emailAccount,
|
||||
email.fromName,
|
||||
email.fromAddress,
|
||||
domain,
|
||||
email.to,
|
||||
email.cc ?? null,
|
||||
email.subject,
|
||||
email.date,
|
||||
email.snippet,
|
||||
email.html ?? null,
|
||||
email.text ?? null,
|
||||
email.attachments.length,
|
||||
labelsToString(email.labels),
|
||||
]);
|
||||
|
||||
db.run(deleteAttachmentsStmt, [email.id]);
|
||||
for (let i = 0; i < email.attachments.length; i++) {
|
||||
const att = email.attachments[i]!;
|
||||
db.run(insertAttachmentStmt, [email.id, i, att.filename, att.size, att.contentType, att.content]);
|
||||
}
|
||||
|
||||
db.exec('COMMIT');
|
||||
} catch (err) {
|
||||
db.exec('ROLLBACK');
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** Upsert a single email from its raw RFC822 text using fast header parsing. */
|
||||
type UpsertFromRawEmlParams = {
|
||||
db: Database;
|
||||
id: string;
|
||||
raw: string;
|
||||
integration: string;
|
||||
emailAccount: string;
|
||||
labels?: string[];
|
||||
};
|
||||
|
||||
export function upsertFromRawEml({ db, id, raw, integration, emailAccount, labels }: UpsertFromRawEmlParams): void {
|
||||
const from = extractHeader(raw, 'From');
|
||||
const { name, address } = extractAddress(from);
|
||||
const to = extractHeader(raw, 'To');
|
||||
const cc = extractHeader(raw, 'Cc') || null;
|
||||
const subject = extractHeader(raw, 'Subject') || '(no subject)';
|
||||
const dateStr = extractHeader(raw, 'Date');
|
||||
const date = dateStr ? new Date(dateStr).toISOString() : new Date(0).toISOString();
|
||||
const snippet = extractSnippet(raw);
|
||||
const attachments = parseAttachments(raw);
|
||||
const domain = extractDomain(address);
|
||||
|
||||
const { html, text } = extractBody(raw);
|
||||
|
||||
db.run(upsertEmailStmt, [
|
||||
id, integration, emailAccount, name, address, domain, to, cc, subject, date, snippet, html, text, attachments.length, labelsToString(labels),
|
||||
]);
|
||||
|
||||
if (attachments.length > 0) {
|
||||
db.run(deleteAttachmentsStmt, [id]);
|
||||
for (let i = 0; i < attachments.length; i++) {
|
||||
const att = attachments[i]!;
|
||||
db.run(insertAttachmentStmt, [id, i, att.filename, att.size, att.contentType, att.content]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Convert a db row to an EmailSummary for the API */
|
||||
export function rowToSummary(row: Record<string, unknown>): EmailSummary {
|
||||
const from = row.from_name
|
||||
? `${row.from_name} <${row.from_address}>`
|
||||
: (row.from_address as string);
|
||||
|
||||
const labels = labelsFromString(row.labels);
|
||||
|
||||
return {
|
||||
id: row.id as string,
|
||||
from,
|
||||
to: row.to_address as string,
|
||||
subject: row.subject as string,
|
||||
date: row.date as string,
|
||||
snippet: row.snippet as string,
|
||||
...(row.attachment_count ? { attachmentCount: row.attachment_count as number } : {}),
|
||||
...(row.read ? { read: true } : {}),
|
||||
...(labels ? { labels } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Sync meta helpers ──
|
||||
|
||||
export function getSyncMeta(db: Database, key: string): string | null {
|
||||
const row = db.query('SELECT value FROM sync_meta WHERE key = ?').get(key) as { value: string } | null;
|
||||
return row?.value ?? null;
|
||||
}
|
||||
|
||||
export function setSyncMeta(db: Database, key: string, value: string): void {
|
||||
db.run('INSERT OR REPLACE INTO sync_meta (key, value) VALUES (?, ?)', [key, value]);
|
||||
}
|
||||
|
||||
export function updateEmailLabels(db: Database, id: string, labels: string[]): void {
|
||||
db.run('UPDATE emails SET labels = ? WHERE id = ?', [labelsToString(labels), id]);
|
||||
}
|
||||
|
||||
// ── Header parsing helpers (same logic as gmail-sync) ──
|
||||
|
||||
function decodeMimeWords(text: string): string {
|
||||
return text.replace(/=\?([^?]+)\?(B|Q)\?([^?]+)\?=/gi, (_, _charset, encoding, encoded) => {
|
||||
try {
|
||||
if (encoding.toUpperCase() === 'B') {
|
||||
return Buffer.from(encoded, 'base64').toString('utf-8');
|
||||
}
|
||||
const bytes: number[] = [];
|
||||
for (let i = 0; i < encoded.length; i++) {
|
||||
if (encoded[i] === '_') {
|
||||
bytes.push(0x20);
|
||||
} else if (encoded[i] === '=' && i + 2 < encoded.length) {
|
||||
bytes.push(parseInt(encoded.slice(i + 1, i + 3), 16));
|
||||
i += 2;
|
||||
} else {
|
||||
bytes.push(encoded.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
return Buffer.from(bytes).toString('utf-8');
|
||||
} catch {
|
||||
return encoded;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function extractHeader(raw: string, name: string): string {
|
||||
const match = raw.match(new RegExp(`^${name}:\\s*(.+)$`, 'mi'));
|
||||
return match?.[1]?.trim() ? decodeMimeWords(match[1].trim()) : '';
|
||||
}
|
||||
|
||||
/** Extract a header value including folded continuation lines (lines starting with whitespace) */
|
||||
function extractFullHeader(raw: string, name: string): string {
|
||||
const headerEnd = findHeaderEnd(raw);
|
||||
const headerBlock = headerEnd !== -1 ? raw.slice(0, headerEnd) : raw.slice(0, 4096);
|
||||
const lines = headerBlock.split(/\r?\n/);
|
||||
let result = '';
|
||||
let capturing = false;
|
||||
for (const line of lines) {
|
||||
if (new RegExp(`^${name}:\\s*`, 'i').test(line)) {
|
||||
result = line.replace(new RegExp(`^${name}:\\s*`, 'i'), '');
|
||||
capturing = true;
|
||||
} else if (capturing && /^[\t ]/.test(line)) {
|
||||
result += ' ' + line.trim();
|
||||
} else if (capturing) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result.trim();
|
||||
}
|
||||
|
||||
function findHeaderEnd(text: string): number {
|
||||
const crlf = text.indexOf('\r\n\r\n');
|
||||
const lf = text.indexOf('\n\n');
|
||||
if (crlf !== -1) return crlf + 4;
|
||||
if (lf !== -1) return lf + 2;
|
||||
return -1;
|
||||
}
|
||||
|
||||
function extractSnippet(raw: string): string {
|
||||
const idx = findHeaderEnd(raw);
|
||||
if (idx === -1) return '';
|
||||
|
||||
let body = raw.slice(idx);
|
||||
|
||||
if (body.trimStart().startsWith('--')) {
|
||||
const afterBoundary = body.slice(body.indexOf('\n') + 1);
|
||||
const partBodyStart = findHeaderEnd(afterBoundary);
|
||||
if (partBodyStart !== -1) body = afterBoundary.slice(partBodyStart);
|
||||
}
|
||||
|
||||
const nextBoundary = body.indexOf('\n--');
|
||||
if (nextBoundary !== -1) body = body.slice(0, nextBoundary);
|
||||
|
||||
return body.replace(/\s+/g, ' ').trim().slice(0, 120);
|
||||
}
|
||||
|
||||
function decodeQuotedPrintable(text: string): string {
|
||||
return text
|
||||
.replace(/=\r?\n/g, '')
|
||||
.replace(/=([0-9A-Fa-f]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
|
||||
}
|
||||
|
||||
function decodePartBody(body: string, encoding: string): string {
|
||||
const enc = encoding.toLowerCase();
|
||||
if (enc === 'base64') return Buffer.from(body.replace(/\s/g, ''), 'base64').toString('utf-8');
|
||||
if (enc === 'quoted-printable') return decodeQuotedPrintable(body);
|
||||
return body;
|
||||
}
|
||||
|
||||
function extractBody(raw: string): { html: string | null; text: string | null } {
|
||||
const headerEnd = findHeaderEnd(raw);
|
||||
if (headerEnd === -1) return { html: null, text: null };
|
||||
|
||||
const topCtRaw = extractFullHeader(raw, 'Content-Type');
|
||||
const topCt = topCtRaw.toLowerCase();
|
||||
const topEncoding = extractFullHeader(raw, 'Content-Transfer-Encoding');
|
||||
|
||||
// Non-multipart: single body
|
||||
if (!topCt.includes('multipart')) {
|
||||
const body = raw.slice(headerEnd);
|
||||
const decoded = decodePartBody(body, topEncoding);
|
||||
if (topCt.includes('text/html')) return { html: decoded, text: null };
|
||||
return { html: null, text: decoded };
|
||||
}
|
||||
|
||||
// Multipart: extract boundary from the raw (case-sensitive) header
|
||||
const boundaryMatch = topCtRaw.match(/boundary=["']?([^"';\s]+)/i);
|
||||
if (!boundaryMatch) return { html: null, text: null };
|
||||
const boundary = boundaryMatch[1]!;
|
||||
|
||||
let html: string | null = null;
|
||||
let text: string | null = null;
|
||||
|
||||
const parts = raw.slice(headerEnd).split(`--${boundary}`);
|
||||
for (const part of parts) {
|
||||
if (part.startsWith('--') || !part.trim()) continue;
|
||||
const partHeaderEnd = findHeaderEnd(part);
|
||||
if (partHeaderEnd === -1) continue;
|
||||
|
||||
const partCt = extractFullHeader(part, 'Content-Type').toLowerCase();
|
||||
const partEnc = extractFullHeader(part, 'Content-Transfer-Encoding');
|
||||
const partBody = part.slice(partHeaderEnd);
|
||||
|
||||
// Recurse into nested multipart (e.g. multipart/alternative inside multipart/mixed)
|
||||
if (partCt.includes('multipart')) {
|
||||
const nested = extractBody(part.trim());
|
||||
if (nested.html && !html) html = nested.html;
|
||||
if (nested.text && !text) text = nested.text;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (partCt.includes('text/html') && !html) {
|
||||
html = decodePartBody(partBody, partEnc);
|
||||
} else if (partCt.includes('text/plain') && !text) {
|
||||
text = decodePartBody(partBody, partEnc);
|
||||
}
|
||||
}
|
||||
|
||||
return { html, text };
|
||||
}
|
||||
|
||||
type AttachmentMeta = { filename: string; size: number; contentType: string; content: string };
|
||||
|
||||
function parseAttachments(raw: string): AttachmentMeta[] {
|
||||
const results: AttachmentMeta[] = [];
|
||||
const regex = /^Content-Disposition:\s*attachment[^\n]*/gim;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = regex.exec(raw)) !== null) {
|
||||
const pos = match.index;
|
||||
|
||||
// Walk backwards to find the start of this MIME part's headers
|
||||
const partStart = raw.lastIndexOf('\n--', pos);
|
||||
const headerBlock = partStart !== -1 ? raw.slice(partStart, pos + 500) : raw.slice(Math.max(0, pos - 500), pos + 500);
|
||||
|
||||
// Extract filename from Content-Disposition or Content-Type
|
||||
const fnMatch = headerBlock.match(/filename\*?=(?:"([^"]+)"|([^\s;]+))/i);
|
||||
const filename = fnMatch ? (fnMatch[1] ?? fnMatch[2] ?? 'unknown') : 'unknown';
|
||||
|
||||
// Extract content-type
|
||||
const ctMatch = headerBlock.match(/^Content-Type:\s*([^\s;]+)/im);
|
||||
const contentType = ctMatch?.[1] ?? 'application/octet-stream';
|
||||
|
||||
// Extract full body content as base64
|
||||
const partHeaderEnd = findHeaderEnd(raw.slice(pos));
|
||||
let content = '';
|
||||
let size = 0;
|
||||
if (partHeaderEnd !== -1) {
|
||||
const bodyStart = pos + partHeaderEnd;
|
||||
const boundaryEnd = raw.indexOf('\n--', bodyStart);
|
||||
const bodyRaw = boundaryEnd !== -1 ? raw.slice(bodyStart, boundaryEnd) : raw.slice(bodyStart);
|
||||
|
||||
// Detect encoding from part headers
|
||||
const encMatch = headerBlock.match(/^Content-Transfer-Encoding:\s*(\S+)/im);
|
||||
const encoding = encMatch?.[1]?.toLowerCase() ?? 'base64';
|
||||
|
||||
if (encoding === 'base64') {
|
||||
content = bodyRaw.replace(/\s/g, '');
|
||||
} else {
|
||||
// For quoted-printable or 7bit/8bit, re-encode to base64
|
||||
const buf = encoding === 'quoted-printable'
|
||||
? Buffer.from(decodeQuotedPrintable(bodyRaw))
|
||||
: Buffer.from(bodyRaw);
|
||||
content = buf.toString('base64');
|
||||
}
|
||||
size = Math.floor(content.length * 3 / 4);
|
||||
}
|
||||
|
||||
results.push({ filename, size, contentType, content });
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
@@ -1,107 +1,67 @@
|
||||
import { mkdir, readdir } from 'node:fs/promises';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { simpleParser } from 'mailparser';
|
||||
import type { EmailSummary, EmailMessage } from 'types';
|
||||
import { rebuildIndex } from '@@/queue/handlers/gmail-sync';
|
||||
import type { EmailMessage } from 'types';
|
||||
import { createRouter } from '../../create-router';
|
||||
import { getUserEmailDir } from '@@/data-path';
|
||||
|
||||
const parseHeadersOnly = async (filePath: string, id: string): Promise<EmailSummary | null> => {
|
||||
try {
|
||||
const file = Bun.file(filePath);
|
||||
const buffer = Buffer.from(await file.arrayBuffer());
|
||||
const parsed = await simpleParser(buffer, { skipHtmlToText: true, skipTextToHtml: true, skipImageLinks: true });
|
||||
|
||||
const text = parsed.text ?? '';
|
||||
const snippet = text.slice(0, 120).replace(/\s+/g, ' ').trim();
|
||||
const attachmentCount = parsed.attachments?.length ?? 0;
|
||||
|
||||
return {
|
||||
id,
|
||||
from: parsed.from?.text ?? '',
|
||||
to: parsed.to ? (Array.isArray(parsed.to) ? parsed.to.map((a) => a.text).join(', ') : parsed.to.text) : '',
|
||||
subject: parsed.subject ?? '(no subject)',
|
||||
date: (parsed.date ?? new Date()).toISOString(),
|
||||
snippet,
|
||||
...(attachmentCount > 0 ? { attachmentCount } : {}),
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
import { DATA_PATH } from '@@/data-path';
|
||||
import { openEmailDb, rowToSummary, getSyncMeta } from './email-db';
|
||||
|
||||
export const emailRouter = createRouter();
|
||||
|
||||
emailRouter.get('/messages', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
const dir = getUserEmailDir(email);
|
||||
|
||||
let filenames: string[];
|
||||
try {
|
||||
const entries = await readdir(dir);
|
||||
filenames = entries.filter((f) => f.endsWith('.eml'));
|
||||
} catch {
|
||||
return ctx.json({ messages: [], total: 0 });
|
||||
}
|
||||
|
||||
const page = Number(ctx.req.query('page') ?? '1');
|
||||
const limit = Number(ctx.req.query('limit') ?? '50');
|
||||
const start = (page - 1) * limit;
|
||||
const offset = (page - 1) * limit;
|
||||
|
||||
const indexFile = Bun.file(join(dir, 'index.json'));
|
||||
if (await indexFile.exists()) {
|
||||
try {
|
||||
const raw = await indexFile.json();
|
||||
const index = Array.isArray(raw) ? null : (raw as { v?: number; entries: EmailSummary[] });
|
||||
if (index?.v === 3 && index.entries.length === filenames.length) {
|
||||
return ctx.json({ messages: index.entries.slice(start, start + limit), total: index.entries.length });
|
||||
}
|
||||
} catch {
|
||||
/* index corrupted, fall through to rebuild */
|
||||
}
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const rows = db.query('SELECT * FROM emails WHERE deleted = 0 ORDER BY date DESC LIMIT ? OFFSET ?').all(limit, offset) as Record<string, unknown>[];
|
||||
const countRow = db.query('SELECT COUNT(*) as total FROM emails WHERE deleted = 0').get() as { total: number };
|
||||
const messages = rows.map(rowToSummary);
|
||||
return ctx.json({ messages, total: countRow.total });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
// Fallback: rebuild index from .eml files
|
||||
const summaries = rebuildIndex(dir);
|
||||
return ctx.json({ messages: summaries.slice(start, start + limit), total: summaries.length });
|
||||
});
|
||||
|
||||
emailRouter.get('/messages/:id', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
const id = ctx.req.param('id');
|
||||
const filePath = join(getUserEmailDir(email), `${id}.eml`);
|
||||
|
||||
const file = Bun.file(filePath);
|
||||
if (!(await file.exists())) {
|
||||
return ctx.text('Not found', 404);
|
||||
}
|
||||
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const buffer = Buffer.from(await file.arrayBuffer());
|
||||
const parsed = await simpleParser(buffer);
|
||||
const row = db.query('SELECT * FROM emails WHERE id = ? AND deleted = 0').get(id) as Record<string, unknown> | null;
|
||||
if (!row) return ctx.text('Not found', 404);
|
||||
|
||||
const attachments = (parsed.attachments ?? []).map((a) => ({
|
||||
filename: a.filename ?? 'unknown',
|
||||
size: a.size,
|
||||
contentType: a.contentType,
|
||||
}));
|
||||
const attachmentRows = db.query('SELECT * FROM attachments WHERE email_id = ? ORDER BY idx').all(id) as Array<Record<string, unknown>>;
|
||||
|
||||
const from = row.from_name
|
||||
? `${row.from_name} <${row.from_address}>`
|
||||
: (row.from_address as string);
|
||||
|
||||
const message: EmailMessage = {
|
||||
id,
|
||||
from: parsed.from?.text ?? '',
|
||||
to: parsed.to ? (Array.isArray(parsed.to) ? parsed.to.map((a) => a.text).join(', ') : parsed.to.text) : '',
|
||||
cc: parsed.cc ? (Array.isArray(parsed.cc) ? parsed.cc.map((a) => a.text).join(', ') : parsed.cc.text) : undefined,
|
||||
subject: parsed.subject ?? '(no subject)',
|
||||
date: (parsed.date ?? new Date()).toISOString(),
|
||||
snippet: (parsed.text ?? '').slice(0, 120).replace(/\s+/g, ' ').trim(),
|
||||
html: parsed.html || undefined,
|
||||
text: parsed.text || undefined,
|
||||
attachments,
|
||||
id: row.id as string,
|
||||
from,
|
||||
to: row.to_address as string,
|
||||
cc: (row.cc as string) ?? undefined,
|
||||
subject: row.subject as string,
|
||||
date: row.date as string,
|
||||
snippet: row.snippet as string,
|
||||
html: (row.html as string) ?? undefined,
|
||||
text: (row.text_body as string) ?? undefined,
|
||||
attachments: attachmentRows.map((a) => ({
|
||||
filename: a.filename as string,
|
||||
size: a.size as number,
|
||||
contentType: a.content_type as string,
|
||||
})),
|
||||
...(row.attachment_count ? { attachmentCount: row.attachment_count as number } : {}),
|
||||
...(row.read ? { read: true } : {}),
|
||||
...(row.labels ? { labels: (row.labels as string).split(',') } : {}),
|
||||
};
|
||||
|
||||
return ctx.json(message);
|
||||
} catch {
|
||||
return ctx.text('Failed to parse email', 500);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -109,33 +69,68 @@ emailRouter.post('/messages/:id/attachments/:index/extract', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
const id = ctx.req.param('id');
|
||||
const index = Number(ctx.req.param('index'));
|
||||
const filePath = join(getUserEmailDir(email), `${id}.eml`);
|
||||
|
||||
const file = Bun.file(filePath);
|
||||
if (!(await file.exists())) {
|
||||
return ctx.text('Not found', 404);
|
||||
}
|
||||
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const buffer = Buffer.from(await file.arrayBuffer());
|
||||
const parsed = await simpleParser(buffer);
|
||||
const attachment = parsed.attachments[index];
|
||||
if (!attachment) {
|
||||
return ctx.text('Attachment not found', 404);
|
||||
}
|
||||
const row = db.query('SELECT filename, content FROM attachments WHERE email_id = ? AND idx = ?').get(id, index) as { filename: string; content: string | null } | null;
|
||||
if (!row || !row.content) return ctx.text('Attachment not found', 404);
|
||||
|
||||
const fileName = attachment.filename ?? 'unknown';
|
||||
const attachDir = join(getUserEmailDir(email), 'attachments');
|
||||
const fileName = row.filename ?? 'unknown';
|
||||
const attachDir = join(DATA_PATH, email, 'Gmail', 'emails', 'attachments');
|
||||
const destPath = join(attachDir, fileName);
|
||||
|
||||
const destFile = Bun.file(destPath);
|
||||
if (!(await destFile.exists())) {
|
||||
await mkdir(attachDir, { recursive: true });
|
||||
await Bun.write(destPath, attachment.content);
|
||||
const binary = Buffer.from(row.content, 'base64');
|
||||
await Bun.write(destPath, binary);
|
||||
}
|
||||
|
||||
return ctx.json({ filePath: `Gmail/emails/attachments/${fileName}`, fileName, root: 'user-data' });
|
||||
} catch {
|
||||
return ctx.text('Failed to extract attachment', 500);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
emailRouter.delete('/messages/:id', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
const id = ctx.req.param('id');
|
||||
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const result = db.run('UPDATE emails SET deleted = 1 WHERE id = ? AND deleted = 0', [id]);
|
||||
if (result.changes === 0) return ctx.text('Not found', 404);
|
||||
return ctx.json({ ok: true });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
emailRouter.get('/sync-status', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const lastSyncAt = getSyncMeta(db, 'last_sync_at');
|
||||
return ctx.json({ lastSyncAt });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
emailRouter.get('/stats', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const total = (db.query('SELECT COUNT(*) as count FROM emails WHERE deleted = 0').get() as { count: number }).count;
|
||||
const byDomain = db.query('SELECT from_domain, COUNT(*) as count FROM emails WHERE deleted = 0 GROUP BY from_domain ORDER BY count DESC LIMIT 20').all() as Array<{ from_domain: string; count: number }>;
|
||||
const bySender = db.query('SELECT from_address, from_name, COUNT(*) as count FROM emails WHERE deleted = 0 GROUP BY from_address ORDER BY count DESC LIMIT 20').all() as Array<{ from_address: string; from_name: string; count: number }>;
|
||||
|
||||
return ctx.json({ total, byDomain, bySender });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -230,6 +230,7 @@ export async function spawnPi(
|
||||
'-e', `OFFICER_RESOURCES=${resourcesEnv}`,
|
||||
'-e', `OFFICER_GOOGLE_CONFIG_PATH=/officer/google-oauth.json`,
|
||||
'-e', `OFFICER_GOOGLE_TOKEN_PATH=/officer/user/integrations/google.json`,
|
||||
'-e', `OFFICER_EMAIL_DB=/officer/emails.db`,
|
||||
];
|
||||
for (const [key, value] of Object.entries(storedKeys)) {
|
||||
if (value?.trim()) envFlags.push('-e', `${key}=${value.trim()}`);
|
||||
@@ -280,7 +281,7 @@ export async function spawnPi(
|
||||
stdin: 'pipe',
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
env: { ...process.env, ...storedKeys, HOME: getHomeDir(email), OFFICER_USER_HOME: getHomeDir(email), OFFICER_USER_ROOT: join(DATA_PATH, email), PI_CODING_AGENT_DIR: PI_CONFIG_DIR, PI_TOOLS_DIRS: toolsDirs, PI_SEARXNG_URL: searxng.url, OFFICER_RESOURCES: buildResourcesEnv(), OFFICER_GOOGLE_CONFIG_PATH: getGoogleConfigPath(), OFFICER_GOOGLE_TOKEN_PATH: getGoogleTokenPath(email) },
|
||||
env: { ...process.env, ...storedKeys, HOME: getHomeDir(email), OFFICER_USER_HOME: getHomeDir(email), OFFICER_USER_ROOT: join(DATA_PATH, email), PI_CODING_AGENT_DIR: PI_CONFIG_DIR, PI_TOOLS_DIRS: toolsDirs, PI_SEARXNG_URL: searxng.url, OFFICER_RESOURCES: buildResourcesEnv(), OFFICER_GOOGLE_CONFIG_PATH: getGoogleConfigPath(), OFFICER_GOOGLE_TOKEN_PATH: getGoogleTokenPath(email), OFFICER_EMAIL_DB: join(DATA_PATH, email, 'emails.db') },
|
||||
});
|
||||
|
||||
logger.info('Spawned Pi locally', {
|
||||
@@ -403,7 +404,13 @@ function parsePiEvent(event: Record<string, unknown>, currentStreamBuffer: strin
|
||||
case 'tool_execution_end': {
|
||||
const toolCallId = (event.toolCallId as string) ?? '';
|
||||
const result = event.result;
|
||||
const isError = (event.isError as boolean) ?? false;
|
||||
let resultObj: Record<string, unknown> | null = null;
|
||||
if (typeof result === 'object' && result !== null) {
|
||||
resultObj = result as Record<string, unknown>;
|
||||
} else if (typeof result === 'string') {
|
||||
try { resultObj = JSON.parse(result); } catch { /* not JSON */ }
|
||||
}
|
||||
const isError = (event.isError as boolean) ?? (resultObj?.isError as boolean) ?? false;
|
||||
const output = result != null ? (typeof result === 'string' ? result : JSON.stringify(result)) : '';
|
||||
|
||||
return {
|
||||
|
||||
@@ -313,6 +313,7 @@ async function handleChat(
|
||||
}
|
||||
|
||||
// Set thinking level if provided
|
||||
console.log(`[pi] model: ${msg.model ?? 'default'}, thinking: ${msg.thinking ?? 'not set'}`);
|
||||
if (msg.thinking) {
|
||||
piBridge.setThinkingLevel(session.piProcess, msg.thinking);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { join } from 'node:path';
|
||||
import { mkdir, copyFile } from 'node:fs/promises';
|
||||
import { mkdir, copyFile, chmod } from 'node:fs/promises';
|
||||
import { PI_CONFIG_DIR, getUserPiConfigDir } from '../../data-path';
|
||||
import { readApiKeys, readAccessPolicy, PROVIDERS } from './pi-mono';
|
||||
import { getUsers } from 'officerdb';
|
||||
@@ -132,7 +132,10 @@ export async function syncUserPiConfig(email: string): Promise<void> {
|
||||
});
|
||||
|
||||
const userDir = getUserPiConfigDir(email);
|
||||
await mkdir(userDir, { recursive: true });
|
||||
const sessionsDir = join(userDir, 'sessions');
|
||||
await mkdir(sessionsDir, { recursive: true });
|
||||
await chmod(userDir, 0o777).catch(() => {});
|
||||
await chmod(sessionsDir, 0o777).catch(() => {});
|
||||
await Bun.write(join(userDir, 'models.json'), JSON.stringify(filtered, null, 2));
|
||||
|
||||
// Copy settings.json from app-level Pi config
|
||||
|
||||
@@ -6,7 +6,7 @@ RUN apt-get update \
|
||||
sudo gosu locales \
|
||||
zip unzip tree btop net-tools tmux \
|
||||
procps psmisc lsof less file man-db \
|
||||
ripgrep fd-find jq htop \
|
||||
ripgrep fd-find jq htop sqlite3 \
|
||||
&& sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen \
|
||||
&& ln -sf /usr/bin/fdfind /usr/local/bin/fd \
|
||||
&& apt-get clean
|
||||
|
||||
@@ -56,5 +56,9 @@ fi
|
||||
mkdir -p /home/$USERNAME/.local/bin
|
||||
chown -R "$USER_UID:$USER_GID" /home/$USERNAME/.local
|
||||
|
||||
# Ensure Pi agent sessions directory exists and is writable
|
||||
mkdir -p /home/$USERNAME/.pi/agent/sessions
|
||||
chown -R "$USER_UID:$USER_GID" /home/$USERNAME/.pi
|
||||
|
||||
# Run sidecar as the user
|
||||
exec gosu "$USER_UID:$USER_GID" node /app/pty-sidecar.mjs
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { ServerWebSocket } from 'bun';
|
||||
import { existsSync, mkdirSync, statSync } from 'node:fs';
|
||||
import { existsSync, mkdirSync, statSync, writeFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@@ -173,6 +173,7 @@ const startDockerSidecar = (port: number, homeDir: string, userId: number, usern
|
||||
'-v', `${join(DATA_PATH, '.generated')}:/officer/generated:ro`,
|
||||
...googleMounts,
|
||||
'-v', `${join(DATA_PATH, email, 'integrations')}:/officer/user/integrations:ro`,
|
||||
'-v', `${join(DATA_PATH, email, 'emails.db')}:/officer/emails.db`,
|
||||
'-w', containerHome,
|
||||
tag,
|
||||
],
|
||||
@@ -242,6 +243,12 @@ export const ensureDockerContainer = async (email: string, userId: number, homeD
|
||||
mkdirSync(getUserToolsDir(email), { recursive: true });
|
||||
mkdirSync(join(DATA_PATH, email, 'integrations'), { recursive: true });
|
||||
|
||||
// Ensure emails.db exists as a file before mount (Docker creates a directory if missing)
|
||||
const emailsDbPath = join(DATA_PATH, email, 'emails.db');
|
||||
if (!existsSync(emailsDbPath)) {
|
||||
writeFileSync(emailsDbPath, '');
|
||||
}
|
||||
|
||||
const map = await loadContainerMap();
|
||||
const existing = map[email];
|
||||
|
||||
@@ -492,3 +499,28 @@ export const terminalWebsocket = {
|
||||
|
||||
drain() {},
|
||||
};
|
||||
|
||||
export const stopAllContainers = async () => {
|
||||
// Stop host sidecar
|
||||
if (hostSidecarProcess) {
|
||||
hostSidecarProcess.kill();
|
||||
await hostSidecarProcess.exited.catch(() => {});
|
||||
hostSidecarProcess = null;
|
||||
console.log('[terminal] host sidecar stopped');
|
||||
}
|
||||
|
||||
// Stop all Docker containers
|
||||
const map = await loadContainerMap();
|
||||
const entries = Object.entries(map);
|
||||
if (entries.length === 0) return;
|
||||
|
||||
const dockerPath = Bun.which('docker') ?? 'docker';
|
||||
for (const [email, info] of entries) {
|
||||
try {
|
||||
Bun.spawnSync({ cmd: [dockerPath, 'stop', '-t', '2', info.dockerId], stdout: 'ignore', stderr: 'ignore' });
|
||||
console.log(`[terminal] stopped container ${info.dockerId} (${email})`);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { homedir } from 'node:os';
|
||||
import type { EmailSummary } from 'types';
|
||||
import type { Database } from 'bun:sqlite';
|
||||
import type { JobHandler } from '../types';
|
||||
import { registerHandler } from '../handler-registry';
|
||||
import { DATA_PATH } from '../../data-path';
|
||||
import { openEmailDb, upsertFromRawEml, getSyncMeta, setSyncMeta, updateEmailLabels } from '../../api/email/email-db';
|
||||
|
||||
type GoogleCredentials = {
|
||||
accessToken: string;
|
||||
@@ -97,176 +97,58 @@ async function gmailGet(token: string, path: string, params?: Record<string, str
|
||||
return res.json();
|
||||
}
|
||||
|
||||
function slugify(text: string, maxLen = 60): string {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.slice(0, maxLen)
|
||||
.replace(/-+$/, '');
|
||||
// ── Pre-flight message count ──
|
||||
|
||||
async function countMessages(token: string, query?: string): Promise<number> {
|
||||
let count = 0;
|
||||
let pageToken: string | undefined;
|
||||
do {
|
||||
const params: Record<string, string> = { maxResults: '500' };
|
||||
if (query) params.q = query;
|
||||
if (pageToken) params.pageToken = pageToken;
|
||||
const list = (await gmailGet(token, '/messages', params)) as {
|
||||
messages?: Array<{ id: string }>;
|
||||
nextPageToken?: string;
|
||||
};
|
||||
count += list.messages?.length ?? 0;
|
||||
if (!list.messages?.length) break;
|
||||
pageToken = list.nextPageToken;
|
||||
} while (pageToken);
|
||||
return count;
|
||||
}
|
||||
|
||||
function buildEmlFilename(id: string, internalDate: string | undefined, rawEmail: string): string {
|
||||
const subjectMatch = rawEmail.match(/^Subject:\s*(.+)$/mi);
|
||||
const subject = subjectMatch?.[1]?.trim() || 'no-subject';
|
||||
const ts = parseInt(internalDate || '0');
|
||||
const d = new Date(ts);
|
||||
const dateStr =
|
||||
ts > 0
|
||||
? `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`
|
||||
: 'unknown-date';
|
||||
return `${dateStr}_${slugify(subject)}_${id}.eml`;
|
||||
}
|
||||
|
||||
function decodeMimeWords(text: string): string {
|
||||
return text.replace(/=\?([^?]+)\?(B|Q)\?([^?]+)\?=/gi, (_, _charset, encoding, encoded) => {
|
||||
try {
|
||||
if (encoding.toUpperCase() === 'B') {
|
||||
return Buffer.from(encoded, 'base64').toString('utf-8');
|
||||
}
|
||||
const bytes: number[] = [];
|
||||
for (let i = 0; i < encoded.length; i++) {
|
||||
if (encoded[i] === '_') {
|
||||
bytes.push(0x20);
|
||||
} else if (encoded[i] === '=' && i + 2 < encoded.length) {
|
||||
bytes.push(parseInt(encoded.slice(i + 1, i + 3), 16));
|
||||
i += 2;
|
||||
} else {
|
||||
bytes.push(encoded.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
return Buffer.from(bytes).toString('utf-8');
|
||||
} catch {
|
||||
return encoded;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function extractHeader(raw: string, name: string): string {
|
||||
const match = raw.match(new RegExp(`^${name}:\\s*(.+)$`, 'mi'));
|
||||
return match?.[1]?.trim() ? decodeMimeWords(match[1].trim()) : '';
|
||||
}
|
||||
|
||||
function findHeaderEnd(text: string): number {
|
||||
const crlf = text.indexOf('\r\n\r\n');
|
||||
const lf = text.indexOf('\n\n');
|
||||
if (crlf !== -1) return crlf + 4;
|
||||
if (lf !== -1) return lf + 2;
|
||||
return -1;
|
||||
}
|
||||
|
||||
function extractSnippet(raw: string): string {
|
||||
const idx = findHeaderEnd(raw);
|
||||
if (idx === -1) return '';
|
||||
|
||||
let body = raw.slice(idx);
|
||||
|
||||
// If multipart, skip boundary line + part headers to reach actual content
|
||||
if (body.trimStart().startsWith('--')) {
|
||||
const afterBoundary = body.slice(body.indexOf('\n') + 1);
|
||||
const partBodyStart = findHeaderEnd(afterBoundary);
|
||||
if (partBodyStart !== -1) body = afterBoundary.slice(partBodyStart);
|
||||
}
|
||||
|
||||
// Stop at next MIME boundary
|
||||
const nextBoundary = body.indexOf('\n--');
|
||||
if (nextBoundary !== -1) body = body.slice(0, nextBoundary);
|
||||
|
||||
return body.replace(/\s+/g, ' ').trim().slice(0, 120);
|
||||
}
|
||||
|
||||
function countAttachments(raw: string): number {
|
||||
const matches = raw.match(/^Content-Disposition:\s*attachment/gim);
|
||||
return matches?.length ?? 0;
|
||||
}
|
||||
|
||||
function parseEmlToSummary(raw: string, id: string): EmailSummary {
|
||||
const dateStr = extractHeader(raw, 'Date');
|
||||
const date = dateStr ? new Date(dateStr).toISOString() : new Date(0).toISOString();
|
||||
const attachmentCount = countAttachments(raw);
|
||||
return {
|
||||
id,
|
||||
from: extractHeader(raw, 'From'),
|
||||
to: extractHeader(raw, 'To'),
|
||||
subject: extractHeader(raw, 'Subject') || '(no subject)',
|
||||
date,
|
||||
snippet: extractSnippet(raw),
|
||||
...(attachmentCount > 0 ? { attachmentCount } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
type EmailIndex = { v: number; entries: EmailSummary[] };
|
||||
const INDEX_VERSION = 3;
|
||||
|
||||
export function rebuildIndex(emailsDir: string): EmailSummary[] {
|
||||
let filenames: string[];
|
||||
try {
|
||||
filenames = readdirSync(emailsDir).filter((f) => f.endsWith('.eml'));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
const indexPath = join(emailsDir, 'index.json');
|
||||
const existing = new Map<string, EmailSummary>();
|
||||
try {
|
||||
const raw = JSON.parse(readFileSync(indexPath, 'utf-8'));
|
||||
const index = (Array.isArray(raw) ? null : raw) as EmailIndex | null;
|
||||
if (index?.v === INDEX_VERSION) {
|
||||
for (const entry of index.entries) {
|
||||
existing.set(entry.id, entry);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* no existing index or corrupted */
|
||||
}
|
||||
|
||||
const entries: EmailSummary[] = [];
|
||||
for (const filename of filenames) {
|
||||
const id = filename.replace(/\.eml$/, '');
|
||||
const cached = existing.get(id);
|
||||
if (cached) {
|
||||
entries.push(cached);
|
||||
} else {
|
||||
try {
|
||||
const raw = readFileSync(join(emailsDir, filename), 'utf-8');
|
||||
entries.push(parseEmlToSummary(raw, id));
|
||||
} catch {
|
||||
/* skip unreadable files */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entries.sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime());
|
||||
writeFileSync(indexPath, JSON.stringify({ v: INDEX_VERSION, entries }));
|
||||
return entries;
|
||||
}
|
||||
// ── Full sync: Gmail API → SQLite directly ──
|
||||
|
||||
type SyncProgress = { saved: number; skipped: number; errors: number; page: number };
|
||||
type OnProgress = (progress: SyncProgress) => void;
|
||||
|
||||
type SyncResult = {
|
||||
saved: number;
|
||||
skipped: number;
|
||||
errors: number;
|
||||
labelMap: Map<string, string[]>;
|
||||
maxHistoryId: string | null;
|
||||
};
|
||||
|
||||
async function syncInbox(
|
||||
token: string,
|
||||
outputDir: string,
|
||||
db: Database,
|
||||
emailAccount: string,
|
||||
query?: string,
|
||||
onProgress?: OnProgress,
|
||||
): Promise<{ saved: number; skipped: number; errors: number }> {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
): Promise<SyncResult> {
|
||||
// Dedup via DB
|
||||
const existingIds = new Set<string>();
|
||||
try {
|
||||
for (const file of readdirSync(outputDir)) {
|
||||
const match = file.match(/_([a-f0-9]+)\.eml$/i);
|
||||
if (match) existingIds.add(match[1]!);
|
||||
}
|
||||
} catch {
|
||||
/* dir might not exist yet */
|
||||
}
|
||||
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
||||
for (const row of rows) existingIds.add(row.id);
|
||||
|
||||
let saved = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
let page = 0;
|
||||
let pageToken: string | undefined;
|
||||
const labelMap = new Map<string, string[]>();
|
||||
let maxHistoryId: bigint | null = null;
|
||||
|
||||
do {
|
||||
const params: Record<string, string> = { maxResults: '100' };
|
||||
@@ -294,12 +176,19 @@ async function syncInbox(
|
||||
id: string;
|
||||
internalDate?: string;
|
||||
raw: string;
|
||||
labelIds?: string[];
|
||||
historyId?: string;
|
||||
};
|
||||
const rawEmail = Buffer.from(msg.raw, 'base64url').toString('utf-8');
|
||||
const filename = buildEmlFilename(msg.id, msg.internalDate, rawEmail);
|
||||
writeFileSync(join(outputDir, filename), rawEmail);
|
||||
upsertFromRawEml({ db, id, raw: rawEmail, integration: 'gmail', emailAccount, labels: msg.labelIds });
|
||||
existingIds.add(id);
|
||||
saved++;
|
||||
|
||||
if (msg.labelIds) labelMap.set(id, msg.labelIds);
|
||||
if (msg.historyId) {
|
||||
const hid = BigInt(msg.historyId);
|
||||
if (maxHistoryId === null || hid > maxHistoryId) maxHistoryId = hid;
|
||||
}
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
@@ -314,7 +203,121 @@ async function syncInbox(
|
||||
pageToken = list.nextPageToken;
|
||||
} while (pageToken);
|
||||
|
||||
return { saved, skipped, errors };
|
||||
return { saved, skipped, errors, labelMap, maxHistoryId: maxHistoryId !== null ? String(maxHistoryId) : null };
|
||||
}
|
||||
|
||||
// ── Gmail History API (incremental sync) ──
|
||||
|
||||
type HistoryMessage = { id: string; labelIds?: string[] };
|
||||
|
||||
type HistoryRecord = {
|
||||
id: string;
|
||||
messagesAdded?: Array<{ message: HistoryMessage }>;
|
||||
messagesDeleted?: Array<{ message: HistoryMessage }>;
|
||||
labelsAdded?: Array<{ message: HistoryMessage; labelIds: string[] }>;
|
||||
labelsRemoved?: Array<{ message: HistoryMessage; labelIds: string[] }>;
|
||||
};
|
||||
|
||||
type HistoryResponse = {
|
||||
history?: HistoryRecord[];
|
||||
nextPageToken?: string;
|
||||
historyId: string;
|
||||
};
|
||||
|
||||
type IncrementalResult =
|
||||
| { stale: false; added: number; deleted: number; relabeled: number; maxHistoryId: string }
|
||||
| { stale: true };
|
||||
|
||||
async function syncIncremental(
|
||||
token: string,
|
||||
db: Database,
|
||||
lastHistoryId: string,
|
||||
emailAccount: string,
|
||||
): Promise<IncrementalResult> {
|
||||
let pageToken: string | undefined;
|
||||
let added = 0;
|
||||
let deleted = 0;
|
||||
let relabeled = 0;
|
||||
let latestHistoryId = lastHistoryId;
|
||||
|
||||
do {
|
||||
const url = new URL(`${GMAIL_BASE}/history`);
|
||||
url.searchParams.set('startHistoryId', lastHistoryId);
|
||||
for (const ht of ['messageAdded', 'messageDeleted', 'labelAdded', 'labelRemoved']) {
|
||||
url.searchParams.append('historyTypes', ht);
|
||||
}
|
||||
if (pageToken) url.searchParams.set('pageToken', pageToken);
|
||||
|
||||
let data: HistoryResponse;
|
||||
try {
|
||||
const res = await fetch(url.toString(), { headers: { Authorization: `Bearer ${token}` } });
|
||||
if (!res.ok) {
|
||||
const error = await res.text().catch(() => '');
|
||||
throw new Error(`Gmail API error (${res.status}): ${error}`);
|
||||
}
|
||||
data = (await res.json()) as HistoryResponse;
|
||||
} catch (err) {
|
||||
// historyId too old — Gmail returns 404
|
||||
if (err instanceof Error && err.message.includes('404')) {
|
||||
return { stale: true };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
latestHistoryId = data.historyId;
|
||||
|
||||
for (const record of data.history ?? []) {
|
||||
// New messages
|
||||
for (const { message } of record.messagesAdded ?? []) {
|
||||
try {
|
||||
const msg = (await gmailGet(token, `/messages/${message.id}`, { format: 'raw' })) as {
|
||||
id: string;
|
||||
internalDate?: string;
|
||||
raw: string;
|
||||
labelIds?: string[];
|
||||
historyId?: string;
|
||||
};
|
||||
const rawEmail = Buffer.from(msg.raw, 'base64url').toString('utf-8');
|
||||
upsertFromRawEml({ db, id: msg.id, raw: rawEmail, integration: 'gmail', emailAccount, labels: msg.labelIds });
|
||||
added++;
|
||||
} catch {
|
||||
/* skip individual failures */
|
||||
}
|
||||
}
|
||||
|
||||
// Deleted messages
|
||||
for (const { message } of record.messagesDeleted ?? []) {
|
||||
db.run('UPDATE emails SET deleted = 1 WHERE id = ?', [message.id]);
|
||||
deleted++;
|
||||
}
|
||||
|
||||
// Labels added
|
||||
for (const { message, labelIds } of record.labelsAdded ?? []) {
|
||||
const row = db.query('SELECT labels FROM emails WHERE id = ?').get(message.id) as { labels: string | null } | null;
|
||||
if (row) {
|
||||
const existing = row.labels ? row.labels.split(',') : [];
|
||||
const merged = [...new Set([...existing, ...labelIds.map((l) => l.toLowerCase())])];
|
||||
updateEmailLabels(db, message.id, merged);
|
||||
relabeled++;
|
||||
}
|
||||
}
|
||||
|
||||
// Labels removed
|
||||
for (const { message, labelIds } of record.labelsRemoved ?? []) {
|
||||
const row = db.query('SELECT labels FROM emails WHERE id = ?').get(message.id) as { labels: string | null } | null;
|
||||
if (row) {
|
||||
const removeSet = new Set(labelIds.map((l) => l.toLowerCase()));
|
||||
const remaining = (row.labels ? row.labels.split(',') : []).filter((l) => !removeSet.has(l));
|
||||
updateEmailLabels(db, message.id, remaining);
|
||||
relabeled++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pageToken = data.nextPageToken;
|
||||
} while (pageToken);
|
||||
|
||||
return { stale: false, added, deleted, relabeled, maxHistoryId: latestHistoryId };
|
||||
}
|
||||
|
||||
const MONTH_NAMES = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'];
|
||||
@@ -341,55 +344,117 @@ const gmailSyncHandler: JobHandler = {
|
||||
run: async (ctx) => {
|
||||
const creds = await loadCredentials(ctx.job.userId);
|
||||
const token = await getValidAccessToken(creds);
|
||||
// Store token in shared meta for the next step
|
||||
ctx.meta.accessToken = token;
|
||||
ctx.meta.outputDir = join(DATA_PATH, ctx.job.userId, 'Gmail', 'emails');
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Sync emails',
|
||||
run: async (ctx) => {
|
||||
const token = ctx.meta.accessToken as string;
|
||||
const outputDir = ctx.meta.outputDir as string;
|
||||
const year = ctx.meta.year as number | undefined;
|
||||
|
||||
let totalSaved = 0;
|
||||
let totalSkipped = 0;
|
||||
let totalErrors = 0;
|
||||
|
||||
if (year) {
|
||||
// Year-scoped sync: month by month with progress
|
||||
const months = buildMonthRanges(year);
|
||||
for (let i = 0; i < months.length; i++) {
|
||||
const month = months[i]!;
|
||||
await ctx.updateProgress({ current: i, total: months.length, label: month.label });
|
||||
const query = `after:${month.after} before:${month.before}`;
|
||||
const { saved, skipped, errors } = await syncInbox(token, outputDir, query, (p) => {
|
||||
const label = `${month.label} — ${p.saved} saved`;
|
||||
ctx.updateProgress({ current: i, total: months.length, label });
|
||||
});
|
||||
totalSaved += saved;
|
||||
totalSkipped += skipped;
|
||||
totalErrors += errors;
|
||||
const db = openEmailDb(ctx.job.userId);
|
||||
try {
|
||||
// Try incremental sync first (only for non-year-scoped syncs)
|
||||
if (!year) {
|
||||
const lastHistoryId = getSyncMeta(db, 'last_history_id');
|
||||
if (lastHistoryId) {
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Incremental sync' });
|
||||
console.log(`[gmail-sync] Attempting incremental sync from historyId ${lastHistoryId}`);
|
||||
const result = await syncIncremental(token, db, lastHistoryId, ctx.job.userId);
|
||||
if (!result.stale) {
|
||||
setSyncMeta(db, 'last_history_id', result.maxHistoryId);
|
||||
setSyncMeta(db, 'last_sync_at', new Date().toISOString());
|
||||
console.log(`[gmail-sync] Incremental: +${result.added} added, -${result.deleted} deleted, ~${result.relabeled} relabeled`);
|
||||
await ctx.updateProgress({ current: 1, total: 1, label: 'Done (incremental)' });
|
||||
return;
|
||||
}
|
||||
console.log('[gmail-sync] historyId stale, falling back to full sync');
|
||||
}
|
||||
}
|
||||
await ctx.updateProgress({ current: months.length, total: months.length, label: 'Done' });
|
||||
} else {
|
||||
// Full sync: all emails with per-page progress
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Starting sync' });
|
||||
const { saved, skipped, errors } = await syncInbox(token, outputDir, undefined, (p) => {
|
||||
const label = `Saved ${p.saved}, skipped ${p.skipped} (page ${p.page})`;
|
||||
ctx.updateProgress({ current: p.saved + p.skipped + p.errors, total: 0, label });
|
||||
});
|
||||
totalSaved = saved;
|
||||
totalSkipped = skipped;
|
||||
totalErrors = errors;
|
||||
await ctx.updateProgress({ current: 1, total: 1, label: 'Done' });
|
||||
|
||||
// Full sync
|
||||
let totalSaved = 0;
|
||||
let totalSkipped = 0;
|
||||
let totalErrors = 0;
|
||||
let allLabelMaps: Map<string, string[]> = new Map();
|
||||
let maxHistoryId: string | null = null;
|
||||
|
||||
if (year) {
|
||||
// Year-scoped sync: count total emails first, then sync month by month
|
||||
const yearQuery = `after:${year}/1/1 before:${year + 1}/1/1`;
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Counting emails...' });
|
||||
const totalEmails = await countMessages(token, yearQuery);
|
||||
console.log(`[gmail-sync] Pre-flight: ${totalEmails} emails for ${year}`);
|
||||
|
||||
const months = buildMonthRanges(year);
|
||||
for (let i = 0; i < months.length; i++) {
|
||||
const month = months[i]!;
|
||||
await ctx.updateProgress({ current: totalSaved + totalSkipped, total: totalEmails, label: month.label });
|
||||
const query = `after:${month.after} before:${month.before}`;
|
||||
const result = await syncInbox(token, db, ctx.job.userId, query, (p) => {
|
||||
const current = totalSaved + p.saved + p.skipped + p.errors;
|
||||
const label = `${month.label} — Saved ${(totalSaved + p.saved).toLocaleString()} of ${totalEmails.toLocaleString()}`;
|
||||
ctx.updateProgress({ current, total: totalEmails, label });
|
||||
});
|
||||
totalSaved += result.saved;
|
||||
totalSkipped += result.skipped;
|
||||
totalErrors += result.errors;
|
||||
for (const [id, labels] of result.labelMap) allLabelMaps.set(id, labels);
|
||||
if (result.maxHistoryId) {
|
||||
if (!maxHistoryId || BigInt(result.maxHistoryId) > BigInt(maxHistoryId)) {
|
||||
maxHistoryId = result.maxHistoryId;
|
||||
}
|
||||
}
|
||||
}
|
||||
await ctx.updateProgress({ current: totalEmails, total: totalEmails, label: 'Done' });
|
||||
} else {
|
||||
// If we have a last_sync_date (e.g. from migration), scope the sync to only newer emails
|
||||
const lastSyncDate = getSyncMeta(db, 'last_sync_date');
|
||||
let syncQuery: string | undefined;
|
||||
if (lastSyncDate) {
|
||||
const d = new Date(lastSyncDate);
|
||||
syncQuery = `after:${d.getFullYear()}/${d.getMonth() + 1}/${d.getDate()}`;
|
||||
console.log(`[gmail-sync] Scoping full sync with query: ${syncQuery}`);
|
||||
}
|
||||
|
||||
// Count total emails for accurate progress
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Counting emails...' });
|
||||
const totalEmails = await countMessages(token, syncQuery);
|
||||
console.log(`[gmail-sync] Pre-flight: ${totalEmails} emails`);
|
||||
|
||||
await ctx.updateProgress({ current: 0, total: totalEmails, label: 'Starting sync' });
|
||||
const result = await syncInbox(token, db, ctx.job.userId, syncQuery, (p) => {
|
||||
const current = p.saved + p.skipped + p.errors;
|
||||
const label = `Saved ${p.saved.toLocaleString()} of ${totalEmails.toLocaleString()}`;
|
||||
ctx.updateProgress({ current, total: totalEmails, label });
|
||||
});
|
||||
totalSaved = result.saved;
|
||||
totalSkipped = result.skipped;
|
||||
totalErrors = result.errors;
|
||||
allLabelMaps = result.labelMap;
|
||||
maxHistoryId = result.maxHistoryId;
|
||||
await ctx.updateProgress({ current: totalEmails, total: totalEmails, label: 'Done' });
|
||||
}
|
||||
|
||||
console.log(`[gmail-sync] Full: saved ${totalSaved}, skipped ${totalSkipped}, errors ${totalErrors}`);
|
||||
|
||||
// Update labels for emails that were skipped but have new label data
|
||||
let labelsUpdated = 0;
|
||||
for (const [id, labels] of allLabelMaps) {
|
||||
updateEmailLabels(db, id, labels);
|
||||
labelsUpdated++;
|
||||
}
|
||||
if (labelsUpdated > 0) console.log(`[gmail-sync] Updated labels for ${labelsUpdated} emails`);
|
||||
|
||||
// Store sync state
|
||||
if (maxHistoryId) {
|
||||
setSyncMeta(db, 'last_history_id', maxHistoryId);
|
||||
}
|
||||
setSyncMeta(db, 'last_sync_at', new Date().toISOString());
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
console.log(`[gmail-sync] Saved ${totalSaved}, skipped ${totalSkipped}, errors ${totalErrors}`);
|
||||
|
||||
rebuildIndex(outputDir);
|
||||
console.log('[gmail-sync] Index rebuilt');
|
||||
},
|
||||
},
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user