email: full-text search (FTS5) — backend index + /email/search + search box in the list
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -82,6 +82,7 @@ export function openEmailDb(email: string): Database {
|
||||
db.exec(SCHEMA_TABLES);
|
||||
migrate(db);
|
||||
db.exec(SCHEMA_INDEXES);
|
||||
ensureFts(db);
|
||||
// Try to chmod, but don't crash if permission denied (e.g., file owned by different user)
|
||||
try {
|
||||
chmodSync(dbPath, 0o666);
|
||||
@@ -119,6 +120,65 @@ function migrate(db: Database): void {
|
||||
}
|
||||
}
|
||||
|
||||
// ── Full-text search (FTS5) ──
|
||||
// A standalone FTS5 index kept in sync with `emails` on every upsert. unicode61 + diacritic folding
|
||||
// gives accent-insensitive matching; per-term prefix queries make it feel incremental.
|
||||
|
||||
function ensureFts(db: Database): void {
|
||||
db.exec(
|
||||
`CREATE VIRTUAL TABLE IF NOT EXISTS emails_fts USING fts5(
|
||||
id UNINDEXED, subject, sender, recipients, snippet, body,
|
||||
tokenize = 'unicode61 remove_diacritics 2'
|
||||
)`,
|
||||
);
|
||||
const fts = (db.query('SELECT count(*) AS c FROM emails_fts').get() as { c: number }).c;
|
||||
const total = (db.query('SELECT count(*) AS c FROM emails').get() as { c: number }).c;
|
||||
if (fts === 0 && total > 0) {
|
||||
// Backfill existing mail (first run after this feature ships).
|
||||
db.exec(
|
||||
`INSERT INTO emails_fts (id, subject, sender, recipients, snippet, body)
|
||||
SELECT id, COALESCE(subject, ''),
|
||||
TRIM(COALESCE(from_name, '') || ' ' || COALESCE(from_address, '')),
|
||||
TRIM(COALESCE(to_address, '') || ' ' || COALESCE(cc, '')),
|
||||
COALESCE(snippet, ''), COALESCE(text_body, '')
|
||||
FROM emails`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const ftsDeleteStmt = 'DELETE FROM emails_fts WHERE id = ?';
|
||||
const ftsInsertStmt = 'INSERT INTO emails_fts (id, subject, sender, recipients, snippet, body) VALUES (?, ?, ?, ?, ?, ?)';
|
||||
|
||||
function syncFtsRow(db: Database, id: string, subject: string, sender: string, recipients: string, snippet: string, body: string): void {
|
||||
db.run(ftsDeleteStmt, [id]);
|
||||
db.run(ftsInsertStmt, [id, subject, sender, recipients, snippet, body]);
|
||||
}
|
||||
|
||||
// Per-term prefix match, ANDed. Each term is quoted (so FTS operators in user input are literal) then
|
||||
// suffixed with * for prefix matching.
|
||||
function toFtsQuery(q: string): string {
|
||||
const terms = q.trim().split(/\s+/).filter(Boolean);
|
||||
return terms.map((t) => `"${t.replace(/"/g, '""')}"*`).join(' ');
|
||||
}
|
||||
|
||||
export function searchEmails(db: Database, q: string, limit: number, offset: number): { rows: Record<string, unknown>[]; total: number } {
|
||||
const match = toFtsQuery(q);
|
||||
if (!match) return { rows: [], total: 0 };
|
||||
const rows = db
|
||||
.query(
|
||||
`SELECT e.* FROM emails_fts JOIN emails e ON e.id = emails_fts.id
|
||||
WHERE emails_fts MATCH ? AND e.deleted = 0
|
||||
ORDER BY e.date DESC LIMIT ? OFFSET ?`,
|
||||
)
|
||||
.all(match, limit, offset) as Record<string, unknown>[];
|
||||
const total = (
|
||||
db.query('SELECT count(*) AS c FROM emails_fts JOIN emails e ON e.id = emails_fts.id WHERE emails_fts MATCH ? AND e.deleted = 0').get(match) as {
|
||||
c: number;
|
||||
}
|
||||
).c;
|
||||
return { rows, total };
|
||||
}
|
||||
|
||||
type ParsedEmail = {
|
||||
id: string;
|
||||
integration: string;
|
||||
@@ -173,6 +233,8 @@ export function upsertEmail(db: Database, email: ParsedEmail): void {
|
||||
db.run(insertAttachmentStmt, [email.id, i, att.filename, att.size, att.contentType, att.content]);
|
||||
}
|
||||
|
||||
syncFtsRow(db, email.id, email.subject ?? '', `${email.fromName ?? ''} ${email.fromAddress ?? ''}`.trim(), `${email.to ?? ''} ${email.cc ?? ''}`.trim(), email.snippet ?? '', email.text ?? '');
|
||||
|
||||
db.exec('COMMIT');
|
||||
} catch (err) {
|
||||
db.exec('ROLLBACK');
|
||||
@@ -215,6 +277,8 @@ export function upsertFromRawEml({ db, id, raw, integration, emailAccount, label
|
||||
db.run(insertAttachmentStmt, [id, i, att.filename, att.size, att.contentType, att.content]);
|
||||
}
|
||||
}
|
||||
|
||||
syncFtsRow(db, id, subject, `${name} ${address}`.trim(), `${to} ${cc ?? ''}`.trim(), snippet, text ?? '');
|
||||
}
|
||||
|
||||
/** Convert a db row to an EmailSummary for the API */
|
||||
|
||||
@@ -3,7 +3,7 @@ import { join } from 'node:path';
|
||||
import type { EmailMessage } from 'types';
|
||||
import { createRouter } from '../../create-router';
|
||||
import { DATA_PATH } from '@@/data-path';
|
||||
import { openEmailDb, rowToSummary, getSyncMeta } from './email-db';
|
||||
import { openEmailDb, rowToSummary, getSyncMeta, searchEmails } from './email-db';
|
||||
import { accountsRouter } from './accounts';
|
||||
|
||||
export const emailRouter = createRouter();
|
||||
@@ -64,6 +64,24 @@ emailRouter.get('/events', (ctx) => {
|
||||
});
|
||||
});
|
||||
|
||||
// Full-text search across all mail (subject / sender / recipients / snippet / body), newest first.
|
||||
emailRouter.get('/search', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
const q = (ctx.req.query('q') ?? '').trim();
|
||||
const page = Math.max(1, Number(ctx.req.query('page') ?? '1') || 1);
|
||||
const limit = Math.min(100, Math.max(1, Number(ctx.req.query('limit') ?? '50') || 50));
|
||||
const offset = (page - 1) * limit;
|
||||
if (!q) return ctx.json({ messages: [], total: 0 });
|
||||
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const { rows, total } = searchEmails(db, q, limit, offset);
|
||||
return ctx.json({ messages: rows.map(rowToSummary), total });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
emailRouter.get('/messages', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
const page = Number(ctx.req.query('page') ?? '1');
|
||||
|
||||
Reference in New Issue
Block a user