diff --git a/scripts/migrate-emails-to-sqlite.ts b/scripts/migrate-emails-to-sqlite.ts new file mode 100644 index 00000000..4558a471 --- /dev/null +++ b/scripts/migrate-emails-to-sqlite.ts @@ -0,0 +1,80 @@ +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { openEmailDb, upsertFromRawEml, setSyncMeta } from '../src/servers/api/email/email-db'; + +const DATA_PATH = process.env.DATA_PATH ?? join(process.cwd(), 'data'); + +// Find all user directories that have Gmail emails +const targetEmail = process.argv[2]; + +if (targetEmail) { + migrate(targetEmail); +} else { + const entries = readdirSync(DATA_PATH, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (!entry.name.includes('@')) continue; + const emailDir = join(DATA_PATH, entry.name, 'Gmail', 'emails'); + try { + const files = readdirSync(emailDir).filter((f) => f.endsWith('.eml')); + if (files.length > 0) migrate(entry.name); + } catch { + // no Gmail dir for this user + } + } +} + +function migrate(userEmail: string): void { + console.log(`Migrating ${userEmail}...`); + const emailDir = join(DATA_PATH, userEmail, 'Gmail', 'emails'); + const db = openEmailDb(userEmail); + + let filenames: string[]; + try { + filenames = readdirSync(emailDir).filter((f) => f.endsWith('.eml')); + } catch { + console.log(' No .eml files found'); + db.close(); + return; + } + + const existingIds = new Set(); + const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>; + for (const row of rows) existingIds.add(row.id); + + let added = 0; + let skipped = 0; + let errors = 0; + + db.exec('BEGIN'); + try { + for (const filename of filenames) { + const id = filename.replace(/\.eml$/, ''); + if (existingIds.has(id)) { + skipped++; + continue; + } + try { + const raw = readFileSync(join(emailDir, filename), 'utf-8'); + upsertFromRawEml({ db, id, raw, integration: 'gmail', emailAccount: userEmail, labels: ['INBOX'] }); + added++; + } catch { + errors++; + } + } + db.exec('COMMIT'); + } catch (err) { + db.exec('ROLLBACK'); + throw err; + } + + console.log(` ${filenames.length} .eml files — ${added} added, ${skipped} skipped, ${errors} errors`); + + // Store the latest email date so the next sync only fetches emails after it + const row = db.query('SELECT date FROM emails ORDER BY date DESC LIMIT 1').get() as { date: string } | null; + if (row?.date) { + setSyncMeta(db, 'last_sync_date', row.date); + console.log(` Stored last_sync_date: ${row.date}`); + } + db.close(); +} diff --git a/scripts/rebuild-email-index.ts b/scripts/rebuild-email-index.ts deleted file mode 100644 index aaa299c8..00000000 --- a/scripts/rebuild-email-index.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { join } from 'node:path'; -import { rebuildIndex } from '../src/servers/queue/handlers/gmail-sync'; - -const email = process.argv[2]; -if (!email) { - console.error('Usage: bun scripts/rebuild-email-index.ts '); - process.exit(1); -} - -const DATA_PATH = process.env.DATA_PATH ?? join(process.cwd(), 'data'); -const dir = join(DATA_PATH, email, 'Gmail', 'emails'); - -console.log(`Rebuilding index for ${dir}...`); -const entries = rebuildIndex(dir); -console.log(`Done — ${entries.length} entries written to index.json`); diff --git a/seed/tools/email-db/TOOL.md b/seed/tools/email-db/TOOL.md new file mode 100644 index 00000000..87511c41 --- /dev/null +++ b/seed/tools/email-db/TOOL.md @@ -0,0 +1,101 @@ +--- +version: 3 +name: email_db +label: Email Database +description: Query, search, aggregate, and manage the user's email database. Use this tool to answer questions about emails, find messages by sender/domain/date/content, get statistics, and delete emails. The database is a local SQLite copy of the user's synced Gmail inbox. +language: typescript +inputs: + action: + type: string + description: "Action to perform: query, search, stats, count, domains, senders, attachments, attachment-types, delete" + sql: + type: string + description: "Raw SQL query for the 'query' action. Only SELECT statements are allowed unless action is 'delete'." + optional: true + search: + type: string + description: "Search term for the 'search' action. Searches subject, from, and snippet fields." + optional: true + domain: + type: string + description: "Domain to filter by (e.g. 'newsletter.example.com') for search, count, or delete actions." + optional: true + sender: + type: string + description: "Sender email address to filter by for search, count, or delete actions." + optional: true + before: + type: string + description: "ISO date string — only include emails before this date." + optional: true + after: + type: string + description: "ISO date string — only include emails after this date." + optional: true + content_type: + type: string + description: "Attachment content type filter. Full MIME type (e.g. 'image/jpeg') or just the type prefix (e.g. 'image' matches all image types). Used with 'attachments' action." + optional: true + limit: + type: number + description: "Maximum number of results to return (default 20, max 100)." + optional: true +--- + +# Email Database Tool + +Query and manage the user's local email database (SQLite). + +## Available Actions + +- **query**: Run a raw SELECT query against the database. Use `sql` parameter. +- **search**: Full-text search across subject, from, and snippet. Use `search` parameter. Combine with `domain`, `sender`, `before`, `after` for filtering. +- **stats**: Get email statistics — total count, top domains, top senders, date range. +- **count**: Count emails matching filters (`domain`, `sender`, `before`, `after`). +- **domains**: List all sender domains with email counts, sorted by frequency. +- **senders**: List all senders with email counts, sorted by frequency. +- **attachments**: Search attachments by type, filename, sender, etc. Use `content_type` for type filtering (e.g. `image` for all images, `image/jpeg` for specific type). Combine with `domain`, `sender`, `before`, `after`, `search`. +- **attachment-types**: List all attachment content types with counts. +- **delete**: Delete emails matching filters. Requires at least one of: `domain`, `sender`, `before`, `after`, or `sql` (with DELETE statement). + +## Database Schema + +```sql +emails ( + id TEXT PRIMARY KEY, + integration TEXT, -- source: 'gmail', 'outlook', etc. + email_account TEXT, -- which account: 'user@gmail.com' + from_name TEXT, + from_address TEXT, + from_domain TEXT, + to_address TEXT, + cc TEXT, + subject TEXT, + date TEXT, -- ISO 8601 + snippet TEXT, + html TEXT, + text_body TEXT, + attachment_count INTEGER, + read INTEGER, + deleted INTEGER +) + +attachments ( + email_id TEXT, + idx INTEGER, + filename TEXT, + size INTEGER, + content_type TEXT +) +``` + +## Examples + +- Search for invoices: `action: "search", search: "invoice"` +- Count emails from a domain: `action: "count", domain: "newsletter.com"` +- Delete all emails from a domain: `action: "delete", domain: "spam.com"` +- Top 10 domains: `action: "domains", limit: 10` +- List attachment types: `action: "attachment-types"` +- Find image attachments: `action: "attachments", content_type: "image"` +- Find PDFs from a sender: `action: "attachments", content_type: "application/pdf", sender: "boss@company.com"` +- Custom query: `action: "query", sql: "SELECT from_domain, COUNT(*) as n FROM emails GROUP BY from_domain HAVING n > 50 ORDER BY n DESC"` diff --git a/seed/tools/email-db/index.ts b/seed/tools/email-db/index.ts new file mode 100644 index 00000000..95f6bd09 --- /dev/null +++ b/seed/tools/email-db/index.ts @@ -0,0 +1,286 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync } from 'node:fs'; + +type ToolResult = { + content: Array<{ type: string; text: string }>; + isError?: boolean; +}; + +type Params = { + action: string; + sql?: string; + search?: string; + domain?: string; + sender?: string; + before?: string; + after?: string; + content_type?: string; + limit?: number; +}; + +const DEFAULT_LIMIT = 20; +const MAX_LIMIT = 100; + +function getDbPath(): string { + return process.env.OFFICER_EMAIL_DB ?? '/officer/emails.db'; +} + +function sqlStr(v: string): string { + return `'${v.replace(/'/g, "''")}'`; +} + +function queryJson(sql: string): Record[] { + const output = execFileSync('sqlite3', ['-json', getDbPath()], { + input: sql, + encoding: 'utf-8', + timeout: 10000, + }); + const trimmed = output.trim(); + if (!trimmed) return []; + return JSON.parse(trimmed); +} + +function execAndCount(sql: string): number { + const output = execFileSync('sqlite3', [getDbPath()], { + input: `${sql};\nSELECT changes();`, + encoding: 'utf-8', + timeout: 10000, + }); + return parseInt(output.trim(), 10) || 0; +} + +function ok(text: string): ToolResult { + return { content: [{ type: 'text', text }] }; +} + +function err(text: string): ToolResult { + return { content: [{ type: 'text', text }], isError: true }; +} + +function formatRows(rows: Record[], limit: number): string { + if (rows.length === 0) return 'No results.'; + + const cols = Object.keys(rows[0]!); + const lines = rows.slice(0, limit).map((row, i) => { + const fields = cols.map((c) => `${c}: ${row[c] ?? ''}`).join(' | '); + return `${i + 1}. ${fields}`; + }); + + const header = `${rows.length} result${rows.length !== 1 ? 's' : ''}${rows.length > limit ? ` (showing first ${limit})` : ''}:`; + return [header, '', ...lines].join('\n'); +} + +// ── Helpers ── + +function buildWhereClause(params: Params): string { + const conditions: string[] = []; + + if (params.domain) { + conditions.push(`from_domain = ${sqlStr(params.domain.toLowerCase())}`); + } + if (params.sender) { + conditions.push(`from_address = ${sqlStr(params.sender.toLowerCase())}`); + } + if (params.before) { + conditions.push(`date < ${sqlStr(params.before)}`); + } + if (params.after) { + conditions.push(`date > ${sqlStr(params.after)}`); + } + if (params.search) { + const escaped = sqlStr(`%${params.search}%`); + conditions.push(`(subject LIKE ${escaped} OR from_address LIKE ${escaped} OR from_name LIKE ${escaped} OR snippet LIKE ${escaped})`); + } + + return conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : ''; +} + +function withActive(where: string): string { + return where ? `${where} AND deleted = 0` : 'WHERE deleted = 0'; +} + +// ── Actions ── + +function runQuery(sql: string, limit: number): string { + const trimmed = sql.trim().toLowerCase(); + if (!trimmed.startsWith('select')) { + throw new Error('Only SELECT statements are allowed in query action.'); + } + const rows = queryJson(sql); + return formatRows(rows, limit); +} + +function searchEmails(params: Params, limit: number): string { + const where = withActive(buildWhereClause(params)); + const rows = queryJson(`SELECT id, from_name, from_address, subject, date, snippet, attachment_count FROM emails ${where} ORDER BY date DESC LIMIT ${limit}`); + return formatRows(rows, limit); +} + +function getStats(): string { + const totalRows = queryJson('SELECT COUNT(*) as count FROM emails WHERE deleted = 0'); + const total = (totalRows[0]?.count as number) ?? 0; + if (total === 0) return 'Email database is empty.'; + + const dateRange = queryJson('SELECT MIN(date) as oldest, MAX(date) as newest FROM emails WHERE deleted = 0'); + const topDomains = queryJson('SELECT from_domain, COUNT(*) as count FROM emails WHERE deleted = 0 GROUP BY from_domain ORDER BY count DESC LIMIT 10'); + const topSenders = queryJson('SELECT from_address, from_name, COUNT(*) as count FROM emails WHERE deleted = 0 GROUP BY from_address ORDER BY count DESC LIMIT 10'); + const attRows = queryJson('SELECT COUNT(*) as count FROM emails WHERE deleted = 0 AND attachment_count > 0'); + const withAttachments = (attRows[0]?.count as number) ?? 0; + + const dr = dateRange[0] ?? {}; + const lines = [ + `**Email Database Statistics**`, + ``, + `Total emails: ${total}`, + `With attachments: ${withAttachments}`, + `Date range: ${(dr.oldest as string)?.slice(0, 10)} to ${(dr.newest as string)?.slice(0, 10)}`, + ``, + `**Top 10 Domains:**`, + ...topDomains.map((d, i) => `${i + 1}. ${d.from_domain} (${d.count})`), + ``, + `**Top 10 Senders:**`, + ...topSenders.map((s, i) => `${i + 1}. ${s.from_name ? `${s.from_name} <${s.from_address}>` : s.from_address} (${s.count})`), + ]; + + return lines.join('\n'); +} + +function countEmails(params: Params): string { + const where = withActive(buildWhereClause(params)); + const rows = queryJson(`SELECT COUNT(*) as count FROM emails ${where}`); + const count = (rows[0]?.count as number) ?? 0; + + const filters: string[] = []; + if (params.domain) filters.push(`domain=${params.domain}`); + if (params.sender) filters.push(`sender=${params.sender}`); + if (params.before) filters.push(`before=${params.before}`); + if (params.after) filters.push(`after=${params.after}`); + if (params.search) filters.push(`search="${params.search}"`); + + const desc = filters.length > 0 ? ` matching ${filters.join(', ')}` : ''; + return `${count} email${count !== 1 ? 's' : ''}${desc}.`; +} + +function listDomains(limit: number): string { + const rows = queryJson(`SELECT from_domain, COUNT(*) as count FROM emails WHERE deleted = 0 GROUP BY from_domain ORDER BY count DESC LIMIT ${limit}`); + if (rows.length === 0) return 'No emails in database.'; + + const lines = rows.map((d, i) => `${i + 1}. ${d.from_domain} — ${d.count} email${(d.count as number) !== 1 ? 's' : ''}`); + return [`**Sender Domains** (${rows.length}):`, '', ...lines].join('\n'); +} + +function listSenders(limit: number): string { + const rows = queryJson(`SELECT from_address, from_name, COUNT(*) as count FROM emails WHERE deleted = 0 GROUP BY from_address ORDER BY count DESC LIMIT ${limit}`); + if (rows.length === 0) return 'No emails in database.'; + + const lines = rows.map((s, i) => { + const display = s.from_name ? `${s.from_name} <${s.from_address}>` : s.from_address; + return `${i + 1}. ${display} — ${s.count} email${(s.count as number) !== 1 ? 's' : ''}`; + }); + return [`**Senders** (${rows.length}):`, '', ...lines].join('\n'); +} + +function listAttachmentTypes(limit: number): string { + const rows = queryJson(`SELECT content_type, COUNT(*) as count FROM attachments a JOIN emails e ON a.email_id = e.id WHERE e.deleted = 0 GROUP BY content_type ORDER BY count DESC LIMIT ${limit}`); + if (rows.length === 0) return 'No attachments in database.'; + + const totalRows = queryJson('SELECT COUNT(*) as count FROM attachments a JOIN emails e ON a.email_id = e.id WHERE e.deleted = 0'); + const total = (totalRows[0]?.count as number) ?? 0; + + const lines = rows.map((r, i) => `${i + 1}. ${r.content_type} — ${r.count}`); + return [`**Attachment Types** (${total} total):`, '', ...lines].join('\n'); +} + +function searchAttachments(params: Params, limit: number): string { + const conditions: string[] = ['e.deleted = 0']; + + if (params.content_type) { + const ct = params.content_type as string; + if (ct.includes('/')) { + conditions.push(`a.content_type = ${sqlStr(ct)}`); + } else { + conditions.push(`a.content_type LIKE ${sqlStr(ct + '/%')}`); + } + } + if (params.domain) conditions.push(`e.from_domain = ${sqlStr(params.domain.toLowerCase())}`); + if (params.sender) conditions.push(`e.from_address = ${sqlStr(params.sender.toLowerCase())}`); + if (params.before) conditions.push(`e.date < ${sqlStr(params.before)}`); + if (params.after) conditions.push(`e.date > ${sqlStr(params.after)}`); + if (params.search) { + const escaped = sqlStr(`%${params.search}%`); + conditions.push(`(a.filename LIKE ${escaped} OR e.subject LIKE ${escaped})`); + } + + const where = `WHERE ${conditions.join(' AND ')}`; + const rows = queryJson(`SELECT a.filename, a.size, a.content_type, e.id as email_id, e.from_address, e.subject, e.date FROM attachments a JOIN emails e ON a.email_id = e.id ${where} ORDER BY e.date DESC LIMIT ${limit}`); + return formatRows(rows, limit); +} + +function deleteEmails(params: Params): string { + const where = buildWhereClause(params); + if (!where) { + throw new Error('Delete requires at least one filter (domain, sender, before, after, or search).'); + } + + const activeWhere = withActive(where); + const countRows = queryJson(`SELECT COUNT(*) as count FROM emails ${activeWhere}`); + const count = (countRows[0]?.count as number) ?? 0; + if (count === 0) return 'No emails match the given filters.'; + + const changes = execAndCount(`UPDATE emails SET deleted = 1 ${activeWhere}`); + return `Deleted ${changes} email${changes !== 1 ? 's' : ''}.`; +} + +// ── Main ── + +export async function execute(_toolCallId: string, params: Params): Promise { + const limit = Math.min(params.limit ?? DEFAULT_LIMIT, MAX_LIMIT); + + const dbPath = getDbPath(); + if (!existsSync(dbPath)) { + return err(`Email database not found at ${dbPath}. Has Gmail been synced?`); + } + + try { + switch (params.action) { + case 'query': + if (!params.sql) return err('sql parameter is required for query action.'); + return ok(runQuery(params.sql, limit)); + + case 'search': + if (!params.search && !params.domain && !params.sender && !params.before && !params.after) { + return err('At least one filter is required: search, domain, sender, before, or after.'); + } + return ok(searchEmails(params, limit)); + + case 'stats': + return ok(getStats()); + + case 'count': + return ok(countEmails(params)); + + case 'domains': + return ok(listDomains(limit)); + + case 'senders': + return ok(listSenders(limit)); + + case 'attachment-types': + return ok(listAttachmentTypes(limit)); + + case 'attachments': + if (!params.content_type && !params.search && !params.domain && !params.sender && !params.before && !params.after) { + return err('At least one filter is required: content_type, search, domain, sender, before, or after.'); + } + return ok(searchAttachments(params, limit)); + + case 'delete': + return ok(deleteEmails(params)); + + default: + return err(`Unknown action: "${params.action}". Available: query, search, stats, count, domains, senders, attachments, attachment-types, delete.`); + } + } catch (e) { + return err(`Email DB error: ${e instanceof Error ? e.message : String(e)}`); + } +} diff --git a/src/apps/officer-web/Screens/Dashboard/Email/EmailChat.tsx b/src/apps/officer-web/Screens/Dashboard/Email/EmailChat.tsx new file mode 100644 index 00000000..39edc0ae --- /dev/null +++ b/src/apps/officer-web/Screens/Dashboard/Email/EmailChat.tsx @@ -0,0 +1,17 @@ +import { EmbeddableChat, usePiChat } from 'officerdev'; + +const PROMPT_PREFIX = `You are an email assistant. The user has a local SQLite email database available via the "email-db" tool — use it for all email queries (search, count, stats, aggregations, deletions) unless the user explicitly asks you to use Gmail. Do not use the Gmail integration for questions about existing emails.`; + +export const EmailChat = () => { + const chat = usePiChat(undefined, undefined, { replaceUrl: false }); + + return ( + + ); +}; diff --git a/src/apps/officer-web/Screens/Dashboard/Email/EmailList.tsx b/src/apps/officer-web/Screens/Dashboard/Email/EmailList.tsx index 8bf7ebdf..dea6f822 100644 --- a/src/apps/officer-web/Screens/Dashboard/Email/EmailList.tsx +++ b/src/apps/officer-web/Screens/Dashboard/Email/EmailList.tsx @@ -1,8 +1,10 @@ -import { useState } from 'react'; -import { useQuery } from '@tanstack/react-query'; -import { ChevronLeft, ChevronRight, Mail, Paperclip } from 'lucide-react'; +import { useEffect, useRef, useState } from 'react'; +import { useQuery, useQueryClient } from '@tanstack/react-query'; +import { ChevronLeft, ChevronRight, Loader2, Mail, Paperclip, RefreshCw } from 'lucide-react'; +import { toast } from 'sonner'; import { useClient } from 'hooks/useClient'; import { useGlobal } from 'hooks/useGlobal'; +import { useJobs } from 'hooks/useJobs'; import type { EmailSummary } from 'types'; const LIMIT = 50; @@ -19,8 +21,11 @@ const formatDate = (iso: string) => { export const EmailList = () => { const client = useClient(); + const queryClient = useQueryClient(); const [selectedId, setSelectedId] = useGlobal('EMAIL_SELECTED', null); const [page, setPage] = useState(1); + const { jobs, createJob } = useJobs({ type: 'gmail-sync' }); + const isSyncing = jobs.some((j) => j.status === 'queued' || j.status === 'running'); const { data, isLoading } = useQuery({ queryKey: ['email-messages', page], @@ -28,6 +33,24 @@ export const EmailList = () => { client.get<{ messages: EmailSummary[]; total: number }>(`/email/messages?page=${page}&limit=${LIMIT}`), }); + const handleSync = async () => { + try { + await createJob({ lane: 'google-api', type: 'gmail-sync' }); + toast.success('Gmail sync started'); + } catch { + toast.error('Failed to start sync'); + } + }; + + // Refresh email list when a sync job completes + const prevSyncing = useRef(false); + useEffect(() => { + if (prevSyncing.current && !isSyncing) { + queryClient.invalidateQueries({ queryKey: ['email-messages'] }); + } + prevSyncing.current = isSyncing; + }, [isSyncing, queryClient]); + const messages = data?.messages ?? []; const total = data?.total ?? 0; const totalPages = Math.max(1, Math.ceil(total / LIMIT)); @@ -55,6 +78,18 @@ export const EmailList = () => { Inbox {total} + {totalPages > 1 && (
)} - {!activeJob && lastJob?.status === 'completed' && ( -
- - Last sync completed {formatTime(lastJob.completedAt!)} -
- )} {!activeJob && lastJob?.status === 'failed' && (
Last sync failed{lastJob.error ? `: ${lastJob.error}` : ''}
)} -
- - -
+ {!activeJob && lastSyncAt && ( +
+ + Last sync completed {formatTime(lastSyncAt)} +
+ )} +