email syncyng
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
import { EmbeddableChat, usePiChat } from 'officerdev';
|
||||
|
||||
const PROMPT_PREFIX = `You are an email assistant. The user has a local SQLite email database available via the "email-db" tool — use it for all email queries (search, count, stats, aggregations, deletions) unless the user explicitly asks you to use Gmail. Do not use the Gmail integration for questions about existing emails.`;
|
||||
|
||||
export const EmailChat = () => {
|
||||
const chat = usePiChat(undefined, undefined, { replaceUrl: false });
|
||||
|
||||
return (
|
||||
<EmbeddableChat
|
||||
className="h-full"
|
||||
chat={chat}
|
||||
sandboxed
|
||||
replaceUrl={false}
|
||||
promptPrefix={PROMPT_PREFIX}
|
||||
/>
|
||||
);
|
||||
};
|
||||
@@ -1,8 +1,10 @@
|
||||
import { useState } from 'react';
|
||||
import { useQuery } from '@tanstack/react-query';
|
||||
import { ChevronLeft, ChevronRight, Mail, Paperclip } from 'lucide-react';
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { useQuery, useQueryClient } from '@tanstack/react-query';
|
||||
import { ChevronLeft, ChevronRight, Loader2, Mail, Paperclip, RefreshCw } from 'lucide-react';
|
||||
import { toast } from 'sonner';
|
||||
import { useClient } from 'hooks/useClient';
|
||||
import { useGlobal } from 'hooks/useGlobal';
|
||||
import { useJobs } from 'hooks/useJobs';
|
||||
import type { EmailSummary } from 'types';
|
||||
|
||||
const LIMIT = 50;
|
||||
@@ -19,8 +21,11 @@ const formatDate = (iso: string) => {
|
||||
|
||||
export const EmailList = () => {
|
||||
const client = useClient();
|
||||
const queryClient = useQueryClient();
|
||||
const [selectedId, setSelectedId] = useGlobal<string | null>('EMAIL_SELECTED', null);
|
||||
const [page, setPage] = useState(1);
|
||||
const { jobs, createJob } = useJobs({ type: 'gmail-sync' });
|
||||
const isSyncing = jobs.some((j) => j.status === 'queued' || j.status === 'running');
|
||||
|
||||
const { data, isLoading } = useQuery({
|
||||
queryKey: ['email-messages', page],
|
||||
@@ -28,6 +33,24 @@ export const EmailList = () => {
|
||||
client.get<{ messages: EmailSummary[]; total: number }>(`/email/messages?page=${page}&limit=${LIMIT}`),
|
||||
});
|
||||
|
||||
const handleSync = async () => {
|
||||
try {
|
||||
await createJob({ lane: 'google-api', type: 'gmail-sync' });
|
||||
toast.success('Gmail sync started');
|
||||
} catch {
|
||||
toast.error('Failed to start sync');
|
||||
}
|
||||
};
|
||||
|
||||
// Refresh email list when a sync job completes
|
||||
const prevSyncing = useRef(false);
|
||||
useEffect(() => {
|
||||
if (prevSyncing.current && !isSyncing) {
|
||||
queryClient.invalidateQueries({ queryKey: ['email-messages'] });
|
||||
}
|
||||
prevSyncing.current = isSyncing;
|
||||
}, [isSyncing, queryClient]);
|
||||
|
||||
const messages = data?.messages ?? [];
|
||||
const total = data?.total ?? 0;
|
||||
const totalPages = Math.max(1, Math.ceil(total / LIMIT));
|
||||
@@ -55,6 +78,18 @@ export const EmailList = () => {
|
||||
<Mail className="h-4 w-4 opacity-60" />
|
||||
<span className="text-sm font-medium">Inbox</span>
|
||||
<span className="text-xs opacity-50">{total}</span>
|
||||
<button
|
||||
onClick={handleSync}
|
||||
disabled={isSyncing}
|
||||
className="p-0.5 rounded hover:bg-black/10 transition-colors cursor-pointer disabled:cursor-default disabled:opacity-50 shrink-0"
|
||||
title={isSyncing ? 'Syncing...' : 'Sync emails'}
|
||||
>
|
||||
{isSyncing ? (
|
||||
<Loader2 className="h-3.5 w-3.5 animate-spin" />
|
||||
) : (
|
||||
<RefreshCw className="h-3.5 w-3.5" />
|
||||
)}
|
||||
</button>
|
||||
{totalPages > 1 && (
|
||||
<div className="ml-auto flex items-center gap-2">
|
||||
<button
|
||||
|
||||
@@ -6,6 +6,7 @@ import { useGlobal } from 'hooks/useGlobal';
|
||||
import { defaultLayout } from './defaultLayout';
|
||||
import { EmailList } from './EmailList';
|
||||
import { EmailReader } from './EmailReader';
|
||||
import { EmailChat } from './EmailChat';
|
||||
|
||||
export const EmailScreen = () => {
|
||||
const isMobile = useIsMobile();
|
||||
@@ -15,6 +16,7 @@ export const EmailScreen = () => {
|
||||
() => ({
|
||||
'email-list': EmailList,
|
||||
'email-reader': EmailReader,
|
||||
'email-chat': EmailChat,
|
||||
}),
|
||||
[],
|
||||
);
|
||||
|
||||
@@ -5,7 +5,18 @@ export const defaultLayout: LayoutNode = {
|
||||
id: 'email-root',
|
||||
direction: 'horizontal',
|
||||
children: [
|
||||
{ node: { type: 'panel', id: 'email-list', appType: null }, size: 30 },
|
||||
{ node: { type: 'panel', id: 'email-reader', appType: null }, size: 70 },
|
||||
{ node: { type: 'panel', id: 'email-list', appType: null }, size: 25 },
|
||||
{
|
||||
node: {
|
||||
type: 'group',
|
||||
id: 'email-right',
|
||||
direction: 'vertical',
|
||||
children: [
|
||||
{ node: { type: 'panel', id: 'email-reader', appType: null }, size: 60 },
|
||||
{ node: { type: 'panel', id: 'email-chat', appType: null }, size: 40 },
|
||||
],
|
||||
},
|
||||
size: 75,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
+29
-29
@@ -12,7 +12,7 @@ type GoogleStatus = {
|
||||
configured: boolean;
|
||||
};
|
||||
|
||||
const formatTime = (ts: number) => {
|
||||
const formatTime = (ts: number | string) => {
|
||||
const d = new Date(ts);
|
||||
return d.toLocaleString(undefined, { dateStyle: 'medium', timeStyle: 'short' });
|
||||
};
|
||||
@@ -21,6 +21,7 @@ export const GoogleAccount = () => {
|
||||
const client = useClient();
|
||||
const [isLoading, setIsLoading] = useState(true);
|
||||
const [status, setStatus] = useState<GoogleStatus>({ connected: false, email: null, picture: null, configured: false });
|
||||
const [lastSyncAt, setLastSyncAt] = useState<string | null>(null);
|
||||
const { jobs, createJob, refetch } = useJobs({ type: 'gmail-sync' });
|
||||
const activeJob = jobs.find((j) => j.status === 'queued' || j.status === 'running');
|
||||
const lastJob = jobs[0];
|
||||
@@ -31,6 +32,10 @@ export const GoogleAccount = () => {
|
||||
.then(setStatus)
|
||||
.catch(() => {})
|
||||
.finally(() => setIsLoading(false));
|
||||
client
|
||||
.get<{ lastSyncAt: string | null }>('/email/sync-status')
|
||||
.then((res) => setLastSyncAt(res.lastSyncAt))
|
||||
.catch(() => {});
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
@@ -47,6 +52,13 @@ export const GoogleAccount = () => {
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Refresh sync status from DB when a job finishes
|
||||
useEffect(() => {
|
||||
if (!activeJob && lastJob?.status === 'completed') {
|
||||
client.get<{ lastSyncAt: string | null }>('/email/sync-status').then((res) => setLastSyncAt(res.lastSyncAt)).catch(() => {});
|
||||
}
|
||||
}, [activeJob, lastJob?.status]);
|
||||
|
||||
const handleSync = async (year?: number) => {
|
||||
try {
|
||||
await createJob({ lane: 'google-api', type: 'gmail-sync', meta: year ? { year } : undefined });
|
||||
@@ -131,40 +143,28 @@ export const GoogleAccount = () => {
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{!activeJob && lastJob?.status === 'completed' && (
|
||||
<div className="flex items-center gap-2 text-xs text-duck-dark/50 dark:text-foreground/50">
|
||||
<CheckCircle2 className="h-3.5 w-3.5 text-green-500 shrink-0" />
|
||||
Last sync completed {formatTime(lastJob.completedAt!)}
|
||||
</div>
|
||||
)}
|
||||
{!activeJob && lastJob?.status === 'failed' && (
|
||||
<div className="flex items-center gap-2 text-xs text-red-500">
|
||||
<XCircle className="h-3.5 w-3.5 shrink-0" />
|
||||
Last sync failed{lastJob.error ? `: ${lastJob.error}` : ''}
|
||||
</div>
|
||||
)}
|
||||
<div className="flex gap-2">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
disabled={!!activeJob}
|
||||
onClick={() => handleSync()}
|
||||
className="flex-1 h-11 cursor-pointer gap-2"
|
||||
>
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
Sync Gmail Inbox
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
disabled={!!activeJob}
|
||||
onClick={() => handleSync(2026)}
|
||||
className="flex-1 h-11 cursor-pointer gap-2"
|
||||
>
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
Test Sync (2026)
|
||||
</Button>
|
||||
</div>
|
||||
{!activeJob && lastSyncAt && (
|
||||
<div className="flex items-center gap-2 text-xs text-duck-dark/50 dark:text-foreground/50">
|
||||
<CheckCircle2 className="h-3.5 w-3.5 text-green-500 shrink-0" />
|
||||
Last sync completed {formatTime(lastSyncAt)}
|
||||
</div>
|
||||
)}
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
disabled={!!activeJob}
|
||||
onClick={() => handleSync()}
|
||||
className="w-full h-11 cursor-pointer gap-2"
|
||||
>
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
Sync Gmail Inbox
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
|
||||
@@ -199,6 +199,7 @@ const server = serve({
|
||||
|
||||
console.log(`🚀 Server running at ${server.url}`);
|
||||
|
||||
|
||||
void initTerminalSidecars();
|
||||
|
||||
// Ensure pi is installed
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
import { Database } from 'bun:sqlite';
|
||||
import { join } from 'node:path';
|
||||
import { chmodSync } from 'node:fs';
|
||||
import { DATA_PATH } from '@@/data-path';
|
||||
import type { EmailSummary } from 'types';
|
||||
|
||||
const SCHEMA_TABLES = `
|
||||
CREATE TABLE IF NOT EXISTS emails (
|
||||
id TEXT PRIMARY KEY,
|
||||
integration TEXT NOT NULL DEFAULT 'gmail',
|
||||
email_account TEXT NOT NULL DEFAULT '',
|
||||
from_name TEXT,
|
||||
from_address TEXT,
|
||||
from_domain TEXT,
|
||||
to_address TEXT,
|
||||
cc TEXT,
|
||||
subject TEXT,
|
||||
date TEXT,
|
||||
snippet TEXT,
|
||||
html TEXT,
|
||||
text_body TEXT,
|
||||
attachment_count INTEGER DEFAULT 0,
|
||||
read INTEGER DEFAULT 0,
|
||||
deleted INTEGER DEFAULT 0,
|
||||
labels TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
email_id TEXT REFERENCES emails(id) ON DELETE CASCADE,
|
||||
idx INTEGER,
|
||||
filename TEXT,
|
||||
size INTEGER,
|
||||
content_type TEXT,
|
||||
content TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sync_meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
`;
|
||||
|
||||
const SCHEMA_INDEXES = `
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_date ON emails(date);
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_from_domain ON emails(from_domain);
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_from_address ON emails(from_address);
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_integration ON emails(integration);
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_email_account ON emails(email_account);
|
||||
CREATE INDEX IF NOT EXISTS idx_emails_labels ON emails(labels);
|
||||
`;
|
||||
|
||||
/** Convert label IDs to lowercase comma-separated string for storage */
|
||||
function labelsToString(labels?: string[]): string | null {
|
||||
if (!labels || labels.length === 0) return null;
|
||||
return labels.map((l) => l.toLowerCase()).join(',');
|
||||
}
|
||||
|
||||
/** Convert stored comma-separated labels back to array */
|
||||
function labelsFromString(value: unknown): string[] | undefined {
|
||||
if (typeof value !== 'string' || !value) return undefined;
|
||||
return value.split(',');
|
||||
}
|
||||
|
||||
function extractAddress(headerValue: string): { name: string; address: string } {
|
||||
const match = headerValue.match(/^"?(.+?)"?\s*<(.+?)>$/);
|
||||
if (match) return { name: match[1]!.trim(), address: match[2]!.toLowerCase() };
|
||||
const bare = headerValue.trim().toLowerCase();
|
||||
return { name: '', address: bare };
|
||||
}
|
||||
|
||||
function extractDomain(address: string): string {
|
||||
const at = address.lastIndexOf('@');
|
||||
return at >= 0 ? address.slice(at + 1) : '';
|
||||
}
|
||||
|
||||
export function openEmailDb(email: string): Database {
|
||||
const dbPath = join(DATA_PATH, email, 'emails.db');
|
||||
const db = new Database(dbPath, { create: true });
|
||||
db.exec('PRAGMA journal_mode = DELETE');
|
||||
db.exec('PRAGMA foreign_keys = ON');
|
||||
db.exec(SCHEMA_TABLES);
|
||||
migrate(db);
|
||||
db.exec(SCHEMA_INDEXES);
|
||||
chmodSync(dbPath, 0o666);
|
||||
return db;
|
||||
}
|
||||
|
||||
function migrate(db: Database): void {
|
||||
const cols = db.query('PRAGMA table_info(emails)').all() as Array<{ name: string }>;
|
||||
const colNames = new Set(cols.map((c) => c.name));
|
||||
|
||||
if (!colNames.has('deleted')) {
|
||||
db.exec('ALTER TABLE emails ADD COLUMN deleted INTEGER DEFAULT 0');
|
||||
}
|
||||
if (!colNames.has('integration')) {
|
||||
db.exec("ALTER TABLE emails ADD COLUMN integration TEXT NOT NULL DEFAULT 'gmail'");
|
||||
}
|
||||
if (!colNames.has('email_account')) {
|
||||
db.exec("ALTER TABLE emails ADD COLUMN email_account TEXT NOT NULL DEFAULT ''");
|
||||
}
|
||||
if (!colNames.has('labels')) {
|
||||
db.exec('ALTER TABLE emails ADD COLUMN labels TEXT');
|
||||
}
|
||||
|
||||
// Ensure sync_meta table exists (for DBs created before it was added to SCHEMA_TABLES)
|
||||
db.exec('CREATE TABLE IF NOT EXISTS sync_meta (key TEXT PRIMARY KEY, value TEXT)');
|
||||
|
||||
// Add content column to attachments if missing
|
||||
const attCols = db.query('PRAGMA table_info(attachments)').all() as Array<{ name: string }>;
|
||||
const attColNames = new Set(attCols.map((c) => c.name));
|
||||
if (!attColNames.has('content')) {
|
||||
db.exec('ALTER TABLE attachments ADD COLUMN content TEXT');
|
||||
}
|
||||
}
|
||||
|
||||
type ParsedEmail = {
|
||||
id: string;
|
||||
integration: string;
|
||||
emailAccount: string;
|
||||
fromName: string;
|
||||
fromAddress: string;
|
||||
to: string;
|
||||
cc?: string;
|
||||
subject: string;
|
||||
date: string;
|
||||
snippet: string;
|
||||
html?: string;
|
||||
text?: string;
|
||||
attachments: Array<{ filename: string; size: number; contentType: string; content: string }>;
|
||||
labels?: string[];
|
||||
};
|
||||
|
||||
const upsertEmailStmt = `
|
||||
INSERT OR REPLACE INTO emails (id, integration, email_account, from_name, from_address, from_domain, to_address, cc, subject, date, snippet, html, text_body, attachment_count, labels)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`;
|
||||
|
||||
const deleteAttachmentsStmt = 'DELETE FROM attachments WHERE email_id = ?';
|
||||
const insertAttachmentStmt = 'INSERT INTO attachments (email_id, idx, filename, size, content_type, content) VALUES (?, ?, ?, ?, ?, ?)';
|
||||
|
||||
export function upsertEmail(db: Database, email: ParsedEmail): void {
|
||||
const domain = extractDomain(email.fromAddress);
|
||||
|
||||
db.exec('BEGIN');
|
||||
try {
|
||||
db.run(upsertEmailStmt, [
|
||||
email.id,
|
||||
email.integration,
|
||||
email.emailAccount,
|
||||
email.fromName,
|
||||
email.fromAddress,
|
||||
domain,
|
||||
email.to,
|
||||
email.cc ?? null,
|
||||
email.subject,
|
||||
email.date,
|
||||
email.snippet,
|
||||
email.html ?? null,
|
||||
email.text ?? null,
|
||||
email.attachments.length,
|
||||
labelsToString(email.labels),
|
||||
]);
|
||||
|
||||
db.run(deleteAttachmentsStmt, [email.id]);
|
||||
for (let i = 0; i < email.attachments.length; i++) {
|
||||
const att = email.attachments[i]!;
|
||||
db.run(insertAttachmentStmt, [email.id, i, att.filename, att.size, att.contentType, att.content]);
|
||||
}
|
||||
|
||||
db.exec('COMMIT');
|
||||
} catch (err) {
|
||||
db.exec('ROLLBACK');
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** Upsert a single email from its raw RFC822 text using fast header parsing. */
|
||||
type UpsertFromRawEmlParams = {
|
||||
db: Database;
|
||||
id: string;
|
||||
raw: string;
|
||||
integration: string;
|
||||
emailAccount: string;
|
||||
labels?: string[];
|
||||
};
|
||||
|
||||
export function upsertFromRawEml({ db, id, raw, integration, emailAccount, labels }: UpsertFromRawEmlParams): void {
|
||||
const from = extractHeader(raw, 'From');
|
||||
const { name, address } = extractAddress(from);
|
||||
const to = extractHeader(raw, 'To');
|
||||
const cc = extractHeader(raw, 'Cc') || null;
|
||||
const subject = extractHeader(raw, 'Subject') || '(no subject)';
|
||||
const dateStr = extractHeader(raw, 'Date');
|
||||
const date = dateStr ? new Date(dateStr).toISOString() : new Date(0).toISOString();
|
||||
const snippet = extractSnippet(raw);
|
||||
const attachments = parseAttachments(raw);
|
||||
const domain = extractDomain(address);
|
||||
|
||||
const { html, text } = extractBody(raw);
|
||||
|
||||
db.run(upsertEmailStmt, [
|
||||
id, integration, emailAccount, name, address, domain, to, cc, subject, date, snippet, html, text, attachments.length, labelsToString(labels),
|
||||
]);
|
||||
|
||||
if (attachments.length > 0) {
|
||||
db.run(deleteAttachmentsStmt, [id]);
|
||||
for (let i = 0; i < attachments.length; i++) {
|
||||
const att = attachments[i]!;
|
||||
db.run(insertAttachmentStmt, [id, i, att.filename, att.size, att.contentType, att.content]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Convert a db row to an EmailSummary for the API */
|
||||
export function rowToSummary(row: Record<string, unknown>): EmailSummary {
|
||||
const from = row.from_name
|
||||
? `${row.from_name} <${row.from_address}>`
|
||||
: (row.from_address as string);
|
||||
|
||||
const labels = labelsFromString(row.labels);
|
||||
|
||||
return {
|
||||
id: row.id as string,
|
||||
from,
|
||||
to: row.to_address as string,
|
||||
subject: row.subject as string,
|
||||
date: row.date as string,
|
||||
snippet: row.snippet as string,
|
||||
...(row.attachment_count ? { attachmentCount: row.attachment_count as number } : {}),
|
||||
...(row.read ? { read: true } : {}),
|
||||
...(labels ? { labels } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Sync meta helpers ──
|
||||
|
||||
export function getSyncMeta(db: Database, key: string): string | null {
|
||||
const row = db.query('SELECT value FROM sync_meta WHERE key = ?').get(key) as { value: string } | null;
|
||||
return row?.value ?? null;
|
||||
}
|
||||
|
||||
export function setSyncMeta(db: Database, key: string, value: string): void {
|
||||
db.run('INSERT OR REPLACE INTO sync_meta (key, value) VALUES (?, ?)', [key, value]);
|
||||
}
|
||||
|
||||
export function updateEmailLabels(db: Database, id: string, labels: string[]): void {
|
||||
db.run('UPDATE emails SET labels = ? WHERE id = ?', [labelsToString(labels), id]);
|
||||
}
|
||||
|
||||
// ── Header parsing helpers (same logic as gmail-sync) ──
|
||||
|
||||
function decodeMimeWords(text: string): string {
|
||||
return text.replace(/=\?([^?]+)\?(B|Q)\?([^?]+)\?=/gi, (_, _charset, encoding, encoded) => {
|
||||
try {
|
||||
if (encoding.toUpperCase() === 'B') {
|
||||
return Buffer.from(encoded, 'base64').toString('utf-8');
|
||||
}
|
||||
const bytes: number[] = [];
|
||||
for (let i = 0; i < encoded.length; i++) {
|
||||
if (encoded[i] === '_') {
|
||||
bytes.push(0x20);
|
||||
} else if (encoded[i] === '=' && i + 2 < encoded.length) {
|
||||
bytes.push(parseInt(encoded.slice(i + 1, i + 3), 16));
|
||||
i += 2;
|
||||
} else {
|
||||
bytes.push(encoded.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
return Buffer.from(bytes).toString('utf-8');
|
||||
} catch {
|
||||
return encoded;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function extractHeader(raw: string, name: string): string {
|
||||
const match = raw.match(new RegExp(`^${name}:\\s*(.+)$`, 'mi'));
|
||||
return match?.[1]?.trim() ? decodeMimeWords(match[1].trim()) : '';
|
||||
}
|
||||
|
||||
/** Extract a header value including folded continuation lines (lines starting with whitespace) */
|
||||
function extractFullHeader(raw: string, name: string): string {
|
||||
const headerEnd = findHeaderEnd(raw);
|
||||
const headerBlock = headerEnd !== -1 ? raw.slice(0, headerEnd) : raw.slice(0, 4096);
|
||||
const lines = headerBlock.split(/\r?\n/);
|
||||
let result = '';
|
||||
let capturing = false;
|
||||
for (const line of lines) {
|
||||
if (new RegExp(`^${name}:\\s*`, 'i').test(line)) {
|
||||
result = line.replace(new RegExp(`^${name}:\\s*`, 'i'), '');
|
||||
capturing = true;
|
||||
} else if (capturing && /^[\t ]/.test(line)) {
|
||||
result += ' ' + line.trim();
|
||||
} else if (capturing) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return result.trim();
|
||||
}
|
||||
|
||||
function findHeaderEnd(text: string): number {
|
||||
const crlf = text.indexOf('\r\n\r\n');
|
||||
const lf = text.indexOf('\n\n');
|
||||
if (crlf !== -1) return crlf + 4;
|
||||
if (lf !== -1) return lf + 2;
|
||||
return -1;
|
||||
}
|
||||
|
||||
function extractSnippet(raw: string): string {
|
||||
const idx = findHeaderEnd(raw);
|
||||
if (idx === -1) return '';
|
||||
|
||||
let body = raw.slice(idx);
|
||||
|
||||
if (body.trimStart().startsWith('--')) {
|
||||
const afterBoundary = body.slice(body.indexOf('\n') + 1);
|
||||
const partBodyStart = findHeaderEnd(afterBoundary);
|
||||
if (partBodyStart !== -1) body = afterBoundary.slice(partBodyStart);
|
||||
}
|
||||
|
||||
const nextBoundary = body.indexOf('\n--');
|
||||
if (nextBoundary !== -1) body = body.slice(0, nextBoundary);
|
||||
|
||||
return body.replace(/\s+/g, ' ').trim().slice(0, 120);
|
||||
}
|
||||
|
||||
function decodeQuotedPrintable(text: string): string {
|
||||
return text
|
||||
.replace(/=\r?\n/g, '')
|
||||
.replace(/=([0-9A-Fa-f]{2})/g, (_, hex) => String.fromCharCode(parseInt(hex, 16)));
|
||||
}
|
||||
|
||||
function decodePartBody(body: string, encoding: string): string {
|
||||
const enc = encoding.toLowerCase();
|
||||
if (enc === 'base64') return Buffer.from(body.replace(/\s/g, ''), 'base64').toString('utf-8');
|
||||
if (enc === 'quoted-printable') return decodeQuotedPrintable(body);
|
||||
return body;
|
||||
}
|
||||
|
||||
function extractBody(raw: string): { html: string | null; text: string | null } {
|
||||
const headerEnd = findHeaderEnd(raw);
|
||||
if (headerEnd === -1) return { html: null, text: null };
|
||||
|
||||
const topCtRaw = extractFullHeader(raw, 'Content-Type');
|
||||
const topCt = topCtRaw.toLowerCase();
|
||||
const topEncoding = extractFullHeader(raw, 'Content-Transfer-Encoding');
|
||||
|
||||
// Non-multipart: single body
|
||||
if (!topCt.includes('multipart')) {
|
||||
const body = raw.slice(headerEnd);
|
||||
const decoded = decodePartBody(body, topEncoding);
|
||||
if (topCt.includes('text/html')) return { html: decoded, text: null };
|
||||
return { html: null, text: decoded };
|
||||
}
|
||||
|
||||
// Multipart: extract boundary from the raw (case-sensitive) header
|
||||
const boundaryMatch = topCtRaw.match(/boundary=["']?([^"';\s]+)/i);
|
||||
if (!boundaryMatch) return { html: null, text: null };
|
||||
const boundary = boundaryMatch[1]!;
|
||||
|
||||
let html: string | null = null;
|
||||
let text: string | null = null;
|
||||
|
||||
const parts = raw.slice(headerEnd).split(`--${boundary}`);
|
||||
for (const part of parts) {
|
||||
if (part.startsWith('--') || !part.trim()) continue;
|
||||
const partHeaderEnd = findHeaderEnd(part);
|
||||
if (partHeaderEnd === -1) continue;
|
||||
|
||||
const partCt = extractFullHeader(part, 'Content-Type').toLowerCase();
|
||||
const partEnc = extractFullHeader(part, 'Content-Transfer-Encoding');
|
||||
const partBody = part.slice(partHeaderEnd);
|
||||
|
||||
// Recurse into nested multipart (e.g. multipart/alternative inside multipart/mixed)
|
||||
if (partCt.includes('multipart')) {
|
||||
const nested = extractBody(part.trim());
|
||||
if (nested.html && !html) html = nested.html;
|
||||
if (nested.text && !text) text = nested.text;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (partCt.includes('text/html') && !html) {
|
||||
html = decodePartBody(partBody, partEnc);
|
||||
} else if (partCt.includes('text/plain') && !text) {
|
||||
text = decodePartBody(partBody, partEnc);
|
||||
}
|
||||
}
|
||||
|
||||
return { html, text };
|
||||
}
|
||||
|
||||
type AttachmentMeta = { filename: string; size: number; contentType: string; content: string };
|
||||
|
||||
function parseAttachments(raw: string): AttachmentMeta[] {
|
||||
const results: AttachmentMeta[] = [];
|
||||
const regex = /^Content-Disposition:\s*attachment[^\n]*/gim;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = regex.exec(raw)) !== null) {
|
||||
const pos = match.index;
|
||||
|
||||
// Walk backwards to find the start of this MIME part's headers
|
||||
const partStart = raw.lastIndexOf('\n--', pos);
|
||||
const headerBlock = partStart !== -1 ? raw.slice(partStart, pos + 500) : raw.slice(Math.max(0, pos - 500), pos + 500);
|
||||
|
||||
// Extract filename from Content-Disposition or Content-Type
|
||||
const fnMatch = headerBlock.match(/filename\*?=(?:"([^"]+)"|([^\s;]+))/i);
|
||||
const filename = fnMatch ? (fnMatch[1] ?? fnMatch[2] ?? 'unknown') : 'unknown';
|
||||
|
||||
// Extract content-type
|
||||
const ctMatch = headerBlock.match(/^Content-Type:\s*([^\s;]+)/im);
|
||||
const contentType = ctMatch?.[1] ?? 'application/octet-stream';
|
||||
|
||||
// Extract full body content as base64
|
||||
const partHeaderEnd = findHeaderEnd(raw.slice(pos));
|
||||
let content = '';
|
||||
let size = 0;
|
||||
if (partHeaderEnd !== -1) {
|
||||
const bodyStart = pos + partHeaderEnd;
|
||||
const boundaryEnd = raw.indexOf('\n--', bodyStart);
|
||||
const bodyRaw = boundaryEnd !== -1 ? raw.slice(bodyStart, boundaryEnd) : raw.slice(bodyStart);
|
||||
|
||||
// Detect encoding from part headers
|
||||
const encMatch = headerBlock.match(/^Content-Transfer-Encoding:\s*(\S+)/im);
|
||||
const encoding = encMatch?.[1]?.toLowerCase() ?? 'base64';
|
||||
|
||||
if (encoding === 'base64') {
|
||||
content = bodyRaw.replace(/\s/g, '');
|
||||
} else {
|
||||
// For quoted-printable or 7bit/8bit, re-encode to base64
|
||||
const buf = encoding === 'quoted-printable'
|
||||
? Buffer.from(decodeQuotedPrintable(bodyRaw))
|
||||
: Buffer.from(bodyRaw);
|
||||
content = buf.toString('base64');
|
||||
}
|
||||
size = Math.floor(content.length * 3 / 4);
|
||||
}
|
||||
|
||||
results.push({ filename, size, contentType, content });
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
@@ -1,107 +1,67 @@
|
||||
import { mkdir, readdir } from 'node:fs/promises';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { simpleParser } from 'mailparser';
|
||||
import type { EmailSummary, EmailMessage } from 'types';
|
||||
import { rebuildIndex } from '@@/queue/handlers/gmail-sync';
|
||||
import type { EmailMessage } from 'types';
|
||||
import { createRouter } from '../../create-router';
|
||||
import { getUserEmailDir } from '@@/data-path';
|
||||
|
||||
const parseHeadersOnly = async (filePath: string, id: string): Promise<EmailSummary | null> => {
|
||||
try {
|
||||
const file = Bun.file(filePath);
|
||||
const buffer = Buffer.from(await file.arrayBuffer());
|
||||
const parsed = await simpleParser(buffer, { skipHtmlToText: true, skipTextToHtml: true, skipImageLinks: true });
|
||||
|
||||
const text = parsed.text ?? '';
|
||||
const snippet = text.slice(0, 120).replace(/\s+/g, ' ').trim();
|
||||
const attachmentCount = parsed.attachments?.length ?? 0;
|
||||
|
||||
return {
|
||||
id,
|
||||
from: parsed.from?.text ?? '',
|
||||
to: parsed.to ? (Array.isArray(parsed.to) ? parsed.to.map((a) => a.text).join(', ') : parsed.to.text) : '',
|
||||
subject: parsed.subject ?? '(no subject)',
|
||||
date: (parsed.date ?? new Date()).toISOString(),
|
||||
snippet,
|
||||
...(attachmentCount > 0 ? { attachmentCount } : {}),
|
||||
};
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
import { DATA_PATH } from '@@/data-path';
|
||||
import { openEmailDb, rowToSummary, getSyncMeta } from './email-db';
|
||||
|
||||
export const emailRouter = createRouter();
|
||||
|
||||
emailRouter.get('/messages', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
const dir = getUserEmailDir(email);
|
||||
|
||||
let filenames: string[];
|
||||
try {
|
||||
const entries = await readdir(dir);
|
||||
filenames = entries.filter((f) => f.endsWith('.eml'));
|
||||
} catch {
|
||||
return ctx.json({ messages: [], total: 0 });
|
||||
}
|
||||
|
||||
const page = Number(ctx.req.query('page') ?? '1');
|
||||
const limit = Number(ctx.req.query('limit') ?? '50');
|
||||
const start = (page - 1) * limit;
|
||||
const offset = (page - 1) * limit;
|
||||
|
||||
const indexFile = Bun.file(join(dir, 'index.json'));
|
||||
if (await indexFile.exists()) {
|
||||
try {
|
||||
const raw = await indexFile.json();
|
||||
const index = Array.isArray(raw) ? null : (raw as { v?: number; entries: EmailSummary[] });
|
||||
if (index?.v === 3 && index.entries.length === filenames.length) {
|
||||
return ctx.json({ messages: index.entries.slice(start, start + limit), total: index.entries.length });
|
||||
}
|
||||
} catch {
|
||||
/* index corrupted, fall through to rebuild */
|
||||
}
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const rows = db.query('SELECT * FROM emails WHERE deleted = 0 ORDER BY date DESC LIMIT ? OFFSET ?').all(limit, offset) as Record<string, unknown>[];
|
||||
const countRow = db.query('SELECT COUNT(*) as total FROM emails WHERE deleted = 0').get() as { total: number };
|
||||
const messages = rows.map(rowToSummary);
|
||||
return ctx.json({ messages, total: countRow.total });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
// Fallback: rebuild index from .eml files
|
||||
const summaries = rebuildIndex(dir);
|
||||
return ctx.json({ messages: summaries.slice(start, start + limit), total: summaries.length });
|
||||
});
|
||||
|
||||
emailRouter.get('/messages/:id', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
const id = ctx.req.param('id');
|
||||
const filePath = join(getUserEmailDir(email), `${id}.eml`);
|
||||
|
||||
const file = Bun.file(filePath);
|
||||
if (!(await file.exists())) {
|
||||
return ctx.text('Not found', 404);
|
||||
}
|
||||
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const buffer = Buffer.from(await file.arrayBuffer());
|
||||
const parsed = await simpleParser(buffer);
|
||||
const row = db.query('SELECT * FROM emails WHERE id = ? AND deleted = 0').get(id) as Record<string, unknown> | null;
|
||||
if (!row) return ctx.text('Not found', 404);
|
||||
|
||||
const attachments = (parsed.attachments ?? []).map((a) => ({
|
||||
filename: a.filename ?? 'unknown',
|
||||
size: a.size,
|
||||
contentType: a.contentType,
|
||||
}));
|
||||
const attachmentRows = db.query('SELECT * FROM attachments WHERE email_id = ? ORDER BY idx').all(id) as Array<Record<string, unknown>>;
|
||||
|
||||
const from = row.from_name
|
||||
? `${row.from_name} <${row.from_address}>`
|
||||
: (row.from_address as string);
|
||||
|
||||
const message: EmailMessage = {
|
||||
id,
|
||||
from: parsed.from?.text ?? '',
|
||||
to: parsed.to ? (Array.isArray(parsed.to) ? parsed.to.map((a) => a.text).join(', ') : parsed.to.text) : '',
|
||||
cc: parsed.cc ? (Array.isArray(parsed.cc) ? parsed.cc.map((a) => a.text).join(', ') : parsed.cc.text) : undefined,
|
||||
subject: parsed.subject ?? '(no subject)',
|
||||
date: (parsed.date ?? new Date()).toISOString(),
|
||||
snippet: (parsed.text ?? '').slice(0, 120).replace(/\s+/g, ' ').trim(),
|
||||
html: parsed.html || undefined,
|
||||
text: parsed.text || undefined,
|
||||
attachments,
|
||||
id: row.id as string,
|
||||
from,
|
||||
to: row.to_address as string,
|
||||
cc: (row.cc as string) ?? undefined,
|
||||
subject: row.subject as string,
|
||||
date: row.date as string,
|
||||
snippet: row.snippet as string,
|
||||
html: (row.html as string) ?? undefined,
|
||||
text: (row.text_body as string) ?? undefined,
|
||||
attachments: attachmentRows.map((a) => ({
|
||||
filename: a.filename as string,
|
||||
size: a.size as number,
|
||||
contentType: a.content_type as string,
|
||||
})),
|
||||
...(row.attachment_count ? { attachmentCount: row.attachment_count as number } : {}),
|
||||
...(row.read ? { read: true } : {}),
|
||||
...(row.labels ? { labels: (row.labels as string).split(',') } : {}),
|
||||
};
|
||||
|
||||
return ctx.json(message);
|
||||
} catch {
|
||||
return ctx.text('Failed to parse email', 500);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -109,33 +69,68 @@ emailRouter.post('/messages/:id/attachments/:index/extract', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
const id = ctx.req.param('id');
|
||||
const index = Number(ctx.req.param('index'));
|
||||
const filePath = join(getUserEmailDir(email), `${id}.eml`);
|
||||
|
||||
const file = Bun.file(filePath);
|
||||
if (!(await file.exists())) {
|
||||
return ctx.text('Not found', 404);
|
||||
}
|
||||
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const buffer = Buffer.from(await file.arrayBuffer());
|
||||
const parsed = await simpleParser(buffer);
|
||||
const attachment = parsed.attachments[index];
|
||||
if (!attachment) {
|
||||
return ctx.text('Attachment not found', 404);
|
||||
}
|
||||
const row = db.query('SELECT filename, content FROM attachments WHERE email_id = ? AND idx = ?').get(id, index) as { filename: string; content: string | null } | null;
|
||||
if (!row || !row.content) return ctx.text('Attachment not found', 404);
|
||||
|
||||
const fileName = attachment.filename ?? 'unknown';
|
||||
const attachDir = join(getUserEmailDir(email), 'attachments');
|
||||
const fileName = row.filename ?? 'unknown';
|
||||
const attachDir = join(DATA_PATH, email, 'Gmail', 'emails', 'attachments');
|
||||
const destPath = join(attachDir, fileName);
|
||||
|
||||
const destFile = Bun.file(destPath);
|
||||
if (!(await destFile.exists())) {
|
||||
await mkdir(attachDir, { recursive: true });
|
||||
await Bun.write(destPath, attachment.content);
|
||||
const binary = Buffer.from(row.content, 'base64');
|
||||
await Bun.write(destPath, binary);
|
||||
}
|
||||
|
||||
return ctx.json({ filePath: `Gmail/emails/attachments/${fileName}`, fileName, root: 'user-data' });
|
||||
} catch {
|
||||
return ctx.text('Failed to extract attachment', 500);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
emailRouter.delete('/messages/:id', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
const id = ctx.req.param('id');
|
||||
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const result = db.run('UPDATE emails SET deleted = 1 WHERE id = ? AND deleted = 0', [id]);
|
||||
if (result.changes === 0) return ctx.text('Not found', 404);
|
||||
return ctx.json({ ok: true });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
emailRouter.get('/sync-status', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const lastSyncAt = getSyncMeta(db, 'last_sync_at');
|
||||
return ctx.json({ lastSyncAt });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
emailRouter.get('/stats', async (ctx) => {
|
||||
const email = ctx.get('user').email;
|
||||
|
||||
const db = openEmailDb(email);
|
||||
try {
|
||||
const total = (db.query('SELECT COUNT(*) as count FROM emails WHERE deleted = 0').get() as { count: number }).count;
|
||||
const byDomain = db.query('SELECT from_domain, COUNT(*) as count FROM emails WHERE deleted = 0 GROUP BY from_domain ORDER BY count DESC LIMIT 20').all() as Array<{ from_domain: string; count: number }>;
|
||||
const bySender = db.query('SELECT from_address, from_name, COUNT(*) as count FROM emails WHERE deleted = 0 GROUP BY from_address ORDER BY count DESC LIMIT 20').all() as Array<{ from_address: string; from_name: string; count: number }>;
|
||||
|
||||
return ctx.json({ total, byDomain, bySender });
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -230,6 +230,7 @@ export async function spawnPi(
|
||||
'-e', `OFFICER_RESOURCES=${resourcesEnv}`,
|
||||
'-e', `OFFICER_GOOGLE_CONFIG_PATH=/officer/google-oauth.json`,
|
||||
'-e', `OFFICER_GOOGLE_TOKEN_PATH=/officer/user/integrations/google.json`,
|
||||
'-e', `OFFICER_EMAIL_DB=/officer/emails.db`,
|
||||
];
|
||||
for (const [key, value] of Object.entries(storedKeys)) {
|
||||
if (value?.trim()) envFlags.push('-e', `${key}=${value.trim()}`);
|
||||
@@ -280,7 +281,7 @@ export async function spawnPi(
|
||||
stdin: 'pipe',
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
env: { ...process.env, ...storedKeys, HOME: getHomeDir(email), OFFICER_USER_HOME: getHomeDir(email), OFFICER_USER_ROOT: join(DATA_PATH, email), PI_CODING_AGENT_DIR: PI_CONFIG_DIR, PI_TOOLS_DIRS: toolsDirs, PI_SEARXNG_URL: searxng.url, OFFICER_RESOURCES: buildResourcesEnv(), OFFICER_GOOGLE_CONFIG_PATH: getGoogleConfigPath(), OFFICER_GOOGLE_TOKEN_PATH: getGoogleTokenPath(email) },
|
||||
env: { ...process.env, ...storedKeys, HOME: getHomeDir(email), OFFICER_USER_HOME: getHomeDir(email), OFFICER_USER_ROOT: join(DATA_PATH, email), PI_CODING_AGENT_DIR: PI_CONFIG_DIR, PI_TOOLS_DIRS: toolsDirs, PI_SEARXNG_URL: searxng.url, OFFICER_RESOURCES: buildResourcesEnv(), OFFICER_GOOGLE_CONFIG_PATH: getGoogleConfigPath(), OFFICER_GOOGLE_TOKEN_PATH: getGoogleTokenPath(email), OFFICER_EMAIL_DB: join(DATA_PATH, email, 'emails.db') },
|
||||
});
|
||||
|
||||
logger.info('Spawned Pi locally', {
|
||||
@@ -403,7 +404,13 @@ function parsePiEvent(event: Record<string, unknown>, currentStreamBuffer: strin
|
||||
case 'tool_execution_end': {
|
||||
const toolCallId = (event.toolCallId as string) ?? '';
|
||||
const result = event.result;
|
||||
const isError = (event.isError as boolean) ?? false;
|
||||
let resultObj: Record<string, unknown> | null = null;
|
||||
if (typeof result === 'object' && result !== null) {
|
||||
resultObj = result as Record<string, unknown>;
|
||||
} else if (typeof result === 'string') {
|
||||
try { resultObj = JSON.parse(result); } catch { /* not JSON */ }
|
||||
}
|
||||
const isError = (event.isError as boolean) ?? (resultObj?.isError as boolean) ?? false;
|
||||
const output = result != null ? (typeof result === 'string' ? result : JSON.stringify(result)) : '';
|
||||
|
||||
return {
|
||||
|
||||
@@ -313,6 +313,7 @@ async function handleChat(
|
||||
}
|
||||
|
||||
// Set thinking level if provided
|
||||
console.log(`[pi] model: ${msg.model ?? 'default'}, thinking: ${msg.thinking ?? 'not set'}`);
|
||||
if (msg.thinking) {
|
||||
piBridge.setThinkingLevel(session.piProcess, msg.thinking);
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { join } from 'node:path';
|
||||
import { mkdir, copyFile } from 'node:fs/promises';
|
||||
import { mkdir, copyFile, chmod } from 'node:fs/promises';
|
||||
import { PI_CONFIG_DIR, getUserPiConfigDir } from '../../data-path';
|
||||
import { readApiKeys, readAccessPolicy, PROVIDERS } from './pi-mono';
|
||||
import { getUsers } from 'officerdb';
|
||||
@@ -132,7 +132,10 @@ export async function syncUserPiConfig(email: string): Promise<void> {
|
||||
});
|
||||
|
||||
const userDir = getUserPiConfigDir(email);
|
||||
await mkdir(userDir, { recursive: true });
|
||||
const sessionsDir = join(userDir, 'sessions');
|
||||
await mkdir(sessionsDir, { recursive: true });
|
||||
await chmod(userDir, 0o777).catch(() => {});
|
||||
await chmod(sessionsDir, 0o777).catch(() => {});
|
||||
await Bun.write(join(userDir, 'models.json'), JSON.stringify(filtered, null, 2));
|
||||
|
||||
// Copy settings.json from app-level Pi config
|
||||
|
||||
@@ -6,7 +6,7 @@ RUN apt-get update \
|
||||
sudo gosu locales \
|
||||
zip unzip tree btop net-tools tmux \
|
||||
procps psmisc lsof less file man-db \
|
||||
ripgrep fd-find jq htop \
|
||||
ripgrep fd-find jq htop sqlite3 \
|
||||
&& sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen \
|
||||
&& ln -sf /usr/bin/fdfind /usr/local/bin/fd \
|
||||
&& apt-get clean
|
||||
|
||||
@@ -56,5 +56,9 @@ fi
|
||||
mkdir -p /home/$USERNAME/.local/bin
|
||||
chown -R "$USER_UID:$USER_GID" /home/$USERNAME/.local
|
||||
|
||||
# Ensure Pi agent sessions directory exists and is writable
|
||||
mkdir -p /home/$USERNAME/.pi/agent/sessions
|
||||
chown -R "$USER_UID:$USER_GID" /home/$USERNAME/.pi
|
||||
|
||||
# Run sidecar as the user
|
||||
exec gosu "$USER_UID:$USER_GID" node /app/pty-sidecar.mjs
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { ServerWebSocket } from 'bun';
|
||||
import { existsSync, mkdirSync, statSync } from 'node:fs';
|
||||
import { existsSync, mkdirSync, statSync, writeFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@@ -173,6 +173,7 @@ const startDockerSidecar = (port: number, homeDir: string, userId: number, usern
|
||||
'-v', `${join(DATA_PATH, '.generated')}:/officer/generated:ro`,
|
||||
...googleMounts,
|
||||
'-v', `${join(DATA_PATH, email, 'integrations')}:/officer/user/integrations:ro`,
|
||||
'-v', `${join(DATA_PATH, email, 'emails.db')}:/officer/emails.db`,
|
||||
'-w', containerHome,
|
||||
tag,
|
||||
],
|
||||
@@ -242,6 +243,12 @@ export const ensureDockerContainer = async (email: string, userId: number, homeD
|
||||
mkdirSync(getUserToolsDir(email), { recursive: true });
|
||||
mkdirSync(join(DATA_PATH, email, 'integrations'), { recursive: true });
|
||||
|
||||
// Ensure emails.db exists as a file before mount (Docker creates a directory if missing)
|
||||
const emailsDbPath = join(DATA_PATH, email, 'emails.db');
|
||||
if (!existsSync(emailsDbPath)) {
|
||||
writeFileSync(emailsDbPath, '');
|
||||
}
|
||||
|
||||
const map = await loadContainerMap();
|
||||
const existing = map[email];
|
||||
|
||||
@@ -492,3 +499,28 @@ export const terminalWebsocket = {
|
||||
|
||||
drain() {},
|
||||
};
|
||||
|
||||
export const stopAllContainers = async () => {
|
||||
// Stop host sidecar
|
||||
if (hostSidecarProcess) {
|
||||
hostSidecarProcess.kill();
|
||||
await hostSidecarProcess.exited.catch(() => {});
|
||||
hostSidecarProcess = null;
|
||||
console.log('[terminal] host sidecar stopped');
|
||||
}
|
||||
|
||||
// Stop all Docker containers
|
||||
const map = await loadContainerMap();
|
||||
const entries = Object.entries(map);
|
||||
if (entries.length === 0) return;
|
||||
|
||||
const dockerPath = Bun.which('docker') ?? 'docker';
|
||||
for (const [email, info] of entries) {
|
||||
try {
|
||||
Bun.spawnSync({ cmd: [dockerPath, 'stop', '-t', '2', info.dockerId], stdout: 'ignore', stderr: 'ignore' });
|
||||
console.log(`[terminal] stopped container ${info.dockerId} (${email})`);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { homedir } from 'node:os';
|
||||
import type { EmailSummary } from 'types';
|
||||
import type { Database } from 'bun:sqlite';
|
||||
import type { JobHandler } from '../types';
|
||||
import { registerHandler } from '../handler-registry';
|
||||
import { DATA_PATH } from '../../data-path';
|
||||
import { openEmailDb, upsertFromRawEml, getSyncMeta, setSyncMeta, updateEmailLabels } from '../../api/email/email-db';
|
||||
|
||||
type GoogleCredentials = {
|
||||
accessToken: string;
|
||||
@@ -97,176 +97,58 @@ async function gmailGet(token: string, path: string, params?: Record<string, str
|
||||
return res.json();
|
||||
}
|
||||
|
||||
function slugify(text: string, maxLen = 60): string {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.slice(0, maxLen)
|
||||
.replace(/-+$/, '');
|
||||
// ── Pre-flight message count ──
|
||||
|
||||
async function countMessages(token: string, query?: string): Promise<number> {
|
||||
let count = 0;
|
||||
let pageToken: string | undefined;
|
||||
do {
|
||||
const params: Record<string, string> = { maxResults: '500' };
|
||||
if (query) params.q = query;
|
||||
if (pageToken) params.pageToken = pageToken;
|
||||
const list = (await gmailGet(token, '/messages', params)) as {
|
||||
messages?: Array<{ id: string }>;
|
||||
nextPageToken?: string;
|
||||
};
|
||||
count += list.messages?.length ?? 0;
|
||||
if (!list.messages?.length) break;
|
||||
pageToken = list.nextPageToken;
|
||||
} while (pageToken);
|
||||
return count;
|
||||
}
|
||||
|
||||
function buildEmlFilename(id: string, internalDate: string | undefined, rawEmail: string): string {
|
||||
const subjectMatch = rawEmail.match(/^Subject:\s*(.+)$/mi);
|
||||
const subject = subjectMatch?.[1]?.trim() || 'no-subject';
|
||||
const ts = parseInt(internalDate || '0');
|
||||
const d = new Date(ts);
|
||||
const dateStr =
|
||||
ts > 0
|
||||
? `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`
|
||||
: 'unknown-date';
|
||||
return `${dateStr}_${slugify(subject)}_${id}.eml`;
|
||||
}
|
||||
|
||||
function decodeMimeWords(text: string): string {
|
||||
return text.replace(/=\?([^?]+)\?(B|Q)\?([^?]+)\?=/gi, (_, _charset, encoding, encoded) => {
|
||||
try {
|
||||
if (encoding.toUpperCase() === 'B') {
|
||||
return Buffer.from(encoded, 'base64').toString('utf-8');
|
||||
}
|
||||
const bytes: number[] = [];
|
||||
for (let i = 0; i < encoded.length; i++) {
|
||||
if (encoded[i] === '_') {
|
||||
bytes.push(0x20);
|
||||
} else if (encoded[i] === '=' && i + 2 < encoded.length) {
|
||||
bytes.push(parseInt(encoded.slice(i + 1, i + 3), 16));
|
||||
i += 2;
|
||||
} else {
|
||||
bytes.push(encoded.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
return Buffer.from(bytes).toString('utf-8');
|
||||
} catch {
|
||||
return encoded;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function extractHeader(raw: string, name: string): string {
|
||||
const match = raw.match(new RegExp(`^${name}:\\s*(.+)$`, 'mi'));
|
||||
return match?.[1]?.trim() ? decodeMimeWords(match[1].trim()) : '';
|
||||
}
|
||||
|
||||
function findHeaderEnd(text: string): number {
|
||||
const crlf = text.indexOf('\r\n\r\n');
|
||||
const lf = text.indexOf('\n\n');
|
||||
if (crlf !== -1) return crlf + 4;
|
||||
if (lf !== -1) return lf + 2;
|
||||
return -1;
|
||||
}
|
||||
|
||||
function extractSnippet(raw: string): string {
|
||||
const idx = findHeaderEnd(raw);
|
||||
if (idx === -1) return '';
|
||||
|
||||
let body = raw.slice(idx);
|
||||
|
||||
// If multipart, skip boundary line + part headers to reach actual content
|
||||
if (body.trimStart().startsWith('--')) {
|
||||
const afterBoundary = body.slice(body.indexOf('\n') + 1);
|
||||
const partBodyStart = findHeaderEnd(afterBoundary);
|
||||
if (partBodyStart !== -1) body = afterBoundary.slice(partBodyStart);
|
||||
}
|
||||
|
||||
// Stop at next MIME boundary
|
||||
const nextBoundary = body.indexOf('\n--');
|
||||
if (nextBoundary !== -1) body = body.slice(0, nextBoundary);
|
||||
|
||||
return body.replace(/\s+/g, ' ').trim().slice(0, 120);
|
||||
}
|
||||
|
||||
function countAttachments(raw: string): number {
|
||||
const matches = raw.match(/^Content-Disposition:\s*attachment/gim);
|
||||
return matches?.length ?? 0;
|
||||
}
|
||||
|
||||
function parseEmlToSummary(raw: string, id: string): EmailSummary {
|
||||
const dateStr = extractHeader(raw, 'Date');
|
||||
const date = dateStr ? new Date(dateStr).toISOString() : new Date(0).toISOString();
|
||||
const attachmentCount = countAttachments(raw);
|
||||
return {
|
||||
id,
|
||||
from: extractHeader(raw, 'From'),
|
||||
to: extractHeader(raw, 'To'),
|
||||
subject: extractHeader(raw, 'Subject') || '(no subject)',
|
||||
date,
|
||||
snippet: extractSnippet(raw),
|
||||
...(attachmentCount > 0 ? { attachmentCount } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
type EmailIndex = { v: number; entries: EmailSummary[] };
|
||||
const INDEX_VERSION = 3;
|
||||
|
||||
export function rebuildIndex(emailsDir: string): EmailSummary[] {
|
||||
let filenames: string[];
|
||||
try {
|
||||
filenames = readdirSync(emailsDir).filter((f) => f.endsWith('.eml'));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
const indexPath = join(emailsDir, 'index.json');
|
||||
const existing = new Map<string, EmailSummary>();
|
||||
try {
|
||||
const raw = JSON.parse(readFileSync(indexPath, 'utf-8'));
|
||||
const index = (Array.isArray(raw) ? null : raw) as EmailIndex | null;
|
||||
if (index?.v === INDEX_VERSION) {
|
||||
for (const entry of index.entries) {
|
||||
existing.set(entry.id, entry);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* no existing index or corrupted */
|
||||
}
|
||||
|
||||
const entries: EmailSummary[] = [];
|
||||
for (const filename of filenames) {
|
||||
const id = filename.replace(/\.eml$/, '');
|
||||
const cached = existing.get(id);
|
||||
if (cached) {
|
||||
entries.push(cached);
|
||||
} else {
|
||||
try {
|
||||
const raw = readFileSync(join(emailsDir, filename), 'utf-8');
|
||||
entries.push(parseEmlToSummary(raw, id));
|
||||
} catch {
|
||||
/* skip unreadable files */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entries.sort((a, b) => new Date(b.date).getTime() - new Date(a.date).getTime());
|
||||
writeFileSync(indexPath, JSON.stringify({ v: INDEX_VERSION, entries }));
|
||||
return entries;
|
||||
}
|
||||
// ── Full sync: Gmail API → SQLite directly ──
|
||||
|
||||
type SyncProgress = { saved: number; skipped: number; errors: number; page: number };
|
||||
type OnProgress = (progress: SyncProgress) => void;
|
||||
|
||||
type SyncResult = {
|
||||
saved: number;
|
||||
skipped: number;
|
||||
errors: number;
|
||||
labelMap: Map<string, string[]>;
|
||||
maxHistoryId: string | null;
|
||||
};
|
||||
|
||||
async function syncInbox(
|
||||
token: string,
|
||||
outputDir: string,
|
||||
db: Database,
|
||||
emailAccount: string,
|
||||
query?: string,
|
||||
onProgress?: OnProgress,
|
||||
): Promise<{ saved: number; skipped: number; errors: number }> {
|
||||
mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
): Promise<SyncResult> {
|
||||
// Dedup via DB
|
||||
const existingIds = new Set<string>();
|
||||
try {
|
||||
for (const file of readdirSync(outputDir)) {
|
||||
const match = file.match(/_([a-f0-9]+)\.eml$/i);
|
||||
if (match) existingIds.add(match[1]!);
|
||||
}
|
||||
} catch {
|
||||
/* dir might not exist yet */
|
||||
}
|
||||
const rows = db.query('SELECT id FROM emails').all() as Array<{ id: string }>;
|
||||
for (const row of rows) existingIds.add(row.id);
|
||||
|
||||
let saved = 0;
|
||||
let skipped = 0;
|
||||
let errors = 0;
|
||||
let page = 0;
|
||||
let pageToken: string | undefined;
|
||||
const labelMap = new Map<string, string[]>();
|
||||
let maxHistoryId: bigint | null = null;
|
||||
|
||||
do {
|
||||
const params: Record<string, string> = { maxResults: '100' };
|
||||
@@ -294,12 +176,19 @@ async function syncInbox(
|
||||
id: string;
|
||||
internalDate?: string;
|
||||
raw: string;
|
||||
labelIds?: string[];
|
||||
historyId?: string;
|
||||
};
|
||||
const rawEmail = Buffer.from(msg.raw, 'base64url').toString('utf-8');
|
||||
const filename = buildEmlFilename(msg.id, msg.internalDate, rawEmail);
|
||||
writeFileSync(join(outputDir, filename), rawEmail);
|
||||
upsertFromRawEml({ db, id, raw: rawEmail, integration: 'gmail', emailAccount, labels: msg.labelIds });
|
||||
existingIds.add(id);
|
||||
saved++;
|
||||
|
||||
if (msg.labelIds) labelMap.set(id, msg.labelIds);
|
||||
if (msg.historyId) {
|
||||
const hid = BigInt(msg.historyId);
|
||||
if (maxHistoryId === null || hid > maxHistoryId) maxHistoryId = hid;
|
||||
}
|
||||
} catch {
|
||||
errors++;
|
||||
}
|
||||
@@ -314,7 +203,121 @@ async function syncInbox(
|
||||
pageToken = list.nextPageToken;
|
||||
} while (pageToken);
|
||||
|
||||
return { saved, skipped, errors };
|
||||
return { saved, skipped, errors, labelMap, maxHistoryId: maxHistoryId !== null ? String(maxHistoryId) : null };
|
||||
}
|
||||
|
||||
// ── Gmail History API (incremental sync) ──
|
||||
|
||||
type HistoryMessage = { id: string; labelIds?: string[] };
|
||||
|
||||
type HistoryRecord = {
|
||||
id: string;
|
||||
messagesAdded?: Array<{ message: HistoryMessage }>;
|
||||
messagesDeleted?: Array<{ message: HistoryMessage }>;
|
||||
labelsAdded?: Array<{ message: HistoryMessage; labelIds: string[] }>;
|
||||
labelsRemoved?: Array<{ message: HistoryMessage; labelIds: string[] }>;
|
||||
};
|
||||
|
||||
type HistoryResponse = {
|
||||
history?: HistoryRecord[];
|
||||
nextPageToken?: string;
|
||||
historyId: string;
|
||||
};
|
||||
|
||||
type IncrementalResult =
|
||||
| { stale: false; added: number; deleted: number; relabeled: number; maxHistoryId: string }
|
||||
| { stale: true };
|
||||
|
||||
async function syncIncremental(
|
||||
token: string,
|
||||
db: Database,
|
||||
lastHistoryId: string,
|
||||
emailAccount: string,
|
||||
): Promise<IncrementalResult> {
|
||||
let pageToken: string | undefined;
|
||||
let added = 0;
|
||||
let deleted = 0;
|
||||
let relabeled = 0;
|
||||
let latestHistoryId = lastHistoryId;
|
||||
|
||||
do {
|
||||
const url = new URL(`${GMAIL_BASE}/history`);
|
||||
url.searchParams.set('startHistoryId', lastHistoryId);
|
||||
for (const ht of ['messageAdded', 'messageDeleted', 'labelAdded', 'labelRemoved']) {
|
||||
url.searchParams.append('historyTypes', ht);
|
||||
}
|
||||
if (pageToken) url.searchParams.set('pageToken', pageToken);
|
||||
|
||||
let data: HistoryResponse;
|
||||
try {
|
||||
const res = await fetch(url.toString(), { headers: { Authorization: `Bearer ${token}` } });
|
||||
if (!res.ok) {
|
||||
const error = await res.text().catch(() => '');
|
||||
throw new Error(`Gmail API error (${res.status}): ${error}`);
|
||||
}
|
||||
data = (await res.json()) as HistoryResponse;
|
||||
} catch (err) {
|
||||
// historyId too old — Gmail returns 404
|
||||
if (err instanceof Error && err.message.includes('404')) {
|
||||
return { stale: true };
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
latestHistoryId = data.historyId;
|
||||
|
||||
for (const record of data.history ?? []) {
|
||||
// New messages
|
||||
for (const { message } of record.messagesAdded ?? []) {
|
||||
try {
|
||||
const msg = (await gmailGet(token, `/messages/${message.id}`, { format: 'raw' })) as {
|
||||
id: string;
|
||||
internalDate?: string;
|
||||
raw: string;
|
||||
labelIds?: string[];
|
||||
historyId?: string;
|
||||
};
|
||||
const rawEmail = Buffer.from(msg.raw, 'base64url').toString('utf-8');
|
||||
upsertFromRawEml({ db, id: msg.id, raw: rawEmail, integration: 'gmail', emailAccount, labels: msg.labelIds });
|
||||
added++;
|
||||
} catch {
|
||||
/* skip individual failures */
|
||||
}
|
||||
}
|
||||
|
||||
// Deleted messages
|
||||
for (const { message } of record.messagesDeleted ?? []) {
|
||||
db.run('UPDATE emails SET deleted = 1 WHERE id = ?', [message.id]);
|
||||
deleted++;
|
||||
}
|
||||
|
||||
// Labels added
|
||||
for (const { message, labelIds } of record.labelsAdded ?? []) {
|
||||
const row = db.query('SELECT labels FROM emails WHERE id = ?').get(message.id) as { labels: string | null } | null;
|
||||
if (row) {
|
||||
const existing = row.labels ? row.labels.split(',') : [];
|
||||
const merged = [...new Set([...existing, ...labelIds.map((l) => l.toLowerCase())])];
|
||||
updateEmailLabels(db, message.id, merged);
|
||||
relabeled++;
|
||||
}
|
||||
}
|
||||
|
||||
// Labels removed
|
||||
for (const { message, labelIds } of record.labelsRemoved ?? []) {
|
||||
const row = db.query('SELECT labels FROM emails WHERE id = ?').get(message.id) as { labels: string | null } | null;
|
||||
if (row) {
|
||||
const removeSet = new Set(labelIds.map((l) => l.toLowerCase()));
|
||||
const remaining = (row.labels ? row.labels.split(',') : []).filter((l) => !removeSet.has(l));
|
||||
updateEmailLabels(db, message.id, remaining);
|
||||
relabeled++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pageToken = data.nextPageToken;
|
||||
} while (pageToken);
|
||||
|
||||
return { stale: false, added, deleted, relabeled, maxHistoryId: latestHistoryId };
|
||||
}
|
||||
|
||||
const MONTH_NAMES = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'];
|
||||
@@ -341,55 +344,117 @@ const gmailSyncHandler: JobHandler = {
|
||||
run: async (ctx) => {
|
||||
const creds = await loadCredentials(ctx.job.userId);
|
||||
const token = await getValidAccessToken(creds);
|
||||
// Store token in shared meta for the next step
|
||||
ctx.meta.accessToken = token;
|
||||
ctx.meta.outputDir = join(DATA_PATH, ctx.job.userId, 'Gmail', 'emails');
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Sync emails',
|
||||
run: async (ctx) => {
|
||||
const token = ctx.meta.accessToken as string;
|
||||
const outputDir = ctx.meta.outputDir as string;
|
||||
const year = ctx.meta.year as number | undefined;
|
||||
|
||||
let totalSaved = 0;
|
||||
let totalSkipped = 0;
|
||||
let totalErrors = 0;
|
||||
|
||||
if (year) {
|
||||
// Year-scoped sync: month by month with progress
|
||||
const months = buildMonthRanges(year);
|
||||
for (let i = 0; i < months.length; i++) {
|
||||
const month = months[i]!;
|
||||
await ctx.updateProgress({ current: i, total: months.length, label: month.label });
|
||||
const query = `after:${month.after} before:${month.before}`;
|
||||
const { saved, skipped, errors } = await syncInbox(token, outputDir, query, (p) => {
|
||||
const label = `${month.label} — ${p.saved} saved`;
|
||||
ctx.updateProgress({ current: i, total: months.length, label });
|
||||
});
|
||||
totalSaved += saved;
|
||||
totalSkipped += skipped;
|
||||
totalErrors += errors;
|
||||
const db = openEmailDb(ctx.job.userId);
|
||||
try {
|
||||
// Try incremental sync first (only for non-year-scoped syncs)
|
||||
if (!year) {
|
||||
const lastHistoryId = getSyncMeta(db, 'last_history_id');
|
||||
if (lastHistoryId) {
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Incremental sync' });
|
||||
console.log(`[gmail-sync] Attempting incremental sync from historyId ${lastHistoryId}`);
|
||||
const result = await syncIncremental(token, db, lastHistoryId, ctx.job.userId);
|
||||
if (!result.stale) {
|
||||
setSyncMeta(db, 'last_history_id', result.maxHistoryId);
|
||||
setSyncMeta(db, 'last_sync_at', new Date().toISOString());
|
||||
console.log(`[gmail-sync] Incremental: +${result.added} added, -${result.deleted} deleted, ~${result.relabeled} relabeled`);
|
||||
await ctx.updateProgress({ current: 1, total: 1, label: 'Done (incremental)' });
|
||||
return;
|
||||
}
|
||||
console.log('[gmail-sync] historyId stale, falling back to full sync');
|
||||
}
|
||||
}
|
||||
await ctx.updateProgress({ current: months.length, total: months.length, label: 'Done' });
|
||||
} else {
|
||||
// Full sync: all emails with per-page progress
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Starting sync' });
|
||||
const { saved, skipped, errors } = await syncInbox(token, outputDir, undefined, (p) => {
|
||||
const label = `Saved ${p.saved}, skipped ${p.skipped} (page ${p.page})`;
|
||||
ctx.updateProgress({ current: p.saved + p.skipped + p.errors, total: 0, label });
|
||||
});
|
||||
totalSaved = saved;
|
||||
totalSkipped = skipped;
|
||||
totalErrors = errors;
|
||||
await ctx.updateProgress({ current: 1, total: 1, label: 'Done' });
|
||||
|
||||
// Full sync
|
||||
let totalSaved = 0;
|
||||
let totalSkipped = 0;
|
||||
let totalErrors = 0;
|
||||
let allLabelMaps: Map<string, string[]> = new Map();
|
||||
let maxHistoryId: string | null = null;
|
||||
|
||||
if (year) {
|
||||
// Year-scoped sync: count total emails first, then sync month by month
|
||||
const yearQuery = `after:${year}/1/1 before:${year + 1}/1/1`;
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Counting emails...' });
|
||||
const totalEmails = await countMessages(token, yearQuery);
|
||||
console.log(`[gmail-sync] Pre-flight: ${totalEmails} emails for ${year}`);
|
||||
|
||||
const months = buildMonthRanges(year);
|
||||
for (let i = 0; i < months.length; i++) {
|
||||
const month = months[i]!;
|
||||
await ctx.updateProgress({ current: totalSaved + totalSkipped, total: totalEmails, label: month.label });
|
||||
const query = `after:${month.after} before:${month.before}`;
|
||||
const result = await syncInbox(token, db, ctx.job.userId, query, (p) => {
|
||||
const current = totalSaved + p.saved + p.skipped + p.errors;
|
||||
const label = `${month.label} — Saved ${(totalSaved + p.saved).toLocaleString()} of ${totalEmails.toLocaleString()}`;
|
||||
ctx.updateProgress({ current, total: totalEmails, label });
|
||||
});
|
||||
totalSaved += result.saved;
|
||||
totalSkipped += result.skipped;
|
||||
totalErrors += result.errors;
|
||||
for (const [id, labels] of result.labelMap) allLabelMaps.set(id, labels);
|
||||
if (result.maxHistoryId) {
|
||||
if (!maxHistoryId || BigInt(result.maxHistoryId) > BigInt(maxHistoryId)) {
|
||||
maxHistoryId = result.maxHistoryId;
|
||||
}
|
||||
}
|
||||
}
|
||||
await ctx.updateProgress({ current: totalEmails, total: totalEmails, label: 'Done' });
|
||||
} else {
|
||||
// If we have a last_sync_date (e.g. from migration), scope the sync to only newer emails
|
||||
const lastSyncDate = getSyncMeta(db, 'last_sync_date');
|
||||
let syncQuery: string | undefined;
|
||||
if (lastSyncDate) {
|
||||
const d = new Date(lastSyncDate);
|
||||
syncQuery = `after:${d.getFullYear()}/${d.getMonth() + 1}/${d.getDate()}`;
|
||||
console.log(`[gmail-sync] Scoping full sync with query: ${syncQuery}`);
|
||||
}
|
||||
|
||||
// Count total emails for accurate progress
|
||||
await ctx.updateProgress({ current: 0, total: 0, label: 'Counting emails...' });
|
||||
const totalEmails = await countMessages(token, syncQuery);
|
||||
console.log(`[gmail-sync] Pre-flight: ${totalEmails} emails`);
|
||||
|
||||
await ctx.updateProgress({ current: 0, total: totalEmails, label: 'Starting sync' });
|
||||
const result = await syncInbox(token, db, ctx.job.userId, syncQuery, (p) => {
|
||||
const current = p.saved + p.skipped + p.errors;
|
||||
const label = `Saved ${p.saved.toLocaleString()} of ${totalEmails.toLocaleString()}`;
|
||||
ctx.updateProgress({ current, total: totalEmails, label });
|
||||
});
|
||||
totalSaved = result.saved;
|
||||
totalSkipped = result.skipped;
|
||||
totalErrors = result.errors;
|
||||
allLabelMaps = result.labelMap;
|
||||
maxHistoryId = result.maxHistoryId;
|
||||
await ctx.updateProgress({ current: totalEmails, total: totalEmails, label: 'Done' });
|
||||
}
|
||||
|
||||
console.log(`[gmail-sync] Full: saved ${totalSaved}, skipped ${totalSkipped}, errors ${totalErrors}`);
|
||||
|
||||
// Update labels for emails that were skipped but have new label data
|
||||
let labelsUpdated = 0;
|
||||
for (const [id, labels] of allLabelMaps) {
|
||||
updateEmailLabels(db, id, labels);
|
||||
labelsUpdated++;
|
||||
}
|
||||
if (labelsUpdated > 0) console.log(`[gmail-sync] Updated labels for ${labelsUpdated} emails`);
|
||||
|
||||
// Store sync state
|
||||
if (maxHistoryId) {
|
||||
setSyncMeta(db, 'last_history_id', maxHistoryId);
|
||||
}
|
||||
setSyncMeta(db, 'last_sync_at', new Date().toISOString());
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
console.log(`[gmail-sync] Saved ${totalSaved}, skipped ${totalSkipped}, errors ${totalErrors}`);
|
||||
|
||||
rebuildIndex(outputDir);
|
||||
console.log('[gmail-sync] Index rebuilt');
|
||||
},
|
||||
},
|
||||
],
|
||||
|
||||
@@ -78,6 +78,7 @@ export function useEmbeddableChat(params: UseEmbeddableChatParams) {
|
||||
if (prefix) prompt = `${prefix}${prompt}`;
|
||||
|
||||
const cwdForFirst = !sessionId ? cwd : undefined;
|
||||
const displayText = promptPrefix ? text : undefined;
|
||||
sendPrompt(
|
||||
prompt,
|
||||
!sessionId && ids.length > 0 ? ids : undefined,
|
||||
@@ -86,6 +87,7 @@ export function useEmbeddableChat(params: UseEmbeddableChatParams) {
|
||||
undefined,
|
||||
sandboxed,
|
||||
thinkingLevel,
|
||||
displayText,
|
||||
);
|
||||
|
||||
attachmentManager.clearAttachments();
|
||||
|
||||
@@ -267,13 +267,14 @@ export function usePiChat(initialSessionId?: string, initialModel?: string | nul
|
||||
groupSlug?: string | null,
|
||||
sandboxed?: boolean,
|
||||
thinking?: string | null,
|
||||
displayText?: string,
|
||||
) {
|
||||
// Mark session as started on first message
|
||||
if (!hasStarted) {
|
||||
setHasStarted(true);
|
||||
}
|
||||
|
||||
setMessages((prev) => [...prev, { role: 'user', text, ...(images?.length ? { images } : {}) }]);
|
||||
|
||||
setMessages((prev) => [...prev, { role: 'user', text: displayText ?? text, ...(images?.length ? { images } : {}) }]);
|
||||
setIsGenerating(true);
|
||||
streamingRef.current = '';
|
||||
setStreamingText('');
|
||||
|
||||
@@ -7,6 +7,8 @@ export type EmailSummary = {
|
||||
snippet: string;
|
||||
attachmentCount?: number;
|
||||
read?: boolean;
|
||||
fromDomain?: string;
|
||||
labels?: string[];
|
||||
};
|
||||
|
||||
export type EmailMessage = EmailSummary & {
|
||||
|
||||
Reference in New Issue
Block a user