officerdev/platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

email: the sidecar schedules its own syncs

Browse Source 
Stage 2, and the end of the inversion. The two sync handlers (1,093 lines) ran in the
platform's queue, which meant the sidecar reached back over its registration socket to ask
the platform to enqueue work, and the credentials travelled through Postgres job metadata to
get there. Option (A) from the plan: they run here now, and the Jobs screen is left to the
things it actually describes.

The handlers moved almost unedited. Their bodies were already a list of steps taking a
context, so sync-runner.ts synthesizes that context and runs them; what went away is the
JobHandler wrapper and the registration. `job.userId` is the OWNER'S EMAIL rather than a
numeric id — the queue's naming — and it resolves the mail store path, so it is called out
in the type. That is the same field whose absence made the mailbox read as empty two commits
ago; it is set from user.email and checked this time.

Deliberately not a queue: one run per account, no persistence, no retry. A failure is picked
up by the ten-minute cron like any other, and a sync interrupted by a restart resumes from
the stored cursor rather than the beginning. PermanentError survives as a local class — it
signalled "do not retry" to the queue and now just carries its message to the sync state.

accounts.ts asks the runner whether an account is syncing instead of scanning job rows, and
the queue-over-WS shim in index.ts is gone: enqueueViaWs, listJobsViaWs, the pending-response
map and the queue branch in the command handler. Nothing but a port crosses that socket now.

The three chat channels stop opening the mail store directly. They each carried their own
copy of count-rows / enqueue / poll / count-again, coupling three chat bridges to the mail
schema — and they enqueued `gmail-sync` unconditionally, the OAuth path, for an
app-password account that syncs over IMAP, so the command was already broken. One shared
helper calls a new POST /sync-now on the sidecar, which syncs and reports what arrived.

queue/handlers/ is now empty; both handlers there were email. The queue is untouched and
still serves the Jobs screen.

Not moved, and fine where they are: scripts/migrate-emails-to-sqlite.ts and
scripts/seed-imap-uids.ts are one-off maintenance scripts that open the store directly.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
pastilhas
2026-07-31 13:33:46 +00:00
co-authored by Claude Opus 5
parent 06cd6bdca3
commit e9e144962d
12 changed files with 236 additions and 298 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/servers/channels/discord/handler.ts
+5 -74
View File
@@ -6,7 +6,6 @@ import { chunkMessage } from './chunker';
import { listChatModels } from '@@/api/chat/list-models';
import { enqueueJob } from '../../queue/init';
import { readJob } from '@@/queue/storage';
import { openUserEmailDb } from '@@/sidecar/email/store';
import type { ModelInfo } from '@@/api/chat/types';
import { toShellUsername } from '@@/data-path';
@@ -36,6 +35,8 @@ async function getVisibleModels(): Promise<ModelInfo[]> {
});
}
import { runEmailSyncCommand } from '../email-sync-command';
type CommandContext = {
content: string;
channel: SendableChannel;
@@ -45,80 +46,10 @@ type CommandContext = {
};
async function handleEmailSync(ctx: CommandContext): Promise<void> {
const { channel, email, userId } = ctx;
// Count emails before sync
let countBefore = 0;
try {
const db = await openUserEmailDb(email, userId);
if (db) {
const row = db.query('SELECT COUNT(*) as count FROM emails WHERE deleted = 0').get() as { count: number };
countBefore = row.count;
db.close();
}
} catch {
// DB might not exist yet
}
const { channel, userId } = ctx;
await channel.send('Syncing emails...');
const job = await enqueueJob({ lane: 'google-api', type: 'gmail-sync', userId: email });
// Poll until done
const poll = async (): Promise<'completed' | 'failed' | 'cancelled'> => {
for (let i = 0; i < 120; i++) {
await new Promise((r) => setTimeout(r, 3000));
const current = await readJob(job.id);
if (!current) return 'failed';
if (current.status === 'completed' || current.status === 'failed' || current.status === 'cancelled') {
return current.status;
}
}
return 'failed';
};
const status = await poll();
if (status !== 'completed') {
await channel.send('Email sync failed. Check the queue dashboard for details.');
return;
}
// Count emails after sync and get newest ones
try {
const db = await openUserEmailDb(email, userId);
if (!db) return;
const row = db.query('SELECT COUNT(*) as count FROM emails WHERE deleted = 0').get() as { count: number };
const countAfter = row.count;
const newCount = countAfter - countBefore;
if (newCount <= 0) {
db.close();
await channel.send('Sync complete — no new emails.');
return;
}
const newest = db
.query('SELECT from_name, from_address, subject FROM emails WHERE deleted = 0 ORDER BY date DESC LIMIT ?')
.all(Math.min(newCount, 20)) as Array<{ from_name: string | null; from_address: string; subject: string }>;
db.close();
const lines = newest.map((e) => {
const sender = e.from_name || e.from_address;
return `- **${sender}**: ${e.subject}`;
});
let text = `Sync complete — **${newCount}** new email${newCount !== 1 ? 's' : ''}`;
if (newCount > 20) text += ` (showing latest 20)`;
text += `:\n\n${lines.join('\n')}`;
const chunks = chunkMessage(text);
for (const chunk of chunks) {
await channel.send(chunk);
}
} catch (err) {
await channel.send(`Sync complete but failed to read results: ${err instanceof Error ? err.message : String(err)}`);
}
const text = await runEmailSyncCommand(userId);
for (const chunk of chunkMessage(text)) await channel.send(chunk);
}
async function handleCommand(ctx: CommandContext): Promise<boolean> {
src/servers/channels/email-sync-command.ts
+45
View File
@@ -0,0 +1,45 @@
import { getEmailServerUrl } from '../api/email/router';
// The "sync my email" chat command, once, for all three channels.
//
// Telegram, Discord and WhatsApp each carried their own copy: open the mail store directly, count rows,
// enqueue a `gmail-sync` job, poll it, count again, diff. That coupled three chat bridges to the mail
// schema, and the job type was hardcoded to the OAuth path even though the account syncs over IMAP — so
// the command was already broken before sync moved into the sidecar. Now it is one HTTP call to the
// sidecar, which does the sync and reports what arrived.
type SyncNowResponse = {
saved: number;
skipped?: number;
errors?: number;
newest: Array<{ from_name: string | null; from_address: string; subject: string }>;
error?: string;
};
/** Runs the sync and returns the message to send back to the user. */
export async function runEmailSyncCommand(userId: number): Promise<string> {
const base = getEmailServerUrl();
if (!base) return 'Email is not available right now — the mail service is starting up.';
let res: Response;
try {
res = await fetch(`${base}/sync-now`, {
method: 'POST',
// Loopback-only, same trust as the platform's own proxy.
headers: { 'X-Officer-User': String(userId) },
});
} catch {
return 'Email sync failed — the mail service is unreachable.';
}
if (!res.ok) return `Email sync failed (${res.status}).`;
const body = (await res.json()) as SyncNowResponse;
if (body.error) return body.error;
if (body.saved <= 0) return 'Sync complete — no new emails.';
const lines = body.newest.map((e) => `- *${e.from_name || e.from_address}*: ${e.subject}`);
let text = `Sync complete — *${body.saved}* new email${body.saved !== 1 ? 's' : ''}`;
if (body.saved > 20) text += ' (showing latest 20)';
return `${text}:\n\n${lines.join('\n')}`;
}
src/servers/channels/telegram/handler.ts
+5 -71
View File
@@ -7,7 +7,6 @@ import { getTelegramBot } from './bot';
import { listChatModels } from '@@/api/chat/list-models';
import { enqueueJob } from '../../queue/init';
import { readJob } from '@@/queue/storage';
import { openUserEmailDb } from '@@/sidecar/email/store';
import type { ModelInfo } from '@@/api/chat/types';
import { toShellUsername } from '@@/data-path';
@@ -37,6 +36,8 @@ async function getVisibleModels(): Promise<ModelInfo[]> {
});
}
import { runEmailSyncCommand } from '../email-sync-command';
type CommandContext = {
content: string;
send: SendFn;
@@ -46,77 +47,10 @@ type CommandContext = {
};
async function handleEmailSync(ctx: CommandContext): Promise<void> {
const { send, email, userId } = ctx;
let countBefore = 0;
try {
const db = await openUserEmailDb(email, userId);
if (db) {
const row = db.query('SELECT COUNT(*) as count FROM emails WHERE deleted = 0').get() as { count: number };
countBefore = row.count;
db.close();
}
} catch {
// DB might not exist yet
}
const { send, userId } = ctx;
await send('Syncing emails...');
const job = await enqueueJob({ lane: 'google-api', type: 'gmail-sync', userId: email });
const poll = async (): Promise<'completed' | 'failed' | 'cancelled'> => {
for (let i = 0; i < 120; i++) {
await new Promise((r) => setTimeout(r, 3000));
const current = await readJob(job.id);
if (!current) return 'failed';
if (current.status === 'completed' || current.status === 'failed' || current.status === 'cancelled') {
return current.status;
}
}
return 'failed';
};
const status = await poll();
if (status !== 'completed') {
await send('Email sync failed. Check the queue dashboard for details.');
return;
}
try {
const db = await openUserEmailDb(email, userId);
if (!db) return;
const row = db.query('SELECT COUNT(*) as count FROM emails WHERE deleted = 0').get() as { count: number };
const countAfter = row.count;
const newCount = countAfter - countBefore;
if (newCount <= 0) {
db.close();
await send('Sync complete — no new emails.');
return;
}
const newest = db
.query('SELECT from_name, from_address, subject FROM emails WHERE deleted = 0 ORDER BY date DESC LIMIT ?')
.all(Math.min(newCount, 20)) as Array<{ from_name: string | null; from_address: string; subject: string }>;
db.close();
const lines = newest.map((e) => {
const sender = e.from_name || e.from_address;
return `- *${sender}*: ${e.subject}`;
});
let text = `Sync complete — *${newCount}* new email${newCount !== 1 ? 's' : ''}`;
if (newCount > 20) text += ` (showing latest 20)`;
text += `:\n\n${lines.join('\n')}`;
const chunks = chunkMessage(text);
for (const chunk of chunks) {
await send(chunk);
}
} catch (err) {
await send(`Sync complete but failed to read results: ${err instanceof Error ? err.message : String(err)}`);
}
const text = await runEmailSyncCommand(userId);
for (const chunk of chunkMessage(text)) await send(chunk);
}
async function handleCommand(ctx: CommandContext): Promise<boolean> {
src/servers/channels/whatsapp/handler.ts
+5 -68
View File
@@ -6,7 +6,6 @@ import { getWhatsAppClient } from './bot';
import { listChatModels } from '@@/api/chat/list-models';
import { enqueueJob } from '../../queue/init';
import { readJob } from '@@/queue/storage';
import { openUserEmailDb } from '@@/sidecar/email/store';
import type { ModelInfo } from '@@/api/chat/types';
import { toShellUsername } from '@@/data-path';
@@ -41,6 +40,8 @@ async function getVisibleModels(): Promise<ModelInfo[]> {
});
}
import { runEmailSyncCommand } from '../email-sync-command';
type CommandContext = {
content: string;
send: SendFn;
@@ -50,74 +51,10 @@ type CommandContext = {
};
async function handleEmailSync(ctx: CommandContext): Promise<void> {
const { send, email, userId } = ctx;
let countBefore = 0;
try {
const db = await openUserEmailDb(email, userId);
if (db) {
const row = db.query('SELECT COUNT(*) as count FROM emails WHERE deleted = 0').get() as { count: number };
countBefore = row.count;
db.close();
}
} catch {
// DB might not exist yet
}
const { send, userId } = ctx;
await send('Syncing emails...');
const job = await enqueueJob({ lane: 'google-api', type: 'gmail-sync', userId: email });
const poll = async (): Promise<'completed' | 'failed' | 'cancelled'> => {
for (let i = 0; i < 120; i++) {
await new Promise((r) => setTimeout(r, 3000));
const current = await readJob(job.id);
if (!current) return 'failed';
if (current.status === 'completed' || current.status === 'failed' || current.status === 'cancelled') {
return current.status;
}
}
return 'failed';
};
const status = await poll();
if (status !== 'completed') {
await send('Email sync failed. Check the queue dashboard for details.');
return;
}
try {
const db = await openUserEmailDb(email, userId);
if (!db) return;
const row = db.query('SELECT COUNT(*) as count FROM emails WHERE deleted = 0').get() as { count: number };
const countAfter = row.count;
const newCount = countAfter - countBefore;
if (newCount <= 0) {
db.close();
await send('Sync complete — no new emails.');
return;
}
const newest = db
.query('SELECT from_name, from_address, subject FROM emails WHERE deleted = 0 ORDER BY date DESC LIMIT ?')
.all(Math.min(newCount, 20)) as Array<{ from_name: string | null; from_address: string; subject: string }>;
db.close();
const lines = newest.map((e) => {
const sender = e.from_name || e.from_address;
return `- *${sender}*: ${e.subject}`;
});
let text = `Sync complete — *${newCount}* new email${newCount !== 1 ? 's' : ''}`;
if (newCount > 20) text += ` (showing latest 20)`;
text += `:\n\n${lines.join('\n')}`;
await send(text);
} catch (err) {
await send(`Sync complete but failed to read results: ${err instanceof Error ? err.message : String(err)}`);
}
const text = await runEmailSyncCommand(userId);
await send(text); // WhatsApp's 65k limit means no chunking is needed
}
async function handleCommand(ctx: CommandContext): Promise<boolean> {