task logs: migrate from filesystem to postgresql; refactor sidecars into submodules

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/servers/api/email/accounts.ts

@@ -9,7 +9,7 @@ import {
   updateEmailAccountStatus,
 } from 'officerdb';
 import { validateImapConnection } from './imap-validate';
-import * as sidecar from '../../sidecar-registry';
+import { enqueueJob, listAllJobs } from '../../queue/init';
 
 type CreateAccountBody = {
   provider: string;
@@ -44,7 +44,7 @@ accountsRouter.get('/', async (ctx) => {
 
   if (hasActiveAccounts) {
     try {
-      const jobs = await sidecar.listJobs();
+      const jobs = await listAllJobs();
       activeJobAccountIds = new Set(
         jobs
           .filter((j) => j.type === 'email-sync' && (j.status === 'queued' || j.status === 'running'))
@@ -148,7 +148,7 @@ accountsRouter.post('/:id/sync', async (ctx) => {
   // Set status immediately so the UI reflects the queued state
   await updateEmailAccountStatus(id, 'queued');
 
-  const job = await sidecar.enqueueJob({
+  const job = await enqueueJob({
     lane: 'email',
     type: 'email-sync',
     userId: user.email,

src/servers/api/queue/queue.ts

@@ -1,5 +1,5 @@
 import { createRouter } from '../../create-router';
-import * as sidecar from '../../sidecar-registry';
+import { enqueueJob, cancelJob, listAllJobs, readJob } from '../../queue/init';
 import { NOT_FOUND } from '../../custom-errors';
 
 export const queueRouter = createRouter();
@@ -10,7 +10,7 @@ queueRouter.get('/jobs', async (ctx) => {
   const type = ctx.req.query('type');
   const status = ctx.req.query('status');
 
-  let jobs = await sidecar.listJobs();
+  let jobs = await listAllJobs();
   jobs = jobs.filter((j) => j.userId === user.email);
 
   if (lane) jobs = jobs.filter((j) => j.lane === lane);
@@ -21,7 +21,7 @@ queueRouter.get('/jobs', async (ctx) => {
 });
 
 queueRouter.get('/jobs/:id', async (ctx) => {
-  const job = await sidecar.getJob(ctx.req.param('id'));
+  const job = await readJob(ctx.req.param('id'));
   if (!job) throw NOT_FOUND('Job not found');
   return ctx.json(job);
 });
@@ -36,12 +36,12 @@ queueRouter.post('/jobs', async (ctx) => {
     notify?: boolean;
   };
 
-  const job = await sidecar.enqueueJob({ lane, type, userId: user.email, meta, notify });
+  const job = await enqueueJob({ lane, type, userId: user.email, meta, notify });
   return ctx.json(job, 201);
 });
 
 queueRouter.delete('/jobs/:id', async (ctx) => {
-  const job = await sidecar.cancelJob(ctx.req.param('id'));
+  const job = await cancelJob(ctx.req.param('id'));
   if (!job) throw NOT_FOUND('Job not found');
   return ctx.json(job);
 });

src/servers/api/task-logger.ts

@@ -1,6 +1,4 @@
-import { mkdir } from 'node:fs/promises';
-import { join } from 'node:path';
-import { getTaskLogsDir } from '@@/data-path';
+import { db, schema } from 'officerdb';
 import type { TaskInfo } from '@@/api/chat-types';
 
 type ChatMessage =
@@ -18,46 +16,41 @@ type ChatMessage =
   | { role: 'error'; text: string };
 
 type TaskLog = {
+  userId: number;
   taskName: string;
   taskDirName: string;
   entryName: string;
   entryType: 'file' | 'directory';
   provider: string;
   model: string;
-  startedAt: string;
-  completedAt: string | null;
+  startedAt: Date;
   messages: ChatMessage[];
 };
 
 type LogEntry = {
-  email: string;
-  filePath: string;
+  userId: number;
   log: TaskLog;
 };
 
 const activeLogs = new Map<string, LogEntry>();
 let logCounter = 0;
 
-export function createTaskLog(email: string, taskInfo: TaskInfo, provider: string, model: string): string {
+export function createTaskLog(userId: number, taskInfo: TaskInfo, provider: string, model: string): string {
   const logId = `log_${Date.now()}_${++logCounter}`;
-  const dir = getTaskLogsDir(email);
-  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
-  const filename = `${timestamp}-${taskInfo.taskDirName}.json`;
-  const filePath = join(dir, filename);
 
   const log: TaskLog = {
+    userId,
     taskName: taskInfo.taskName,
     taskDirName: taskInfo.taskDirName,
     entryName: taskInfo.entryName,
     entryType: taskInfo.entryType,
     provider,
     model,
-    startedAt: new Date().toISOString(),
-    completedAt: null,
+    startedAt: new Date(),
     messages: [],
   };
 
-  activeLogs.set(logId, { email, filePath, log });
+  activeLogs.set(logId, { userId, log });
   return logId;
 }
 
@@ -82,11 +75,24 @@ export async function finalizeLog(logId: string) {
   const entry = activeLogs.get(logId);
   if (!entry) return;
 
-  entry.log.completedAt = new Date().toISOString();
+  const { log } = entry;
+  const lastMessage = log.messages[log.messages.length - 1];
+  const isError = lastMessage?.role === 'result' ? lastMessage.isError : lastMessage?.role === 'error';
 
   try {
-    await mkdir(join(entry.filePath, '..'), { recursive: true });
-    await Bun.write(entry.filePath, JSON.stringify(entry.log, null, 2));
+    await db.insert(schema.taskLogs).values({
+      userId: log.userId,
+      taskName: log.taskName,
+      taskDirName: log.taskDirName,
+      entryName: log.entryName,
+      entryType: log.entryType,
+      provider: log.provider,
+      model: log.model,
+      isError: !!isError,
+      messages: log.messages,
+      startedAt: log.startedAt,
+      completedAt: new Date(),
+    });
   } catch (err) {
     console.error('[task-logger] Failed to write log:', err);
   }

src/servers/api/task-logs/task-logs.ts

@@ -1,79 +1,47 @@
-import { readdir } from 'node:fs/promises';
-import { join } from 'node:path';
+import { desc, eq } from 'drizzle-orm';
+import { db, schema } from 'officerdb';
 import { createRouter } from '../../create-router';
-import { getTaskLogsDir } from '../../data-path';
-
-type LogMetadata = {
-  filename: string;
-  taskName: string;
-  taskDirName: string;
-  entryName: string;
-  entryType: 'file' | 'directory';
-  provider: string;
-  model: string;
-  startedAt: string;
-  completedAt: string | null;
-  isError: boolean;
-};
 
 export const taskLogsRouter = createRouter();
 
-// GET / — list all log files (metadata only, no messages)
+// GET / — list all logs (metadata only, no messages)
 taskLogsRouter.get('/', async (ctx) => {
-  const email = ctx.get('user').email;
-  const dir = getTaskLogsDir(email);
+  const userId = ctx.get('user').id;
 
-  let files: string[];
-  try {
-    files = (await readdir(dir)).filter((f) => f.endsWith('.json'));
-  } catch {
-    return ctx.json([]);
-  }
-
-  // Sort by filename descending (newest first since filenames start with timestamp)
-  files.sort((a, b) => b.localeCompare(a));
-
-  const logs: LogMetadata[] = [];
-  for (const filename of files) {
-    try {
-      const raw = await Bun.file(join(dir, filename)).json();
-      const lastMessage = Array.isArray(raw.messages) ? raw.messages[raw.messages.length - 1] : null;
-      const isError = lastMessage?.role === 'result' ? lastMessage.isError : lastMessage?.role === 'error';
-      logs.push({
-        filename,
-        taskName: raw.taskName ?? '',
-        taskDirName: raw.taskDirName ?? '',
-        entryName: raw.entryName ?? '',
-        entryType: raw.entryType ?? 'file',
-        provider: raw.provider ?? '',
-        model: raw.model ?? '',
-        startedAt: raw.startedAt ?? '',
-        completedAt: raw.completedAt ?? null,
-        isError: !!isError,
-      });
-    } catch {
-      // Skip unreadable files
-    }
-  }
+  const logs = await db
+    .select({
+      id: schema.taskLogs.id,
+      taskName: schema.taskLogs.taskName,
+      taskDirName: schema.taskLogs.taskDirName,
+      entryName: schema.taskLogs.entryName,
+      entryType: schema.taskLogs.entryType,
+      provider: schema.taskLogs.provider,
+      model: schema.taskLogs.model,
+      isError: schema.taskLogs.isError,
+      startedAt: schema.taskLogs.startedAt,
+      completedAt: schema.taskLogs.completedAt,
+    })
+    .from(schema.taskLogs)
+    .where(eq(schema.taskLogs.userId, userId))
+    .orderBy(desc(schema.taskLogs.startedAt));
 
   return ctx.json(logs);
 });
 
-// GET /:filename — return full log file content
-taskLogsRouter.get('/:filename', async (ctx) => {
-  const email = ctx.get('user').email;
-  const filename = ctx.req.param('filename');
+// GET /:id — return full log with messages
+taskLogsRouter.get('/:id', async (ctx) => {
+  const userId = ctx.get('user').id;
+  const id = Number(ctx.req.param('id'));
 
-  if (!filename.endsWith('.json') || filename.includes('/') || filename.includes('..')) {
-    return ctx.text('Invalid filename', 400);
+  if (Number.isNaN(id)) {
+    return ctx.text('Invalid id', 400);
   }
 
-  const filePath = join(getTaskLogsDir(email), filename);
+  const [log] = await db.select().from(schema.taskLogs).where(eq(schema.taskLogs.id, id));
 
-  try {
-    const data = await Bun.file(filePath).json();
-    return ctx.json(data);
-  } catch {
+  if (!log || log.userId !== userId) {
     return ctx.text('Not found', 404);
   }
+
+  return ctx.json(log);
 });