jobs: queue scheduler + REST job API (phase 1c + 2)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/servers/api/tasks/pipeline-job-manager.ts

@@ -6,8 +6,11 @@ import {
   getPipelineJob,
   updatePipelineJob,
   getPipelineJobsForUser,
+  getOldestPendingJob,
   markInterruptedJobs,
+  getUserById,
 } from 'officerdb';
+import { toShellUsername } from '../../data-path';
 import { executePipeline } from './pipeline-executor';
 import type { OutMessage } from './pipeline-executor';
 import { executeScript, type ScriptEvent } from './execute-script';
@@ -79,24 +82,39 @@ type StartJobParams = {
   startAt?: number;
 };
 
-export async function startJob(params: StartJobParams): Promise<string> {
+export function runningCount(): number {
+  return liveJobs.size;
+}
+
+// Create a job. action 'start' runs it now; 'queue' runs it only if nothing is running, else it stays
+// 'pending' and gets promoted when the running job finishes. (Single user → one global queue.)
+export async function enqueueJob(params: StartJobParams, action: 'start' | 'queue'): Promise<{ jobId: string; status: 'running' | 'pending' }> {
   const jobId = randomUUID();
   const mode: JobMode = params.mode ?? 'pipeline';
-
-  // Create DB record
+  const run = action === 'start' || runningCount() === 0;
   await createPipelineJob({
     id: jobId,
     userId: params.userId,
     taskDirName: params.taskDirName,
     taskName: params.taskName,
     mode,
-    status: 'running',
+    status: run ? 'running' : 'pending',
     inputs: params.inputs,
     cwd: params.cwd ?? null,
     config: params.config,
-    startedAt: new Date(),
+    startedAt: run ? new Date() : null,
   });
+  if (run) launch(jobId, mode, params);
+  return { jobId, status: run ? 'running' : 'pending' };
+}
 
+// Back-compat: the pipeline WS caller expects startJob(params) -> jobId, running immediately.
+export async function startJob(params: StartJobParams): Promise<string> {
+  return (await enqueueJob(params, 'start')).jobId;
+}
+
+// Build live state + run the executor for a job whose DB row is already status 'running'.
+function launch(jobId: string, mode: JobMode, params: StartJobParams) {
   // Create live state
   const job: LiveJob = {
     jobId,
@@ -191,6 +209,7 @@ export async function startJob(params: StartJobParams): Promise<string> {
       completedAt: new Date(),
     }).catch((err) => console.error(`[pipeline-jobs] failed to finalize ${jobId}:`, err));
     liveJobs.delete(jobId);
+    void promoteNext();
   }).catch(async (err) => {
     clearInterval(flushInterval);
     const message = err instanceof Error ? err.message : String(err);
@@ -204,9 +223,36 @@ export async function startJob(params: StartJobParams): Promise<string> {
       completedAt: new Date(),
     }).catch((e) => console.error(`[pipeline-jobs] failed to finalize ${jobId}:`, e));
     liveJobs.delete(jobId);
+    void promoteNext();
   });
+}
 
-  return jobId;
+// When a job finishes (and nothing else is running), promote the oldest queued job. Also called on
+// startup to resume a backlog. Rebuilds the user context from userId since there's no request here.
+async function promoteNext(): Promise<void> {
+  if (runningCount() > 0) return;
+  const next = await getOldestPendingJob();
+  if (!next) return;
+  const user = await getUserById(next.userId);
+  if (!user) {
+    await updatePipelineJob(next.id, { status: 'failed', error: 'user not found', completedAt: new Date() }).catch(() => {});
+    return promoteNext();
+  }
+  await updatePipelineJob(next.id, { status: 'running', startedAt: new Date() });
+  const nextMode = (next.mode as JobMode) ?? 'pipeline';
+  launch(next.id, nextMode, {
+    userId: next.userId,
+    email: user.email,
+    username: toShellUsername(user.username ?? '', user.email),
+    role: user.role ?? '',
+    mode: nextMode,
+    sandboxed: (user.role ?? '') !== 'Super Admin',
+    taskDirName: next.taskDirName,
+    taskName: next.taskName,
+    inputs: next.inputs as Record<string, string>,
+    cwd: next.cwd ?? undefined,
+    config: next.config,
+  });
 }
 
 export function attachViewer(jobId: string, ws: ServerWebSocket<WSData>) {
@@ -271,6 +317,8 @@ export async function cleanupOnStartup() {
   if (count > 0) {
     console.log(`[pipeline-jobs] marked ${count} interrupted jobs on startup`);
   }
+  // Resume any queued backlog that survived the restart.
+  await promoteNext();
 }
 
 /** Extract minimal progress info from events for DB storage. Script events fall through to `prev`. */