add pipeline job management with /jobs pages and per-step output viewer
- Pipeline jobs now persist to DB with progress tracking and cost accumulation - Jobs survive WebSocket disconnects with in-memory event buffer replay - New /jobs list page with search, status badges, and cost display - New /jobs/:id detail page with live WebSocket attachment and REST fallback - Two-column layout using WorkspaceLayout for resizable steps/output panels - Streaming messages tagged with stepIndex/iterationLabel for per-step output grouping - TaskRunnerModal links to job detail page once job is created - Dock entry added for Jobs page Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
@@ -0,0 +1,293 @@
|
||||
import type { ServerWebSocket } from 'bun';
|
||||
import { randomUUID } from 'crypto';
|
||||
import { EventEmitter } from 'events';
|
||||
import {
|
||||
createPipelineJob,
|
||||
getPipelineJob,
|
||||
updatePipelineJob,
|
||||
getPipelineJobsForUser,
|
||||
markInterruptedJobs,
|
||||
} from 'officerdb';
|
||||
import { executePipeline } from './pipeline-executor';
|
||||
import type { OutMessage } from './pipeline-executor';
|
||||
|
||||
type WSData = {
|
||||
userId: number;
|
||||
email: string;
|
||||
username: string;
|
||||
role: string;
|
||||
sandboxed: boolean;
|
||||
};
|
||||
|
||||
type LiveJob = {
|
||||
jobId: string;
|
||||
userId: number;
|
||||
abortSignal: { aborted: boolean };
|
||||
emitter: EventEmitter;
|
||||
eventBuffer: OutMessage[];
|
||||
viewers: Set<ServerWebSocket<WSData>>;
|
||||
progressDirty: boolean;
|
||||
lastProgress: unknown;
|
||||
lastCost: unknown;
|
||||
};
|
||||
|
||||
const EVENT_BUFFER_SIZE = 500;
|
||||
const PROGRESS_FLUSH_MS = 3000;
|
||||
|
||||
const liveJobs = new Map<string, LiveJob>();
|
||||
|
||||
function sendToViewer(ws: ServerWebSocket<WSData>, jobId: string, event: OutMessage) {
|
||||
if (ws.readyState === 1) {
|
||||
ws.send(JSON.stringify({ jobId, ...event }));
|
||||
}
|
||||
}
|
||||
|
||||
function broadcast(job: LiveJob, event: OutMessage) {
|
||||
// Buffer for replay — skip high-frequency deltas (only useful for live streaming)
|
||||
if (event.type !== 'assistant:delta') {
|
||||
job.eventBuffer.push(event);
|
||||
if (job.eventBuffer.length > EVENT_BUFFER_SIZE) {
|
||||
job.eventBuffer.shift();
|
||||
}
|
||||
}
|
||||
|
||||
// Forward to all viewers
|
||||
for (const ws of job.viewers) {
|
||||
sendToViewer(ws, job.jobId, event);
|
||||
}
|
||||
}
|
||||
|
||||
type StartJobParams = {
|
||||
userId: number;
|
||||
email: string;
|
||||
username: string;
|
||||
role: string;
|
||||
taskDirName: string;
|
||||
taskName: string;
|
||||
inputs: Record<string, string>;
|
||||
cwd?: string;
|
||||
config: unknown;
|
||||
};
|
||||
|
||||
export async function startJob(params: StartJobParams): Promise<string> {
|
||||
const jobId = randomUUID();
|
||||
|
||||
// Create DB record
|
||||
await createPipelineJob({
|
||||
id: jobId,
|
||||
userId: params.userId,
|
||||
taskDirName: params.taskDirName,
|
||||
taskName: params.taskName,
|
||||
status: 'running',
|
||||
inputs: params.inputs,
|
||||
cwd: params.cwd ?? null,
|
||||
config: params.config,
|
||||
startedAt: new Date(),
|
||||
});
|
||||
|
||||
// Create live state
|
||||
const job: LiveJob = {
|
||||
jobId,
|
||||
userId: params.userId,
|
||||
abortSignal: { aborted: false },
|
||||
emitter: new EventEmitter(),
|
||||
eventBuffer: [],
|
||||
viewers: new Set(),
|
||||
progressDirty: false,
|
||||
lastProgress: null,
|
||||
lastCost: null,
|
||||
};
|
||||
liveJobs.set(jobId, job);
|
||||
|
||||
// Emit callback — broadcasts to viewers and tracks progress
|
||||
const emit = (event: OutMessage) => {
|
||||
broadcast(job, event);
|
||||
|
||||
// Track progress for DB persistence (skip streaming messages — only structural events)
|
||||
if (event.type === 'pipeline:init' || event.type.startsWith('step:') || event.type.startsWith('iteration:')) {
|
||||
job.progressDirty = true;
|
||||
job.lastProgress = extractProgress(event, job.lastProgress);
|
||||
}
|
||||
if (event.type === 'step:complete' || event.type === 'iteration:complete') {
|
||||
const cost = 'cost' in event ? event.cost : undefined;
|
||||
if (cost) {
|
||||
const prev = (job.lastCost as { inputTokens: number; outputTokens: number; totalUSD: number } | null) ?? { inputTokens: 0, outputTokens: 0, totalUSD: 0 };
|
||||
job.lastCost = {
|
||||
inputTokens: prev.inputTokens + cost.inputTokens,
|
||||
outputTokens: prev.outputTokens + cost.outputTokens,
|
||||
totalUSD: prev.totalUSD + cost.totalUSD,
|
||||
};
|
||||
job.progressDirty = true;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Periodic progress flush to DB
|
||||
const flushInterval = setInterval(async () => {
|
||||
if (job.progressDirty) {
|
||||
job.progressDirty = false;
|
||||
try {
|
||||
await updatePipelineJob(jobId, {
|
||||
progress: job.lastProgress as Record<string, unknown>,
|
||||
totalCost: job.lastCost as Record<string, unknown>,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error(`[pipeline-jobs] failed to flush progress for ${jobId}:`, err);
|
||||
}
|
||||
}
|
||||
}, PROGRESS_FLUSH_MS);
|
||||
|
||||
// Run the pipeline in the background
|
||||
executePipeline({
|
||||
userId: params.userId,
|
||||
email: params.email,
|
||||
username: params.username,
|
||||
role: params.role,
|
||||
taskDirName: params.taskDirName,
|
||||
inputs: params.inputs,
|
||||
cwd: params.cwd,
|
||||
abortSignal: job.abortSignal,
|
||||
emit,
|
||||
}).then(async () => {
|
||||
clearInterval(flushInterval);
|
||||
await updatePipelineJob(jobId, {
|
||||
status: 'completed',
|
||||
progress: job.lastProgress as Record<string, unknown>,
|
||||
totalCost: job.lastCost as Record<string, unknown>,
|
||||
completedAt: new Date(),
|
||||
}).catch((err) => console.error(`[pipeline-jobs] failed to finalize ${jobId}:`, err));
|
||||
liveJobs.delete(jobId);
|
||||
}).catch(async (err) => {
|
||||
clearInterval(flushInterval);
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const isStopped = job.abortSignal.aborted;
|
||||
await updatePipelineJob(jobId, {
|
||||
status: isStopped ? 'stopped' : 'failed',
|
||||
progress: job.lastProgress as Record<string, unknown>,
|
||||
totalCost: job.lastCost as Record<string, unknown>,
|
||||
error: isStopped ? undefined : message,
|
||||
completedAt: new Date(),
|
||||
}).catch((e) => console.error(`[pipeline-jobs] failed to finalize ${jobId}:`, e));
|
||||
liveJobs.delete(jobId);
|
||||
});
|
||||
|
||||
return jobId;
|
||||
}
|
||||
|
||||
export function attachViewer(jobId: string, ws: ServerWebSocket<WSData>) {
|
||||
const job = liveJobs.get(jobId);
|
||||
if (!job) return false;
|
||||
|
||||
job.viewers.add(ws);
|
||||
|
||||
// Replay buffered events so the viewer catches up
|
||||
for (const event of job.eventBuffer) {
|
||||
sendToViewer(ws, jobId, event);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
export function detachViewer(jobId: string, ws: ServerWebSocket<WSData>) {
|
||||
const job = liveJobs.get(jobId);
|
||||
if (job) job.viewers.delete(ws);
|
||||
}
|
||||
|
||||
export function detachAllForViewer(ws: ServerWebSocket<WSData>) {
|
||||
for (const job of liveJobs.values()) {
|
||||
job.viewers.delete(ws);
|
||||
}
|
||||
}
|
||||
|
||||
export function stopJob(jobId: string): boolean {
|
||||
const job = liveJobs.get(jobId);
|
||||
if (!job) return false;
|
||||
job.abortSignal.aborted = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
export function isJobLive(jobId: string): boolean {
|
||||
return liveJobs.has(jobId);
|
||||
}
|
||||
|
||||
export function getLiveJobForUser(userId: number): string | null {
|
||||
for (const [jobId, job] of liveJobs) {
|
||||
if (job.userId === userId) return jobId;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function getJobsForUser(userId: number) {
|
||||
const jobs = await getPipelineJobsForUser(userId);
|
||||
return jobs.map((j) => ({
|
||||
...j,
|
||||
isLive: liveJobs.has(j.id),
|
||||
}));
|
||||
}
|
||||
|
||||
export async function getJob(jobId: string) {
|
||||
const job = await getPipelineJob(jobId);
|
||||
if (!job) return null;
|
||||
return { ...job, isLive: liveJobs.has(job.id) };
|
||||
}
|
||||
|
||||
export async function cleanupOnStartup() {
|
||||
const count = await markInterruptedJobs();
|
||||
if (count > 0) {
|
||||
console.log(`[pipeline-jobs] marked ${count} interrupted jobs on startup`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Extract minimal progress info from events for DB storage. */
|
||||
function extractProgress(event: OutMessage, prev: unknown): unknown {
|
||||
const p = (prev as Record<string, unknown>) ?? {};
|
||||
|
||||
switch (event.type) {
|
||||
case 'pipeline:init':
|
||||
return { ...p, steps: event.steps, currentStepIndex: -1 };
|
||||
|
||||
case 'step:start':
|
||||
return { ...p, currentStepIndex: event.stepIndex, currentIteration: event.iteration ?? null };
|
||||
|
||||
case 'step:complete':
|
||||
return { ...p, currentStepIndex: event.stepIndex };
|
||||
|
||||
case 'step:parallel':
|
||||
return {
|
||||
...p,
|
||||
currentStepIndex: event.stepIndex,
|
||||
parallel: { taskName: event.taskName, concurrency: event.concurrency, iterations: event.iterations.map((l) => ({ label: l, status: 'pending' })) },
|
||||
};
|
||||
|
||||
case 'iteration:start':
|
||||
if (p.parallel && Array.isArray((p.parallel as Record<string, unknown>).iterations)) {
|
||||
const parallel = p.parallel as { iterations: Array<{ label: string; status: string }> };
|
||||
return {
|
||||
...p,
|
||||
parallel: {
|
||||
...parallel,
|
||||
iterations: parallel.iterations.map((it) => it.label === event.label ? { ...it, status: 'running' } : it),
|
||||
},
|
||||
};
|
||||
}
|
||||
return p;
|
||||
|
||||
case 'iteration:complete':
|
||||
case 'iteration:error':
|
||||
if (p.parallel && Array.isArray((p.parallel as Record<string, unknown>).iterations)) {
|
||||
const parallel = p.parallel as { iterations: Array<{ label: string; status: string }> };
|
||||
const status = event.type === 'iteration:complete' ? 'complete' : 'error';
|
||||
return {
|
||||
...p,
|
||||
parallel: {
|
||||
...parallel,
|
||||
iterations: parallel.iterations.map((it) => it.label === event.label ? { ...it, status } : it),
|
||||
},
|
||||
};
|
||||
}
|
||||
return p;
|
||||
|
||||
default:
|
||||
return p;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user