replace hardcoded download job with download-media script capability
move video/audio downloads off the dedicated ReClip download lane onto the generic script-job path: - delete execute-download.ts, reclip-client.ts and the POST /jobs/download endpoint; drop the 'download' mode from the pipeline_jobs enum (legacy rows tolerated) - execute-script.ts: strip the @@officer:progress@@ sentinel from the log, emit progress events, and isolate viewer/log writes (safeEmit/safeLog) so a broadcast or log throw can't wedge the stdout pump - pipeline-job-manager.ts: persist latest progress; guard sendToViewer sends - ScriptJobDetail: render the two progress bars; DownloadJobDetail kept for legacy history rows - TaskRunnerModal: ScriptRunner descends into the triggered directory - VideoDownloadPanel: rewire startJob to the script-job path Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -18,16 +18,10 @@ import { toShellUsername } from '../../data-path';
|
||||
import { executePipeline } from './pipeline-executor';
|
||||
import type { OutMessage } from './pipeline-executor';
|
||||
import { executeScript, jobLogPath, type ScriptEvent } from './execute-script';
|
||||
import { executeDownload, type DownloadEvent } from './execute-download';
|
||||
|
||||
// Everything a job can stream — pipeline structural events, script stdout/stderr/exit, download progress.
|
||||
type JobEvent = OutMessage | ScriptEvent | DownloadEvent;
|
||||
type JobMode = 'pipeline' | 'script' | 'agentic' | 'download';
|
||||
|
||||
// Execution lanes: downloads run independently of agentic/script jobs so a multi-hour playlist doesn't
|
||||
// block capability jobs. Each lane runs one job at a time (single user).
|
||||
type Lane = 'download' | 'default';
|
||||
const laneOf = (mode: JobMode): Lane => (mode === 'download' ? 'download' : 'default');
|
||||
// Everything a job can stream — pipeline structural events, script stdout/stderr/exit/progress.
|
||||
type JobEvent = OutMessage | ScriptEvent;
|
||||
type JobMode = 'pipeline' | 'script' | 'agentic';
|
||||
|
||||
type WSData = {
|
||||
userId: number;
|
||||
@@ -38,7 +32,6 @@ type WSData = {
|
||||
type LiveJob = {
|
||||
jobId: string;
|
||||
userId: number;
|
||||
lane: Lane;
|
||||
abortSignal: { aborted: boolean };
|
||||
emitter: EventEmitter;
|
||||
eventBuffer: JobEvent[];
|
||||
@@ -53,15 +46,15 @@ const PROGRESS_FLUSH_MS = 3000;
|
||||
|
||||
const liveJobs = new Map<string, LiveJob>();
|
||||
|
||||
const runningInLane = (lane: Lane): number => {
|
||||
let n = 0;
|
||||
for (const job of liveJobs.values()) if (job.lane === lane) n++;
|
||||
return n;
|
||||
};
|
||||
|
||||
function sendToViewer(ws: ServerWebSocket<WSData>, jobId: string, event: JobEvent) {
|
||||
// readyState can flip to closing between this check and send (e.g. a page refresh), so send() may
|
||||
// still throw. A dead viewer must never propagate into the job's output pump, so swallow it here.
|
||||
if (ws.readyState === 1) {
|
||||
ws.send(JSON.stringify({ jobId, ...event }));
|
||||
try {
|
||||
ws.send(JSON.stringify({ jobId, ...event }));
|
||||
} catch {
|
||||
/* viewer went away mid-send */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,8 +100,8 @@ export async function enqueueJob(
|
||||
): Promise<{ jobId: string; status: 'running' | 'pending' }> {
|
||||
const jobId = randomUUID();
|
||||
const mode: JobMode = params.mode ?? 'pipeline';
|
||||
// Queue within the job's lane: run now if forced, or if that lane is idle.
|
||||
const run = action === 'start' || runningInLane(laneOf(mode)) === 0;
|
||||
// One global queue (single user): run now if forced, or if nothing is currently running.
|
||||
const run = action === 'start' || liveJobs.size === 0;
|
||||
await createPipelineJob({
|
||||
id: jobId,
|
||||
userId: params.userId,
|
||||
@@ -136,7 +129,6 @@ function launch(jobId: string, mode: JobMode, params: StartJobParams) {
|
||||
const job: LiveJob = {
|
||||
jobId,
|
||||
userId: params.userId,
|
||||
lane: laneOf(mode),
|
||||
abortSignal: { aborted: false },
|
||||
emitter: new EventEmitter(),
|
||||
eventBuffer: [],
|
||||
@@ -156,7 +148,7 @@ function launch(jobId: string, mode: JobMode, params: StartJobParams) {
|
||||
job.progressDirty = true;
|
||||
job.lastProgress = extractProgress(event, job.lastProgress);
|
||||
}
|
||||
if (event.type === 'download:progress') {
|
||||
if (event.type === 'progress') {
|
||||
job.progressDirty = true;
|
||||
job.lastProgress = event.progress;
|
||||
}
|
||||
@@ -195,41 +187,30 @@ function launch(jobId: string, mode: JobMode, params: StartJobParams) {
|
||||
|
||||
// Run the job in the background — dispatch by mode.
|
||||
const runner: Promise<void | { exitCode: number }> =
|
||||
mode === 'download'
|
||||
? executeDownload({
|
||||
mode === 'script'
|
||||
? executeScript({
|
||||
jobId,
|
||||
userId: params.userId,
|
||||
email: params.email,
|
||||
username: params.username,
|
||||
taskDirName: params.taskDirName,
|
||||
inputs: params.inputs,
|
||||
cwd: params.cwd,
|
||||
abortSignal: job.abortSignal,
|
||||
emit,
|
||||
})
|
||||
: mode === 'script'
|
||||
? executeScript({
|
||||
jobId,
|
||||
userId: params.userId,
|
||||
email: params.email,
|
||||
username: params.username,
|
||||
taskDirName: params.taskDirName,
|
||||
inputs: params.inputs,
|
||||
cwd: params.cwd,
|
||||
abortSignal: job.abortSignal,
|
||||
emit,
|
||||
})
|
||||
: executePipeline({
|
||||
userId: params.userId,
|
||||
email: params.email,
|
||||
username: params.username,
|
||||
taskDirName: params.taskDirName,
|
||||
inputs: params.inputs,
|
||||
cwd: params.cwd,
|
||||
model: params.model,
|
||||
startAt: params.startAt,
|
||||
abortSignal: job.abortSignal,
|
||||
emit,
|
||||
});
|
||||
: executePipeline({
|
||||
userId: params.userId,
|
||||
email: params.email,
|
||||
username: params.username,
|
||||
taskDirName: params.taskDirName,
|
||||
inputs: params.inputs,
|
||||
cwd: params.cwd,
|
||||
model: params.model,
|
||||
startAt: params.startAt,
|
||||
abortSignal: job.abortSignal,
|
||||
emit,
|
||||
});
|
||||
|
||||
runner
|
||||
.then(async (result) => {
|
||||
@@ -265,39 +246,33 @@ function launch(jobId: string, mode: JobMode, params: StartJobParams) {
|
||||
});
|
||||
}
|
||||
|
||||
// When a job finishes (or on startup), fill each idle lane with its oldest queued job. Rebuilds the user
|
||||
// context from userId since there's no request here. Lanes are independent — a running download doesn't
|
||||
// hold back an agentic job and vice-versa.
|
||||
// When a job finishes (or on startup), launch the oldest queued job — one global queue, one job at a
|
||||
// time (single user). Rebuilds the user context from userId since there's no request here.
|
||||
async function promoteNext(): Promise<void> {
|
||||
if (liveJobs.size > 0) return; // something is already running
|
||||
const pending = await getPendingJobs(); // oldest first
|
||||
const launchedNow = new Set<string>();
|
||||
for (const lane of ['default', 'download'] as Lane[]) {
|
||||
if (runningInLane(lane) > 0) continue;
|
||||
for (const next of pending) {
|
||||
const nextMode = (next.mode as JobMode) ?? 'pipeline';
|
||||
if (laneOf(nextMode) !== lane || launchedNow.has(next.id)) continue;
|
||||
const user = await getUserById(next.userId);
|
||||
if (!user) {
|
||||
await updatePipelineJob(next.id, { status: 'failed', error: 'user not found', completedAt: new Date() }).catch(
|
||||
() => {},
|
||||
);
|
||||
continue; // skip to the next candidate in this lane
|
||||
}
|
||||
await updatePipelineJob(next.id, { status: 'running', startedAt: new Date() });
|
||||
launchedNow.add(next.id);
|
||||
launch(next.id, nextMode, {
|
||||
userId: next.userId,
|
||||
email: user.email,
|
||||
username: toShellUsername(user.username ?? '', user.email),
|
||||
mode: nextMode,
|
||||
taskDirName: next.taskDirName,
|
||||
taskName: next.taskName,
|
||||
inputs: next.inputs as Record<string, string>,
|
||||
cwd: next.cwd ?? undefined,
|
||||
config: next.config,
|
||||
});
|
||||
break; // lane filled
|
||||
for (const next of pending) {
|
||||
const nextMode = (next.mode as JobMode) ?? 'pipeline';
|
||||
const user = await getUserById(next.userId);
|
||||
if (!user) {
|
||||
await updatePipelineJob(next.id, { status: 'failed', error: 'user not found', completedAt: new Date() }).catch(
|
||||
() => {},
|
||||
);
|
||||
continue; // skip to the next candidate
|
||||
}
|
||||
await updatePipelineJob(next.id, { status: 'running', startedAt: new Date() });
|
||||
launch(next.id, nextMode, {
|
||||
userId: next.userId,
|
||||
email: user.email,
|
||||
username: toShellUsername(user.username ?? '', user.email),
|
||||
mode: nextMode,
|
||||
taskDirName: next.taskDirName,
|
||||
taskName: next.taskName,
|
||||
inputs: next.inputs as Record<string, string>,
|
||||
cwd: next.cwd ?? undefined,
|
||||
config: next.config,
|
||||
});
|
||||
return; // one at a time
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user