jobs: video/audio download as a two-phase job (backend)

Turns the downloader into a server-side job on the existing jobs spine (Postgres
persistence, live WS viewers + replay, abort, /jobs UI) — but with its own
executor and its own lane, since it's deterministic scripting, not an agent, and
a multi-hour playlist mustn't block agentic jobs.

- reclip-client.ts (new, shared): reclipInfo / reclipPlaylist / reclipDownloadOne
  (single download → streams the file to a dir, abort-aware). Extracted so both
  the file-browser endpoints and the job executor use one client.
- execute-download.ts (new): the two-phase executor —
  phase 1 metadata (expand playlist, fetch each info, keep survivors, skip
  errors), phase 2 download (each survivor in the chosen format; skip download
  errors). Emits a compact `download:progress` snapshot (counters, not per-item
  events — playlists are thousands of items). Throws on abort / fatal.
- job manager: `download` mode dispatch → executeDownload; persists
  download:progress; adds execution LANES (download vs default) so the two run
  independently and each serializes on its own; promoteNext fills both lanes.
- POST /api/tasks/jobs/download { url, format, dir, root?, label? } — enqueues a
  download job (own lane, no capability task needed; traversal-guarded target).
- schema: `download` added to the mode enum (drizzle text-enum — no DB migration);
  getPendingJobs() query for lane filling.

Verified the executor with a mocked ReClip client: two-phase filtering, skip-on-
error counts, and abort-throws all correct.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/servers/api/tasks/pipeline-job-manager.ts

@@ -6,7 +6,7 @@ import {
   getPipelineJob,
   updatePipelineJob,
   getPipelineJobsForUser,
-  getOldestPendingJob,
+  getPendingJobs,
   countPendingJobs,
   deletePipelineJob,
   deleteTerminalJobsForUser,
@@ -18,10 +18,16 @@ import { toShellUsername } from '../../data-path';
 import { executePipeline } from './pipeline-executor';
 import type { OutMessage } from './pipeline-executor';
 import { executeScript, jobLogPath, type ScriptEvent } from './execute-script';
+import { executeDownload, type DownloadEvent } from './execute-download';
 
-// Everything a job can stream — pipeline structural events plus script stdout/stderr/exit.
-type JobEvent = OutMessage | ScriptEvent;
-type JobMode = 'pipeline' | 'script' | 'agentic';
+// Everything a job can stream — pipeline structural events, script stdout/stderr/exit, download progress.
+type JobEvent = OutMessage | ScriptEvent | DownloadEvent;
+type JobMode = 'pipeline' | 'script' | 'agentic' | 'download';
+
+// Execution lanes: downloads run independently of agentic/script jobs so a multi-hour playlist doesn't
+// block capability jobs. Each lane runs one job at a time (single user).
+type Lane = 'download' | 'default';
+const laneOf = (mode: JobMode): Lane => (mode === 'download' ? 'download' : 'default');
 
 type WSData = {
   userId: number;
@@ -32,6 +38,7 @@ type WSData = {
 type LiveJob = {
   jobId: string;
   userId: number;
+  lane: Lane;
   abortSignal: { aborted: boolean };
   emitter: EventEmitter;
   eventBuffer: JobEvent[];
@@ -46,6 +53,12 @@ const PROGRESS_FLUSH_MS = 3000;
 
 const liveJobs = new Map<string, LiveJob>();
 
+const runningInLane = (lane: Lane): number => {
+  let n = 0;
+  for (const job of liveJobs.values()) if (job.lane === lane) n++;
+  return n;
+};
+
 function sendToViewer(ws: ServerWebSocket<WSData>, jobId: string, event: JobEvent) {
   if (ws.readyState === 1) {
     ws.send(JSON.stringify({ jobId, ...event }));
@@ -94,7 +107,8 @@ export async function enqueueJob(
 ): Promise<{ jobId: string; status: 'running' | 'pending' }> {
   const jobId = randomUUID();
   const mode: JobMode = params.mode ?? 'pipeline';
-  const run = action === 'start' || runningCount() === 0;
+  // Queue within the job's lane: run now if forced, or if that lane is idle.
+  const run = action === 'start' || runningInLane(laneOf(mode)) === 0;
   await createPipelineJob({
     id: jobId,
     userId: params.userId,
@@ -122,6 +136,7 @@ function launch(jobId: string, mode: JobMode, params: StartJobParams) {
   const job: LiveJob = {
     jobId,
     userId: params.userId,
+    lane: laneOf(mode),
     abortSignal: { aborted: false },
     emitter: new EventEmitter(),
     eventBuffer: [],
@@ -141,6 +156,10 @@ function launch(jobId: string, mode: JobMode, params: StartJobParams) {
       job.progressDirty = true;
       job.lastProgress = extractProgress(event, job.lastProgress);
     }
+    if (event.type === 'download:progress') {
+      job.progressDirty = true;
+      job.lastProgress = event.progress;
+    }
     if (event.type === 'step:complete' || event.type === 'iteration:complete') {
       const cost = 'cost' in event ? event.cost : undefined;
       if (cost) {
@@ -176,30 +195,41 @@ function launch(jobId: string, mode: JobMode, params: StartJobParams) {
 
   // Run the job in the background — dispatch by mode.
   const runner: Promise<void | { exitCode: number }> =
-    mode === 'script'
-      ? executeScript({
+    mode === 'download'
+      ? executeDownload({
           jobId,
           userId: params.userId,
           email: params.email,
           username: params.username,
-          taskDirName: params.taskDirName,
           inputs: params.inputs,
           cwd: params.cwd,
           abortSignal: job.abortSignal,
           emit,
         })
-      : executePipeline({
-          userId: params.userId,
-          email: params.email,
-          username: params.username,
-          taskDirName: params.taskDirName,
-          inputs: params.inputs,
-          cwd: params.cwd,
-          model: params.model,
-          startAt: params.startAt,
-          abortSignal: job.abortSignal,
-          emit,
-        });
+      : mode === 'script'
+        ? executeScript({
+            jobId,
+            userId: params.userId,
+            email: params.email,
+            username: params.username,
+            taskDirName: params.taskDirName,
+            inputs: params.inputs,
+            cwd: params.cwd,
+            abortSignal: job.abortSignal,
+            emit,
+          })
+        : executePipeline({
+            userId: params.userId,
+            email: params.email,
+            username: params.username,
+            taskDirName: params.taskDirName,
+            inputs: params.inputs,
+            cwd: params.cwd,
+            model: params.model,
+            startAt: params.startAt,
+            abortSignal: job.abortSignal,
+            emit,
+          });
 
   runner
     .then(async (result) => {
@@ -235,32 +265,40 @@ function launch(jobId: string, mode: JobMode, params: StartJobParams) {
     });
 }
 
-// When a job finishes (and nothing else is running), promote the oldest queued job. Also called on
-// startup to resume a backlog. Rebuilds the user context from userId since there's no request here.
+// When a job finishes (or on startup), fill each idle lane with its oldest queued job. Rebuilds the user
+// context from userId since there's no request here. Lanes are independent — a running download doesn't
+// hold back an agentic job and vice-versa.
 async function promoteNext(): Promise<void> {
-  if (runningCount() > 0) return;
-  const next = await getOldestPendingJob();
-  if (!next) return;
-  const user = await getUserById(next.userId);
-  if (!user) {
-    await updatePipelineJob(next.id, { status: 'failed', error: 'user not found', completedAt: new Date() }).catch(
-      () => {},
-    );
-    return promoteNext();
+  const pending = await getPendingJobs(); // oldest first
+  const launchedNow = new Set<string>();
+  for (const lane of ['default', 'download'] as Lane[]) {
+    if (runningInLane(lane) > 0) continue;
+    for (const next of pending) {
+      const nextMode = (next.mode as JobMode) ?? 'pipeline';
+      if (laneOf(nextMode) !== lane || launchedNow.has(next.id)) continue;
+      const user = await getUserById(next.userId);
+      if (!user) {
+        await updatePipelineJob(next.id, { status: 'failed', error: 'user not found', completedAt: new Date() }).catch(
+          () => {},
+        );
+        continue; // skip to the next candidate in this lane
+      }
+      await updatePipelineJob(next.id, { status: 'running', startedAt: new Date() });
+      launchedNow.add(next.id);
+      launch(next.id, nextMode, {
+        userId: next.userId,
+        email: user.email,
+        username: toShellUsername(user.username ?? '', user.email),
+        mode: nextMode,
+        taskDirName: next.taskDirName,
+        taskName: next.taskName,
+        inputs: next.inputs as Record<string, string>,
+        cwd: next.cwd ?? undefined,
+        config: next.config,
+      });
+      break; // lane filled
+    }
   }
-  await updatePipelineJob(next.id, { status: 'running', startedAt: new Date() });
-  const nextMode = (next.mode as JobMode) ?? 'pipeline';
-  launch(next.id, nextMode, {
-    userId: next.userId,
-    email: user.email,
-    username: toShellUsername(user.username ?? '', user.email),
-    mode: nextMode,
-    taskDirName: next.taskDirName,
-    taskName: next.taskName,
-    inputs: next.inputs as Record<string, string>,
-    cwd: next.cwd ?? undefined,
-    config: next.config,
-  });
 }
 
 export function attachViewer(jobId: string, ws: ServerWebSocket<WSData>) {