jobs: video/audio download as a two-phase job (backend)
Turns the downloader into a server-side job on the existing jobs spine (Postgres
persistence, live WS viewers + replay, abort, /jobs UI) — but with its own
executor and its own lane, since it's deterministic scripting, not an agent, and
a multi-hour playlist mustn't block agentic jobs.
- reclip-client.ts (new, shared): reclipInfo / reclipPlaylist / reclipDownloadOne
(single download → streams the file to a dir, abort-aware). Extracted so both
the file-browser endpoints and the job executor use one client.
- execute-download.ts (new): the two-phase executor —
phase 1 metadata (expand playlist, fetch each info, keep survivors, skip
errors), phase 2 download (each survivor in the chosen format; skip download
errors). Emits a compact `download:progress` snapshot (counters, not per-item
events — playlists are thousands of items). Throws on abort / fatal.
- job manager: `download` mode dispatch → executeDownload; persists
download:progress; adds execution LANES (download vs default) so the two run
independently and each serializes on its own; promoteNext fills both lanes.
- POST /api/tasks/jobs/download { url, format, dir, root?, label? } — enqueues a
download job (own lane, no capability task needed; traversal-guarded target).
- schema: `download` added to the mode enum (drizzle text-enum — no DB migration);
getPendingJobs() query for lane filling.
Verified the executor with a mocked ReClip client: two-phase filtering, skip-on-
error counts, and abort-throws all correct.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,110 @@
|
||||
import { reclipInfo, reclipPlaylist, reclipDownloadOne, type ReclipInfo } from '../../reclip-client';
|
||||
|
||||
// The download-job executor — pure scripting, no agent. Two phases:
|
||||
// 1. metadata — expand the playlist, fetch each item's info sequentially, keep the ones that resolve
|
||||
// (skip the errors: private/deleted/unavailable).
|
||||
// 2. download — download every survivor in the chosen format (audio/video); skip anything that fails.
|
||||
// Emits a compact `download:progress` snapshot (counters, not per-item events — a playlist can be
|
||||
// thousands of items). Throws on abort or a fatal error (playlist expansion) so the job manager marks it
|
||||
// stopped/failed; per-item errors are counted and skipped, never fatal.
|
||||
|
||||
type Counts = { done: number; failed: number; total: number };
|
||||
export type DownloadProgress = {
|
||||
phase: 'expanding' | 'metadata' | 'download' | 'done';
|
||||
meta: Counts;
|
||||
dl: Counts;
|
||||
current?: string; // title of the item currently downloading
|
||||
};
|
||||
|
||||
export type DownloadEvent = { type: 'download:progress'; progress: DownloadProgress };
|
||||
|
||||
export type ExecuteDownloadParams = {
|
||||
jobId: string;
|
||||
userId: number;
|
||||
email: string;
|
||||
username?: string | null;
|
||||
inputs: Record<string, string>; // { url, format: 'audio'|'video', absDir }
|
||||
cwd?: string;
|
||||
abortSignal: { aborted: boolean };
|
||||
emit: (event: DownloadEvent) => void;
|
||||
};
|
||||
|
||||
const EMIT_THROTTLE_MS = 750;
|
||||
|
||||
export async function executeDownload(params: ExecuteDownloadParams): Promise<void> {
|
||||
const { url, format, absDir } = params.inputs;
|
||||
if (!url || !absDir) throw new Error('download job missing url or target directory');
|
||||
const audioOnly = format !== 'video'; // default to audio
|
||||
|
||||
const progress: DownloadProgress = {
|
||||
phase: 'expanding',
|
||||
meta: { done: 0, failed: 0, total: 0 },
|
||||
dl: { done: 0, failed: 0, total: 0 },
|
||||
};
|
||||
let lastEmit = 0;
|
||||
const emit = (force = false) => {
|
||||
const now = Date.now();
|
||||
if (!force && now - lastEmit < EMIT_THROTTLE_MS) return;
|
||||
lastEmit = now;
|
||||
params.emit({
|
||||
type: 'download:progress',
|
||||
progress: { ...progress, meta: { ...progress.meta }, dl: { ...progress.dl } },
|
||||
});
|
||||
};
|
||||
const checkAbort = () => {
|
||||
if (params.abortSignal.aborted) throw new Error('aborted');
|
||||
};
|
||||
|
||||
emit(true);
|
||||
|
||||
// ── Expand ──
|
||||
let urls = [url];
|
||||
if (url.includes('list=')) {
|
||||
const pl = await reclipPlaylist(url);
|
||||
if (pl.error) throw new Error(pl.error);
|
||||
if (pl.urls?.length) urls = pl.urls;
|
||||
}
|
||||
|
||||
// ── Phase 1: metadata (keep survivors) ──
|
||||
progress.phase = 'metadata';
|
||||
progress.meta.total = urls.length;
|
||||
emit(true);
|
||||
const valid: Array<{ url: string; title: string }> = [];
|
||||
for (const u of urls) {
|
||||
checkAbort();
|
||||
const info = await reclipInfo(u).catch((): ReclipInfo => ({ error: 'fetch failed' }));
|
||||
if (info && !info.error) valid.push({ url: u, title: info.title ?? '' });
|
||||
else progress.meta.failed++;
|
||||
progress.meta.done++;
|
||||
emit();
|
||||
}
|
||||
emit(true);
|
||||
|
||||
// ── Phase 2: download survivors ──
|
||||
progress.phase = 'download';
|
||||
progress.dl.total = valid.length;
|
||||
emit(true);
|
||||
for (const item of valid) {
|
||||
checkAbort();
|
||||
progress.current = item.title || item.url;
|
||||
emit(true);
|
||||
try {
|
||||
await reclipDownloadOne({
|
||||
url: item.url,
|
||||
destDir: absDir,
|
||||
audioOnly,
|
||||
title: item.title,
|
||||
signal: params.abortSignal,
|
||||
});
|
||||
progress.dl.done++;
|
||||
} catch {
|
||||
checkAbort(); // an abort surfaces as a throw here — re-check so it stops instead of counting as a skip
|
||||
progress.dl.failed++;
|
||||
}
|
||||
emit();
|
||||
}
|
||||
|
||||
progress.phase = 'done';
|
||||
progress.current = undefined;
|
||||
emit(true);
|
||||
}
|
||||
Reference in New Issue
Block a user