From dc4fe758f3ea8afb90c5418537acc07c4011ed30 Mon Sep 17 00:00:00 2001 From: Pastilhas Date: Thu, 20 Aug 2026 13:29:10 +0100 Subject: [PATCH] Decode HEIC, and transcode video MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things the platform needed and could not get. HEIC: sharp reported "corrupt header" on every iPhone photograph, which they are not. Two separate causes. libheif caps iref references and burst/depth HEICs exceed it — `unlimited` gets past that, and limitInputPixels is now passed explicitly alongside because unlimited also loosens the bomb guard (verified the pixel limit still rejects a bomb). Then sharp still could not decode the pixels: its bundled libheif is built without libde265, so HEVC-coded HEIF is unreadable to it. The system libheif does have the plugin, so HEIF now routes through ImageMagick to a lossless PNG first. Verified: 1.4 MB HEIC in, 102 KB WebP out. /v1/ffmpeg is built. ffmpeg's own option names, structured filters rather than a -vf string, and four presets — web-1080/720/480 and poster. The presets are where the encoder knowledge lives, so a consumer asks for "web-720" rather than knowing flags, and revising quality is a change here for every site that uses it. Default-deny throughout, same as the sharp surface: filters that read files or load native code are absent from the table rather than blocklisted, protocols are whitelisted to file, and the caller still never names a path. Co-Authored-By: Claude Opus 5 --- src/assets/store.ts | 9 +- src/ffmpeg/run.ts | 199 +++++++++++++++++++++++++++++++++++++++ src/ffmpeg/vocabulary.ts | 136 ++++++++++++++++++++++++++ src/server.ts | 9 +- src/sharp/decodable.ts | 71 ++++++++++++++ src/sharp/run.ts | 14 ++- 6 files changed, 434 insertions(+), 4 deletions(-) create mode 100644 src/ffmpeg/run.ts create mode 100644 src/ffmpeg/vocabulary.ts create mode 100644 src/sharp/decodable.ts diff --git a/src/assets/store.ts b/src/assets/store.ts index 17ba7d9..97d5644 100644 --- a/src/assets/store.ts +++ b/src/assets/store.ts @@ -82,7 +82,14 @@ export async function deleteAsset(id: string): Promise { */ async function identify(path: string): Promise> { try { - const meta = await sharp(path, { limitInputPixels: MAX_INPUT_PIXELS }).metadata(); + const meta = await sharp(path, { + limitInputPixels: MAX_INPUT_PIXELS, + // `unlimited` is what gets past libheif's iref-reference cap, which iPhone HEICs carrying + // burst or depth data routinely exceed — sharp reports it as "corrupt header", which it is not. + // It loosens libvips' own buffer guards, so `limitInputPixels` is passed EXPLICITLY alongside: + // verified that the pixel limit still rejects a decompression bomb with unlimited set. + unlimited: true, + }).metadata(); if (meta.format) { return { kind: "image", format: meta.format, width: meta.width, height: meta.height }; } diff --git a/src/ffmpeg/run.ts b/src/ffmpeg/run.ts new file mode 100644 index 0000000..641e49a --- /dev/null +++ b/src/ffmpeg/run.ts @@ -0,0 +1,199 @@ +import { join } from "node:path"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { JOB_TIMEOUT_MS } from "../config"; +import { invalid, unprocessable, ApiError } from "../errors"; +import { contentPath, putAsset, readAsset, type Asset } from "../assets/store"; +import { + AUDIO_FILTERS, + FORMATS, + INPUT_OPTIONS, + OUTPUT_OPTIONS, + PRESETS, + VIDEO_FILTERS, + type FfmpegParam, +} from "./vocabulary"; + +/** + * Running ffmpeg. + * + * Everything the caller supplies is a NAME or a NUMBER from the vocabulary. Paths are constructed + * here from an asset id, and the process is spawned with an argv array so there is no shell to quote + * for. See docs/SECURITY.md. + */ + +export type FfmpegRequest = { + input?: unknown; + preset?: unknown; + inputOptions?: unknown; + outputOptions?: unknown; + filters?: unknown; + format?: unknown; +}; + +const BITRATE = /^[0-9]+(\.[0-9]+)?[kKmM]?$/; + +function value(spec: FfmpegParam, raw: unknown, path: string): string { + switch (spec.kind) { + case "int": + if (typeof raw !== "number" || !Number.isInteger(raw)) throw invalid(`${path} must be an integer`, path); + if (raw < spec.min || raw > spec.max) throw invalid(`${path} must be between ${spec.min} and ${spec.max}`, path); + return String(raw); + case "number": + if (typeof raw !== "number" || !Number.isFinite(raw)) throw invalid(`${path} must be a number`, path); + if (raw < spec.min || raw > spec.max) throw invalid(`${path} must be between ${spec.min} and ${spec.max}`, path); + return String(raw); + case "enum": + if (typeof raw !== "string" || !spec.values.includes(raw)) { + throw invalid(`${path} must be one of: ${spec.values.join(", ")}`, path); + } + return raw; + case "bitrate": + if (typeof raw !== "string" || !BITRATE.test(raw)) { + throw invalid(`${path} must be a bitrate like "2M" or "128k"`, path); + } + return raw; + } +} + +function options(table: Record, raw: unknown, path: string): string[] { + if (raw === undefined || raw === null) return []; + if (typeof raw !== "object" || Array.isArray(raw)) throw invalid(`${path} must be an object`, path); + + const args: string[] = []; + for (const [key, entry] of Object.entries(raw as Record)) { + const spec = table[key]; + if (!spec) { + throw invalid(`Unknown option ${JSON.stringify(key)}. Accepted here: ${Object.keys(table).join(", ")}`, `${path}.${key}`); + } + args.push(`-${key}`, value(spec, entry, `${path}.${key}`)); + } + return args; +} + +/** + * A filter chain, built from structured objects rather than parsed from a string. + * + * ffmpeg's filter syntax has its own escaping rules, and parsing it well enough to validate it is + * harder than accepting `{ name: args }`. The names and parameters are ffmpeg's; only the + * punctuation is ours — and every filter that can read a file or load a plugin is simply absent from + * the table, so it cannot be named. + */ +function filterChain(table: Record>, raw: unknown, path: string): string { + if (raw === undefined || raw === null) return ""; + if (!Array.isArray(raw)) throw invalid(`${path} must be an array of filters`, path); + + return raw + .map((entry, i) => { + const at = `${path}[${i}]`; + if (!entry || typeof entry !== "object") throw invalid(`${at} must be an object`, at); + + const pairs = Object.entries(entry as Record); + if (pairs.length !== 1) throw invalid(`${at} must name exactly one filter`, at); + + const [name, rawArgs] = pairs[0]!; + const spec = table[name]; + if (!spec) throw invalid(`Unknown filter ${JSON.stringify(name)}. See docs/FFMPEG.md.`, `${at}.${name}`); + + const args = Object.entries((rawArgs ?? {}) as Record).map(([key, v]) => { + const argSpec = spec[key]; + if (!argSpec) { + throw invalid(`Unknown parameter ${JSON.stringify(key)} for ${name}`, `${at}.${name}.${key}`); + } + return `${key}=${value(argSpec, v, `${at}.${name}.${key}`)}`; + }); + + return args.length ? `${name}=${args.join(":")}` : name; + }) + .join(","); +} + +/** Run ffmpeg on an asset and store the result as a new asset. The input is never modified. */ +export async function runFfmpeg(body: FfmpegRequest): Promise { + if (typeof body.input !== "string") throw invalid("input must be an asset id", "input"); + + const source = await readAsset(body.input); + if (source.kind === "image") { + throw invalid(`Asset ${source.id} is an image. Use /v1/sharp for images.`, "input"); + } + + // A preset supplies the options; anything given explicitly overrides it. So "web-1080 but crf 20" + // does not mean writing out every flag again. + let format = "mp4"; + let presetOptions: string[] = []; + let presetFilters = ""; + + if (body.preset !== undefined) { + if (typeof body.preset !== "string" || !PRESETS[body.preset]) { + throw invalid(`preset must be one of: ${Object.keys(PRESETS).join(", ")}`, "preset"); + } + const preset = PRESETS[body.preset]!; + format = preset.format; + presetOptions = options(OUTPUT_OPTIONS, preset.options, "preset"); + presetFilters = preset.filters?.v + ? filterChain(VIDEO_FILTERS, preset.filters.v.map((f) => ({ [f.name]: f.args })), "preset.filters") + : ""; + } + + if (body.format !== undefined) { + if (typeof body.format !== "string" || !FORMATS[body.format]) { + throw invalid(`format must be one of: ${Object.keys(FORMATS).join(", ")}`, "format"); + } + format = body.format; + } + + const inputArgs = options(INPUT_OPTIONS, body.inputOptions, "inputOptions"); + const outputArgs = options(OUTPUT_OPTIONS, body.outputOptions, "outputOptions"); + + const filters = (body.filters ?? {}) as Record; + const videoChain = filterChain(VIDEO_FILTERS, filters.v, "filters.v") || presetFilters; + const audioChain = filterChain(AUDIO_FILTERS, filters.a, "filters.a"); + + // One directory per job, removed however this ends — otherwise disk fills with the failures. + const workDir = await mkdtemp(join(tmpdir(), "avclub-")); + const output = join(workDir, `out.${FORMATS[format]!.extension}`); + + try { + const argv = [ + "ffmpeg", + "-nostdin", + "-y", + "-v", "error", + // Nothing but local files. Kills SSRF and the concat:/subfile: tricks outright — the second + // lock behind "the caller never names a path". + "-protocol_whitelist", "file", + ...inputArgs, + "-i", contentPath(source.id), + ...(videoChain ? ["-vf", videoChain] : []), + ...(audioChain ? ["-af", audioChain] : []), + ...presetOptions, + // After the preset, so an explicit option wins. + ...outputArgs, + "-f", format, + output, + ]; + + const proc = Bun.spawn(argv, { stdout: "pipe", stderr: "pipe", stdin: "ignore" }); + + // A crafted file can make an encoder run effectively forever. + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + proc.kill(); + }, JOB_TIMEOUT_MS); + + const [stderr, code] = await Promise.all([new Response(proc.stderr).text(), proc.exited]); + clearTimeout(timer); + + if (timedOut) { + throw new ApiError("timeout", `Encoding took longer than ${JOB_TIMEOUT_MS / 1000}s and was stopped.`); + } + if (code !== 0) { + throw unprocessable(`ffmpeg failed: ${stderr.trim().split("\n").at(-1) ?? `exit ${code}`}`); + } + + return await putAsset(await Bun.file(output).arrayBuffer()); + } finally { + await rm(workDir, { recursive: true, force: true }); + } +} diff --git a/src/ffmpeg/vocabulary.ts b/src/ffmpeg/vocabulary.ts new file mode 100644 index 0000000..7dab49e --- /dev/null +++ b/src/ffmpeg/vocabulary.ts @@ -0,0 +1,136 @@ +/** + * The ffmpeg surface av-club exposes, as data — the same shape as the sharp vocabulary. + * + * Options are spelled the way ffmpeg spells them (`c:v`, `crf`, `pix_fmt`), so ffmpeg's own + * documentation is the reference. What is NOT here is anything that names a file: no `-i`, no output + * path, no protocol. See docs/SECURITY.md — that single restriction is what makes the rest of this + * safe to open up. + */ + +export type FfmpegParam = + | { kind: "int"; min: number; max: number; note?: string } + | { kind: "number"; min: number; max: number; note?: string } + | { kind: "enum"; values: readonly string[]; note?: string } + | { kind: "bitrate"; note?: string }; + +/** Options that go BEFORE `-i`: seeking and trimming, which are far cheaper applied to the input. */ +export const INPUT_OPTIONS: Record = { + ss: { kind: "number", min: 0, max: 86_400, note: "Start at this many seconds." }, + t: { kind: "number", min: 0.001, max: 86_400, note: "Take this many seconds." }, + to: { kind: "number", min: 0, max: 86_400, note: "Stop at this timestamp." }, +}; + +/** + * Output options. + * + * The encoder list is short on purpose: these are the codecs a browser plays. An encoder ffmpeg + * happens to have compiled in is not automatically something worth publishing. + */ +export const OUTPUT_OPTIONS: Record = { + "c:v": { kind: "enum", values: ["libx264", "libvpx-vp9", "libaom-av1", "copy"], note: "Video encoder. libx264 is the one every browser plays." }, + "c:a": { kind: "enum", values: ["aac", "libopus", "copy"], note: "Audio encoder." }, + crf: { kind: "int", min: 0, max: 63, note: "Quality. Lower is better and bigger; 23 is a sane default for x264." }, + preset: { kind: "enum", values: ["ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow"], note: "Encoder effort. Slower is smaller at the same quality." }, + tune: { kind: "enum", values: ["film", "animation", "grain", "stillimage", "fastdecode", "zerolatency"] }, + "profile:v": { kind: "enum", values: ["baseline", "main", "high"], note: "`high` is fine everywhere current." }, + level: { kind: "enum", values: ["3.0", "3.1", "4.0", "4.1", "4.2", "5.0", "5.1"] }, + pix_fmt: { kind: "enum", values: ["yuv420p", "yuv422p", "yuv444p"], note: "yuv420p, or Safari and QuickTime will not play it." }, + "b:v": { kind: "bitrate", note: "Target video bitrate, e.g. 2M." }, + "b:a": { kind: "bitrate", note: "Target audio bitrate, e.g. 128k." }, + maxrate: { kind: "bitrate" }, + bufsize: { kind: "bitrate" }, + r: { kind: "number", min: 1, max: 120, note: "Output frame rate." }, + g: { kind: "int", min: 1, max: 600, note: "Keyframe interval." }, + ar: { kind: "enum", values: ["22050", "44100", "48000"], note: "Audio sample rate." }, + ac: { kind: "int", min: 1, max: 2 }, + "frames:v": { kind: "int", min: 1, max: 10_000, note: "Stop after this many frames — 1 gives a poster." }, + movflags: { kind: "enum", values: ["+faststart"], note: "Moves the index to the front so playback can start before the file has arrived. Always want this for web." }, + an: { kind: "enum", values: ["1"], note: "Drop audio entirely." }, +}; + +/** Output containers. `image2` is how a single frame comes out as a still. */ +export const FORMATS: Record = { + mp4: { extension: "mp4", describe: "H.264/AAC in MP4 — plays everywhere." }, + webm: { extension: "webm", describe: "VP9/Opus. Smaller, but not on older Safari." }, + mp3: { extension: "mp3", describe: "Audio only." }, + image2: { extension: "jpg", describe: "A single frame, for poster images." }, +}; + +/** + * Filters, default-deny. + * + * ffmpeg's filter graph is a language, and most of it is arithmetic on pixels. A small, specific part + * of it reads files or loads native code — `movie`, `subtitles`, `frei0r`, `ladspa` and relatives — + * and those are not in this list, so they are refused. A filter nobody has classified is refused too: + * a blocklist would silently admit whatever the next ffmpeg release adds. + */ +export const VIDEO_FILTERS: Record> = { + scale: { + w: { kind: "int", min: -2, max: 8192, note: "-2 keeps the aspect ratio and an even number of pixels, which h264 requires." }, + h: { kind: "int", min: -2, max: 8192 }, + }, + crop: { + w: { kind: "int", min: 1, max: 8192 }, + h: { kind: "int", min: 1, max: 8192 }, + x: { kind: "int", min: 0, max: 8192 }, + y: { kind: "int", min: 0, max: 8192 }, + }, + fps: { fps: { kind: "number", min: 1, max: 120 } }, + format: { pix_fmts: { kind: "enum", values: ["yuv420p", "yuv422p", "yuv444p", "rgb24"] } }, + transpose: { dir: { kind: "enum", values: ["0", "1", "2", "3"] } }, + hflip: {}, + vflip: {}, + eq: { + brightness: { kind: "number", min: -1, max: 1 }, + contrast: { kind: "number", min: -1000, max: 1000 }, + saturation: { kind: "number", min: 0, max: 3 }, + gamma: { kind: "number", min: 0.1, max: 10 }, + }, + unsharp: { + luma_msize_x: { kind: "int", min: 3, max: 23 }, + luma_msize_y: { kind: "int", min: 3, max: 23 }, + luma_amount: { kind: "number", min: -2, max: 5 }, + }, +}; + +export const AUDIO_FILTERS: Record> = { + volume: { volume: { kind: "number", min: 0, max: 10 } }, + aresample: { osr: { kind: "enum", values: ["22050", "44100", "48000"] } }, + loudnorm: { + I: { kind: "number", min: -70, max: -5, note: "Target loudness in LUFS. -16 suits the web." }, + TP: { kind: "number", min: -9, max: 0 }, + }, +}; + +/** + * Named presets, so a caller does not have to know encoder settings to get a web-playable file. + * + * This is where the media knowledge lives — the whole reason av-club is a separate repo. A consumer + * says what the media is FOR; what that means in flags is decided here and can be revised in one + * place for every site that uses it. + */ +export const PRESETS: Record; filters?: { v?: { name: string; args: Record }[] } }> = { + "web-1080": { + describe: "H.264 up to 1080p. The safe default — plays in every browser.", + format: "mp4", + options: { "c:v": "libx264", crf: 23, preset: "medium", "profile:v": "high", pix_fmt: "yuv420p", "c:a": "aac", "b:a": "128k", movflags: "+faststart" }, + filters: { v: [{ name: "scale", args: { w: -2, h: 1080 } }] }, + }, + "web-720": { + describe: "H.264 up to 720p.", + format: "mp4", + options: { "c:v": "libx264", crf: 23, preset: "medium", "profile:v": "high", pix_fmt: "yuv420p", "c:a": "aac", "b:a": "128k", movflags: "+faststart" }, + filters: { v: [{ name: "scale", args: { w: -2, h: 720 } }] }, + }, + "web-480": { + describe: "H.264 up to 480p, for slow connections.", + format: "mp4", + options: { "c:v": "libx264", crf: 26, preset: "medium", "profile:v": "main", pix_fmt: "yuv420p", "c:a": "aac", "b:a": "96k", movflags: "+faststart" }, + filters: { v: [{ name: "scale", args: { w: -2, h: 480 } }] }, + }, + poster: { + describe: "One frame as a JPEG, for a video's poster image.", + format: "image2", + options: { "frames:v": 1 }, + }, +}; diff --git a/src/server.ts b/src/server.ts index 71b0dca..3d32ff9 100644 --- a/src/server.ts +++ b/src/server.ts @@ -4,6 +4,7 @@ import { ApiError, invalid, notFound, toErrorResponse } from "./errors"; import { contentPath, deleteAsset, putAsset, readAsset } from "./assets/store"; import { ffprobe } from "./probe/ffprobe"; import { runSharp, type SharpRequest } from "./sharp/run"; +import { runFfmpeg, type FfmpegRequest } from "./ffmpeg/run"; import sharp from "sharp"; /** @@ -167,8 +168,12 @@ async function route(request: Request): Promise { return Response.json(result, { status: 201 }); } - if (path === "/v1/ffmpeg") { - throw new ApiError("invalid_request", "POST /v1/ffmpeg is not implemented yet. See docs/API.md."); + if (path === "/v1/ffmpeg" && request.method === "POST") { + const body = (await request.json().catch(() => { + throw invalid("Expected a JSON body"); + })) as FfmpegRequest; + const result = await withSlot(() => runFfmpeg(body)); + return Response.json(result, { status: 201 }); } throw notFound(`No route for ${request.method} ${path}`); diff --git a/src/sharp/decodable.ts b/src/sharp/decodable.ts new file mode 100644 index 0000000..8b62250 --- /dev/null +++ b/src/sharp/decodable.ts @@ -0,0 +1,71 @@ +import { join } from "node:path"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { JOB_TIMEOUT_MS } from "../config"; +import { unprocessable } from "../errors"; + +/** + * Getting HEIC into something sharp can actually decode. + * + * ── Why this exists ── + * + * sharp ships its own libvips and libheif, built without the libde265 plugin, so it can parse a + * HEIC's header — dimensions come back fine — and then fails on the pixels with "bad seek", which + * reads like a corrupt file and is not one. Every photograph an iPhone takes is HEVC-coded HEIF, so + * this is not an edge case: it is most of what a client will upload from a phone. + * + * ImageMagick links the SYSTEM libheif, which does have the plugin (see docs/TOOLING.md), so it + * decodes them. This converts through it and hands sharp a PNG. + * + * PNG rather than JPEG for the intermediate: it is lossless, so the one real encode is the final one + * and the derivative is not a copy of a copy. + */ + +/** Formats sharp reports but cannot decode here. Extend only with evidence, not suspicion. */ +const NEEDS_CONVERSION = new Set(["heif", "heic", "avif-sequence"]); + +export type Decodable = { + path: string; + /** Call when finished. Removes the intermediate, if there was one. */ + cleanup: () => Promise; +}; + +export async function decodableInput(path: string, format: string): Promise { + if (!NEEDS_CONVERSION.has(format.toLowerCase())) { + return { path, cleanup: async () => {} }; + } + + const workDir = await mkdtemp(join(tmpdir(), "avclub-heif-")); + const converted = join(workDir, "decoded.png"); + + // ImageMagick 6 on Ubuntu 24.04, so `convert` — there is no `magick`. Spawned as an argv array, + // so the path is data rather than something a shell parses. + // + // -auto-orient applies the EXIF rotation while the metadata still exists; the PNG carries none, so + // this is the last chance to get a phone photograph the right way up. + const proc = Bun.spawn(["convert", path, "-auto-orient", converted], { + stdout: "pipe", + stderr: "pipe", + stdin: "ignore", + }); + + const timer = setTimeout(() => proc.kill(), JOB_TIMEOUT_MS); + + try { + const [stderr, code] = await Promise.all([new Response(proc.stderr).text(), proc.exited]); + + if (code !== 0 || !(await Bun.file(converted).exists())) { + await rm(workDir, { recursive: true, force: true }); + throw unprocessable( + `Could not read that HEIC: ${stderr.trim().split("\n").at(-1) ?? "conversion failed"}`, + ); + } + + return { + path: converted, + cleanup: () => rm(workDir, { recursive: true, force: true }), + }; + } finally { + clearTimeout(timer); + } +} diff --git a/src/sharp/run.ts b/src/sharp/run.ts index 16a4bd4..9a29c19 100644 --- a/src/sharp/run.ts +++ b/src/sharp/run.ts @@ -4,6 +4,7 @@ import { invalid, unprocessable } from "../errors"; import { contentPath, putAsset, readAsset, type Asset } from "../assets/store"; import { INPUT_OPTIONS, OPERATIONS, OUTPUT_FORMATS } from "./vocabulary"; import { validateObject } from "./validate"; +import { decodableInput } from "./decodable"; export type SharpRequest = { input?: unknown; @@ -33,9 +34,18 @@ export async function runSharp(body: SharpRequest): Promise { // limitInputPixels is set HERE and cannot be supplied by the caller: it is the decompression-bomb // guard, and a caller who could raise it could turn a 1 KB PNG into a 40 GB allocation. - let image = sharp(contentPath(source.id), { + // HEIC has to go through ImageMagick first — sharp's bundled libheif cannot decode it. A no-op + // for every other format. See decodable.ts. + const decoded = await decodableInput(contentPath(source.id), source.format); + + let image = sharp(decoded.path, { ...inputOptions, limitInputPixels: MAX_INPUT_PIXELS, + // `unlimited` is what gets past libheif's iref-reference cap, which iPhone HEICs carrying + // burst or depth data routinely exceed — sharp reports it as "corrupt header", which it is not. + // It loosens libvips' own buffer guards, so `limitInputPixels` is passed EXPLICITLY alongside: + // verified that the pixel limit still rejects a decompression bomb with unlimited set. + unlimited: true, }); // ── The pipeline ───────────────────────────────────────────────────────────────────────────── @@ -123,6 +133,7 @@ export async function runSharp(body: SharpRequest): Promise { try { buffer = await encode.call(image, encoderOptions).toBuffer(); } catch (error) { + await decoded.cleanup(); // A failure here is usually the source file, not the request — a truncated JPEG, a PNG whose // declared dimensions disagree with its data. throw unprocessable( @@ -130,5 +141,6 @@ export async function runSharp(body: SharpRequest): Promise { ); } + await decoded.cleanup(); return putAsset(buffer); }