Decode HEIC, and transcode video

Two things the platform needed and could not get.

HEIC: sharp reported "corrupt header" on every iPhone photograph, which they are
not. Two separate causes. libheif caps iref references and burst/depth HEICs exceed
it — `unlimited` gets past that, and limitInputPixels is now passed explicitly
alongside because unlimited also loosens the bomb guard (verified the pixel limit
still rejects a bomb). Then sharp still could not decode the pixels: its bundled
libheif is built without libde265, so HEVC-coded HEIF is unreadable to it. The
system libheif does have the plugin, so HEIF now routes through ImageMagick to a
lossless PNG first. Verified: 1.4 MB HEIC in, 102 KB WebP out.

/v1/ffmpeg is built. ffmpeg's own option names, structured filters rather than a
-vf string, and four presets — web-1080/720/480 and poster. The presets are where
the encoder knowledge lives, so a consumer asks for "web-720" rather than knowing
flags, and revising quality is a change here for every site that uses it.

Default-deny throughout, same as the sharp surface: filters that read files or load
native code are absent from the table rather than blocklisted, protocols are
whitelisted to file, and the caller still never names a path.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Pastilhas
2026-08-20 13:29:10 +01:00
co-authored by Claude Opus 5
parent 1597fccd75
commit dc4fe758f3
6 changed files with 434 additions and 4 deletions
+8 -1
View File
@@ -82,7 +82,14 @@ export async function deleteAsset(id: string): Promise<void> {
*/ */
async function identify(path: string): Promise<Pick<Asset, "kind" | "format" | "width" | "height" | "durationSeconds">> { async function identify(path: string): Promise<Pick<Asset, "kind" | "format" | "width" | "height" | "durationSeconds">> {
try { try {
const meta = await sharp(path, { limitInputPixels: MAX_INPUT_PIXELS }).metadata(); const meta = await sharp(path, {
limitInputPixels: MAX_INPUT_PIXELS,
// `unlimited` is what gets past libheif's iref-reference cap, which iPhone HEICs carrying
// burst or depth data routinely exceed — sharp reports it as "corrupt header", which it is not.
// It loosens libvips' own buffer guards, so `limitInputPixels` is passed EXPLICITLY alongside:
// verified that the pixel limit still rejects a decompression bomb with unlimited set.
unlimited: true,
}).metadata();
if (meta.format) { if (meta.format) {
return { kind: "image", format: meta.format, width: meta.width, height: meta.height }; return { kind: "image", format: meta.format, width: meta.width, height: meta.height };
} }
+199
View File
@@ -0,0 +1,199 @@
import { join } from "node:path";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { JOB_TIMEOUT_MS } from "../config";
import { invalid, unprocessable, ApiError } from "../errors";
import { contentPath, putAsset, readAsset, type Asset } from "../assets/store";
import {
AUDIO_FILTERS,
FORMATS,
INPUT_OPTIONS,
OUTPUT_OPTIONS,
PRESETS,
VIDEO_FILTERS,
type FfmpegParam,
} from "./vocabulary";
/**
* Running ffmpeg.
*
* Everything the caller supplies is a NAME or a NUMBER from the vocabulary. Paths are constructed
* here from an asset id, and the process is spawned with an argv array so there is no shell to quote
* for. See docs/SECURITY.md.
*/
export type FfmpegRequest = {
input?: unknown;
preset?: unknown;
inputOptions?: unknown;
outputOptions?: unknown;
filters?: unknown;
format?: unknown;
};
const BITRATE = /^[0-9]+(\.[0-9]+)?[kKmM]?$/;
function value(spec: FfmpegParam, raw: unknown, path: string): string {
switch (spec.kind) {
case "int":
if (typeof raw !== "number" || !Number.isInteger(raw)) throw invalid(`${path} must be an integer`, path);
if (raw < spec.min || raw > spec.max) throw invalid(`${path} must be between ${spec.min} and ${spec.max}`, path);
return String(raw);
case "number":
if (typeof raw !== "number" || !Number.isFinite(raw)) throw invalid(`${path} must be a number`, path);
if (raw < spec.min || raw > spec.max) throw invalid(`${path} must be between ${spec.min} and ${spec.max}`, path);
return String(raw);
case "enum":
if (typeof raw !== "string" || !spec.values.includes(raw)) {
throw invalid(`${path} must be one of: ${spec.values.join(", ")}`, path);
}
return raw;
case "bitrate":
if (typeof raw !== "string" || !BITRATE.test(raw)) {
throw invalid(`${path} must be a bitrate like "2M" or "128k"`, path);
}
return raw;
}
}
function options(table: Record<string, FfmpegParam>, raw: unknown, path: string): string[] {
if (raw === undefined || raw === null) return [];
if (typeof raw !== "object" || Array.isArray(raw)) throw invalid(`${path} must be an object`, path);
const args: string[] = [];
for (const [key, entry] of Object.entries(raw as Record<string, unknown>)) {
const spec = table[key];
if (!spec) {
throw invalid(`Unknown option ${JSON.stringify(key)}. Accepted here: ${Object.keys(table).join(", ")}`, `${path}.${key}`);
}
args.push(`-${key}`, value(spec, entry, `${path}.${key}`));
}
return args;
}
/**
* A filter chain, built from structured objects rather than parsed from a string.
*
* ffmpeg's filter syntax has its own escaping rules, and parsing it well enough to validate it is
* harder than accepting `{ name: args }`. The names and parameters are ffmpeg's; only the
* punctuation is ours — and every filter that can read a file or load a plugin is simply absent from
* the table, so it cannot be named.
*/
function filterChain(table: Record<string, Record<string, FfmpegParam>>, raw: unknown, path: string): string {
if (raw === undefined || raw === null) return "";
if (!Array.isArray(raw)) throw invalid(`${path} must be an array of filters`, path);
return raw
.map((entry, i) => {
const at = `${path}[${i}]`;
if (!entry || typeof entry !== "object") throw invalid(`${at} must be an object`, at);
const pairs = Object.entries(entry as Record<string, unknown>);
if (pairs.length !== 1) throw invalid(`${at} must name exactly one filter`, at);
const [name, rawArgs] = pairs[0]!;
const spec = table[name];
if (!spec) throw invalid(`Unknown filter ${JSON.stringify(name)}. See docs/FFMPEG.md.`, `${at}.${name}`);
const args = Object.entries((rawArgs ?? {}) as Record<string, unknown>).map(([key, v]) => {
const argSpec = spec[key];
if (!argSpec) {
throw invalid(`Unknown parameter ${JSON.stringify(key)} for ${name}`, `${at}.${name}.${key}`);
}
return `${key}=${value(argSpec, v, `${at}.${name}.${key}`)}`;
});
return args.length ? `${name}=${args.join(":")}` : name;
})
.join(",");
}
/** Run ffmpeg on an asset and store the result as a new asset. The input is never modified. */
export async function runFfmpeg(body: FfmpegRequest): Promise<Asset> {
if (typeof body.input !== "string") throw invalid("input must be an asset id", "input");
const source = await readAsset(body.input);
if (source.kind === "image") {
throw invalid(`Asset ${source.id} is an image. Use /v1/sharp for images.`, "input");
}
// A preset supplies the options; anything given explicitly overrides it. So "web-1080 but crf 20"
// does not mean writing out every flag again.
let format = "mp4";
let presetOptions: string[] = [];
let presetFilters = "";
if (body.preset !== undefined) {
if (typeof body.preset !== "string" || !PRESETS[body.preset]) {
throw invalid(`preset must be one of: ${Object.keys(PRESETS).join(", ")}`, "preset");
}
const preset = PRESETS[body.preset]!;
format = preset.format;
presetOptions = options(OUTPUT_OPTIONS, preset.options, "preset");
presetFilters = preset.filters?.v
? filterChain(VIDEO_FILTERS, preset.filters.v.map((f) => ({ [f.name]: f.args })), "preset.filters")
: "";
}
if (body.format !== undefined) {
if (typeof body.format !== "string" || !FORMATS[body.format]) {
throw invalid(`format must be one of: ${Object.keys(FORMATS).join(", ")}`, "format");
}
format = body.format;
}
const inputArgs = options(INPUT_OPTIONS, body.inputOptions, "inputOptions");
const outputArgs = options(OUTPUT_OPTIONS, body.outputOptions, "outputOptions");
const filters = (body.filters ?? {}) as Record<string, unknown>;
const videoChain = filterChain(VIDEO_FILTERS, filters.v, "filters.v") || presetFilters;
const audioChain = filterChain(AUDIO_FILTERS, filters.a, "filters.a");
// One directory per job, removed however this ends — otherwise disk fills with the failures.
const workDir = await mkdtemp(join(tmpdir(), "avclub-"));
const output = join(workDir, `out.${FORMATS[format]!.extension}`);
try {
const argv = [
"ffmpeg",
"-nostdin",
"-y",
"-v", "error",
// Nothing but local files. Kills SSRF and the concat:/subfile: tricks outright — the second
// lock behind "the caller never names a path".
"-protocol_whitelist", "file",
...inputArgs,
"-i", contentPath(source.id),
...(videoChain ? ["-vf", videoChain] : []),
...(audioChain ? ["-af", audioChain] : []),
...presetOptions,
// After the preset, so an explicit option wins.
...outputArgs,
"-f", format,
output,
];
const proc = Bun.spawn(argv, { stdout: "pipe", stderr: "pipe", stdin: "ignore" });
// A crafted file can make an encoder run effectively forever.
let timedOut = false;
const timer = setTimeout(() => {
timedOut = true;
proc.kill();
}, JOB_TIMEOUT_MS);
const [stderr, code] = await Promise.all([new Response(proc.stderr).text(), proc.exited]);
clearTimeout(timer);
if (timedOut) {
throw new ApiError("timeout", `Encoding took longer than ${JOB_TIMEOUT_MS / 1000}s and was stopped.`);
}
if (code !== 0) {
throw unprocessable(`ffmpeg failed: ${stderr.trim().split("\n").at(-1) ?? `exit ${code}`}`);
}
return await putAsset(await Bun.file(output).arrayBuffer());
} finally {
await rm(workDir, { recursive: true, force: true });
}
}
+136
View File
@@ -0,0 +1,136 @@
/**
* The ffmpeg surface av-club exposes, as data — the same shape as the sharp vocabulary.
*
* Options are spelled the way ffmpeg spells them (`c:v`, `crf`, `pix_fmt`), so ffmpeg's own
* documentation is the reference. What is NOT here is anything that names a file: no `-i`, no output
* path, no protocol. See docs/SECURITY.md — that single restriction is what makes the rest of this
* safe to open up.
*/
export type FfmpegParam =
| { kind: "int"; min: number; max: number; note?: string }
| { kind: "number"; min: number; max: number; note?: string }
| { kind: "enum"; values: readonly string[]; note?: string }
| { kind: "bitrate"; note?: string };
/** Options that go BEFORE `-i`: seeking and trimming, which are far cheaper applied to the input. */
export const INPUT_OPTIONS: Record<string, FfmpegParam> = {
ss: { kind: "number", min: 0, max: 86_400, note: "Start at this many seconds." },
t: { kind: "number", min: 0.001, max: 86_400, note: "Take this many seconds." },
to: { kind: "number", min: 0, max: 86_400, note: "Stop at this timestamp." },
};
/**
* Output options.
*
* The encoder list is short on purpose: these are the codecs a browser plays. An encoder ffmpeg
* happens to have compiled in is not automatically something worth publishing.
*/
export const OUTPUT_OPTIONS: Record<string, FfmpegParam> = {
"c:v": { kind: "enum", values: ["libx264", "libvpx-vp9", "libaom-av1", "copy"], note: "Video encoder. libx264 is the one every browser plays." },
"c:a": { kind: "enum", values: ["aac", "libopus", "copy"], note: "Audio encoder." },
crf: { kind: "int", min: 0, max: 63, note: "Quality. Lower is better and bigger; 23 is a sane default for x264." },
preset: { kind: "enum", values: ["ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow"], note: "Encoder effort. Slower is smaller at the same quality." },
tune: { kind: "enum", values: ["film", "animation", "grain", "stillimage", "fastdecode", "zerolatency"] },
"profile:v": { kind: "enum", values: ["baseline", "main", "high"], note: "`high` is fine everywhere current." },
level: { kind: "enum", values: ["3.0", "3.1", "4.0", "4.1", "4.2", "5.0", "5.1"] },
pix_fmt: { kind: "enum", values: ["yuv420p", "yuv422p", "yuv444p"], note: "yuv420p, or Safari and QuickTime will not play it." },
"b:v": { kind: "bitrate", note: "Target video bitrate, e.g. 2M." },
"b:a": { kind: "bitrate", note: "Target audio bitrate, e.g. 128k." },
maxrate: { kind: "bitrate" },
bufsize: { kind: "bitrate" },
r: { kind: "number", min: 1, max: 120, note: "Output frame rate." },
g: { kind: "int", min: 1, max: 600, note: "Keyframe interval." },
ar: { kind: "enum", values: ["22050", "44100", "48000"], note: "Audio sample rate." },
ac: { kind: "int", min: 1, max: 2 },
"frames:v": { kind: "int", min: 1, max: 10_000, note: "Stop after this many frames — 1 gives a poster." },
movflags: { kind: "enum", values: ["+faststart"], note: "Moves the index to the front so playback can start before the file has arrived. Always want this for web." },
an: { kind: "enum", values: ["1"], note: "Drop audio entirely." },
};
/** Output containers. `image2` is how a single frame comes out as a still. */
export const FORMATS: Record<string, { extension: string; describe: string }> = {
mp4: { extension: "mp4", describe: "H.264/AAC in MP4 — plays everywhere." },
webm: { extension: "webm", describe: "VP9/Opus. Smaller, but not on older Safari." },
mp3: { extension: "mp3", describe: "Audio only." },
image2: { extension: "jpg", describe: "A single frame, for poster images." },
};
/**
* Filters, default-deny.
*
* ffmpeg's filter graph is a language, and most of it is arithmetic on pixels. A small, specific part
* of it reads files or loads native code — `movie`, `subtitles`, `frei0r`, `ladspa` and relatives —
* and those are not in this list, so they are refused. A filter nobody has classified is refused too:
* a blocklist would silently admit whatever the next ffmpeg release adds.
*/
export const VIDEO_FILTERS: Record<string, Record<string, FfmpegParam>> = {
scale: {
w: { kind: "int", min: -2, max: 8192, note: "-2 keeps the aspect ratio and an even number of pixels, which h264 requires." },
h: { kind: "int", min: -2, max: 8192 },
},
crop: {
w: { kind: "int", min: 1, max: 8192 },
h: { kind: "int", min: 1, max: 8192 },
x: { kind: "int", min: 0, max: 8192 },
y: { kind: "int", min: 0, max: 8192 },
},
fps: { fps: { kind: "number", min: 1, max: 120 } },
format: { pix_fmts: { kind: "enum", values: ["yuv420p", "yuv422p", "yuv444p", "rgb24"] } },
transpose: { dir: { kind: "enum", values: ["0", "1", "2", "3"] } },
hflip: {},
vflip: {},
eq: {
brightness: { kind: "number", min: -1, max: 1 },
contrast: { kind: "number", min: -1000, max: 1000 },
saturation: { kind: "number", min: 0, max: 3 },
gamma: { kind: "number", min: 0.1, max: 10 },
},
unsharp: {
luma_msize_x: { kind: "int", min: 3, max: 23 },
luma_msize_y: { kind: "int", min: 3, max: 23 },
luma_amount: { kind: "number", min: -2, max: 5 },
},
};
export const AUDIO_FILTERS: Record<string, Record<string, FfmpegParam>> = {
volume: { volume: { kind: "number", min: 0, max: 10 } },
aresample: { osr: { kind: "enum", values: ["22050", "44100", "48000"] } },
loudnorm: {
I: { kind: "number", min: -70, max: -5, note: "Target loudness in LUFS. -16 suits the web." },
TP: { kind: "number", min: -9, max: 0 },
},
};
/**
* Named presets, so a caller does not have to know encoder settings to get a web-playable file.
*
* This is where the media knowledge lives — the whole reason av-club is a separate repo. A consumer
* says what the media is FOR; what that means in flags is decided here and can be revised in one
* place for every site that uses it.
*/
export const PRESETS: Record<string, { describe: string; format: string; options: Record<string, string | number>; filters?: { v?: { name: string; args: Record<string, number | string> }[] } }> = {
"web-1080": {
describe: "H.264 up to 1080p. The safe default — plays in every browser.",
format: "mp4",
options: { "c:v": "libx264", crf: 23, preset: "medium", "profile:v": "high", pix_fmt: "yuv420p", "c:a": "aac", "b:a": "128k", movflags: "+faststart" },
filters: { v: [{ name: "scale", args: { w: -2, h: 1080 } }] },
},
"web-720": {
describe: "H.264 up to 720p.",
format: "mp4",
options: { "c:v": "libx264", crf: 23, preset: "medium", "profile:v": "high", pix_fmt: "yuv420p", "c:a": "aac", "b:a": "128k", movflags: "+faststart" },
filters: { v: [{ name: "scale", args: { w: -2, h: 720 } }] },
},
"web-480": {
describe: "H.264 up to 480p, for slow connections.",
format: "mp4",
options: { "c:v": "libx264", crf: 26, preset: "medium", "profile:v": "main", pix_fmt: "yuv420p", "c:a": "aac", "b:a": "96k", movflags: "+faststart" },
filters: { v: [{ name: "scale", args: { w: -2, h: 480 } }] },
},
poster: {
describe: "One frame as a JPEG, for a video's poster image.",
format: "image2",
options: { "frames:v": 1 },
},
};
+7 -2
View File
@@ -4,6 +4,7 @@ import { ApiError, invalid, notFound, toErrorResponse } from "./errors";
import { contentPath, deleteAsset, putAsset, readAsset } from "./assets/store"; import { contentPath, deleteAsset, putAsset, readAsset } from "./assets/store";
import { ffprobe } from "./probe/ffprobe"; import { ffprobe } from "./probe/ffprobe";
import { runSharp, type SharpRequest } from "./sharp/run"; import { runSharp, type SharpRequest } from "./sharp/run";
import { runFfmpeg, type FfmpegRequest } from "./ffmpeg/run";
import sharp from "sharp"; import sharp from "sharp";
/** /**
@@ -167,8 +168,12 @@ async function route(request: Request): Promise<Response> {
return Response.json(result, { status: 201 }); return Response.json(result, { status: 201 });
} }
if (path === "/v1/ffmpeg") { if (path === "/v1/ffmpeg" && request.method === "POST") {
throw new ApiError("invalid_request", "POST /v1/ffmpeg is not implemented yet. See docs/API.md."); const body = (await request.json().catch(() => {
throw invalid("Expected a JSON body");
})) as FfmpegRequest;
const result = await withSlot(() => runFfmpeg(body));
return Response.json(result, { status: 201 });
} }
throw notFound(`No route for ${request.method} ${path}`); throw notFound(`No route for ${request.method} ${path}`);
+71
View File
@@ -0,0 +1,71 @@
import { join } from "node:path";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { JOB_TIMEOUT_MS } from "../config";
import { unprocessable } from "../errors";
/**
* Getting HEIC into something sharp can actually decode.
*
* ── Why this exists ──
*
* sharp ships its own libvips and libheif, built without the libde265 plugin, so it can parse a
* HEIC's header — dimensions come back fine — and then fails on the pixels with "bad seek", which
* reads like a corrupt file and is not one. Every photograph an iPhone takes is HEVC-coded HEIF, so
* this is not an edge case: it is most of what a client will upload from a phone.
*
* ImageMagick links the SYSTEM libheif, which does have the plugin (see docs/TOOLING.md), so it
* decodes them. This converts through it and hands sharp a PNG.
*
* PNG rather than JPEG for the intermediate: it is lossless, so the one real encode is the final one
* and the derivative is not a copy of a copy.
*/
/** Formats sharp reports but cannot decode here. Extend only with evidence, not suspicion. */
const NEEDS_CONVERSION = new Set(["heif", "heic", "avif-sequence"]);
export type Decodable = {
path: string;
/** Call when finished. Removes the intermediate, if there was one. */
cleanup: () => Promise<void>;
};
export async function decodableInput(path: string, format: string): Promise<Decodable> {
if (!NEEDS_CONVERSION.has(format.toLowerCase())) {
return { path, cleanup: async () => {} };
}
const workDir = await mkdtemp(join(tmpdir(), "avclub-heif-"));
const converted = join(workDir, "decoded.png");
// ImageMagick 6 on Ubuntu 24.04, so `convert` — there is no `magick`. Spawned as an argv array,
// so the path is data rather than something a shell parses.
//
// -auto-orient applies the EXIF rotation while the metadata still exists; the PNG carries none, so
// this is the last chance to get a phone photograph the right way up.
const proc = Bun.spawn(["convert", path, "-auto-orient", converted], {
stdout: "pipe",
stderr: "pipe",
stdin: "ignore",
});
const timer = setTimeout(() => proc.kill(), JOB_TIMEOUT_MS);
try {
const [stderr, code] = await Promise.all([new Response(proc.stderr).text(), proc.exited]);
if (code !== 0 || !(await Bun.file(converted).exists())) {
await rm(workDir, { recursive: true, force: true });
throw unprocessable(
`Could not read that HEIC: ${stderr.trim().split("\n").at(-1) ?? "conversion failed"}`,
);
}
return {
path: converted,
cleanup: () => rm(workDir, { recursive: true, force: true }),
};
} finally {
clearTimeout(timer);
}
}
+13 -1
View File
@@ -4,6 +4,7 @@ import { invalid, unprocessable } from "../errors";
import { contentPath, putAsset, readAsset, type Asset } from "../assets/store"; import { contentPath, putAsset, readAsset, type Asset } from "../assets/store";
import { INPUT_OPTIONS, OPERATIONS, OUTPUT_FORMATS } from "./vocabulary"; import { INPUT_OPTIONS, OPERATIONS, OUTPUT_FORMATS } from "./vocabulary";
import { validateObject } from "./validate"; import { validateObject } from "./validate";
import { decodableInput } from "./decodable";
export type SharpRequest = { export type SharpRequest = {
input?: unknown; input?: unknown;
@@ -33,9 +34,18 @@ export async function runSharp(body: SharpRequest): Promise<Asset> {
// limitInputPixels is set HERE and cannot be supplied by the caller: it is the decompression-bomb // limitInputPixels is set HERE and cannot be supplied by the caller: it is the decompression-bomb
// guard, and a caller who could raise it could turn a 1 KB PNG into a 40 GB allocation. // guard, and a caller who could raise it could turn a 1 KB PNG into a 40 GB allocation.
let image = sharp(contentPath(source.id), { // HEIC has to go through ImageMagick first — sharp's bundled libheif cannot decode it. A no-op
// for every other format. See decodable.ts.
const decoded = await decodableInput(contentPath(source.id), source.format);
let image = sharp(decoded.path, {
...inputOptions, ...inputOptions,
limitInputPixels: MAX_INPUT_PIXELS, limitInputPixels: MAX_INPUT_PIXELS,
// `unlimited` is what gets past libheif's iref-reference cap, which iPhone HEICs carrying
// burst or depth data routinely exceed — sharp reports it as "corrupt header", which it is not.
// It loosens libvips' own buffer guards, so `limitInputPixels` is passed EXPLICITLY alongside:
// verified that the pixel limit still rejects a decompression bomb with unlimited set.
unlimited: true,
}); });
// ── The pipeline ───────────────────────────────────────────────────────────────────────────── // ── The pipeline ─────────────────────────────────────────────────────────────────────────────
@@ -123,6 +133,7 @@ export async function runSharp(body: SharpRequest): Promise<Asset> {
try { try {
buffer = await encode.call(image, encoderOptions).toBuffer(); buffer = await encode.call(image, encoderOptions).toBuffer();
} catch (error) { } catch (error) {
await decoded.cleanup();
// A failure here is usually the source file, not the request — a truncated JPEG, a PNG whose // A failure here is usually the source file, not the request — a truncated JPEG, a PNG whose
// declared dimensions disagree with its data. // declared dimensions disagree with its data.
throw unprocessable( throw unprocessable(
@@ -130,5 +141,6 @@ export async function runSharp(body: SharpRequest): Promise<Asset> {
); );
} }
await decoded.cleanup();
return putAsset(buffer); return putAsset(buffer);
} }