fofos-admin/api/derive.ts

import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
import { dirname, join, relative } from 'node:path';
import { sidecarPort } from '@@/sidecar-registry';
import { mediaPath } from './media';

/**
 * Building `derived/` from `originals/`.
 *
 * ── The mapping ──
 *
 * `derived/` mirrors `originals/` directory for directory. Where originals holds a FILE, derived holds
 * a DIRECTORY of the same name — extension included — and the renditions live inside it:
 *
 *   originals/homepage_galeria01.jpg           →  derived/homepage_galeria01.jpg/320.webp
 *                                                 derived/homepage_galeria01.jpg/640.webp  …
 *   originals/ABOUT US/portrait.HEIC           →  derived/ABOUT US/portrait.HEIC/…
 *   originals/homepage_video.mp4               →  derived/homepage_video.mp4/1080.mp4
 *                                                 derived/homepage_video.mp4/poster.webp
 *
 * Keeping the extension is what makes the mapping total: `a.jpg` and `a.png` are different originals
 * and must not share a directory. It also makes the inverse pure string work — no index, no table, no
 * second copy of the truth. **The filesystem is the record.**
 *
 * ── The ladder is fixed here, and the SITE chooses from it ──
 *
 * The widths below cover every slot fofos has at up to 3× pixel density — `website/src/media/slots.ts`
 * puts the largest at 1232 CSS px, so `minUpload` is 2464 and 2560 is the top rung. But this file does
 * not read that one, and must not: a plugin that knows a website's layout is a plugin for one website.
 * It emits a ladder; `srcset` and the `sizes` already written in `website/src/media/render.ts` do the
 * choosing, because the site is the only thing that knows what space an image occupies.
 *
 * That boundary is the difference between this being a template and being fofos-shaped.
 *
 * ── Nothing is ever upscaled ──
 *
 * A rung wider than the original is not written. Inventing a 2560 from a 900px photograph produces a
 * soft file that `srcset` would then confidently prefer over the sharp one — worse than not having it.
 *
 * ── Video rungs are named by height, images by width ──
 *
 * Deliberate, not an oversight. `srcset` descriptors are widths, so an image file called `1280` has to
 * be 1280 pixels wide or the browser is being lied to. Video has no such mechanism, and every person
 * reading `1080.mp4` knows what it means. Each unit matches its consumer.
 *
 * ── Every byte is written by av-club ──
 *
 * This module decides WHAT should exist and av-club decides how to make it. The video presets are its
 * `web-1080` / `web-720` / `poster`, by name — the encoder flags behind them are its business, and
 * revising them for every site at once is the reason it is a separate service.
 */

/**
 * Image rungs, in pixels of width. Rungs wider than the original are skipped — see above.
 *
 * 96, 128, 160 and 256 are the FLOOR, added because there was none. A logo renders at 80 CSS pixels
 * — 140 device pixels on an ordinary phone — and the
 * smallest rung was 320, so the ladder had nothing to offer it: `fofos-logo-black` came out with
 * one 12,350-byte rung against a 12,678-byte original, a ladder that saved 328 bytes. Anything shown
 * smaller than a thumbnail — a logo, an icon, an avatar — lives down here.
 *
 * 480 was added on 2026-08-29 because the gap between 320 and 640 is where a phone lands. Lighthouse
 * measured the home gallery's slots at 454 device pixels and the browser, having only those two rungs
 * to choose from, correctly took 640 — a 184 KB file for a 454-pixel slot, and the same story on
 * every image in that carousel.
 *
 * A rung is cheap: it is one more WebP beside files that already exist, chosen only when it fits
 * better than its neighbours. The cost of NOT having one is paid on every visit.
 */
export const LADDER = [96, 128, 160, 256, 320, 480, 640, 960, 1280, 1920, 2560] as const;

/**
 * Square rungs, cropped rather than scaled — `320sq.webp`, and so on.
 *
 * ── Why a second, differently shaped ladder ──
 *
 * A gallery grid is square slots. The ladder above is the picture's own shape, so a 480x720 portrait
 * arrives, `object-cover` crops it to 480x480 in the browser, and a third of every file that was
 * downloaded is discarded before anyone sees it. Measured on this gallery: 108,388 bytes shipped where
 * a 480x480 crop is 70,070.
 *
 * Cropping in the browser is the right default when the shape is unknown. It is the wrong one when the
 * shape is known at build time, and for a carousel of square thumbnails it always is.
 *
 * ── The rungs are chosen against real slots and real screens ──
 *
 * A gallery slot measures 193 CSS pixels on a phone and about 475 on a wide desktop, so the widths
 * that matter are those times the pixel density somebody is actually holding:
 *
 *   193 x 1     = 193   ->  256
 *   193 x 1.75  = 338   ->  384      (Lighthouse's emulated phone, and most Android)
 *   193 x 3     = 579   ->  640      (a dense phone)
 *   475 x 1     = 475   ->  480      (desktop)
 *   475 x 2     = 950   ->  960      (retina desktop)
 *
 * It was 320, 480, 640, and the middle of that is the problem: a phone needing 338 could not have it,
 * so the browser correctly took 480 and downloaded 1.4x the pixels it could use. A rung is a few
 * kilobytes of square WebP; a missing rung is paid for on every visit by everyone whose screen falls
 * in the gap.
 *
 * 960 is here for the opposite reason — the ladder stopped at 640 and a retina desktop wanting 950 had
 * nothing, so it took 640 and showed it soft.
 *
 * Above this, a picture is being looked at rather than browsed, which is what the aspect ladder and the
 * lightbox are for. A square crop is lost information and only makes sense where the slot demands it.
 *
 * ── Centre, always ──
 *
 * sharp offers `attention`, which keeps the region it judges busiest. It is rejected deliberately: it
 * decides differently for every picture, so a row of thumbnails framed by an algorithm is a row with
 * no common logic to it, and the one it gets wrong is impossible to predict or explain. Centre is what
 * a photographer framing for a square already assumes, and it is the same answer every time.
 *
 * It is a DEFAULT. A placement that sets its own crop keeps the aspect file and frames it in CSS —
 * see how the site chooses in src/media/render.ts. Nothing here knows about placements, and should not.
 */
export const SQUARE_LADDER = [128, 192, 256, 384, 480, 640, 960] as const;

/**
 * Poster rungs, in pixels of width — `poster-768.webp`, and so on.
 *
 * A video's poster was one file at the source's full size: 1280x720 for a hero displayed 412 CSS
 * pixels wide. That is three times the pixels a phone can show, on the element that IS the page's
 * Largest Contentful Paint — the most expensive place on the site to be careless.
 *
 * A hero is full-bleed, so the widths are viewport widths times the density behind them:
 *
 *   412 x 1     =  412  ->  448      (a phone, at the density the hero is CAPPED to — see below)
 *   768 x 1     =  768  ->  768      (a tablet)
 *   1280 x 1    = 1280  -> 1280      (a desktop)
 *
 * The hero declares a `sizes` well under its real width on phones, deliberately, so the browser stops
 * multiplying by the device pixel ratio and asks for roughly one image pixel per screen pixel. 448 is
 * the rung that lands on a 412-wide phone once that cap is applied. Without it the nearest was 512.
 *
 * `poster.webp` keeps its name and stays the widest rung. Everything already refers to it — the
 * placeholder is cut from it, the site falls back to it, and a directory built before this has only
 * that one.
 */
export const POSTER_LADDER = [448, 512, 768, 1024] as const;

/**
 * WebP quality, by rung.
 *
 * 82 is the usual knee for a photograph being LOOKED at — the lightbox, a detail page, anything at or
 * near its real size. It was applied to every rung, which meant a thumbnail 173 CSS pixels wide in a
 * gallery grid carried the same fidelity as the copy someone opens full screen.
 *
 * Artefacts scale with display size. At thumbnail size 74 is indistinguishable from 82 and 15% smaller,
 * measured on the worst image in this gallery: 108,388 bytes to 92,568. Nine of those on a page is
 * most of what an audit calls "increasing the compression factor".
 *
 * 640 is the line because it is the widest rung a phone ever picks for a grid slot. Anything above it
 * is being chosen because something is showing the picture large.
 */
const WEBP_QUALITY_SMALL = 74;
const WEBP_QUALITY = 82;

/*
  A poster is a FIRST PAINT, not a photograph, and is encoded accordingly.

  `WEBP_QUALITY` is described below as what a rung is worth, and 82 is reserved for the file somebody
  actually opens and looks at. Nobody opens a poster: it fills the frame for about a second and is then
  covered by the video it belongs to. Paying photograph quality for that is paying for a second that
  ends.

  Measured on the same pixels with the same encoder, at 768 wide: 22,906 bytes at 82, 17,738 at 74 —
  23% of the LARGEST CONTENTFUL PAINT, for a difference nothing is on screen long enough to show.

  Not lower than the small rungs use. It IS the LCP, so it is the one image whose first impression is
  the page's, and 74 is the quality every thumbnail on the site already ships at.
*/
const WEBP_QUALITY_POSTER = WEBP_QUALITY_SMALL;
const THUMB_MAX_WIDTH = 640;

/**
 * How far from square still counts as square.
 *
 * 2% of the longer side. A 1000x1002 photograph is square for every purpose this serves, and building
 * a parallel ladder to crop two pixels off it is work nobody benefits from.
 */
const SQUARE_TOLERANCE = 0.02;

/** Where the drawing-or-photograph question is asked. See prefersLossless for why not smaller. */
const PROBE_WIDTH = 480;

/**
 * Is this a DRAWING rather than a photograph?
 *
 * ── Why it has to be asked at all ──
 *
 * Lossy WebP is built for photographs: it throws away detail the eye does not miss in a picture full
 * of texture and noise. A logo is the opposite — flat areas, hard edges, a handful of colours — and
 * lossy encoding is worse than useless on it twice over. It softens the edges, and it makes the file
 * BIGGER, because smooth flat regions are exactly what lossless compression is good at and exactly
 * what lossy encoding turns into gradients it then has to describe.
 *
 * Measured here: `fofos-logo-saffron.png` is 300x300 and became a 22,432-byte 256x256 rung beside a
 * 9,144-byte full-size one. A smaller picture, two and a half times the file.
 *
 * ── Measured, not guessed ──
 *
 * The obvious approach is a heuristic — the file extension, an alpha channel, a colour count — and
 * every one of them is wrong somewhere. A PNG can hold a photograph, a JPEG can hold a diagram, and
 * plenty of logos are opaque.
 *
 * So it encodes the smallest rung BOTH ways and keeps the answer. One extra encode of a 160-pixel
 * image per original, and what comes out is not an opinion about the file: it is the file telling us
 * which encoder suits it. The decision is then applied to every rung, because an image does not stop
 * being a drawing at 640 pixels.
 *
 * Ties and failures go to lossy, which is the right default for a library that is mostly photographs.
 */
async function prefersLossless(
  sourcePath: string,
  staging: string,
  email: string,
  sourceWidth: number,
): Promise<boolean> {
  /*
    Measured near the source's own size, NOT at the smallest rung.

    The first attempt probed at 96 pixels and got the answer wrong on exactly the file it was written
    for. Downscaling a logo to 96 resamples its flat areas into gradients, so by the time the probe
    sees it, it already looks like a photograph — lossy wins the comparison, and that verdict is then
    applied to every rung above, where the flat areas are still flat and lossless would have won by a
    wide margin. fofos-logo-saffron came out with a 13,074-byte 160-pixel rung beside a 9,144-byte
    300-pixel original.

    480 rather than the full size: a picture is still recognisably itself at 480 — flat stays flat and
    photographic stays photographic — and probing a 4000x6000 photograph losslessly to learn something
    already obvious is minutes of encoding across a library. Anything smaller than 480 is probed at its
    own size, which is the case that matters, because that is what a logo is.
  */
  const probe = Math.min(PROBE_WIDTH, sourceWidth);
  const lossyPath = join(staging, '.probe-lossy.webp');
  const losslessPath = join(staging, '.probe-lossless.webp');

  try {
    const [lossy, lossless] = await Promise.all([
      post<{ bytes: number }>('/v1/sharp', email, {
        inputPath: sourcePath,
        outputPath: lossyPath,
        pipeline: [{ resize: { width: probe, fit: 'inside', withoutEnlargement: true } }],
        output: { format: 'webp', options: { quality: WEBP_QUALITY_SMALL } },
      }),
      post<{ bytes: number }>('/v1/sharp', email, {
        inputPath: sourcePath,
        outputPath: losslessPath,
        pipeline: [{ resize: { width: probe, fit: 'inside', withoutEnlargement: true } }],
        output: { format: 'webp', options: { lossless: true } },
      }),
    ]);

    return lossless.bytes < lossy.bytes;
  } catch {
    // A probe that could not run is not a reason to refuse the ladder. Lossy is the right default.
    return false;
  } finally {
    await rm(lossyPath, { force: true }).catch(() => {});
    await rm(losslessPath, { force: true }).catch(() => {});
  }
}

/** The quality a rung is worth. `null` is `full.webp`, which is the one people open. */
const qualityFor = (rung: number | null): number =>
  rung !== null && rung <= THUMB_MAX_WIDTH ? WEBP_QUALITY_SMALL : WEBP_QUALITY;

/**
 * The WebP settings for one rung, given what the picture turned out to be.
 *
 * `lossless` and `quality` are not combined: sharp ignores quality in lossless mode, so passing both
 * would read as a setting that does something and does not.
 */
const webpOptions = (rung: number | null, lossless: boolean): Record<string, unknown> =>
  lossless ? { lossless: true } : { quality: qualityFor(rung) };

/**
 * How to resample, given what the picture is.
 *
 * ── Why a drawing needs a different filter ──
 *
 * A smooth kernel interpolates. On a photograph that is exactly right; on a drawing it is destructive
 * in a way that is easy to miss, because the damage shows up as SIZE rather than as blur. A logo with
 * 174 flat colours downscaled with lanczos3 comes out with thousands of interpolated ones, and
 * lossless compression — which was winning until that moment — then has to encode every one of them.
 *
 * Measured on fofos-logo-saffron at 160 pixels, all lossless:
 *
 *   lanczos3   13,356        mitchell + nearLossless   7,982
 *   mitchell    9,854        NEAREST                   3,436
 *
 * The whole 300-pixel original is 8,886. Every smooth option produced a 160-pixel file heavier than
 * the full-size one; nearest produced one a third of its size.
 *
 * ── The cost, stated ──
 *
 * Nearest at a fractional scale drops rows and columns rather than blending them, so edges alias. That
 * is the real trade and it is accepted here for one reason: these renditions are shown SMALLER than
 * they are, and the browser's own downscale smooths what nearest left behind. A 160-pixel logo in an
 * 80-pixel slot is where this lives.
 */
const kernelFor = (lossless: boolean): Record<string, unknown> => (lossless ? { kernel: 'nearest' } : {});

/**
 * The placeholder: a picture small enough to travel INSIDE the manifest.
 *
 * 16 pixels wide, which is about 300–500 bytes of WebP. The site inlines it as the image's own
 * background, so it costs no request at all — the point of a placeholder is to be there before
 * anything has been fetched, and a placeholder you have to fetch is just a smaller version of the
 * problem.
 *
 * It is never displayed at 16px. Stretched over the real box the browser's own smoothing does the
 * blurring, so there is no filter to apply and nothing to undo: when the real photograph arrives it
 * paints over its own background and the placeholder is simply gone. No JavaScript, no load handler,
 * nothing to clean up if the script fails to run.
 *
 * Quality 50 because it is going to be scaled up two hundred times. Detail is not the job — colour
 * and where the light is, is.
 */
const PLACEHOLDER_WIDTH = 16;
const PLACEHOLDER_QUALITY = 50;

/**
 * Video renditions, tallest first. `height` is what the preset scales TO.
 *
 * A rung is made only when the source is at least that tall — the same no-upscaling rule the image
 * ladder follows, and it was got wrong here first: `web-1080` was being asked for from a 720p source,
 * which ffmpeg obligingly did. The result was a 14.6 MB file that looks worse than the 7.8 MB one
 * beside it, and 84 seconds of a single core to make it. Under a running sync that overran av-club's
 * idle timeout and the video failed outright, which is how the bug surfaced at all.
 *
 * `web-480` is BOTH a rung and the floor: a phone is served it whatever the source, and a source
 * smaller than 720 still gets something,
 * rather than nothing or an upscale.
 */
const VIDEO_RUNGS = [
  { file: '1080.mp4', preset: 'web-1080', height: 1080, options: undefined },
  { file: '720.mp4', preset: 'web-720', height: 720, options: undefined },
  /*
    480 is a real rung now, not only a floor.

    It was reachable únicamente when the SOURCE was smaller than 720, which is backwards: the rung a
    phone wants is the one a 720p master never produced. The hero here is 1280x720 and the only file
    that existed for it was 7.8 MB of 720p, served to everything.

    `b:a` overrides the preset's 96k. The audio on a hero loop is ambience — the sound of a tufting
    gun — and 64k AAC carries that. It is the same argument as crf 26 for the picture: the rung exists
    for a phone on a mobile connection, and every kilobit is one it is paying for.
  */
  { file: '480.mp4', preset: 'web-480', height: 480, options: { 'b:a': '64k' } },
] as const;

/**
 * When the source is smaller than every rung.
 *
 * A 360p upload still needs one servable file. Kept separate from the ladder because it is a fallback
 * rather than a choice — it is what happens when the filter above matches nothing.
 */
const VIDEO_FLOOR = { file: '480.mp4', preset: 'web-480', height: 480, options: { 'b:a': '64k' } } as const;

const IMAGE_EXT = new Set(['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'avif', 'tif', 'tiff', 'heic', 'heif']);
const VIDEO_EXT = new Set(['mp4', 'mov', 'm4v', 'webm', 'mkv', 'avi', 'wmv', 'flv', 'mpg', 'mpeg', '3gp', 'm2ts', 'mts']);

export type Kind = 'image' | 'video';

export type Variant = {
  file: string;
  width: number | null;
  height: number | null;
  bytes: number;
};

/**
 * What a derived directory says about itself.
 *
 * Written last, and it is also the idempotency record: a source whose size and mtime still match the
 * one recorded here has nothing to redo. That is what makes the initial sync resumable after a
 * restart, and what makes the watcher's per-file call the same code path as the full pass.
 *
 * It is a manifest rather than a database row for the same reason the mapping is string work: a
 * `derived/` you can delete and rebuild must not leave a row behind claiming it is still there.
 */
export type Manifest = {
  kind: Kind;
  source: { name: string; bytes: number; mtimeMs: number; width: number | null; height: number | null };
  variants: Variant[];
  /**
   * A `data:` URI of the picture at 16px, or absent on a manifest built before this existed.
   *
   * In the manifest rather than beside it as a file, because it exists to be inlined — a rendition
   * this small is smaller than the request that would fetch it.
   */
  placeholder?: string;
  /**
   * True when this picture encoded smaller losslessly — a drawing rather than a photograph.
   *
   * Recorded because it is a measurement, not a setting. Somebody looking at a logo's directory and
   * wondering why its files are shaped differently from every other one finds the answer here rather
   * than having to re-derive it to find out.
   */
  lossless?: boolean;
  /** ISO. For a person reading the folder, not for the code. */
  builtAt: string;
};

const MANIFEST = 'manifest.json';

export const extensionOf = (name: string): string => {
  const dot = name.lastIndexOf('.');
  return dot <= 0 ? '' : name.slice(dot + 1).toLowerCase();
};

export const kindOf = (name: string): Kind | null => {
  const ext = extensionOf(name);
  if (IMAGE_EXT.has(ext)) return 'image';
  if (VIDEO_EXT.has(ext)) return 'video';
  return null;
};

export const originalsDir = async (email: string): Promise<string> => join(await mediaPath(email), 'originals');
export const derivedDir = async (email: string): Promise<string> => join(await mediaPath(email), 'derived');

/**
 * Talk to av-club.
 *
 * Straight to the sidecar's loopback port rather than back through the platform's own HTTP: this code
 * IS the platform, and making it authenticate to itself to reach a service already on the machine adds
 * a credential and a hop to buy nothing. `x-officer-user` is the sidecar's whole authentication and is
 * exactly what its proxy would have sent — see `src/servers/api/file-browser/plugin-file-types.ts`,
 * which reaches it the same way and explains why that is sound.
 */
class AvClubUnavailable extends Error {}

async function avClub<T>(path: string, email: string, init?: RequestInit): Promise<T> {
  const port = sidecarPort('av-club');
  if (!port) throw new AvClubUnavailable('av-club is not running. Install and start the AV Club plugin.');

  const response = await fetch(`http://127.0.0.1:${port}${path}`, {
    ...init,
    headers: { 'x-officer-user': email, ...(init?.body ? { 'content-type': 'application/json' } : {}), ...init?.headers },
  });

  if (!response.ok) {
    const body = (await response.json().catch(() => null)) as { error?: { message?: string } } | null;
    throw new Error(body?.error?.message ?? `av-club answered ${response.status}`);
  }
  return (await response.json()) as T;
}

const post = <T>(path: string, email: string, body: unknown): Promise<T> =>
  avClub<T>(path, email, { method: 'POST', body: JSON.stringify(body) });

/**
 * Is this derived directory already correct for this original?
 *
 * Exported because the panel asks the same question the sync does. Two implementations of "current"
 * is how a screen comes to say everything is in step while the next run rebuilds half of it.
 */
export async function upToDate(dir: string, source: { bytes: number; mtimeMs: number }): Promise<boolean> {
  const raw = await readFile(join(dir, MANIFEST), 'utf8').catch(() => null);
  if (!raw) return false;
  try {
    const manifest = JSON.parse(raw) as Manifest;
    return manifest.source.bytes === source.bytes && manifest.source.mtimeMs === source.mtimeMs;
  } catch {
    return false;
  }
}

/**
 * Add what a correct-but-older directory is missing, without rebuilding it.
 *
 * ── Why this is not just "mark it stale" ──
 *
 * Placeholders arrived after 62 ladders had already been built. The obvious move is to make `upToDate`
 * insist on one, which turns every existing directory stale and re-derives the whole library: minutes
 * of CPU and 67 MB rewritten, to add about 400 bytes to each of them. Every rung in those directories
 * is already correct.
 *
 * So the ladders are left alone and only the missing piece is made.
 *
 * ── It reads a RUNG, never the original ──
 *
 * `320.webp` — or `poster.webp` for a video — is already small, already browser-readable, and already
 * has the EXIF rotation baked in. Going back to the original would mean decoding a 4000×6000 HEIC to
 * produce a 16-pixel picture, which is the expensive half of a full rebuild and the thing this exists
 * to avoid.
 *
 * A picture that had no small rung is skipped rather than forced: it keeps no placeholder, the page
 * renders exactly as it did before, and the next real rebuild will make one.
 */
async function topUp(dir: string, manifest: Manifest, email: string): Promise<Manifest> {
  if (manifest.placeholder) return manifest;

  const smallest = manifest.variants
    .filter((variant) => variant.file.endsWith('.webp') && variant.file !== 'full.webp')
    .sort((a, b) => (a.width ?? Number.MAX_SAFE_INTEGER) - (b.width ?? Number.MAX_SAFE_INTEGER))[0];
  if (!smallest) return manifest;

  const placeholder = await encodePlaceholder(join(dir, smallest.file), join(dir, '.placeholder.webp'), email);
  if (!placeholder) return manifest;

  const updated = { ...manifest, placeholder };

  /*
    Rewritten in place rather than through the build-beside-and-swap dance the rest of this file uses.
    Nothing else in the directory is changing, and the manifest is a single small write — the swap
    exists to stop a half-built LADDER being published, which is not what is happening here.
  */
  await writeFile(join(dir, MANIFEST), `${JSON.stringify(updated, null, 2)}\n`, 'utf8');
  return updated;
}

/**
 * Build one original's directory of renditions.
 *
 * ── It builds beside, then swaps ──
 *
 * Into `<name>.partial`, renamed over the real one only once everything succeeded. A sync interrupted
 * halfway — a restart, a kill, ffmpeg dying on one file — must never leave a directory that LOOKS
 * finished, because the manifest is what everything else trusts. Half a ladder with a manifest is
 * worse than no ladder at all.
 */
export async function deriveOne(
  sourcePath: string,
  targetDir: string,
  email: string,
  options: { force?: boolean } = {},
): Promise<{ built: boolean; manifest: Manifest }> {
  const name = sourcePath.slice(sourcePath.lastIndexOf('/') + 1);
  const kind = kindOf(name);
  if (!kind) throw new Error(`${name} is neither an image nor a video this understands.`);

  const info = await stat(sourcePath);
  const source = { bytes: info.size, mtimeMs: Math.round(info.mtimeMs) };

  if (!options.force && (await upToDate(targetDir, source))) {
    const manifest = JSON.parse(await readFile(join(targetDir, MANIFEST), 'utf8')) as Manifest;
    return { built: false, manifest: await topUp(targetDir, manifest, email) };
  }

  const staging = `${targetDir}.partial`;
  await rm(staging, { recursive: true, force: true });
  await mkdir(staging, { recursive: true });

  try {
    const manifest =
      kind === 'image'
        ? await buildImage(sourcePath, staging, name, source, email)
        : await buildVideo(sourcePath, staging, name, source, email);

    await writeFile(join(staging, MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');

    // `rename` cannot replace a non-empty directory, so the old one goes first. The window between the
    // two is the reason this is not the step that produces the files.
    await rm(targetDir, { recursive: true, force: true });
    await rename(staging, targetDir);

    return { built: true, manifest };
  } catch (error) {
    await rm(staging, { recursive: true, force: true }).catch(() => {});
    throw error;
  }
}

/**
 * The intermediate a ladder is built from, when reading the original repeatedly would be the expensive
 * part.
 *
 * Never compressed — `compressionLevel: 0`. It lives for seconds inside the staging directory and is
 * then deleted, so every cycle spent shrinking it is spent twice for nothing: once packing, once
 * unpacking, seven times over.
 */
const MASTER = '.master.png';

async function buildImage(
  sourcePath: string,
  staging: string,
  name: string,
  source: { bytes: number; mtimeMs: number },
  email: string,
): Promise<Manifest> {
  const meta = await avClub<{ width: number | null; height: number | null; needsConversion: boolean }>(
    `/v1/image?path=${encodeURIComponent(sourcePath)}`,
    email,
  );
  const width = meta.width ?? 0;
  const height = meta.height ?? 0;
  if (!width || !height) throw new Error(`Could not read the dimensions of ${name}.`);

  /*
    DECODE ONCE.

    av-club is stateless per request, so seven rungs asked for from the original are seven decodes of
    it. For a JPEG that is fine — sharp reads one cheaply, and writing a full-resolution intermediate
    to save six cheap reads would cost more than it saved. For a HEIC it is not: every request spawns
    ImageMagick to produce a full-resolution PNG, measured on fofos at 8.5 s against 1.1 s for the
    resize and encode that follow. Seven rungs spent about a minute making the same intermediate over
    and over and throwing six of them away.

    So when av-club says this format needs converting, it is converted ONCE, here, into the staging
    directory nobody else can see, and every rung is cut from that. 67 seconds becomes 16.

    `needsConversion` is asked rather than guessed. Which formats are expensive is av-club's fact and
    changes when its build changes; a list of them kept here would go on paying for a conversion that
    had stopped being necessary.
  */
  const master = meta.needsConversion ? join(staging, MASTER) : sourcePath;
  if (meta.needsConversion) {
    await post('/v1/sharp', email, {
      inputPath: sourcePath,
      outputPath: master,
      pipeline: [{ rotate: {} }],
      output: { format: 'png', options: { compressionLevel: 0 } },
    });
  }

  /*
    Asked once, of the picture itself, before anything is built. See prefersLossless: a drawing and a
    photograph want opposite encoders, and getting it wrong on a logo produced a smaller rung two and
    a half times the size of the original.
  */
  const lossless = await prefersLossless(master, staging, email, width);

  const variants: Variant[] = [];

  /*
    `full.webp` first, and always.

    It is the top of the ladder, and for a HEIC it is the only browser-readable copy of the picture that
    exists at all — the original renders nowhere. A lightbox, a zoom, or handing the client their own
    photograph back all need it.
  */
  variants.push(await encodeWebp(master, join(staging, 'full.webp'), null, email, width, height, lossless));

  // Strictly less than the original: a rung at or above it would duplicate `full.webp` byte for byte.
  for (const rung of LADDER) {
    if (rung >= width) continue;
    variants.push(await encodeWebp(master, join(staging, `${rung}.webp`), rung, email, width, height, lossless));
  }

  /*
    The square crops — but only if the picture is not already square.

    A 300x300 logo was getting a `256sq.webp` that cropped nothing: the same image at a different name,
    a second file to store, serve and keep in step. The aspect ladder above already produces squares
    for a square source, so there is nothing a square rung could add.

    `SQUARE_TOLERANCE` rather than an exact match, because a 1000x1002 photograph is square for every
    purpose this serves and cropping two pixels off it is not worth a parallel ladder.

    Skipped too when the source is smaller than the rung — enlarging to make a thumbnail is the one
    thing worse than cropping in the browser.
  */
  const squareEnough = Math.abs(width - height) / Math.max(width, height) <= SQUARE_TOLERANCE;
  if (!squareEnough) {
    for (const rung of SQUARE_LADDER) {
      if (rung > Math.min(width, height)) continue;
      variants.push(await encodeSquare(master, join(staging, `${rung}sq.webp`), rung, email, lossless));
    }
  }

  const placeholder = await encodePlaceholder(master, join(staging, 'placeholder.webp'), email);

  // Before the manifest is written, so a directory that has one never contains this.
  if (meta.needsConversion) await rm(master, { force: true });

  return {
    kind: 'image',
    source: { name, bytes: source.bytes, mtimeMs: source.mtimeMs, width, height },
    variants,
    placeholder,
    lossless,
    builtAt: new Date().toISOString(),
  };
}

/**
 * The inline placeholder, as a `data:` URI.
 *
 * Written to disk and read back rather than asked for as bytes, because the sharp sidecar's contract
 * is file-in, file-out — the same call every other rung makes. The file is removed immediately: the
 * staging directory becomes the published one, and a `placeholder.webp` sitting in it would be a
 * rendition nothing references and everything would have to learn to ignore.
 *
 * Returns undefined rather than throwing. A ladder that built is worth keeping even if this one
 * 500-byte extra did not, and the site treats a missing placeholder as "no placeholder" already.
 */
async function encodePlaceholder(
  sourcePath: string,
  outputPath: string,
  email: string,
): Promise<string | undefined> {
  try {
    await post<{ path: string; bytes: number }>('/v1/sharp', email, {
      inputPath: sourcePath,
      outputPath,
      pipeline: [{ rotate: {} }, { resize: { width: PLACEHOLDER_WIDTH, fit: 'inside', withoutEnlargement: true } }],
      output: { format: 'webp', options: { quality: PLACEHOLDER_QUALITY } },
    });

    const bytes = await readFile(outputPath);
    await rm(outputPath, { force: true });
    return `data:image/webp;base64,${bytes.toString('base64')}`;
  } catch {
    await rm(outputPath, { force: true }).catch(() => {});
    return undefined;
  }
}

/**
 * One square crop.
 *
 * `fit: cover` with `position: centre` is sharp doing the work the browser would otherwise do with
 * object-cover, except the bytes never travel. Quality follows the thumbnail rung, because that is
 * exactly what this is.
 */
async function encodeSquare(
  sourcePath: string,
  outputPath: string,
  rung: number,
  email: string,
  lossless: boolean,
): Promise<Variant> {
  const result = await post<{ path: string; bytes: number }>('/v1/sharp', email, {
    inputPath: sourcePath,
    outputPath,
    pipeline: [
      { rotate: {} },
      {
        resize: { width: rung, height: rung, fit: 'cover', position: 'centre', withoutEnlargement: true, ...kernelFor(lossless) },
      },
    ],
    output: { format: 'webp', options: webpOptions(rung, lossless) },
  });

  return {
    file: outputPath.slice(outputPath.lastIndexOf('/') + 1),
    width: rung,
    height: rung,
    bytes: result.bytes,
  };
}

async function encodeWebp(
  sourcePath: string,
  outputPath: string,
  rung: number | null,
  email: string,
  sourceWidth: number,
  sourceHeight: number,
  lossless: boolean,
): Promise<Variant> {
  const result = await post<{ path: string; bytes: number }>('/v1/sharp', email, {
    inputPath: sourcePath,
    outputPath,
    // `rotate` with no angle applies the EXIF orientation, which phone photographs rely on entirely. A
    // ladder built without it is a set of sideways pictures. Kept on every rung including those cut
    // from the master, where it is a no-op — a PNG carries no EXIF, and the master was written with the
    // rotation already applied. One code path beats two that can disagree about which way up a photo is.
    pipeline: [
      { rotate: {} },
      ...(rung === null
        ? []
        : [{ resize: { width: rung, fit: 'inside', withoutEnlargement: true, ...kernelFor(lossless) } }]),
    ],
    output: { format: 'webp', options: webpOptions(rung, lossless) },
  });

  const width = rung ?? sourceWidth;
  return {
    file: outputPath.slice(outputPath.lastIndexOf('/') + 1),
    width,
    height: Math.round((width / sourceWidth) * sourceHeight),
    bytes: result.bytes,
  };
}

async function buildVideo(
  sourcePath: string,
  staging: string,
  name: string,
  source: { bytes: number; mtimeMs: number },
  email: string,
): Promise<Manifest> {
  const probe = await avClub<{ streams?: Array<{ codec_type?: string; width?: number; height?: number }> }>(
    `/v1/probe?path=${encodeURIComponent(sourcePath)}`,
    email,
  );
  const video = probe.streams?.find((stream) => stream.codec_type === 'video');
  const height = video?.height ?? 0;
  const width = video?.width ?? 0;

  const variants: Variant[] = [];

  /*
    Every rung the source is actually tall enough for, and the floor when it is tall enough for none.

    An unreadable height (0) is treated as "make everything": a probe that told us nothing is not
    evidence the video is small, and the encoder will decline to enlarge on its own long before we
    should decide on its behalf.
  */
  const wanted = height ? VIDEO_RUNGS.filter((rung) => height >= rung.height) : [...VIDEO_RUNGS];
  const rungs = wanted.length ? wanted : [VIDEO_FLOOR];

  for (const rung of rungs) {
    const result = await post<{ path: string; bytes: number }>('/v1/ffmpeg', email, {
      inputPath: sourcePath,
      outputPath: join(staging, rung.file),
      preset: rung.preset,
      // The preset supplies the options and anything given here overrides it — av-club's contract,
      // see sidecar/ffmpeg/run.ts. Used only to lighten the audio on the 480 rung.
      ...(rung.options ? { options: rung.options } : {}),
    });
    variants.push({ file: rung.file, width: null, height: null, bytes: result.bytes });
  }

  /*
    The poster, in two steps.

    av-club's `poster` preset writes a JPEG, which is its call and the right one for a general service.
    Here everything a browser is handed is WebP, so it goes through sharp on the way out and the JPEG
    is dropped. Two calls per video, once, at build time.
  */
  const stillJpg = join(staging, 'poster.jpg');
  await post('/v1/ffmpeg', email, {
    inputPath: sourcePath,
    outputPath: stillJpg,
    preset: 'poster',
    // A frame from a second in, because frame zero of a great many videos is black.
    inputOptions: { ss: 1 },
  }).catch(() => post('/v1/ffmpeg', email, { inputPath: sourcePath, outputPath: stillJpg, preset: 'poster' }));

  const poster = await post<{ path: string; bytes: number }>('/v1/sharp', email, {
    inputPath: stillJpg,
    outputPath: join(staging, 'poster.webp'),
    // The widest rung, and a poster like all the others — see WEBP_QUALITY_POSTER.
    output: { format: 'webp', options: { quality: WEBP_QUALITY_POSTER } },
  });
  /*
    From the STILL, not from the video.

    The same frame the poster comes from, so the two agree — and sharp can read a JPEG where it cannot
    read an MP4. Before the still is deleted, which is the only reason these three lines are in this
    order rather than the other one.
  */
  const placeholder = await encodePlaceholder(stillJpg, join(staging, 'placeholder.webp'), email);
  variants.push({ file: 'poster.webp', width: width || null, height: height || null, bytes: poster.bytes });

  /*
    The poster's own ladder, cut from the same still.

    From `stillJpg` rather than from `poster.webp`, so each rung is scaled once from the original frame
    instead of being a re-encode of an encode.

    EVERYTHING that reads the still happens above the line that removes it, and that ordering is the
    only thing holding this together — the placeholder, then poster.webp, then these. This loop was
    added below the removal on 2026-08-30 and every video failed to convert until it moved. The comment
    warning about exactly this was already three lines up.
  */
  for (const rung of POSTER_LADDER) {
    if (!width || rung >= width) continue;
    const cut = await post<{ path: string; bytes: number }>('/v1/sharp', email, {
      inputPath: stillJpg,
      outputPath: join(staging, `poster-${rung}.webp`),
      pipeline: [{ resize: { width: rung, fit: 'inside', withoutEnlargement: true } }],
      output: { format: 'webp', options: { quality: WEBP_QUALITY_POSTER } },
    });
    variants.push({
      file: `poster-${rung}.webp`,
      width: rung,
      height: height ? Math.round((rung / width) * height) : null,
      bytes: cut.bytes,
    });
  }

  // Last, once nothing else needs it. It is an intermediate: a JPEG in a directory where everything
  // served is WebP, and leaving it would publish a rendition nothing references.
  await rm(stillJpg, { force: true });

  return {
    kind: 'video',
    source: { name, bytes: source.bytes, mtimeMs: source.mtimeMs, width: width || null, height: height || null },
    variants,
    placeholder,
    builtAt: new Date().toISOString(),
  };
}

/** Every file under `originals/`, relative to it, depth first. Directories are structure, not work. */
export async function listOriginals(root: string, prefix = ''): Promise<string[]> {
  /*
    Sorted, because `readdir` is not.

    It returns entries in the filesystem's own order — inode order on ext4 — which is arbitrary and
    stable only until somebody moves a file. That is harmless for the sync, which does the same work in
    any order, and it is not harmless for "Create Gallery": this list becomes the gallery's initial
    order, and nine photographs arriving as 05, 08, 01, 07 reads as a bug even though nothing is wrong.

    `numeric` so `galeria2` sorts before `galeria10`, which is what anybody naming files that way means.
  */
  const entries = (await readdir(join(root, prefix), { withFileTypes: true }).catch(() => [])).sort((a, b) =>
    a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: 'base' }),
  );
  const found: string[] = [];

  for (const entry of entries) {
    // Dotfiles are `.DS_Store` and friends. Nothing under here is a deliberate hidden file.
    if (entry.name.startsWith('.')) continue;
    const rel = prefix ? `${prefix}/${entry.name}` : entry.name;
    if (entry.isDirectory()) found.push(...(await listOriginals(root, rel)));
    else if (kindOf(entry.name)) found.push(rel);
  }

  return found;
}

/**
 * A caller-supplied subtree of `originals/`, made safe to walk.
 *
 * Relative, always, and never able to climb out: `..` in any segment is refused rather than resolved,
 * because a scope that escapes `originals/` would have the sync building derived folders for files
 * nobody put there. Leading and trailing slashes are trimmed so `/GALLERY/` and `GALLERY` mean one
 * thing — the caller is naming a folder, not writing a path.
 */
export function safeScope(scope: string): string {
  const trimmed = scope.replace(/^\/+|\/+$/g, '');
  if (!trimmed) return '';
  if (trimmed.split('/').includes('..')) throw new Error('That is not a folder inside originals.');
  return trimmed;
}

/** Where a relative original ends up. Exported because the watcher needs the same answer. */
export const targetFor = (derivedRoot: string, relativePath: string): string => join(derivedRoot, relativePath);

export { relative, AvClubUnavailable };

export type Reconciliation = {
  /** Convertible files under `originals/`. */
  originals: number;
  /** Built from the current version of their source. */
  current: number;
  /** Never built, or built and since lost. Names, so the panel can say WHICH. */
  missing: string[];
  /** Built, but from a version of the source that has changed since. */
  stale: string[];
};

/**
 * What `derived/` would still have to do to be in step with `originals/`.
 *
 * Answered from the filesystem every time rather than cached, because it is the question "is what I am
 * looking at current" and a cached answer to that is worse than none.
 *
 * It uses `upToDate` — the SAME predicate the sync uses to decide whether to skip a file. That is the
 * whole point: a panel reporting "62 current" and a run then rebuilding all 62 would be two different
 * definitions of the word, and the one on screen would be the one nobody could debug.
 *
 * `missing` and `stale` are separated because they mean different things to a person. Missing is a file
 * that has never been converted or whose output was deleted; stale is one that was replaced with a new
 * version. Both are work, but only the second means somebody changed something.
 */
export async function reconcile(email: string, scope = ''): Promise<Reconciliation> {
  const from = await originalsDir(email);
  const to = await derivedDir(email);
  const files = await listOriginals(from, safeScope(scope));

  const missing: string[] = [];
  const stale: string[] = [];
  let current = 0;

  for (const file of files) {
    const info = await stat(join(from, file)).catch(() => null);
    if (!info) continue;

    const target = targetFor(to, file);
    const manifest = await readFile(join(target, MANIFEST), 'utf8').catch(() => null);
    if (!manifest) {
      missing.push(file);
      continue;
    }

    if (await upToDate(target, { bytes: info.size, mtimeMs: Math.round(info.mtimeMs) })) current++;
    else stale.push(file);
  }

  return { originals: files.length, current, missing, stale };
}

/**
 * Every derived folder that already exists, indexed by what it was built FROM.
 *
 * ── What this is for ──
 *
 * The mapping from original to derived is by PATH, so moving a file into a folder makes the sync look
 * somewhere new, find nothing, and convert it again — minutes of work to produce bytes that already
 * exist a directory away, while the old folder stays behind forever because nothing deletes it.
 *
 * A move preserves size and mtime, so the pair identifies the same file at a new path. That is what
 * this index is: build it once before a sync, and a file that has moved can have its ladder RENAMED
 * into place instead of rebuilt.
 *
 * ── Why the pair, and not a hash ──
 *
 * A hash would be exact and would also mean reading every byte of every original on every sync, which
 * is the cost this is trying to avoid. Size and mtime are already in the manifest and already in the
 * stat, so the index is free.
 *
 * It is not unique in principle — two files can share both — so an ambiguous match is REFUSED below
 * rather than guessed at. Rebuilding is slow; attaching the wrong pictures to a name is wrong.
 */
export type DerivedIndex = Map<string, string[]>;

const indexKey = (bytes: number, mtimeMs: number) => `${bytes}:${mtimeMs}`;

/** Walk `derived/`, reading each manifest. Directories are the unit — a manifest is one built original. */
export async function indexDerived(root: string, prefix = '', into: DerivedIndex = new Map()): Promise<DerivedIndex> {
  const entries = await readdir(join(root, prefix), { withFileTypes: true }).catch(() => []);

  for (const entry of entries) {
    if (!entry.isDirectory() || entry.name.startsWith('.')) continue;
    const rel = prefix ? `${prefix}/${entry.name}` : entry.name;

    const raw = await readFile(join(root, rel, MANIFEST), 'utf8').catch(() => null);
    if (raw) {
      try {
        const manifest = JSON.parse(raw) as Manifest;
        const key = indexKey(manifest.source.bytes, manifest.source.mtimeMs);
        into.set(key, [...(into.get(key) ?? []), rel]);
      } catch {
        // Unreadable manifest: not indexed, so the file it belongs to rebuilds. One corrupt folder
        // costs one conversion rather than making the whole index untrustworthy.
      }
      // A built original is a leaf — its contents are renditions, not more originals.
      continue;
    }

    await indexDerived(root, rel, into);
  }

  return into;
}

/**
 * Has this original simply MOVED? If so, put its ladder where the file now is.
 *
 * Returns true when the folder was relocated and is now correct for this path.
 *
 * Refuses an ambiguous match. Two originals sharing a size and an mtime is unlikely and not
 * impossible — a camera dump can write a whole card in one second — and attaching the wrong
 * photographs to a name is a worse outcome than converting again.
 */
export async function relocateDerived(
  index: DerivedIndex,
  derivedRoot: string,
  relativePath: string,
  source: { bytes: number; mtimeMs: number },
): Promise<boolean> {
  const candidates = index.get(indexKey(source.bytes, source.mtimeMs)) ?? [];
  if (candidates.length !== 1) return false;

  const from = candidates[0]!;
  if (from === relativePath) return false;

  const fromDir = join(derivedRoot, from);
  const toDir = join(derivedRoot, relativePath);

  // Only if the destination is genuinely empty. A folder already there is either current — in which
  // case nothing needed doing — or half-built, and overwriting it with a rename is not a repair.
  if (await stat(toDir).catch(() => null)) return false;

  try {
    await mkdir(dirname(toDir), { recursive: true });
    await rename(fromDir, toDir);
    // Claimed, so a second original with the same size and mtime cannot take it too.
    index.delete(indexKey(source.bytes, source.mtimeMs));
    return true;
  } catch {
    return false;
  }
}

/**
 * Remove derived folders whose original is gone.
 *
 * ── Only for a whole-tree sync ──
 *
 * A scoped sync knows about one folder, so everything outside it would look abandoned. Deleting on
 * that basis would take out the entire library the first time somebody re-synced a subfolder, which is
 * the kind of mistake that is only noticed later.
 *
 * Safe to run because `derived/` is reproducible by definition — that is what its name has meant since
 * the day the two folders were called `originals` and `derived` rather than `user` and `treated`.
 */
export async function pruneDerived(originalsRoot: string, derivedRoot: string): Promise<string[]> {
  const alive = new Set(await listOriginals(originalsRoot));
  const built = await indexDerived(derivedRoot);

  const removed: string[] = [];
  for (const paths of built.values()) {
    for (const path of paths) {
      if (alive.has(path)) continue;
      await rm(join(derivedRoot, path), { recursive: true, force: true }).catch(() => {});
      removed.push(path);
    }
  }
  return removed;
}