fofos-admin/api/derive.ts

import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
import { dirname, join, relative } from 'node:path';
import { sidecarPort } from '@@/sidecar-registry';
import { mediaPath } from './media';

/**
 * Building `derived/` from `originals/`.
 *
 * ── The mapping ──
 *
 * `derived/` mirrors `originals/` directory for directory. Where originals holds a FILE, derived holds
 * a DIRECTORY of the same name — extension included — and the renditions live inside it:
 *
 *   originals/homepage_galeria01.jpg           →  derived/homepage_galeria01.jpg/320.webp
 *                                                 derived/homepage_galeria01.jpg/640.webp  …
 *   originals/ABOUT US/portrait.HEIC           →  derived/ABOUT US/portrait.HEIC/…
 *   originals/homepage_video.mp4               →  derived/homepage_video.mp4/1080.mp4
 *                                                 derived/homepage_video.mp4/poster.webp
 *
 * Keeping the extension is what makes the mapping total: `a.jpg` and `a.png` are different originals
 * and must not share a directory. It also makes the inverse pure string work — no index, no table, no
 * second copy of the truth. **The filesystem is the record.**
 *
 * ── The ladder is fixed here, and the SITE chooses from it ──
 *
 * The widths below cover every slot fofos has at up to 3× pixel density — `website/src/media/slots.ts`
 * puts the largest at 1232 CSS px, so `minUpload` is 2464 and 2560 is the top rung. But this file does
 * not read that one, and must not: a plugin that knows a website's layout is a plugin for one website.
 * It emits a ladder; `srcset` and the `sizes` already written in `website/src/media/render.ts` do the
 * choosing, because the site is the only thing that knows what space an image occupies.
 *
 * That boundary is the difference between this being a template and being fofos-shaped.
 *
 * ── Nothing is ever upscaled ──
 *
 * A rung wider than the original is not written. Inventing a 2560 from a 900px photograph produces a
 * soft file that `srcset` would then confidently prefer over the sharp one — worse than not having it.
 *
 * ── Video rungs are named by height, images by width ──
 *
 * Deliberate, not an oversight. `srcset` descriptors are widths, so an image file called `1280` has to
 * be 1280 pixels wide or the browser is being lied to. Video has no such mechanism, and every person
 * reading `1080.mp4` knows what it means. Each unit matches its consumer.
 *
 * ── Every byte is written by av-club ──
 *
 * This module decides WHAT should exist and av-club decides how to make it. The video presets are its
 * `web-1080` / `web-720` / `poster`, by name — the encoder flags behind them are its business, and
 * revising them for every site at once is the reason it is a separate service.
 */

/** Image rungs, in pixels of width. Rungs wider than the original are skipped — see above. */
export const LADDER = [320, 640, 960, 1280, 1920, 2560] as const;

/** WebP quality. 82 is the usual knee: visually clean on photographs, well off the top of the curve. */
const WEBP_QUALITY = 82;

/**
 * The placeholder: a picture small enough to travel INSIDE the manifest.
 *
 * 16 pixels wide, which is about 300–500 bytes of WebP. The site inlines it as the image's own
 * background, so it costs no request at all — the point of a placeholder is to be there before
 * anything has been fetched, and a placeholder you have to fetch is just a smaller version of the
 * problem.
 *
 * It is never displayed at 16px. Stretched over the real box the browser's own smoothing does the
 * blurring, so there is no filter to apply and nothing to undo: when the real photograph arrives it
 * paints over its own background and the placeholder is simply gone. No JavaScript, no load handler,
 * nothing to clean up if the script fails to run.
 *
 * Quality 50 because it is going to be scaled up two hundred times. Detail is not the job — colour
 * and where the light is, is.
 */
const PLACEHOLDER_WIDTH = 16;
const PLACEHOLDER_QUALITY = 50;

/**
 * Video renditions, tallest first. `height` is what the preset scales TO.
 *
 * A rung is made only when the source is at least that tall — the same no-upscaling rule the image
 * ladder follows, and it was got wrong here first: `web-1080` was being asked for from a 720p source,
 * which ffmpeg obligingly did. The result was a 14.6 MB file that looks worse than the 7.8 MB one
 * beside it, and 84 seconds of a single core to make it. Under a running sync that overran av-club's
 * idle timeout and the video failed outright, which is how the bug surfaced at all.
 *
 * `web-480` is the floor, not a rung: it exists so a source smaller than 720 still gets something,
 * rather than nothing or an upscale.
 */
const VIDEO_RUNGS = [
  { file: '1080.mp4', preset: 'web-1080', height: 1080 },
  { file: '720.mp4', preset: 'web-720', height: 720 },
] as const;

const VIDEO_FLOOR = { file: '480.mp4', preset: 'web-480', height: 480 } as const;

const IMAGE_EXT = new Set(['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'avif', 'tif', 'tiff', 'heic', 'heif']);
const VIDEO_EXT = new Set(['mp4', 'mov', 'm4v', 'webm', 'mkv', 'avi', 'wmv', 'flv', 'mpg', 'mpeg', '3gp', 'm2ts', 'mts']);

export type Kind = 'image' | 'video';

export type Variant = {
  file: string;
  width: number | null;
  height: number | null;
  bytes: number;
};

/**
 * What a derived directory says about itself.
 *
 * Written last, and it is also the idempotency record: a source whose size and mtime still match the
 * one recorded here has nothing to redo. That is what makes the initial sync resumable after a
 * restart, and what makes the watcher's per-file call the same code path as the full pass.
 *
 * It is a manifest rather than a database row for the same reason the mapping is string work: a
 * `derived/` you can delete and rebuild must not leave a row behind claiming it is still there.
 */
export type Manifest = {
  kind: Kind;
  source: { name: string; bytes: number; mtimeMs: number; width: number | null; height: number | null };
  variants: Variant[];
  /**
   * A `data:` URI of the picture at 16px, or absent on a manifest built before this existed.
   *
   * In the manifest rather than beside it as a file, because it exists to be inlined — a rendition
   * this small is smaller than the request that would fetch it.
   */
  placeholder?: string;
  /** ISO. For a person reading the folder, not for the code. */
  builtAt: string;
};

const MANIFEST = 'manifest.json';

export const extensionOf = (name: string): string => {
  const dot = name.lastIndexOf('.');
  return dot <= 0 ? '' : name.slice(dot + 1).toLowerCase();
};

export const kindOf = (name: string): Kind | null => {
  const ext = extensionOf(name);
  if (IMAGE_EXT.has(ext)) return 'image';
  if (VIDEO_EXT.has(ext)) return 'video';
  return null;
};

export const originalsDir = async (email: string): Promise<string> => join(await mediaPath(email), 'originals');
export const derivedDir = async (email: string): Promise<string> => join(await mediaPath(email), 'derived');

/**
 * Talk to av-club.
 *
 * Straight to the sidecar's loopback port rather than back through the platform's own HTTP: this code
 * IS the platform, and making it authenticate to itself to reach a service already on the machine adds
 * a credential and a hop to buy nothing. `x-officer-user` is the sidecar's whole authentication and is
 * exactly what its proxy would have sent — see `src/servers/api/file-browser/plugin-file-types.ts`,
 * which reaches it the same way and explains why that is sound.
 */
class AvClubUnavailable extends Error {}

async function avClub<T>(path: string, email: string, init?: RequestInit): Promise<T> {
  const port = sidecarPort('av-club');
  if (!port) throw new AvClubUnavailable('av-club is not running. Install and start the AV Club plugin.');

  const response = await fetch(`http://127.0.0.1:${port}${path}`, {
    ...init,
    headers: { 'x-officer-user': email, ...(init?.body ? { 'content-type': 'application/json' } : {}), ...init?.headers },
  });

  if (!response.ok) {
    const body = (await response.json().catch(() => null)) as { error?: { message?: string } } | null;
    throw new Error(body?.error?.message ?? `av-club answered ${response.status}`);
  }
  return (await response.json()) as T;
}

const post = <T>(path: string, email: string, body: unknown): Promise<T> =>
  avClub<T>(path, email, { method: 'POST', body: JSON.stringify(body) });

/**
 * Is this derived directory already correct for this original?
 *
 * Exported because the panel asks the same question the sync does. Two implementations of "current"
 * is how a screen comes to say everything is in step while the next run rebuilds half of it.
 */
export async function upToDate(dir: string, source: { bytes: number; mtimeMs: number }): Promise<boolean> {
  const raw = await readFile(join(dir, MANIFEST), 'utf8').catch(() => null);
  if (!raw) return false;
  try {
    const manifest = JSON.parse(raw) as Manifest;
    return manifest.source.bytes === source.bytes && manifest.source.mtimeMs === source.mtimeMs;
  } catch {
    return false;
  }
}

/**
 * Add what a correct-but-older directory is missing, without rebuilding it.
 *
 * ── Why this is not just "mark it stale" ──
 *
 * Placeholders arrived after 62 ladders had already been built. The obvious move is to make `upToDate`
 * insist on one, which turns every existing directory stale and re-derives the whole library: minutes
 * of CPU and 67 MB rewritten, to add about 400 bytes to each of them. Every rung in those directories
 * is already correct.
 *
 * So the ladders are left alone and only the missing piece is made.
 *
 * ── It reads a RUNG, never the original ──
 *
 * `320.webp` — or `poster.webp` for a video — is already small, already browser-readable, and already
 * has the EXIF rotation baked in. Going back to the original would mean decoding a 4000×6000 HEIC to
 * produce a 16-pixel picture, which is the expensive half of a full rebuild and the thing this exists
 * to avoid.
 *
 * A picture that had no small rung is skipped rather than forced: it keeps no placeholder, the page
 * renders exactly as it did before, and the next real rebuild will make one.
 */
async function topUp(dir: string, manifest: Manifest, email: string): Promise<Manifest> {
  if (manifest.placeholder) return manifest;

  const smallest = manifest.variants
    .filter((variant) => variant.file.endsWith('.webp') && variant.file !== 'full.webp')
    .sort((a, b) => (a.width ?? Number.MAX_SAFE_INTEGER) - (b.width ?? Number.MAX_SAFE_INTEGER))[0];
  if (!smallest) return manifest;

  const placeholder = await encodePlaceholder(join(dir, smallest.file), join(dir, '.placeholder.webp'), email);
  if (!placeholder) return manifest;

  const updated = { ...manifest, placeholder };

  /*
    Rewritten in place rather than through the build-beside-and-swap dance the rest of this file uses.
    Nothing else in the directory is changing, and the manifest is a single small write — the swap
    exists to stop a half-built LADDER being published, which is not what is happening here.
  */
  await writeFile(join(dir, MANIFEST), `${JSON.stringify(updated, null, 2)}\n`, 'utf8');
  return updated;
}

/**
 * Build one original's directory of renditions.
 *
 * ── It builds beside, then swaps ──
 *
 * Into `<name>.partial`, renamed over the real one only once everything succeeded. A sync interrupted
 * halfway — a restart, a kill, ffmpeg dying on one file — must never leave a directory that LOOKS
 * finished, because the manifest is what everything else trusts. Half a ladder with a manifest is
 * worse than no ladder at all.
 */
export async function deriveOne(
  sourcePath: string,
  targetDir: string,
  email: string,
  options: { force?: boolean } = {},
): Promise<{ built: boolean; manifest: Manifest }> {
  const name = sourcePath.slice(sourcePath.lastIndexOf('/') + 1);
  const kind = kindOf(name);
  if (!kind) throw new Error(`${name} is neither an image nor a video this understands.`);

  const info = await stat(sourcePath);
  const source = { bytes: info.size, mtimeMs: Math.round(info.mtimeMs) };

  if (!options.force && (await upToDate(targetDir, source))) {
    const manifest = JSON.parse(await readFile(join(targetDir, MANIFEST), 'utf8')) as Manifest;
    return { built: false, manifest: await topUp(targetDir, manifest, email) };
  }

  const staging = `${targetDir}.partial`;
  await rm(staging, { recursive: true, force: true });
  await mkdir(staging, { recursive: true });

  try {
    const manifest =
      kind === 'image'
        ? await buildImage(sourcePath, staging, name, source, email)
        : await buildVideo(sourcePath, staging, name, source, email);

    await writeFile(join(staging, MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`, 'utf8');

    // `rename` cannot replace a non-empty directory, so the old one goes first. The window between the
    // two is the reason this is not the step that produces the files.
    await rm(targetDir, { recursive: true, force: true });
    await rename(staging, targetDir);

    return { built: true, manifest };
  } catch (error) {
    await rm(staging, { recursive: true, force: true }).catch(() => {});
    throw error;
  }
}

/**
 * The intermediate a ladder is built from, when reading the original repeatedly would be the expensive
 * part.
 *
 * Never compressed — `compressionLevel: 0`. It lives for seconds inside the staging directory and is
 * then deleted, so every cycle spent shrinking it is spent twice for nothing: once packing, once
 * unpacking, seven times over.
 */
const MASTER = '.master.png';

async function buildImage(
  sourcePath: string,
  staging: string,
  name: string,
  source: { bytes: number; mtimeMs: number },
  email: string,
): Promise<Manifest> {
  const meta = await avClub<{ width: number | null; height: number | null; needsConversion: boolean }>(
    `/v1/image?path=${encodeURIComponent(sourcePath)}`,
    email,
  );
  const width = meta.width ?? 0;
  const height = meta.height ?? 0;
  if (!width || !height) throw new Error(`Could not read the dimensions of ${name}.`);

  /*
    DECODE ONCE.

    av-club is stateless per request, so seven rungs asked for from the original are seven decodes of
    it. For a JPEG that is fine — sharp reads one cheaply, and writing a full-resolution intermediate
    to save six cheap reads would cost more than it saved. For a HEIC it is not: every request spawns
    ImageMagick to produce a full-resolution PNG, measured on fofos at 8.5 s against 1.1 s for the
    resize and encode that follow. Seven rungs spent about a minute making the same intermediate over
    and over and throwing six of them away.

    So when av-club says this format needs converting, it is converted ONCE, here, into the staging
    directory nobody else can see, and every rung is cut from that. 67 seconds becomes 16.

    `needsConversion` is asked rather than guessed. Which formats are expensive is av-club's fact and
    changes when its build changes; a list of them kept here would go on paying for a conversion that
    had stopped being necessary.
  */
  const master = meta.needsConversion ? join(staging, MASTER) : sourcePath;
  if (meta.needsConversion) {
    await post('/v1/sharp', email, {
      inputPath: sourcePath,
      outputPath: master,
      pipeline: [{ rotate: {} }],
      output: { format: 'png', options: { compressionLevel: 0 } },
    });
  }

  const variants: Variant[] = [];

  /*
    `full.webp` first, and always.

    It is the top of the ladder, and for a HEIC it is the only browser-readable copy of the picture that
    exists at all — the original renders nowhere. A lightbox, a zoom, or handing the client their own
    photograph back all need it.
  */
  variants.push(await encodeWebp(master, join(staging, 'full.webp'), null, email, width, height));

  // Strictly less than the original: a rung at or above it would duplicate `full.webp` byte for byte.
  for (const rung of LADDER) {
    if (rung >= width) continue;
    variants.push(await encodeWebp(master, join(staging, `${rung}.webp`), rung, email, width, height));
  }

  const placeholder = await encodePlaceholder(master, join(staging, 'placeholder.webp'), email);

  // Before the manifest is written, so a directory that has one never contains this.
  if (meta.needsConversion) await rm(master, { force: true });

  return {
    kind: 'image',
    source: { name, bytes: source.bytes, mtimeMs: source.mtimeMs, width, height },
    variants,
    placeholder,
    builtAt: new Date().toISOString(),
  };
}

/**
 * The inline placeholder, as a `data:` URI.
 *
 * Written to disk and read back rather than asked for as bytes, because the sharp sidecar's contract
 * is file-in, file-out — the same call every other rung makes. The file is removed immediately: the
 * staging directory becomes the published one, and a `placeholder.webp` sitting in it would be a
 * rendition nothing references and everything would have to learn to ignore.
 *
 * Returns undefined rather than throwing. A ladder that built is worth keeping even if this one
 * 500-byte extra did not, and the site treats a missing placeholder as "no placeholder" already.
 */
async function encodePlaceholder(
  sourcePath: string,
  outputPath: string,
  email: string,
): Promise<string | undefined> {
  try {
    await post<{ path: string; bytes: number }>('/v1/sharp', email, {
      inputPath: sourcePath,
      outputPath,
      pipeline: [{ rotate: {} }, { resize: { width: PLACEHOLDER_WIDTH, fit: 'inside', withoutEnlargement: true } }],
      output: { format: 'webp', options: { quality: PLACEHOLDER_QUALITY } },
    });

    const bytes = await readFile(outputPath);
    await rm(outputPath, { force: true });
    return `data:image/webp;base64,${bytes.toString('base64')}`;
  } catch {
    await rm(outputPath, { force: true }).catch(() => {});
    return undefined;
  }
}

async function encodeWebp(
  sourcePath: string,
  outputPath: string,
  rung: number | null,
  email: string,
  sourceWidth: number,
  sourceHeight: number,
): Promise<Variant> {
  const result = await post<{ path: string; bytes: number }>('/v1/sharp', email, {
    inputPath: sourcePath,
    outputPath,
    // `rotate` with no angle applies the EXIF orientation, which phone photographs rely on entirely. A
    // ladder built without it is a set of sideways pictures. Kept on every rung including those cut
    // from the master, where it is a no-op — a PNG carries no EXIF, and the master was written with the
    // rotation already applied. One code path beats two that can disagree about which way up a photo is.
    pipeline: [
      { rotate: {} },
      ...(rung === null ? [] : [{ resize: { width: rung, fit: 'inside', withoutEnlargement: true } }]),
    ],
    output: { format: 'webp', options: { quality: WEBP_QUALITY } },
  });

  const width = rung ?? sourceWidth;
  return {
    file: outputPath.slice(outputPath.lastIndexOf('/') + 1),
    width,
    height: Math.round((width / sourceWidth) * sourceHeight),
    bytes: result.bytes,
  };
}

async function buildVideo(
  sourcePath: string,
  staging: string,
  name: string,
  source: { bytes: number; mtimeMs: number },
  email: string,
): Promise<Manifest> {
  const probe = await avClub<{ streams?: Array<{ codec_type?: string; width?: number; height?: number }> }>(
    `/v1/probe?path=${encodeURIComponent(sourcePath)}`,
    email,
  );
  const video = probe.streams?.find((stream) => stream.codec_type === 'video');
  const height = video?.height ?? 0;
  const width = video?.width ?? 0;

  const variants: Variant[] = [];

  /*
    Every rung the source is actually tall enough for, and the floor when it is tall enough for none.

    An unreadable height (0) is treated as "make everything": a probe that told us nothing is not
    evidence the video is small, and the encoder will decline to enlarge on its own long before we
    should decide on its behalf.
  */
  const wanted = height ? VIDEO_RUNGS.filter((rung) => height >= rung.height) : [...VIDEO_RUNGS];
  const rungs = wanted.length ? wanted : [VIDEO_FLOOR];

  for (const rung of rungs) {
    const result = await post<{ path: string; bytes: number }>('/v1/ffmpeg', email, {
      inputPath: sourcePath,
      outputPath: join(staging, rung.file),
      preset: rung.preset,
    });
    variants.push({ file: rung.file, width: null, height: null, bytes: result.bytes });
  }

  /*
    The poster, in two steps.

    av-club's `poster` preset writes a JPEG, which is its call and the right one for a general service.
    Here everything a browser is handed is WebP, so it goes through sharp on the way out and the JPEG
    is dropped. Two calls per video, once, at build time.
  */
  const stillJpg = join(staging, 'poster.jpg');
  await post('/v1/ffmpeg', email, {
    inputPath: sourcePath,
    outputPath: stillJpg,
    preset: 'poster',
    // A frame from a second in, because frame zero of a great many videos is black.
    inputOptions: { ss: 1 },
  }).catch(() => post('/v1/ffmpeg', email, { inputPath: sourcePath, outputPath: stillJpg, preset: 'poster' }));

  const poster = await post<{ path: string; bytes: number }>('/v1/sharp', email, {
    inputPath: stillJpg,
    outputPath: join(staging, 'poster.webp'),
    output: { format: 'webp', options: { quality: WEBP_QUALITY } },
  });
  /*
    From the STILL, not from the video.

    The same frame the poster comes from, so the two agree — and sharp can read a JPEG where it cannot
    read an MP4. Before the still is deleted, which is the only reason these three lines are in this
    order rather than the other one.
  */
  const placeholder = await encodePlaceholder(stillJpg, join(staging, 'placeholder.webp'), email);
  await rm(stillJpg, { force: true });
  variants.push({ file: 'poster.webp', width: null, height: null, bytes: poster.bytes });

  return {
    kind: 'video',
    source: { name, bytes: source.bytes, mtimeMs: source.mtimeMs, width: width || null, height: height || null },
    variants,
    placeholder,
    builtAt: new Date().toISOString(),
  };
}

/** Every file under `originals/`, relative to it, depth first. Directories are structure, not work. */
export async function listOriginals(root: string, prefix = ''): Promise<string[]> {
  /*
    Sorted, because `readdir` is not.

    It returns entries in the filesystem's own order — inode order on ext4 — which is arbitrary and
    stable only until somebody moves a file. That is harmless for the sync, which does the same work in
    any order, and it is not harmless for "Create Gallery": this list becomes the gallery's initial
    order, and nine photographs arriving as 05, 08, 01, 07 reads as a bug even though nothing is wrong.

    `numeric` so `galeria2` sorts before `galeria10`, which is what anybody naming files that way means.
  */
  const entries = (await readdir(join(root, prefix), { withFileTypes: true }).catch(() => [])).sort((a, b) =>
    a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: 'base' }),
  );
  const found: string[] = [];

  for (const entry of entries) {
    // Dotfiles are `.DS_Store` and friends. Nothing under here is a deliberate hidden file.
    if (entry.name.startsWith('.')) continue;
    const rel = prefix ? `${prefix}/${entry.name}` : entry.name;
    if (entry.isDirectory()) found.push(...(await listOriginals(root, rel)));
    else if (kindOf(entry.name)) found.push(rel);
  }

  return found;
}

/**
 * A caller-supplied subtree of `originals/`, made safe to walk.
 *
 * Relative, always, and never able to climb out: `..` in any segment is refused rather than resolved,
 * because a scope that escapes `originals/` would have the sync building derived folders for files
 * nobody put there. Leading and trailing slashes are trimmed so `/GALLERY/` and `GALLERY` mean one
 * thing — the caller is naming a folder, not writing a path.
 */
export function safeScope(scope: string): string {
  const trimmed = scope.replace(/^\/+|\/+$/g, '');
  if (!trimmed) return '';
  if (trimmed.split('/').includes('..')) throw new Error('That is not a folder inside originals.');
  return trimmed;
}

/** Where a relative original ends up. Exported because the watcher needs the same answer. */
export const targetFor = (derivedRoot: string, relativePath: string): string => join(derivedRoot, relativePath);

export { relative, AvClubUnavailable };

export type Reconciliation = {
  /** Convertible files under `originals/`. */
  originals: number;
  /** Built from the current version of their source. */
  current: number;
  /** Never built, or built and since lost. Names, so the panel can say WHICH. */
  missing: string[];
  /** Built, but from a version of the source that has changed since. */
  stale: string[];
};

/**
 * What `derived/` would still have to do to be in step with `originals/`.
 *
 * Answered from the filesystem every time rather than cached, because it is the question "is what I am
 * looking at current" and a cached answer to that is worse than none.
 *
 * It uses `upToDate` — the SAME predicate the sync uses to decide whether to skip a file. That is the
 * whole point: a panel reporting "62 current" and a run then rebuilding all 62 would be two different
 * definitions of the word, and the one on screen would be the one nobody could debug.
 *
 * `missing` and `stale` are separated because they mean different things to a person. Missing is a file
 * that has never been converted or whose output was deleted; stale is one that was replaced with a new
 * version. Both are work, but only the second means somebody changed something.
 */
export async function reconcile(email: string, scope = ''): Promise<Reconciliation> {
  const from = await originalsDir(email);
  const to = await derivedDir(email);
  const files = await listOriginals(from, safeScope(scope));

  const missing: string[] = [];
  const stale: string[] = [];
  let current = 0;

  for (const file of files) {
    const info = await stat(join(from, file)).catch(() => null);
    if (!info) continue;

    const target = targetFor(to, file);
    const manifest = await readFile(join(target, MANIFEST), 'utf8').catch(() => null);
    if (!manifest) {
      missing.push(file);
      continue;
    }

    if (await upToDate(target, { bytes: info.size, mtimeMs: Math.round(info.mtimeMs) })) current++;
    else stale.push(file);
  }

  return { originals: files.length, current, missing, stale };
}

/**
 * Every derived folder that already exists, indexed by what it was built FROM.
 *
 * ── What this is for ──
 *
 * The mapping from original to derived is by PATH, so moving a file into a folder makes the sync look
 * somewhere new, find nothing, and convert it again — minutes of work to produce bytes that already
 * exist a directory away, while the old folder stays behind forever because nothing deletes it.
 *
 * A move preserves size and mtime, so the pair identifies the same file at a new path. That is what
 * this index is: build it once before a sync, and a file that has moved can have its ladder RENAMED
 * into place instead of rebuilt.
 *
 * ── Why the pair, and not a hash ──
 *
 * A hash would be exact and would also mean reading every byte of every original on every sync, which
 * is the cost this is trying to avoid. Size and mtime are already in the manifest and already in the
 * stat, so the index is free.
 *
 * It is not unique in principle — two files can share both — so an ambiguous match is REFUSED below
 * rather than guessed at. Rebuilding is slow; attaching the wrong pictures to a name is wrong.
 */
export type DerivedIndex = Map<string, string[]>;

const indexKey = (bytes: number, mtimeMs: number) => `${bytes}:${mtimeMs}`;

/** Walk `derived/`, reading each manifest. Directories are the unit — a manifest is one built original. */
export async function indexDerived(root: string, prefix = '', into: DerivedIndex = new Map()): Promise<DerivedIndex> {
  const entries = await readdir(join(root, prefix), { withFileTypes: true }).catch(() => []);

  for (const entry of entries) {
    if (!entry.isDirectory() || entry.name.startsWith('.')) continue;
    const rel = prefix ? `${prefix}/${entry.name}` : entry.name;

    const raw = await readFile(join(root, rel, MANIFEST), 'utf8').catch(() => null);
    if (raw) {
      try {
        const manifest = JSON.parse(raw) as Manifest;
        const key = indexKey(manifest.source.bytes, manifest.source.mtimeMs);
        into.set(key, [...(into.get(key) ?? []), rel]);
      } catch {
        // Unreadable manifest: not indexed, so the file it belongs to rebuilds. One corrupt folder
        // costs one conversion rather than making the whole index untrustworthy.
      }
      // A built original is a leaf — its contents are renditions, not more originals.
      continue;
    }

    await indexDerived(root, rel, into);
  }

  return into;
}

/**
 * Has this original simply MOVED? If so, put its ladder where the file now is.
 *
 * Returns true when the folder was relocated and is now correct for this path.
 *
 * Refuses an ambiguous match. Two originals sharing a size and an mtime is unlikely and not
 * impossible — a camera dump can write a whole card in one second — and attaching the wrong
 * photographs to a name is a worse outcome than converting again.
 */
export async function relocateDerived(
  index: DerivedIndex,
  derivedRoot: string,
  relativePath: string,
  source: { bytes: number; mtimeMs: number },
): Promise<boolean> {
  const candidates = index.get(indexKey(source.bytes, source.mtimeMs)) ?? [];
  if (candidates.length !== 1) return false;

  const from = candidates[0]!;
  if (from === relativePath) return false;

  const fromDir = join(derivedRoot, from);
  const toDir = join(derivedRoot, relativePath);

  // Only if the destination is genuinely empty. A folder already there is either current — in which
  // case nothing needed doing — or half-built, and overwriting it with a rename is not a repair.
  if (await stat(toDir).catch(() => null)) return false;

  try {
    await mkdir(dirname(toDir), { recursive: true });
    await rename(fromDir, toDir);
    // Claimed, so a second original with the same size and mtime cannot take it too.
    index.delete(indexKey(source.bytes, source.mtimeMs));
    return true;
  } catch {
    return false;
  }
}

/**
 * Remove derived folders whose original is gone.
 *
 * ── Only for a whole-tree sync ──
 *
 * A scoped sync knows about one folder, so everything outside it would look abandoned. Deleting on
 * that basis would take out the entire library the first time somebody re-synced a subfolder, which is
 * the kind of mistake that is only noticed later.
 *
 * Safe to run because `derived/` is reproducible by definition — that is what its name has meant since
 * the day the two folders were called `originals` and `derived` rather than `user` and `treated`.
 */
export async function pruneDerived(originalsRoot: string, derivedRoot: string): Promise<string[]> {
  const alive = new Set(await listOriginals(originalsRoot));
  const built = await indexDerived(derivedRoot);

  const removed: string[] = [];
  for (const paths of built.values()) {
    for (const path of paths) {
      if (alive.has(path)) continue;
      await rm(join(derivedRoot, path), { recursive: true, force: true }).catch(() => {});
      removed.push(path);
    }
  }
  return removed;
}