fofos-admin/api/sync.ts

import { mkdir, stat } from 'node:fs/promises';
import { dirname } from 'node:path';
import {
  deriveOne,
  derivedDir,
  indexDerived,
  listOriginals,
  originalsDir,
  pruneDerived,
  relocateDerived,
  safeScope,
  targetFor,
} from './derive';
import { appendRun, type RunTrigger } from './ledger';

/**
 * The sync: bring `derived/` into line with `originals/`.
 *
 * ── One job, in memory, per user ──
 *
 * Not a `pipeline_jobs` row. That table belongs to the task runner, which runs a script in a temp
 * directory with a cwd and a log — a good fit for what a user wrote and a poor one for a plugin
 * calling a service in-process. So the state lives here, and the cost is stated plainly: **a restart
 * loses the job.**
 *
 * That cost is small because the work is idempotent. Every derived directory carries a manifest naming
 * the size and mtime it was built from, so a sync that died halfway is finished by running it again —
 * it skips everything already correct and picks up where the interruption left it. There is nothing to
 * reconcile and nothing to clean up, which is the property that made an in-memory job acceptable
 * rather than merely convenient.
 *
 * ── The same code path as a single file ──
 *
 * `deriveOne` is what the watcher will call for one upload. This walks and calls it; it decides
 * nothing about how a file is built. A full sync and a single arrival must never be two implementations
 * that can disagree about what a derived directory should contain.
 *
 * ── One at a time ──
 *
 * It was two, to match av-club's own concurrency. That is the right number on a machine with room and
 * the wrong one here: converting a 12-megapixel HEIC holds a full-resolution bitmap, and fofos has
 * 1.87 GiB of RAM. Two at once left about 520 MiB free with 648 MB already paged out — CPU pinned at
 * 99%, and an out-of-memory kill one large photograph away.
 *
 * Nothing is lost by going single-file. The work is CPU-bound on a box that was already saturated, so
 * the second worker was not buying throughput; it was buying a second resident bitmap. It also makes
 * "what is happening now" one filename instead of two, which is the more useful thing to read.
 */

export type SyncError = { file: string; message: string };

export type SyncState = {
  running: boolean;
  /** How many originals the walk found. Null until the walk finishes. */
  total: number | null;
  /** Attempted — built, skipped and failed together. What a progress bar divides by `total`. */
  done: number;
  built: number;
  skipped: number;
  /** Ladders that were RENAMED into place because their original moved, rather than rebuilt. */
  moved: number;
  /** Derived folders removed because their original is gone. Only on a whole-tree sync. */
  pruned: number;
  errors: SyncError[];
  /** The files being worked on right now, for somebody watching. Empty when idle. */
  current: string[];
  startedAt: string | null;
  finishedAt: string | null;
  /** Set when the run stopped for a reason that was not about any single file — av-club being down. */
  fatal: string | null;
  /**
   * The subtree this run covered, relative to `originals/`. Empty means all of it.
   *
   * Recorded rather than implied, because "3 built" means something different depending on whether it
   * was the whole library or one folder somebody had just dropped photographs into.
   */
  scope: string;
  trigger: RunTrigger;
};

/** One. See the note above — this is a memory limit, not a politeness. */
const CONCURRENCY = 1;

const idle = (): SyncState => ({
  running: false,
  total: null,
  done: 0,
  built: 0,
  skipped: 0,
  moved: 0,
  pruned: 0,
  errors: [],
  current: [],
  startedAt: null,
  finishedAt: null,
  fatal: null,
  scope: '',
  trigger: 'manual',
});

const states = new Map<string, SyncState>();

export const readSync = (email: string): SyncState => states.get(email) ?? idle();

/**
 * Start one, unless one is already running.
 *
 * Returns the state either way, so the caller does not have to decide what "already running" means to
 * a person: the panel shows the run that is in progress, which is the honest answer to "sync now".
 */
export function startSync(
  email: string,
  options: { scope?: string; trigger?: RunTrigger } = {},
): { started: boolean; state: SyncState } {
  const existing = states.get(email);
  if (existing?.running) return { started: false, state: existing };

  const state: SyncState = {
    ...idle(),
    running: true,
    startedAt: new Date().toISOString(),
    scope: safeScope(options.scope ?? ''),
    trigger: options.trigger ?? 'manual',
  };
  states.set(email, state);

  // Deliberately not awaited. The route answers immediately with the job's opening state and the panel
  // polls; an await here would be the request timing out on a folder of any size.
  void run(email, state);

  return { started: true, state };
}

async function run(email: string, state: SyncState): Promise<void> {
  try {
    const from = await originalsDir(email);
    const to = await derivedDir(email);

    const files = await listOriginals(from, state.scope);
    state.total = files.length;

    /*
      What is already built, indexed by what it was built FROM.

      Built once, before the walk, so a file that has simply MOVED can have its ladder renamed into
      place instead of converted again — minutes of work to produce bytes that already exist a
      directory away. Size and mtime survive a move and are already in both the manifest and the stat,
      so the index costs one pass over derived/ and nothing else.
    */
    const derivedIndex = await indexDerived(to);

    let next = 0;
    const worker = async (): Promise<void> => {
      for (;;) {
        const index = next++;
        const file = files[index];
        if (file === undefined) return;

        state.current.push(file);
        try {
          const target = targetFor(to, file);
          await mkdir(dirname(target), { recursive: true });

          /*
            A move, before a build.

            `deriveOne` decides by looking for a manifest at the target, so a file that changed folder
            has none there and would convert from scratch. This asks the index whether the same bytes
            were already built somewhere else and, if exactly one folder matches, renames it across.
            An ambiguous match is refused inside `relocateDerived` — rebuilding is slow, attaching the
            wrong photographs to a name is wrong.
          */
          const info = await stat(`${from}/${file}`).catch(() => null);
          const relocated = info
            ? await relocateDerived(derivedIndex, to, file, { bytes: info.size, mtimeMs: Math.round(info.mtimeMs) })
            : false;

          const { built } = await deriveOne(`${from}/${file}`, target, email);
          if (relocated && !built) state.moved++;
          else if (built) state.built++;
          else state.skipped++;
        } catch (error) {
          const message = (error as Error).message;
          /*
            av-club being unreachable is not a fact about this file, and recording it once per file
            would bury the one thing worth reading under four hundred copies of it. Stop the run and
            say so once.
          */
          if (message.includes('av-club is not running')) {
            state.fatal = message;
            return;
          }
          state.errors.push({ file, message });
        } finally {
          state.current = state.current.filter((name) => name !== file);
          state.done++;
        }

        if (state.fatal) return;
      }
    };

    await Promise.all(Array.from({ length: CONCURRENCY }, worker));

    /*
      Then the orphans — derived folders whose original no longer exists.

      Only on a WHOLE-TREE sync. A scoped one knows about a single folder, so everything outside it
      looks abandoned, and deleting on that basis would take out the library the first time somebody
      re-synced one subfolder. Safe otherwise because derived/ is reproducible by definition, which is
      what the name has meant since these two folders stopped being called `user` and `treated`.
    */
    if (!state.scope && !state.fatal) {
      state.pruned = (await pruneDerived(from, to)).length;
    }
  } catch (error) {
    state.fatal = (error as Error).message;
  } finally {
    state.running = false;
    state.current = [];
    state.finishedAt = new Date().toISOString();

    /*
      The one durable record this leaves.

      In `finally`, so a run that died still writes a line — a failure nobody was watching is the whole
      reason the ledger exists, and the runs that go unrecorded would be exactly the ones worth having.
    */
    await appendRun(email, {
      startedAt: state.startedAt ?? state.finishedAt,
      finishedAt: state.finishedAt,
      trigger: state.trigger,
      scope: state.scope,
      total: state.total ?? state.done,
      built: state.built,
      skipped: state.skipped,
      moved: state.moved,
      pruned: state.pruned,
      errors: state.errors,
      fatal: state.fatal,
    });
  }
}