import { mkdir } from 'node:fs/promises'; import { dirname } from 'node:path'; import { deriveOne, derivedDir, listOriginals, originalsDir, safeScope, targetFor } from './derive'; import { appendRun, type RunTrigger } from './ledger'; /** * The sync: bring `derived/` into line with `originals/`. * * ── One job, in memory, per user ── * * Not a `pipeline_jobs` row. That table belongs to the task runner, which runs a script in a temp * directory with a cwd and a log — a good fit for what a user wrote and a poor one for a plugin * calling a service in-process. So the state lives here, and the cost is stated plainly: **a restart * loses the job.** * * That cost is small because the work is idempotent. Every derived directory carries a manifest naming * the size and mtime it was built from, so a sync that died halfway is finished by running it again — * it skips everything already correct and picks up where the interruption left it. There is nothing to * reconcile and nothing to clean up, which is the property that made an in-memory job acceptable * rather than merely convenient. * * ── The same code path as a single file ── * * `deriveOne` is what the watcher will call for one upload. This walks and calls it; it decides * nothing about how a file is built. A full sync and a single arrival must never be two implementations * that can disagree about what a derived directory should contain. * * ── One at a time ── * * It was two, to match av-club's own concurrency. That is the right number on a machine with room and * the wrong one here: converting a 12-megapixel HEIC holds a full-resolution bitmap, and fofos has * 1.87 GiB of RAM. Two at once left about 520 MiB free with 648 MB already paged out — CPU pinned at * 99%, and an out-of-memory kill one large photograph away. * * Nothing is lost by going single-file. The work is CPU-bound on a box that was already saturated, so * the second worker was not buying throughput; it was buying a second resident bitmap. It also makes * "what is happening now" one filename instead of two, which is the more useful thing to read. */ export type SyncError = { file: string; message: string }; export type SyncState = { running: boolean; /** How many originals the walk found. Null until the walk finishes. */ total: number | null; /** Attempted — built, skipped and failed together. What a progress bar divides by `total`. */ done: number; built: number; skipped: number; errors: SyncError[]; /** The files being worked on right now, for somebody watching. Empty when idle. */ current: string[]; startedAt: string | null; finishedAt: string | null; /** Set when the run stopped for a reason that was not about any single file — av-club being down. */ fatal: string | null; /** * The subtree this run covered, relative to `originals/`. Empty means all of it. * * Recorded rather than implied, because "3 built" means something different depending on whether it * was the whole library or one folder somebody had just dropped photographs into. */ scope: string; trigger: RunTrigger; }; /** One. See the note above — this is a memory limit, not a politeness. */ const CONCURRENCY = 1; const idle = (): SyncState => ({ running: false, total: null, done: 0, built: 0, skipped: 0, errors: [], current: [], startedAt: null, finishedAt: null, fatal: null, scope: '', trigger: 'manual', }); const states = new Map(); export const readSync = (email: string): SyncState => states.get(email) ?? idle(); /** * Start one, unless one is already running. * * Returns the state either way, so the caller does not have to decide what "already running" means to * a person: the panel shows the run that is in progress, which is the honest answer to "sync now". */ export function startSync( email: string, options: { scope?: string; trigger?: RunTrigger } = {}, ): { started: boolean; state: SyncState } { const existing = states.get(email); if (existing?.running) return { started: false, state: existing }; const state: SyncState = { ...idle(), running: true, startedAt: new Date().toISOString(), scope: safeScope(options.scope ?? ''), trigger: options.trigger ?? 'manual', }; states.set(email, state); // Deliberately not awaited. The route answers immediately with the job's opening state and the panel // polls; an await here would be the request timing out on a folder of any size. void run(email, state); return { started: true, state }; } async function run(email: string, state: SyncState): Promise { try { const from = await originalsDir(email); const to = await derivedDir(email); const files = await listOriginals(from, state.scope); state.total = files.length; let next = 0; const worker = async (): Promise => { for (;;) { const index = next++; const file = files[index]; if (file === undefined) return; state.current.push(file); try { const target = targetFor(to, file); await mkdir(dirname(target), { recursive: true }); const { built } = await deriveOne(`${from}/${file}`, target, email); if (built) state.built++; else state.skipped++; } catch (error) { const message = (error as Error).message; /* av-club being unreachable is not a fact about this file, and recording it once per file would bury the one thing worth reading under four hundred copies of it. Stop the run and say so once. */ if (message.includes('av-club is not running')) { state.fatal = message; return; } state.errors.push({ file, message }); } finally { state.current = state.current.filter((name) => name !== file); state.done++; } if (state.fatal) return; } }; await Promise.all(Array.from({ length: CONCURRENCY }, worker)); } catch (error) { state.fatal = (error as Error).message; } finally { state.running = false; state.current = []; state.finishedAt = new Date().toISOString(); /* The one durable record this leaves. In `finally`, so a run that died still writes a line — a failure nobody was watching is the whole reason the ledger exists, and the runs that go unrecorded would be exactly the ones worth having. */ await appendRun(email, { startedAt: state.startedAt ?? state.finishedAt, finishedAt: state.finishedAt, trigger: state.trigger, scope: state.scope, total: state.total ?? state.done, built: state.built, skipped: state.skipped, errors: state.errors, fatal: state.fatal, }); } }