diff --git a/api/derive.ts b/api/derive.ts index 2244b1c..30e6ec2 100644 --- a/api/derive.ts +++ b/api/derive.ts @@ -151,8 +151,13 @@ async function avClub(path: string, email: string, init?: RequestInit): Promi const post = (path: string, email: string, body: unknown): Promise => avClub(path, email, { method: 'POST', body: JSON.stringify(body) }); -/** Is this derived directory already correct for this original? */ -async function upToDate(dir: string, source: { bytes: number; mtimeMs: number }): Promise { +/** + * Is this derived directory already correct for this original? + * + * Exported because the panel asks the same question the sync does. Two implementations of "current" + * is how a screen comes to say everything is in step while the next run rebuilds half of it. + */ +export async function upToDate(dir: string, source: { bytes: number; mtimeMs: number }): Promise { const raw = await readFile(join(dir, MANIFEST), 'utf8').catch(() => null); if (!raw) return false; try { @@ -410,7 +415,74 @@ export async function listOriginals(root: string, prefix = ''): Promise join(derivedRoot, relativePath); export { relative, AvClubUnavailable }; + +export type Reconciliation = { + /** Convertible files under `originals/`. */ + originals: number; + /** Built from the current version of their source. */ + current: number; + /** Never built, or built and since lost. Names, so the panel can say WHICH. */ + missing: string[]; + /** Built, but from a version of the source that has changed since. */ + stale: string[]; +}; + +/** + * What `derived/` would still have to do to be in step with `originals/`. + * + * Answered from the filesystem every time rather than cached, because it is the question "is what I am + * looking at current" and a cached answer to that is worse than none. + * + * It uses `upToDate` — the SAME predicate the sync uses to decide whether to skip a file. That is the + * whole point: a panel reporting "62 current" and a run then rebuilding all 62 would be two different + * definitions of the word, and the one on screen would be the one nobody could debug. + * + * `missing` and `stale` are separated because they mean different things to a person. Missing is a file + * that has never been converted or whose output was deleted; stale is one that was replaced with a new + * version. Both are work, but only the second means somebody changed something. + */ +export async function reconcile(email: string, scope = ''): Promise { + const from = await originalsDir(email); + const to = await derivedDir(email); + const files = await listOriginals(from, safeScope(scope)); + + const missing: string[] = []; + const stale: string[] = []; + let current = 0; + + for (const file of files) { + const info = await stat(join(from, file)).catch(() => null); + if (!info) continue; + + const target = targetFor(to, file); + const manifest = await readFile(join(target, MANIFEST), 'utf8').catch(() => null); + if (!manifest) { + missing.push(file); + continue; + } + + if (await upToDate(target, { bytes: info.size, mtimeMs: Math.round(info.mtimeMs) })) current++; + else stale.push(file); + } + + return { originals: files.length, current, missing, stale }; +} diff --git a/api/ledger.ts b/api/ledger.ts new file mode 100644 index 0000000..8c3c2f8 --- /dev/null +++ b/api/ledger.ts @@ -0,0 +1,98 @@ +import { appendFile, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { mediaPath } from './media'; +import type { SyncError } from './sync'; + +/** + * What has been synced, and when. + * + * ── The hole this fills ── + * + * A run's state lives in memory, so a restart erases it — and with it any record that a file FAILED. + * That is the part the manifests cannot cover: a derived folder records the size, mtime and variants it + * was built from, so provenance is already answerable precisely, but a file that did not build has no + * folder and leaves no trace at all. With the watcher running, a failure at three in the morning would + * simply never be discovered. + * + * So the ledger's job is RUNS AND FAILURES, not provenance. The manifests keep the latter and are + * better at it. + * + * ── A file, not a table ── + * + * `db/schema.ts` says the connection is the only thing this plugin stores, and gives the reason: the + * installer drops a plugin's tables when "delete its data" is ticked, so the strongest thing that can + * be said about this plugin is that no operation on it can destroy the client's work. A ledger is + * losable — re-run and it rebuilds — so it does not need that protection, and a table would be a + * schema, a migration and a third thing to keep in step for something a line of JSON answers. + * + * It also matches the design already in play next door: the filesystem is the record. + * + * ── Beside `derived/`, not inside it ── + * + * Wiping `derived/` is a normal thing to do and must not erase the history of what happened before it — + * that history is exactly the comparison this exists for. + * + * ── Append-only, and why that is safe ── + * + * One line per run, written with a single `appendFile` on an O_APPEND handle. Writes under a page are + * atomic on Linux, and these are a couple of hundred bytes, so two writers cannot interleave a line. + * Today there is only ever one run at a time; that will stop being true when the watcher lands, which + * is why it is worth being true by construction rather than by luck. + */ + +const FILE = '.sync-log.jsonl'; + +export type RunTrigger = 'manual' | 'watcher'; + +export type Run = { + startedAt: string; + finishedAt: string; + trigger: RunTrigger; + /** The subtree covered, relative to `originals/`. Empty means all of it. */ + scope: string; + /** Convertible files the walk found. */ + total: number; + built: number; + skipped: number; + errors: SyncError[]; + /** Set when the run stopped for a reason that was not about any single file. */ + fatal: string | null; +}; + +const ledgerPath = async (email: string): Promise => join(await mediaPath(email), FILE); + +export async function appendRun(email: string, run: Run): Promise { + try { + await appendFile(await ledgerPath(email), `${JSON.stringify(run)}\n`, 'utf8'); + } catch { + /* + Swallowed on purpose. Failing to RECORD a sync must never fail the sync — the files are the + product and this is the note about them. The most likely cause is `media/` not existing, which + the panel already reports far more usefully than an exception thrown from here would. + */ + } +} + +/** + * The most recent runs, newest first. + * + * Read whole and sliced. The file is a couple of hundred bytes a line and would take years of a + * watcher firing on every upload to become worth streaming; when it does, that is the moment to + * change this, and not before. + */ +export async function readRuns(email: string, limit = 20): Promise { + const raw = await readFile(await ledgerPath(email), 'utf8').catch(() => ''); + + const runs: Run[] = []; + for (const line of raw.split('\n')) { + if (!line.trim()) continue; + try { + runs.push(JSON.parse(line) as Run); + } catch { + // A truncated last line — a crash mid-append. Skipped rather than fatal: one unreadable entry + // must not make the whole history unreadable. + } + } + + return runs.reverse().slice(0, limit); +} diff --git a/api/router.ts b/api/router.ts index 2ad2fed..997091a 100644 --- a/api/router.ts +++ b/api/router.ts @@ -6,6 +6,8 @@ import { readSettings, saveRootPath } from './settings'; import { readWebsiteStatus, cloneWebsite, updateWebsite } from './website'; import { readMediaStatus, createMediaDir } from './media'; import { readSync, startSync } from './sync'; +import { readRuns } from './ledger'; +import { reconcile } from './derive'; import { fofosRequest, FofosError, probe, type FofosResponse } from './fofos-client'; /** @@ -243,7 +245,14 @@ router.post('/_config/media/create', async (ctx) => { */ router.post('/_config/media/sync', async (ctx) => { const user = ctx.get('user'); - return ctx.json(startSync(user.email)); + // A folder inside `originals/`, or nothing for all of it. Named `path` because that is what the file + // browser calls it, and this is the value it will hand over when a folder is synced from there. + const { path } = (ctx.get('body') ?? {}) as { path?: string }; + try { + return ctx.json(startSync(user.email, { scope: path })); + } catch (error) { + return ctx.json({ error: (error as Error).message }, 400); + } }); router.get('/_config/media/sync', async (ctx) => { @@ -251,6 +260,24 @@ router.get('/_config/media/sync', async (ctx) => { return ctx.json(readSync(user.email)); }); +/** + * What has been done, and what is left. + * + * Separate from the live job state on purpose: that one is polled every second while a run is in + * flight, and this one walks the whole tree reading manifests. Answering both from one route would + * make every tick of a progress bar re-read the library. + */ +router.get('/_config/media/ledger', async (ctx) => { + const user = ctx.get('user'); + const scope = ctx.req.query('path') ?? ''; + try { + const [runs, state] = await Promise.all([readRuns(user.email), reconcile(user.email, scope)]); + return ctx.json({ runs, reconciliation: state }); + } catch (error) { + return ctx.json({ error: (error as Error).message }, 400); + } +}); + /** * Is this plugin usable? Asked once by the detail panel rather than by all nine sections. * diff --git a/api/sync.ts b/api/sync.ts index c1ac7bc..4a314ab 100644 --- a/api/sync.ts +++ b/api/sync.ts @@ -1,6 +1,7 @@ import { mkdir } from 'node:fs/promises'; import { dirname } from 'node:path'; -import { deriveOne, derivedDir, listOriginals, originalsDir, targetFor } from './derive'; +import { deriveOne, derivedDir, listOriginals, originalsDir, safeScope, targetFor } from './derive'; +import { appendRun, type RunTrigger } from './ledger'; /** * The sync: bring `derived/` into line with `originals/`. @@ -53,6 +54,14 @@ export type SyncState = { finishedAt: string | null; /** Set when the run stopped for a reason that was not about any single file — av-club being down. */ fatal: string | null; + /** + * The subtree this run covered, relative to `originals/`. Empty means all of it. + * + * Recorded rather than implied, because "3 built" means something different depending on whether it + * was the whole library or one folder somebody had just dropped photographs into. + */ + scope: string; + trigger: RunTrigger; }; /** One. See the note above — this is a memory limit, not a politeness. */ @@ -69,6 +78,8 @@ const idle = (): SyncState => ({ startedAt: null, finishedAt: null, fatal: null, + scope: '', + trigger: 'manual', }); const states = new Map(); @@ -81,11 +92,20 @@ export const readSync = (email: string): SyncState => states.get(email) ?? idle( * Returns the state either way, so the caller does not have to decide what "already running" means to * a person: the panel shows the run that is in progress, which is the honest answer to "sync now". */ -export function startSync(email: string): { started: boolean; state: SyncState } { +export function startSync( + email: string, + options: { scope?: string; trigger?: RunTrigger } = {}, +): { started: boolean; state: SyncState } { const existing = states.get(email); if (existing?.running) return { started: false, state: existing }; - const state: SyncState = { ...idle(), running: true, startedAt: new Date().toISOString() }; + const state: SyncState = { + ...idle(), + running: true, + startedAt: new Date().toISOString(), + scope: safeScope(options.scope ?? ''), + trigger: options.trigger ?? 'manual', + }; states.set(email, state); // Deliberately not awaited. The route answers immediately with the job's opening state and the panel @@ -100,7 +120,7 @@ async function run(email: string, state: SyncState): Promise { const from = await originalsDir(email); const to = await derivedDir(email); - const files = await listOriginals(from); + const files = await listOriginals(from, state.scope); state.total = files.length; let next = 0; @@ -145,5 +165,23 @@ async function run(email: string, state: SyncState): Promise { state.running = false; state.current = []; state.finishedAt = new Date().toISOString(); + + /* + The one durable record this leaves. + + In `finally`, so a run that died still writes a line — a failure nobody was watching is the whole + reason the ledger exists, and the runs that go unrecorded would be exactly the ones worth having. + */ + await appendRun(email, { + startedAt: state.startedAt ?? state.finishedAt, + finishedAt: state.finishedAt, + trigger: state.trigger, + scope: state.scope, + total: state.total ?? state.done, + built: state.built, + skipped: state.skipped, + errors: state.errors, + fatal: state.fatal, + }); } } diff --git a/web/MediaSync.tsx b/web/MediaSync.tsx index 2ceb25a..70b5bf5 100644 --- a/web/MediaSync.tsx +++ b/web/MediaSync.tsx @@ -53,15 +53,46 @@ type SyncState = { fatal: string | null; }; +type Run = { + startedAt: string; + finishedAt: string; + trigger: 'manual' | 'watcher'; + scope: string; + total: number; + built: number; + skipped: number; + errors: SyncError[]; + fatal: string | null; +}; + +type Reconciliation = { originals: number; current: number; missing: string[]; stale: string[] }; + +type Ledger = { runs: Run[]; reconciliation: Reconciliation }; + /** How often to ask, while something is happening. Fast enough to look live, slow enough to be free. */ const POLL_MS = 1000; +/** Recent runs worth showing. Enough to see a pattern, few enough to stay a footnote. */ +const SHOWN_RUNS = 5; + +const clock = (iso: string) => + new Date(iso).toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit' }); + +const day = (iso: string) => new Date(iso).toLocaleDateString(undefined, { day: 'numeric', month: 'short' }); + +/** `1m 12s`, or `8s`. Minutes matter here and hours have never happened. */ +const took = (from: string, to: string) => { + const seconds = Math.max(0, Math.round((Date.parse(to) - Date.parse(from)) / 1000)); + return seconds < 60 ? `${seconds}s` : `${Math.floor(seconds / 60)}m ${seconds % 60}s`; +}; + export const MediaSync = () => { const client = useClient(); const clientRef = useRef(client); clientRef.current = client; const [state, setState] = useState(null); + const [ledger, setLedger] = useState(null); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); @@ -74,9 +105,25 @@ export const MediaSync = () => { } }, []); + /* + The ledger is NOT polled. + + It walks the whole tree reading manifests, so asking for it once a second beside a progress bar + would have every tick re-read the library. It is read on arrival and again when a run stops, which + are the only two moments its answer can have changed. + */ + const loadLedger = useCallback(async () => { + try { + setLedger(await clientRef.current.get('/fofos-admin/_config/media/ledger')); + } catch { + setLedger(null); + } + }, []); + useEffect(() => { void load(); - }, [load]); + void loadLedger(); + }, [load, loadLedger]); useEffect(() => { if (!state?.running) return; @@ -84,6 +131,15 @@ export const MediaSync = () => { return () => clearInterval(timer); }, [state?.running, load]); + // Re-read when a run STOPS. `running` going true→false is the edge, so a finished sync updates what + // is left to do without the panel being reopened. + const wasRunning = useRef(false); + useEffect(() => { + const running = state?.running ?? false; + if (wasRunning.current && !running) void loadLedger(); + wasRunning.current = running; + }, [state?.running, loadLedger]); + const start = async () => { setBusy(true); setError(null); @@ -105,25 +161,35 @@ export const MediaSync = () => { const done = state?.done ?? 0; // Null total means the walk has not finished, so there is nothing honest to divide by yet. const percent = total && total > 0 ? Math.round((done / total) * 100) : null; + const reconciliation = ledger?.reconciliation ?? null; + const outstanding = (reconciliation?.missing.length ?? 0) + (reconciliation?.stale.length ?? 0); return (
+ {/* Not what the last run did — what is TRUE NOW. A run's counts age the moment somebody uploads + a file, and "62 built" three days ago says nothing about whether derived/ is current. This is + recomputed from the filesystem, using the same freshness test the sync itself uses. */}

{running ? total === null ? 'Reading originals…' : `${done} of ${total}${state?.current.length ? ` · ${state.current.join(', ')}` : ''}` - : state?.finishedAt - ? `${state.built} built · ${state.skipped} already current${ - state.errors.length ? ` · ${state.errors.length} failed` : '' - }` + : reconciliation + ? outstanding === 0 + ? `${reconciliation.current} of ${reconciliation.originals} current` + : [ + reconciliation.missing.length ? `${reconciliation.missing.length} missing` : '', + reconciliation.stale.length ? `${reconciliation.stale.length} changed` : '', + ] + .filter(Boolean) + .join(' · ') : 'Web-ready copies of everything in originals.'}

@@ -157,6 +223,32 @@ export const MediaSync = () => { )} {error &&

{error}

} + + {/* + The ledger. Survives a restart, which the live state above does not — and it is the only place a + failure from a run nobody watched is ever recorded, since a file that did not build leaves no + folder and no manifest behind it. + */} + {ledger && ledger.runs.length > 0 && ( +
    + {ledger.runs.slice(0, SHOWN_RUNS).map((run) => ( +
  • + + {day(run.finishedAt)} {clock(run.finishedAt)} + + + {run.scope ? `${run.scope} · ` : ''} + {run.built} built + {run.skipped ? `, ${run.skipped} current` : ''} + {run.errors.length ? `, ${run.errors.length} failed` : ''} + {run.fatal ? ` — ${run.fatal}` : ''} + {run.trigger === 'watcher' ? ' · on upload' : ''} + + {took(run.startedAt, run.finishedAt)} +
    • + ))} +
+ )}
); };