From f30d40446a2eef3522064930d2ee6d8ef0ddd3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Sat, 29 Aug 2026 11:47:32 +0100 Subject: [PATCH] a ledger of runs, and what is still outstanding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things the panel could not answer. A run's state lives in a Map, so a restart erased it — and with it any record that a file FAILED. That is the gap the manifests structurally cannot cover: a derived folder records the size, mtime and variants it was built from, so provenance is already precise, but a file that did not build has no folder and leaves no trace. Once the watcher is running, a failure at three in the morning would never be discovered. So one JSON line per run, appended to media/.sync-log.jsonl. A file rather than a table because db/schema.ts is emphatic that the connection is the only thing this plugin stores, and a ledger is losable — re-run and it rebuilds. Beside derived/ rather than inside it, so wiping derived does not erase the history of what happened before, which is the comparison it exists for. Written in a finally, so a run that died still records that it did. And the line above the button now says what is TRUE rather than what a run once did: reconcile() walks originals, reads each manifest, and reports current, missing and changed. It uses upToDate — the same predicate the sync uses to skip a file — because two definitions of "current" is how a panel comes to say everything is in step while the next run rebuilds half of it. Missing and changed are separated because only the second means somebody edited something. The ledger is not polled. It reads the whole tree, so it loads on arrival and again when a run stops — the only two moments its answer can change. Sync also takes an optional folder now, so a subtree can be re-synced on its own rather than the whole library. The scope is recorded on each run, because "3 built" means different things for one folder and for everything. Co-Authored-By: Claude Opus 5 --- api/derive.ts | 76 ++++++++++++++++++++++++++++++++- api/ledger.ts | 98 +++++++++++++++++++++++++++++++++++++++++++ api/router.ts | 29 ++++++++++++- api/sync.ts | 46 ++++++++++++++++++-- web/MediaSync.tsx | 104 +++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 340 insertions(+), 13 deletions(-) create mode 100644 api/ledger.ts diff --git a/api/derive.ts b/api/derive.ts index 2244b1c..30e6ec2 100644 --- a/api/derive.ts +++ b/api/derive.ts @@ -151,8 +151,13 @@ async function avClub(path: string, email: string, init?: RequestInit): Promi const post = (path: string, email: string, body: unknown): Promise => avClub(path, email, { method: 'POST', body: JSON.stringify(body) }); -/** Is this derived directory already correct for this original? */ -async function upToDate(dir: string, source: { bytes: number; mtimeMs: number }): Promise { +/** + * Is this derived directory already correct for this original? + * + * Exported because the panel asks the same question the sync does. Two implementations of "current" + * is how a screen comes to say everything is in step while the next run rebuilds half of it. + */ +export async function upToDate(dir: string, source: { bytes: number; mtimeMs: number }): Promise { const raw = await readFile(join(dir, MANIFEST), 'utf8').catch(() => null); if (!raw) return false; try { @@ -410,7 +415,74 @@ export async function listOriginals(root: string, prefix = ''): Promise join(derivedRoot, relativePath); export { relative, AvClubUnavailable }; + +export type Reconciliation = { + /** Convertible files under `originals/`. */ + originals: number; + /** Built from the current version of their source. */ + current: number; + /** Never built, or built and since lost. Names, so the panel can say WHICH. */ + missing: string[]; + /** Built, but from a version of the source that has changed since. */ + stale: string[]; +}; + +/** + * What `derived/` would still have to do to be in step with `originals/`. + * + * Answered from the filesystem every time rather than cached, because it is the question "is what I am + * looking at current" and a cached answer to that is worse than none. + * + * It uses `upToDate` — the SAME predicate the sync uses to decide whether to skip a file. That is the + * whole point: a panel reporting "62 current" and a run then rebuilding all 62 would be two different + * definitions of the word, and the one on screen would be the one nobody could debug. + * + * `missing` and `stale` are separated because they mean different things to a person. Missing is a file + * that has never been converted or whose output was deleted; stale is one that was replaced with a new + * version. Both are work, but only the second means somebody changed something. + */ +export async function reconcile(email: string, scope = ''): Promise { + const from = await originalsDir(email); + const to = await derivedDir(email); + const files = await listOriginals(from, safeScope(scope)); + + const missing: string[] = []; + const stale: string[] = []; + let current = 0; + + for (const file of files) { + const info = await stat(join(from, file)).catch(() => null); + if (!info) continue; + + const target = targetFor(to, file); + const manifest = await readFile(join(target, MANIFEST), 'utf8').catch(() => null); + if (!manifest) { + missing.push(file); + continue; + } + + if (await upToDate(target, { bytes: info.size, mtimeMs: Math.round(info.mtimeMs) })) current++; + else stale.push(file); + } + + return { originals: files.length, current, missing, stale }; +} diff --git a/api/ledger.ts b/api/ledger.ts new file mode 100644 index 0000000..8c3c2f8 --- /dev/null +++ b/api/ledger.ts @@ -0,0 +1,98 @@ +import { appendFile, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { mediaPath } from './media'; +import type { SyncError } from './sync'; + +/** + * What has been synced, and when. + * + * ── The hole this fills ── + * + * A run's state lives in memory, so a restart erases it — and with it any record that a file FAILED. + * That is the part the manifests cannot cover: a derived folder records the size, mtime and variants it + * was built from, so provenance is already answerable precisely, but a file that did not build has no + * folder and leaves no trace at all. With the watcher running, a failure at three in the morning would + * simply never be discovered. + * + * So the ledger's job is RUNS AND FAILURES, not provenance. The manifests keep the latter and are + * better at it. + * + * ── A file, not a table ── + * + * `db/schema.ts` says the connection is the only thing this plugin stores, and gives the reason: the + * installer drops a plugin's tables when "delete its data" is ticked, so the strongest thing that can + * be said about this plugin is that no operation on it can destroy the client's work. A ledger is + * losable — re-run and it rebuilds — so it does not need that protection, and a table would be a + * schema, a migration and a third thing to keep in step for something a line of JSON answers. + * + * It also matches the design already in play next door: the filesystem is the record. + * + * ── Beside `derived/`, not inside it ── + * + * Wiping `derived/` is a normal thing to do and must not erase the history of what happened before it — + * that history is exactly the comparison this exists for. + * + * ── Append-only, and why that is safe ── + * + * One line per run, written with a single `appendFile` on an O_APPEND handle. Writes under a page are + * atomic on Linux, and these are a couple of hundred bytes, so two writers cannot interleave a line. + * Today there is only ever one run at a time; that will stop being true when the watcher lands, which + * is why it is worth being true by construction rather than by luck. + */ + +const FILE = '.sync-log.jsonl'; + +export type RunTrigger = 'manual' | 'watcher'; + +export type Run = { + startedAt: string; + finishedAt: string; + trigger: RunTrigger; + /** The subtree covered, relative to `originals/`. Empty means all of it. */ + scope: string; + /** Convertible files the walk found. */ + total: number; + built: number; + skipped: number; + errors: SyncError[]; + /** Set when the run stopped for a reason that was not about any single file. */ + fatal: string | null; +}; + +const ledgerPath = async (email: string): Promise => join(await mediaPath(email), FILE); + +export async function appendRun(email: string, run: Run): Promise { + try { + await appendFile(await ledgerPath(email), `${JSON.stringify(run)}\n`, 'utf8'); + } catch { + /* + Swallowed on purpose. Failing to RECORD a sync must never fail the sync — the files are the + product and this is the note about them. The most likely cause is `media/` not existing, which + the panel already reports far more usefully than an exception thrown from here would. + */ + } +} + +/** + * The most recent runs, newest first. + * + * Read whole and sliced. The file is a couple of hundred bytes a line and would take years of a + * watcher firing on every upload to become worth streaming; when it does, that is the moment to + * change this, and not before. + */ +export async function readRuns(email: string, limit = 20): Promise { + const raw = await readFile(await ledgerPath(email), 'utf8').catch(() => ''); + + const runs: Run[] = []; + for (const line of raw.split('\n')) { + if (!line.trim()) continue; + try { + runs.push(JSON.parse(line) as Run); + } catch { + // A truncated last line — a crash mid-append. Skipped rather than fatal: one unreadable entry + // must not make the whole history unreadable. + } + } + + return runs.reverse().slice(0, limit); +} diff --git a/api/router.ts b/api/router.ts index 2ad2fed..997091a 100644 --- a/api/router.ts +++ b/api/router.ts @@ -6,6 +6,8 @@ import { readSettings, saveRootPath } from './settings'; import { readWebsiteStatus, cloneWebsite, updateWebsite } from './website'; import { readMediaStatus, createMediaDir } from './media'; import { readSync, startSync } from './sync'; +import { readRuns } from './ledger'; +import { reconcile } from './derive'; import { fofosRequest, FofosError, probe, type FofosResponse } from './fofos-client'; /** @@ -243,7 +245,14 @@ router.post('/_config/media/create', async (ctx) => { */ router.post('/_config/media/sync', async (ctx) => { const user = ctx.get('user'); - return ctx.json(startSync(user.email)); + // A folder inside `originals/`, or nothing for all of it. Named `path` because that is what the file + // browser calls it, and this is the value it will hand over when a folder is synced from there. + const { path } = (ctx.get('body') ?? {}) as { path?: string }; + try { + return ctx.json(startSync(user.email, { scope: path })); + } catch (error) { + return ctx.json({ error: (error as Error).message }, 400); + } }); router.get('/_config/media/sync', async (ctx) => { @@ -251,6 +260,24 @@ router.get('/_config/media/sync', async (ctx) => { return ctx.json(readSync(user.email)); }); +/** + * What has been done, and what is left. + * + * Separate from the live job state on purpose: that one is polled every second while a run is in + * flight, and this one walks the whole tree reading manifests. Answering both from one route would + * make every tick of a progress bar re-read the library. + */ +router.get('/_config/media/ledger', async (ctx) => { + const user = ctx.get('user'); + const scope = ctx.req.query('path') ?? ''; + try { + const [runs, state] = await Promise.all([readRuns(user.email), reconcile(user.email, scope)]); + return ctx.json({ runs, reconciliation: state }); + } catch (error) { + return ctx.json({ error: (error as Error).message }, 400); + } +}); + /** * Is this plugin usable? Asked once by the detail panel rather than by all nine sections. * diff --git a/api/sync.ts b/api/sync.ts index c1ac7bc..4a314ab 100644 --- a/api/sync.ts +++ b/api/sync.ts @@ -1,6 +1,7 @@ import { mkdir } from 'node:fs/promises'; import { dirname } from 'node:path'; -import { deriveOne, derivedDir, listOriginals, originalsDir, targetFor } from './derive'; +import { deriveOne, derivedDir, listOriginals, originalsDir, safeScope, targetFor } from './derive'; +import { appendRun, type RunTrigger } from './ledger'; /** * The sync: bring `derived/` into line with `originals/`. @@ -53,6 +54,14 @@ export type SyncState = { finishedAt: string | null; /** Set when the run stopped for a reason that was not about any single file — av-club being down. */ fatal: string | null; + /** + * The subtree this run covered, relative to `originals/`. Empty means all of it. + * + * Recorded rather than implied, because "3 built" means something different depending on whether it + * was the whole library or one folder somebody had just dropped photographs into. + */ + scope: string; + trigger: RunTrigger; }; /** One. See the note above — this is a memory limit, not a politeness. */ @@ -69,6 +78,8 @@ const idle = (): SyncState => ({ startedAt: null, finishedAt: null, fatal: null, + scope: '', + trigger: 'manual', }); const states = new Map(); @@ -81,11 +92,20 @@ export const readSync = (email: string): SyncState => states.get(email) ?? idle( * Returns the state either way, so the caller does not have to decide what "already running" means to * a person: the panel shows the run that is in progress, which is the honest answer to "sync now". */ -export function startSync(email: string): { started: boolean; state: SyncState } { +export function startSync( + email: string, + options: { scope?: string; trigger?: RunTrigger } = {}, +): { started: boolean; state: SyncState } { const existing = states.get(email); if (existing?.running) return { started: false, state: existing }; - const state: SyncState = { ...idle(), running: true, startedAt: new Date().toISOString() }; + const state: SyncState = { + ...idle(), + running: true, + startedAt: new Date().toISOString(), + scope: safeScope(options.scope ?? ''), + trigger: options.trigger ?? 'manual', + }; states.set(email, state); // Deliberately not awaited. The route answers immediately with the job's opening state and the panel @@ -100,7 +120,7 @@ async function run(email: string, state: SyncState): Promise { const from = await originalsDir(email); const to = await derivedDir(email); - const files = await listOriginals(from); + const files = await listOriginals(from, state.scope); state.total = files.length; let next = 0; @@ -145,5 +165,23 @@ async function run(email: string, state: SyncState): Promise { state.running = false; state.current = []; state.finishedAt = new Date().toISOString(); + + /* + The one durable record this leaves. + + In `finally`, so a run that died still writes a line — a failure nobody was watching is the whole + reason the ledger exists, and the runs that go unrecorded would be exactly the ones worth having. + */ + await appendRun(email, { + startedAt: state.startedAt ?? state.finishedAt, + finishedAt: state.finishedAt, + trigger: state.trigger, + scope: state.scope, + total: state.total ?? state.done, + built: state.built, + skipped: state.skipped, + errors: state.errors, + fatal: state.fatal, + }); } } diff --git a/web/MediaSync.tsx b/web/MediaSync.tsx index 2ceb25a..70b5bf5 100644 --- a/web/MediaSync.tsx +++ b/web/MediaSync.tsx @@ -53,15 +53,46 @@ type SyncState = { fatal: string | null; }; +type Run = { + startedAt: string; + finishedAt: string; + trigger: 'manual' | 'watcher'; + scope: string; + total: number; + built: number; + skipped: number; + errors: SyncError[]; + fatal: string | null; +}; + +type Reconciliation = { originals: number; current: number; missing: string[]; stale: string[] }; + +type Ledger = { runs: Run[]; reconciliation: Reconciliation }; + /** How often to ask, while something is happening. Fast enough to look live, slow enough to be free. */ const POLL_MS = 1000; +/** Recent runs worth showing. Enough to see a pattern, few enough to stay a footnote. */ +const SHOWN_RUNS = 5; + +const clock = (iso: string) => + new Date(iso).toLocaleTimeString(undefined, { hour: '2-digit', minute: '2-digit' }); + +const day = (iso: string) => new Date(iso).toLocaleDateString(undefined, { day: 'numeric', month: 'short' }); + +/** `1m 12s`, or `8s`. Minutes matter here and hours have never happened. */ +const took = (from: string, to: string) => { + const seconds = Math.max(0, Math.round((Date.parse(to) - Date.parse(from)) / 1000)); + return seconds < 60 ? `${seconds}s` : `${Math.floor(seconds / 60)}m ${seconds % 60}s`; +}; + export const MediaSync = () => { const client = useClient(); const clientRef = useRef(client); clientRef.current = client; const [state, setState] = useState(null); + const [ledger, setLedger] = useState(null); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); @@ -74,9 +105,25 @@ export const MediaSync = () => { } }, []); + /* + The ledger is NOT polled. + + It walks the whole tree reading manifests, so asking for it once a second beside a progress bar + would have every tick re-read the library. It is read on arrival and again when a run stops, which + are the only two moments its answer can have changed. + */ + const loadLedger = useCallback(async () => { + try { + setLedger(await clientRef.current.get('/fofos-admin/_config/media/ledger')); + } catch { + setLedger(null); + } + }, []); + useEffect(() => { void load(); - }, [load]); + void loadLedger(); + }, [load, loadLedger]); useEffect(() => { if (!state?.running) return; @@ -84,6 +131,15 @@ export const MediaSync = () => { return () => clearInterval(timer); }, [state?.running, load]); + // Re-read when a run STOPS. `running` going true→false is the edge, so a finished sync updates what + // is left to do without the panel being reopened. + const wasRunning = useRef(false); + useEffect(() => { + const running = state?.running ?? false; + if (wasRunning.current && !running) void loadLedger(); + wasRunning.current = running; + }, [state?.running, loadLedger]); + const start = async () => { setBusy(true); setError(null); @@ -105,25 +161,35 @@ export const MediaSync = () => { const done = state?.done ?? 0; // Null total means the walk has not finished, so there is nothing honest to divide by yet. const percent = total && total > 0 ? Math.round((done / total) * 100) : null; + const reconciliation = ledger?.reconciliation ?? null; + const outstanding = (reconciliation?.missing.length ?? 0) + (reconciliation?.stale.length ?? 0); return (
+ {/* Not what the last run did — what is TRUE NOW. A run's counts age the moment somebody uploads + a file, and "62 built" three days ago says nothing about whether derived/ is current. This is + recomputed from the filesystem, using the same freshness test the sync itself uses. */}

{running ? total === null ? 'Reading originals…' : `${done} of ${total}${state?.current.length ? ` · ${state.current.join(', ')}` : ''}` - : state?.finishedAt - ? `${state.built} built · ${state.skipped} already current${ - state.errors.length ? ` · ${state.errors.length} failed` : '' - }` + : reconciliation + ? outstanding === 0 + ? `${reconciliation.current} of ${reconciliation.originals} current` + : [ + reconciliation.missing.length ? `${reconciliation.missing.length} missing` : '', + reconciliation.stale.length ? `${reconciliation.stale.length} changed` : '', + ] + .filter(Boolean) + .join(' · ') : 'Web-ready copies of everything in originals.'}

@@ -157,6 +223,32 @@ export const MediaSync = () => { )} {error &&

{error}

} + + {/* + The ledger. Survives a restart, which the live state above does not — and it is the only place a + failure from a run nobody watched is ever recorded, since a file that did not build leaves no + folder and no manifest behind it. + */} + {ledger && ledger.runs.length > 0 && ( +
    + {ledger.runs.slice(0, SHOWN_RUNS).map((run) => ( +
  • + + {day(run.finishedAt)} {clock(run.finishedAt)} + + + {run.scope ? `${run.scope} · ` : ''} + {run.built} built + {run.skipped ? `, ${run.skipped} current` : ''} + {run.errors.length ? `, ${run.errors.length} failed` : ''} + {run.fatal ? ` — ${run.fatal}` : ''} + {run.trigger === 'watcher' ? ' · on upload' : ''} + + {took(run.startedAt, run.finishedAt)} +
    • + ))} +
+ )}
); };