From 01652f3480e3e490768c51e67309776c742209e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Sat, 29 Aug 2026 20:42:20 +0100 Subject: [PATCH] a moved file relocates its ladder instead of rebuilding it, and orphans go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving a file into a folder made the sync look at a new path, find nothing, and convert it again — minutes of work to produce bytes that already existed a directory away — while the old folder stayed behind forever, because nothing ever deleted one. A move preserves size and mtime, and both are already in the manifest and in the stat. So the sync indexes what is already derived on that pair, and a file that has moved has its ladder RENAMED across. An ambiguous match is refused rather than guessed. Two originals can share a size and an mtime — a camera dump can write a whole card in one second — and rebuilding is merely slow where attaching the wrong photographs to a name is wrong. Orphans are removed afterwards, and only on a WHOLE-TREE sync. A scoped run knows about one folder, so everything outside it looks abandoned; deleting on that basis would take out the library the first time somebody re-synced a subfolder. Safe otherwise because derived/ is reproducible by definition, which is what its name has meant since these two folders stopped being called `user` and `treated`. Both are counted, recorded in the ledger and shown only when they happened, so an ordinary run stays one short line. Co-Authored-By: Claude Opus 5 --- api/derive.ts | 122 +++++++++++++++++++++++++++++++++++++++++++++- api/ledger.ts | 4 ++ api/sync.ts | 62 +++++++++++++++++++++-- web/MediaSync.tsx | 9 ++++ 4 files changed, 193 insertions(+), 4 deletions(-) diff --git a/api/derive.ts b/api/derive.ts index 30e6ec2..b5de482 100644 --- a/api/derive.ts +++ b/api/derive.ts @@ -1,5 +1,5 @@ import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'; -import { join, relative } from 'node:path'; +import { dirname, join, relative } from 'node:path'; import { sidecarPort } from '@@/sidecar-registry'; import { mediaPath } from './media'; @@ -486,3 +486,123 @@ export async function reconcile(email: string, scope = ''): Promise; + +const indexKey = (bytes: number, mtimeMs: number) => `${bytes}:${mtimeMs}`; + +/** Walk `derived/`, reading each manifest. Directories are the unit — a manifest is one built original. */ +export async function indexDerived(root: string, prefix = '', into: DerivedIndex = new Map()): Promise { + const entries = await readdir(join(root, prefix), { withFileTypes: true }).catch(() => []); + + for (const entry of entries) { + if (!entry.isDirectory() || entry.name.startsWith('.')) continue; + const rel = prefix ? `${prefix}/${entry.name}` : entry.name; + + const raw = await readFile(join(root, rel, MANIFEST), 'utf8').catch(() => null); + if (raw) { + try { + const manifest = JSON.parse(raw) as Manifest; + const key = indexKey(manifest.source.bytes, manifest.source.mtimeMs); + into.set(key, [...(into.get(key) ?? []), rel]); + } catch { + // Unreadable manifest: not indexed, so the file it belongs to rebuilds. One corrupt folder + // costs one conversion rather than making the whole index untrustworthy. + } + // A built original is a leaf — its contents are renditions, not more originals. + continue; + } + + await indexDerived(root, rel, into); + } + + return into; +} + +/** + * Has this original simply MOVED? If so, put its ladder where the file now is. + * + * Returns true when the folder was relocated and is now correct for this path. + * + * Refuses an ambiguous match. Two originals sharing a size and an mtime is unlikely and not + * impossible — a camera dump can write a whole card in one second — and attaching the wrong + * photographs to a name is a worse outcome than converting again. + */ +export async function relocateDerived( + index: DerivedIndex, + derivedRoot: string, + relativePath: string, + source: { bytes: number; mtimeMs: number }, +): Promise { + const candidates = index.get(indexKey(source.bytes, source.mtimeMs)) ?? []; + if (candidates.length !== 1) return false; + + const from = candidates[0]!; + if (from === relativePath) return false; + + const fromDir = join(derivedRoot, from); + const toDir = join(derivedRoot, relativePath); + + // Only if the destination is genuinely empty. A folder already there is either current — in which + // case nothing needed doing — or half-built, and overwriting it with a rename is not a repair. + if (await stat(toDir).catch(() => null)) return false; + + try { + await mkdir(dirname(toDir), { recursive: true }); + await rename(fromDir, toDir); + // Claimed, so a second original with the same size and mtime cannot take it too. + index.delete(indexKey(source.bytes, source.mtimeMs)); + return true; + } catch { + return false; + } +} + +/** + * Remove derived folders whose original is gone. + * + * ── Only for a whole-tree sync ── + * + * A scoped sync knows about one folder, so everything outside it would look abandoned. Deleting on + * that basis would take out the entire library the first time somebody re-synced a subfolder, which is + * the kind of mistake that is only noticed later. + * + * Safe to run because `derived/` is reproducible by definition — that is what its name has meant since + * the day the two folders were called `originals` and `derived` rather than `user` and `treated`. + */ +export async function pruneDerived(originalsRoot: string, derivedRoot: string): Promise { + const alive = new Set(await listOriginals(originalsRoot)); + const built = await indexDerived(derivedRoot); + + const removed: string[] = []; + for (const paths of built.values()) { + for (const path of paths) { + if (alive.has(path)) continue; + await rm(join(derivedRoot, path), { recursive: true, force: true }).catch(() => {}); + removed.push(path); + } + } + return removed; +} diff --git a/api/ledger.ts b/api/ledger.ts index 8c3c2f8..1638089 100644 --- a/api/ledger.ts +++ b/api/ledger.ts @@ -54,6 +54,10 @@ export type Run = { total: number; built: number; skipped: number; + /** Renamed into place after a move, rather than rebuilt. */ + moved: number; + /** Removed because the original is gone. */ + pruned: number; errors: SyncError[]; /** Set when the run stopped for a reason that was not about any single file. */ fatal: string | null; diff --git a/api/sync.ts b/api/sync.ts index 4a314ab..8360e7a 100644 --- a/api/sync.ts +++ b/api/sync.ts @@ -1,6 +1,16 @@ -import { mkdir } from 'node:fs/promises'; +import { mkdir, stat } from 'node:fs/promises'; import { dirname } from 'node:path'; -import { deriveOne, derivedDir, listOriginals, originalsDir, safeScope, targetFor } from './derive'; +import { + deriveOne, + derivedDir, + indexDerived, + listOriginals, + originalsDir, + pruneDerived, + relocateDerived, + safeScope, + targetFor, +} from './derive'; import { appendRun, type RunTrigger } from './ledger'; /** @@ -47,6 +57,10 @@ export type SyncState = { done: number; built: number; skipped: number; + /** Ladders that were RENAMED into place because their original moved, rather than rebuilt. */ + moved: number; + /** Derived folders removed because their original is gone. Only on a whole-tree sync. */ + pruned: number; errors: SyncError[]; /** The files being worked on right now, for somebody watching. Empty when idle. */ current: string[]; @@ -73,6 +87,8 @@ const idle = (): SyncState => ({ done: 0, built: 0, skipped: 0, + moved: 0, + pruned: 0, errors: [], current: [], startedAt: null, @@ -123,6 +139,16 @@ async function run(email: string, state: SyncState): Promise { const files = await listOriginals(from, state.scope); state.total = files.length; + /* + What is already built, indexed by what it was built FROM. + + Built once, before the walk, so a file that has simply MOVED can have its ladder renamed into + place instead of converted again — minutes of work to produce bytes that already exist a + directory away. Size and mtime survive a move and are already in both the manifest and the stat, + so the index costs one pass over derived/ and nothing else. + */ + const derivedIndex = await indexDerived(to); + let next = 0; const worker = async (): Promise => { for (;;) { @@ -134,8 +160,24 @@ async function run(email: string, state: SyncState): Promise { try { const target = targetFor(to, file); await mkdir(dirname(target), { recursive: true }); + + /* + A move, before a build. + + `deriveOne` decides by looking for a manifest at the target, so a file that changed folder + has none there and would convert from scratch. This asks the index whether the same bytes + were already built somewhere else and, if exactly one folder matches, renames it across. + An ambiguous match is refused inside `relocateDerived` — rebuilding is slow, attaching the + wrong photographs to a name is wrong. + */ + const info = await stat(`${from}/${file}`).catch(() => null); + const relocated = info + ? await relocateDerived(derivedIndex, to, file, { bytes: info.size, mtimeMs: Math.round(info.mtimeMs) }) + : false; + const { built } = await deriveOne(`${from}/${file}`, target, email); - if (built) state.built++; + if (relocated && !built) state.moved++; + else if (built) state.built++; else state.skipped++; } catch (error) { const message = (error as Error).message; @@ -159,6 +201,18 @@ async function run(email: string, state: SyncState): Promise { }; await Promise.all(Array.from({ length: CONCURRENCY }, worker)); + + /* + Then the orphans — derived folders whose original no longer exists. + + Only on a WHOLE-TREE sync. A scoped one knows about a single folder, so everything outside it + looks abandoned, and deleting on that basis would take out the library the first time somebody + re-synced one subfolder. Safe otherwise because derived/ is reproducible by definition, which is + what the name has meant since these two folders stopped being called `user` and `treated`. + */ + if (!state.scope && !state.fatal) { + state.pruned = (await pruneDerived(from, to)).length; + } } catch (error) { state.fatal = (error as Error).message; } finally { @@ -180,6 +234,8 @@ async function run(email: string, state: SyncState): Promise { total: state.total ?? state.done, built: state.built, skipped: state.skipped, + moved: state.moved, + pruned: state.pruned, errors: state.errors, fatal: state.fatal, }); diff --git a/web/MediaSync.tsx b/web/MediaSync.tsx index 173d9aa..4d55342 100644 --- a/web/MediaSync.tsx +++ b/web/MediaSync.tsx @@ -46,6 +46,10 @@ type SyncState = { done: number; built: number; skipped: number; + /** Ladders renamed into place because their original moved, rather than rebuilt. */ + moved: number; + /** Derived folders removed because their original is gone. Whole-tree syncs only. */ + pruned: number; errors: SyncError[]; current: string[]; startedAt: string | null; @@ -61,6 +65,8 @@ type Run = { total: number; built: number; skipped: number; + moved: number; + pruned: number; errors: SyncError[]; fatal: string | null; }; @@ -251,6 +257,9 @@ export const MediaSync = ({ onSettled }: { onSettled?: (settled: boolean) => voi {run.scope ? `${run.scope} · ` : ''} {run.built} built {run.skipped ? `, ${run.skipped} current` : ''} + {/* Only when they happened, so an ordinary run stays one short line. */} + {run.moved ? `, ${run.moved} moved` : ''} + {run.pruned ? `, ${run.pruned} orphans removed` : ''} {run.errors.length ? `, ${run.errors.length} failed` : ''} {run.fatal ? ` — ${run.fatal}` : ''} {run.trigger === 'watcher' ? ' · on upload' : ''}