From b1d1c91968d9ba65368496d52168db4bb9543cfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Mon, 27 Jul 2026 16:29:41 +0000 Subject: [PATCH] music: staging-slot cache with atomic symlink swap for full reindex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation for a safe from-scratch reindex. The live cache path is now a SYMLINK to a slot dir; readers + incremental writes follow it. - ensureCacheSetup() (run at sidecar boot): makes `cache` a symlink to a slot, migrating an existing real cache dir once (a fast rename, not a copy). - runBuild(outRoot, prev): the build now writes to a given root and returns the manifest without touching live serving state. - reindexNow(): incremental, in-place live build (manual + localized updates). - reindexFull(): builds a complete index into a FRESH slot without touching the live one, then activateSlot() swaps the symlink atomically (rename-over) ONLY on success — a failed rebuild leaves the live index untouched; old slots pruned. walk() takes the output root. Verified end-to-end: fresh setup, full build + swap + prune, incremental-through-symlink, and the one-time real-dir migration (content preserved). Restart officer-music to apply (triggers the migration). Co-Authored-By: Claude Opus 4.8 --- src/servers/sidecar/music/index.ts | 4 + src/servers/sidecar/music/indexer.ts | 118 ++++++++++++++++++++++----- 2 files changed, 102 insertions(+), 20 deletions(-) diff --git a/src/servers/sidecar/music/index.ts b/src/servers/sidecar/music/index.ts index 59196b6b..00eb1d54 100644 --- a/src/servers/sidecar/music/index.ts +++ b/src/servers/sidecar/music/index.ts @@ -5,6 +5,7 @@ import { createSidecarConnector } from '../connect'; import { streamAudioFile } from './stream-audio'; import { reindexNow, + ensureCacheSetup, getIndexStatus, getManifest, albumVersion, @@ -70,6 +71,9 @@ function getFreePort(): number { const port = getFreePort(); +// Ensure the cache is a symlink-to-slot before serving/building, so full reindexes can swap atomically. +await ensureCacheSetup(); + const server = Bun.serve({ port, hostname: '127.0.0.1', diff --git a/src/servers/sidecar/music/indexer.ts b/src/servers/sidecar/music/indexer.ts index 09ad06ff..3c22eb95 100644 --- a/src/servers/sidecar/music/indexer.ts +++ b/src/servers/sidecar/music/indexer.ts @@ -1,6 +1,6 @@ -import { readdir, stat, mkdir, writeFile, readFile, rm } from 'node:fs/promises'; +import { readdir, stat, mkdir, writeFile, readFile, rm, symlink, rename, lstat } from 'node:fs/promises'; import { existsSync } from 'node:fs'; -import { join, relative, basename } from 'node:path'; +import { join, relative, basename, dirname } from 'node:path'; import { homedir } from 'node:os'; // Server-side music library indexer — the platform counterpart of the app's music-index.ts. Walks the @@ -40,6 +40,44 @@ const COVER_MAX_PX = 600; // v1 → initial (meta + cover) v2 → + posters/ + lyrics/ const CACHE_VERSION = 2; +// ── Staging slots + atomic symlink swap ── +// The live cache path (CACHE_ROOT) is a SYMLINK to a slot dir; all readers + incremental writes follow +// it. A full rebuild is built into a FRESH slot and swapped in atomically (rename over the symlink) only +// on success — so the live index is never half-built and a failed rebuild leaves it untouched. + +const MUSIC_DIR = dirname(CACHE_ROOT); +const SLOT_PREFIX = 'cache.store-'; +const slotPath = (id: string) => join(MUSIC_DIR, `${SLOT_PREFIX}${id}`); + +/** Ensure CACHE_ROOT is a symlink to a slot dir, migrating an existing real cache dir once. Idempotent. */ +export async function ensureCacheSetup(): Promise { + const st = await lstat(CACHE_ROOT).catch(() => null); + if (st?.isSymbolicLink()) return; // already set up + const slot = slotPath('initial'); + if (st?.isDirectory()) { + await rm(slot, { recursive: true, force: true }).catch(() => {}); + await rename(CACHE_ROOT, slot); // move the existing cache into a slot (a rename, not a copy — fast) + console.log('[music] migrated cache/ into a slot for symlink swapping'); + } else { + await mkdir(slot, { recursive: true }); + } + await symlink(basename(slot), CACHE_ROOT); // relative symlink: cache -> cache.store-initial + console.log(`[music] cache -> ${basename(slot)}`); +} + +/** Atomically repoint the cache symlink to `slot`, then remove every other slot. */ +async function activateSlot(slot: string): Promise { + const tmp = join(MUSIC_DIR, `.cache.swap-${Date.now()}`); + await rm(tmp, { force: true }).catch(() => {}); + await symlink(basename(slot), tmp); // temp relative symlink → new slot + await rename(tmp, CACHE_ROOT); // atomic replace of the live cache symlink + for (const name of await readdir(MUSIC_DIR).catch(() => [] as string[])) { + if (name.startsWith(SLOT_PREFIX) && name !== basename(slot)) { + await rm(join(MUSIC_DIR, name), { recursive: true, force: true }).catch(() => {}); + } + } +} + // ── Types (meta.json matches the app's IndexMeta/IndexTrack) ── export type IndexTrack = { @@ -443,8 +481,11 @@ async function resolveTrackLyrics( // ── Build ── -export async function buildMusicIndex(): Promise { - if (status.running) return getIndexStatus(); +// Core build: writes a full or incremental index into `outRoot` (+ its manifest.json), then returns the +// built manifest. Does NOT touch the live serving state — the caller decides when/if it goes live. +// `prev` drives the incremental skip: a populated prev skips unchanged albums (in-place live build); an +// empty prev rebuilds everything (into a fresh staging slot for a full reindex). +async function runBuild(outRoot: string, prev: Manifest): Promise { Object.assign(status, { running: true, startedAt: Date.now(), @@ -455,31 +496,28 @@ export async function buildMusicIndex(): Promise { tracksIndexed: 0, videosIndexed: 0, coversSaved: 0, - postersSaved: 0, - lyricsIndexed: 0, + postersSaved: 0, + lyricsIndexed: 0, discographies: 0, currentPath: '', error: null, }); console.log('[music] resync started'); - const prev = await loadManifest(); const next: Manifest = { version: CACHE_VERSION, generatedAt: status.startedAt!, albums: {} }; try { - await mkdir(CACHE_ROOT, { recursive: true }); - await walk(MUSIC_ROOT, prev, next); + await mkdir(outRoot, { recursive: true }); + await walk(MUSIC_ROOT, prev, next, outRoot); - // Prune cache dirs for albums that vanished from the library. + // Prune cache dirs for albums that vanished from the library (only meaningful when prev is populated). for (const rel of Object.keys(prev.albums)) { if (!next.albums[rel]) { - await rm(join(CACHE_ROOT, rel), { recursive: true, force: true }).catch(() => {}); + await rm(join(outRoot, rel), { recursive: true, force: true }).catch(() => {}); } } - await writeFile(MANIFEST_PATH, JSON.stringify(next)); - manifest = next; - manifestLoaded = true; + await writeFile(join(outRoot, 'manifest.json'), JSON.stringify(next)); } catch (err) { status.error = err instanceof Error ? err.message : String(err); } finally { @@ -496,7 +534,7 @@ export async function buildMusicIndex(): Promise { ); } } - return getIndexStatus(); + return next; } // ── Coalesced build entry point ── @@ -506,16 +544,56 @@ export async function buildMusicIndex(): Promise { let inflightBuild: Promise | null = null; -/** Run a build, joining an in-flight one instead of starting a second; resolves when it completes. */ +/** + * Incremental, live: rebuild only changed albums (unchanged ones skip by version stamp), updating the + * live cache + manifest in place. Fast — the manual reindex + localized watcher updates use this. + * Joins an in-flight build rather than starting a second. + */ export function reindexNow(): Promise { if (inflightBuild) return inflightBuild; - inflightBuild = buildMusicIndex().finally(() => { + inflightBuild = (async () => { + const next = await runBuild(CACHE_ROOT, await loadManifest()); + if (!status.error) { + manifest = next; + manifestLoaded = true; + } + return getIndexStatus(); + })().finally(() => { inflightBuild = null; }); return inflightBuild; } -async function walk(dirAbs: string, prev: Manifest, next: Manifest): Promise { +/** + * Full, from scratch: build a complete index into a FRESH slot without touching the live one, then swap + * it in atomically only on success — a failed rebuild leaves the live index untouched. For the nightly + * cron. Joins an in-flight build (so it won't stampede a running one). + */ +export function reindexFull(): Promise { + if (inflightBuild) return inflightBuild; + inflightBuild = (async () => { + await ensureCacheSetup(); + const slot = slotPath(String(Date.now())); + await rm(slot, { recursive: true, force: true }).catch(() => {}); + // Empty prev ⇒ everything rebuilds into the fresh slot. + const next = await runBuild(slot, { version: CACHE_VERSION, generatedAt: 0, albums: {} }); + if (status.error) { + await rm(slot, { recursive: true, force: true }).catch(() => {}); + console.error('[music] full reindex failed — live index left untouched'); + } else { + await activateSlot(slot); + manifest = next; + manifestLoaded = true; + console.log('[music] full reindex swapped in as the live index'); + } + return getIndexStatus(); + })().finally(() => { + inflightBuild = null; + }); + return inflightBuild; +} + +async function walk(dirAbs: string, prev: Manifest, next: Manifest, outRoot: string): Promise { status.currentPath = relative(MUSIC_ROOT, dirAbs) || '.'; status.foldersScanned += 1; emitProgress(); @@ -572,7 +650,7 @@ async function walk(dirAbs: string, prev: Manifest, next: Manifest): Promise