music: SSE reindex progress stream + reindex-music CLI
Adds a live progress channel for the library index: - indexer.ts: progress subscribers (onIndexProgress) + throttled emit during the walk, and buildReport() for a final summary. - sidecar: GET /reindex/stream (SSE) — triggers a build if idle (?trigger=0 to watch only), streams `progress` events, ends with a `done` event carrying the report; auto-proxied at /api/music/reindex/stream for the app. Sidecar also writes DATA_PATH/music/.server (its port) for local tooling. - scripts/reindex-music.ts: CLI that reads the port file, follows the SSE, prints live progress + a final report. Run: bun scripts/reindex-music.ts Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,118 @@
|
|||||||
|
#!/usr/bin/env bun
|
||||||
|
/**
|
||||||
|
* Trigger the music library index on the officer-music sidecar and follow its progress live,
|
||||||
|
* ending with a summary report. Run from the platform repo: bun scripts/reindex-music.ts
|
||||||
|
*
|
||||||
|
* It reads the sidecar's port from DATA_PATH/music/.server (written by the sidecar on startup) and
|
||||||
|
* consumes its /reindex/stream SSE endpoint — the same stream the app subscribes to.
|
||||||
|
*/
|
||||||
|
import { readFileSync } from 'node:fs';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
|
||||||
|
const DATA_PATH = process.env.DATA_PATH ?? join(process.cwd(), 'data');
|
||||||
|
const PORT_FILE = join(DATA_PATH, 'music', '.server');
|
||||||
|
|
||||||
|
type Progress = {
|
||||||
|
running: boolean;
|
||||||
|
foldersScanned: number;
|
||||||
|
albumsBuilt: number;
|
||||||
|
albumsSkipped: number;
|
||||||
|
tracksIndexed: number;
|
||||||
|
coversSaved: number;
|
||||||
|
currentPath: string;
|
||||||
|
};
|
||||||
|
type Report = {
|
||||||
|
albums: number;
|
||||||
|
built: number;
|
||||||
|
skipped: number;
|
||||||
|
foldersScanned: number;
|
||||||
|
tracksIndexed: number;
|
||||||
|
coversSaved: number;
|
||||||
|
elapsedSec: number;
|
||||||
|
error: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
function readPort(): number {
|
||||||
|
try {
|
||||||
|
const p = parseInt(readFileSync(PORT_FILE, 'utf8').trim(), 10);
|
||||||
|
if (Number.isInteger(p) && p > 0) return p;
|
||||||
|
} catch {
|
||||||
|
/* fall through */
|
||||||
|
}
|
||||||
|
console.error(`✗ Could not read the sidecar port from ${PORT_FILE}.`);
|
||||||
|
console.error(' Is officer-music running? pm2 restart officer-music');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const isTTY = Boolean(process.stdout.isTTY);
|
||||||
|
const cols = () => (process.stdout.columns && process.stdout.columns > 0 ? process.stdout.columns : 100);
|
||||||
|
|
||||||
|
function printProgress(p: Progress): void {
|
||||||
|
const line =
|
||||||
|
`♪ indexing… folders ${p.foldersScanned} · tracks ${p.tracksIndexed} · ` +
|
||||||
|
`built ${p.albumsBuilt} · skipped ${p.albumsSkipped} · covers ${p.coversSaved}` +
|
||||||
|
(p.currentPath ? ` · ${p.currentPath}` : '');
|
||||||
|
if (isTTY) {
|
||||||
|
const clipped = line.length > cols() - 1 ? line.slice(0, cols() - 2) + '…' : line;
|
||||||
|
process.stdout.write('\r\x1b[2K' + clipped);
|
||||||
|
} else {
|
||||||
|
process.stdout.write(line + '\n');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function printReport(r: Report): void {
|
||||||
|
if (isTTY) process.stdout.write('\r\x1b[2K');
|
||||||
|
const l = (k: string, v: string | number) => console.log(` ${k.padEnd(9)} ${v}`);
|
||||||
|
console.log('\n─── Music index complete ───');
|
||||||
|
l('Albums:', `${r.albums} (${r.built} built, ${r.skipped} unchanged)`);
|
||||||
|
l('Tracks:', `${r.tracksIndexed} indexed`);
|
||||||
|
l('Covers:', `${r.coversSaved} compressed`);
|
||||||
|
l('Folders:', `${r.foldersScanned} scanned`);
|
||||||
|
l('Elapsed:', `${r.elapsedSec}s`);
|
||||||
|
if (r.error) l('Error:', r.error);
|
||||||
|
console.log('────────────────────────────\n');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main(): Promise<void> {
|
||||||
|
const port = readPort();
|
||||||
|
const url = `http://127.0.0.1:${port}/reindex/stream`;
|
||||||
|
console.log(`Triggering music index via ${url}\n`);
|
||||||
|
|
||||||
|
let res: Response;
|
||||||
|
try {
|
||||||
|
res = await fetch(url);
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`✗ Could not reach the sidecar at 127.0.0.1:${port}: ${String(err)}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
if (!res.ok || !res.body) {
|
||||||
|
console.error(`✗ Sidecar returned ${res.status}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
const reader = res.body.getReader();
|
||||||
|
const decoder = new TextDecoder();
|
||||||
|
let buf = '';
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const { done, value } = await reader.read();
|
||||||
|
if (done) break;
|
||||||
|
buf += decoder.decode(value, { stream: true });
|
||||||
|
let sep: number;
|
||||||
|
while ((sep = buf.indexOf('\n\n')) >= 0) {
|
||||||
|
const frame = buf.slice(0, sep);
|
||||||
|
buf = buf.slice(sep + 2);
|
||||||
|
let event = 'message';
|
||||||
|
let data = '';
|
||||||
|
for (const line of frame.split('\n')) {
|
||||||
|
if (line.startsWith('event:')) event = line.slice(6).trim();
|
||||||
|
else if (line.startsWith('data:')) data += line.slice(5).trim();
|
||||||
|
}
|
||||||
|
if (!data) continue;
|
||||||
|
if (event === 'progress') printProgress(JSON.parse(data) as Progress);
|
||||||
|
else if (event === 'done') printReport(JSON.parse(data) as Report);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main();
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { mkdirSync, writeFileSync } from 'node:fs';
|
||||||
|
import { join } from 'node:path';
|
||||||
import type { SidecarCommand, SidecarEvent } from '../protocol';
|
import type { SidecarCommand, SidecarEvent } from '../protocol';
|
||||||
import { createSidecarConnector } from '../connect';
|
import { createSidecarConnector } from '../connect';
|
||||||
import { streamAudioFile } from './stream-audio';
|
import { streamAudioFile } from './stream-audio';
|
||||||
@@ -8,8 +10,12 @@ import {
|
|||||||
albumVersion,
|
albumVersion,
|
||||||
metaFilePath,
|
metaFilePath,
|
||||||
coverFilePath,
|
coverFilePath,
|
||||||
|
onIndexProgress,
|
||||||
|
buildReport,
|
||||||
} from './indexer';
|
} from './indexer';
|
||||||
|
|
||||||
|
const DATA_PATH = process.env.DATA_PATH ?? join(process.cwd(), 'data');
|
||||||
|
|
||||||
// The music sidecar (officer-music). Same philosophy as the other officer-* sidecars: a singleton
|
// The music sidecar (officer-music). Same philosophy as the other officer-* sidecars: a singleton
|
||||||
// process that registers with the API server. It OWNS an audio-streaming HTTP server (all path
|
// process that registers with the API server. It OWNS an audio-streaming HTTP server (all path
|
||||||
// resolution + streaming + ffprobe duration happens here); the platform API is just a thin proxy that
|
// resolution + streaming + ffprobe duration happens here); the platform API is just a thin proxy that
|
||||||
@@ -56,6 +62,54 @@ const server = Bun.serve({
|
|||||||
}
|
}
|
||||||
if (url.pathname === '/reindex/status') return json(getIndexStatus());
|
if (url.pathname === '/reindex/status') return json(getIndexStatus());
|
||||||
|
|
||||||
|
// SSE progress stream (for the app + the CLI). Triggers a build if idle (unless ?trigger=0), then
|
||||||
|
// streams `progress` events until the build finishes, ending with a `done` event carrying the report.
|
||||||
|
if (url.pathname === '/reindex/stream') {
|
||||||
|
const trigger = url.searchParams.get('trigger') !== '0';
|
||||||
|
if (trigger && !getIndexStatus().running) void buildMusicIndex();
|
||||||
|
|
||||||
|
const encoder = new TextEncoder();
|
||||||
|
const stream = new ReadableStream({
|
||||||
|
start(controller) {
|
||||||
|
let closed = false;
|
||||||
|
let unsub = () => {};
|
||||||
|
const send = (event: string, data: unknown) => {
|
||||||
|
if (closed) return;
|
||||||
|
try {
|
||||||
|
controller.enqueue(encoder.encode(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`));
|
||||||
|
} catch {
|
||||||
|
/* stream closed */
|
||||||
|
}
|
||||||
|
};
|
||||||
|
const finish = (s: ReturnType<typeof getIndexStatus>) => {
|
||||||
|
send('done', buildReport(s));
|
||||||
|
unsub();
|
||||||
|
closed = true;
|
||||||
|
try {
|
||||||
|
controller.close();
|
||||||
|
} catch {
|
||||||
|
/* already closed */
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
send('progress', getIndexStatus());
|
||||||
|
const cur = getIndexStatus();
|
||||||
|
if (!cur.running) {
|
||||||
|
finish(cur); // nothing running → emit the last report and close
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
unsub = onIndexProgress((s) => {
|
||||||
|
send('progress', s);
|
||||||
|
if (!s.running && s.finishedAt) finish(s);
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return new Response(stream, {
|
||||||
|
headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', Connection: 'keep-alive' },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// ── Sync surface ──
|
// ── Sync surface ──
|
||||||
if (url.pathname === '/manifest') return json(await getManifest());
|
if (url.pathname === '/manifest') return json(await getManifest());
|
||||||
|
|
||||||
@@ -83,6 +137,14 @@ const server = Bun.serve({
|
|||||||
|
|
||||||
console.log(`[music] audio server listening on http://127.0.0.1:${port}`);
|
console.log(`[music] audio server listening on http://127.0.0.1:${port}`);
|
||||||
|
|
||||||
|
// Write the port to a well-known file so local tooling (scripts/reindex-music.ts) can find the server.
|
||||||
|
try {
|
||||||
|
mkdirSync(join(DATA_PATH, 'music'), { recursive: true });
|
||||||
|
writeFileSync(join(DATA_PATH, 'music', '.server'), String(port));
|
||||||
|
} catch {
|
||||||
|
/* best-effort */
|
||||||
|
}
|
||||||
|
|
||||||
// ── Command handlers ──
|
// ── Command handlers ──
|
||||||
|
|
||||||
type ReplyFn = (msg: SidecarEvent) => void;
|
type ReplyFn = (msg: SidecarEvent) => void;
|
||||||
|
|||||||
@@ -101,6 +101,57 @@ export function getIndexStatus(): IndexStatus {
|
|||||||
return { ...status };
|
return { ...status };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type IndexReport = {
|
||||||
|
albums: number; // albums with content (built + skipped)
|
||||||
|
built: number;
|
||||||
|
skipped: number;
|
||||||
|
foldersScanned: number;
|
||||||
|
tracksIndexed: number;
|
||||||
|
coversSaved: number;
|
||||||
|
elapsedSec: number;
|
||||||
|
error: string | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export function buildReport(s: IndexStatus = status): IndexReport {
|
||||||
|
const elapsed = s.startedAt && s.finishedAt ? (s.finishedAt - s.startedAt) / 1000 : 0;
|
||||||
|
return {
|
||||||
|
albums: s.albumsBuilt + s.albumsSkipped,
|
||||||
|
built: s.albumsBuilt,
|
||||||
|
skipped: s.albumsSkipped,
|
||||||
|
foldersScanned: s.foldersScanned,
|
||||||
|
tracksIndexed: s.tracksIndexed,
|
||||||
|
coversSaved: s.coversSaved,
|
||||||
|
elapsedSec: Math.round(elapsed * 10) / 10,
|
||||||
|
error: s.error,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Progress subscribers (for SSE) ──
|
||||||
|
|
||||||
|
type ProgressListener = (s: IndexStatus) => void;
|
||||||
|
const listeners = new Set<ProgressListener>();
|
||||||
|
let lastEmit = 0;
|
||||||
|
|
||||||
|
export function onIndexProgress(cb: ProgressListener): () => void {
|
||||||
|
listeners.add(cb);
|
||||||
|
return () => listeners.delete(cb);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Notify subscribers of progress; throttled to ~200ms unless `force` (e.g. terminal 'done'). */
|
||||||
|
function emitProgress(force = false): void {
|
||||||
|
const now = Date.now();
|
||||||
|
if (!force && now - lastEmit < 200) return;
|
||||||
|
lastEmit = now;
|
||||||
|
const snap = getIndexStatus();
|
||||||
|
for (const cb of listeners) {
|
||||||
|
try {
|
||||||
|
cb(snap);
|
||||||
|
} catch {
|
||||||
|
/* ignore listener errors */
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── Helpers ──
|
// ── Helpers ──
|
||||||
|
|
||||||
function isAudio(name: string): boolean {
|
function isAudio(name: string): boolean {
|
||||||
@@ -226,6 +277,7 @@ export async function buildMusicIndex(): Promise<IndexStatus> {
|
|||||||
status.running = false;
|
status.running = false;
|
||||||
status.finishedAt = Date.now();
|
status.finishedAt = Date.now();
|
||||||
status.currentPath = '';
|
status.currentPath = '';
|
||||||
|
emitProgress(true); // final push — signals 'done' to SSE subscribers
|
||||||
}
|
}
|
||||||
return getIndexStatus();
|
return getIndexStatus();
|
||||||
}
|
}
|
||||||
@@ -233,6 +285,7 @@ export async function buildMusicIndex(): Promise<IndexStatus> {
|
|||||||
async function walk(dirAbs: string, prev: Manifest, next: Manifest): Promise<void> {
|
async function walk(dirAbs: string, prev: Manifest, next: Manifest): Promise<void> {
|
||||||
status.currentPath = relative(MUSIC_ROOT, dirAbs) || '.';
|
status.currentPath = relative(MUSIC_ROOT, dirAbs) || '.';
|
||||||
status.foldersScanned += 1;
|
status.foldersScanned += 1;
|
||||||
|
emitProgress();
|
||||||
|
|
||||||
let entries: import('node:fs').Dirent[];
|
let entries: import('node:fs').Dirent[];
|
||||||
try {
|
try {
|
||||||
@@ -276,6 +329,7 @@ async function walk(dirAbs: string, prev: Manifest, next: Manifest): Promise<voi
|
|||||||
const tracks = await mapPool(audio, TRACK_CONCURRENCY, async (name) => {
|
const tracks = await mapPool(audio, TRACK_CONCURRENCY, async (name) => {
|
||||||
const t = await ffprobeTrack(join(dirAbs, name), name);
|
const t = await ffprobeTrack(join(dirAbs, name), name);
|
||||||
status.tracksIndexed += 1;
|
status.tracksIndexed += 1;
|
||||||
|
emitProgress();
|
||||||
return t;
|
return t;
|
||||||
});
|
});
|
||||||
const meta: IndexMeta = { path: rel, cover: coverName ? 'cover.jpg' : undefined, tracks };
|
const meta: IndexMeta = { path: rel, cover: coverName ? 'cover.jpg' : undefined, tracks };
|
||||||
|
|||||||
Reference in New Issue
Block a user