soulseek: cache peer share trees server-side

Browsing a peer inline could never work. slskd answers GET /users/{u}/browse with
the entire tree in one blocking response — measured at 59 MB / 18k folders / 284k
files for a single real peer — and it takes minutes because it round-trips to that
peer. The browser was made to wait for that, so navigating away threw the whole
thing out and the panel showed an error more often than a tree.

So the fetch moves into the sidecar and the result into Postgres. Clicking "fetch
shares" returns 202 and the job keeps running without the tab; the UI polls the
snapshot row and reads back pages. Folders are rows and files ride along as jsonb
on their folder, because folders are what you filter and page through while files
are only ever read for the one folder you opened — a row per file would be 284k
rows per peer for no gain.

A failed or in-flight refresh deliberately leaves the previous folders in place: a
peer going offline shouldn't cost you a good cache, so the panel drives off rows
existing rather than off status. Interrupted 'pending' snapshots are failed at
sidecar boot, since the job died with the process and would otherwise spin forever.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/servers/sidecar/slskd/browse.ts

@@ -0,0 +1,104 @@
+import type { BrowseDirInput, BrowsedFile } from 'officerdb';
+import { startSoulseekBrowse, finishSoulseekBrowse, failSoulseekBrowse } from 'officerdb';
+import { getSlskdBase, getSlskdApiKey } from './upstream';
+
+// Background share-tree fetches.
+//
+// slskd's GET /users/{u}/browse is a single blocking call that inlines every file of every folder —
+// measured at 59 MB / 18k folders / 284k files for one real peer, and it can take minutes because it
+// round-trips to that peer over the Soulseek network. Doing it from the browser means a tab that hangs
+// and throws the result away on navigation.
+//
+// So it runs HERE instead: the route kicks this off and returns immediately, the fetch outlives the
+// request (and the tab), and the result lands in Postgres for the UI to page through. The sidecar is a
+// long-lived PM2 process, which is what makes "come back later and it's there" work at all.
+
+// slskd's wire shape. Files carry `attributes` (bitrate/duration) that real peers leave empty, plus
+// `code`/`attributeCount` noise; all of it is dropped at ingest.
+type SlskdBrowseFile = { filename: string; size?: number; extension?: string };
+type SlskdBrowseDir = { name: string; fileCount?: number; files?: SlskdBrowseFile[] };
+type SlskdBrowsePayload = { directories?: SlskdBrowseDir[] } | SlskdBrowseDir[];
+
+// One browse per (user, peer) at a time. A second click while one is running is a no-op rather than a
+// duplicate multi-minute fetch of the same tree.
+const inFlight = new Map<string, Promise<void>>();
+const keyOf = (userId: number, username: string) => `${userId}:${username}`;
+
+/** Peers currently being fetched, so a caller can report "already running" without racing the map. */
+export const isBrowseRunning = (userId: number, username: string): boolean => inFlight.has(keyOf(userId, username));
+
+// Generous, but not unbounded: a peer that accepts the connection and then stalls shouldn't pin a
+// snapshot in 'pending' forever.
+const BROWSE_TIMEOUT_MS = 10 * 60 * 1000;
+
+const extensionOf = (file: SlskdBrowseFile): string => {
+  const raw = file.extension?.trim();
+  if (raw) return raw.replace(/^\./, '').toLowerCase();
+  const dot = file.filename.lastIndexOf('.');
+  return dot > 0 ? file.filename.slice(dot + 1).toLowerCase() : '';
+};
+
+/**
+ * Strip the response to what the UI actually renders. Locked directories are skipped: they're shares the
+ * peer won't serve us, so listing them would only offer downloads that always fail.
+ */
+function slim(payload: SlskdBrowsePayload): BrowseDirInput[] {
+  const dirs = Array.isArray(payload) ? payload : (payload?.directories ?? []);
+  return dirs.map((dir) => {
+    const files: BrowsedFile[] = (dir.files ?? []).map((f) => ({
+      // Browse gives a basename only, unlike search results' full remote path. Downloading one means
+      // rejoining it to the directory name with a backslash.
+      name: f.filename,
+      size: f.size ?? 0,
+      extension: extensionOf(f),
+    }));
+    return { name: dir.name, files };
+  });
+}
+
+async function run(userId: number, username: string, snapshotId: number): Promise<void> {
+  const base = getSlskdBase();
+  if (!base) {
+    await failSoulseekBrowse(snapshotId, 'slskd upstream not configured');
+    return;
+  }
+  const started = Date.now();
+  try {
+    const headers = new Headers();
+    const apiKey = getSlskdApiKey();
+    if (apiKey) headers.set('X-API-Key', apiKey);
+
+    const res = await fetch(`${base}/api/v0/users/${encodeURIComponent(username)}/browse`, {
+      headers,
+      signal: AbortSignal.timeout(BROWSE_TIMEOUT_MS),
+    });
+    if (!res.ok) {
+      const detail = (await res.text().catch(() => '')).slice(0, 200);
+      throw new Error(`slskd returned ${res.status}${detail ? `: ${detail}` : ''}`);
+    }
+
+    const dirs = slim((await res.json()) as SlskdBrowsePayload);
+    await finishSoulseekBrowse(snapshotId, dirs);
+    const files = dirs.reduce((n, d) => n + d.files.length, 0);
+    console.log(`[slskd] browse ${username} -> ${dirs.length} dirs / ${files} files in ${Date.now() - started}ms`);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    console.error(`[slskd] browse ${username} failed after ${Date.now() - started}ms: ${message}`);
+    await failSoulseekBrowse(snapshotId, message);
+  }
+}
+
+/**
+ * Start caching a peer's share tree and return once it's *registered*, not once it's done — the caller
+ * responds 202 and the fetch continues in the background. Resolves false if one was already running.
+ */
+export async function startBrowse(userId: number, username: string): Promise<boolean> {
+  const key = keyOf(userId, username);
+  if (inFlight.has(key)) return false;
+
+  const snapshotId = await startSoulseekBrowse(userId, username);
+  // Registered before awaiting anything inside run(), so two rapid clicks can't both get past the check.
+  const task = run(userId, username, snapshotId).finally(() => inFlight.delete(key));
+  inFlight.set(key, task);
+  return true;
+}

src/servers/sidecar/slskd/index.ts

@@ -1,5 +1,6 @@
 import type { SidecarCommand, SidecarEvent } from '../protocol';
 import { createSidecarConnector } from '../connect';
+import { resetStaleSoulseekBrowses } from 'officerdb';
 import { getSlskdBase, getSlskdApiKey, stripHopByHop } from './upstream';
 import { handleOfficerRoute } from './officer';
 
@@ -107,6 +108,12 @@ const server = Bun.serve({
 
 console.log(`[slskd] reverse-proxy listening on 127.0.0.1:${port} -> ${getSlskdBase() ?? '(SLSKD_URL unset)'}`);
 
+// A background share-tree fetch dies with this process, so anything left 'pending' from the previous life
+// would spin in the UI forever. Clear it once, at boot, before serving.
+resetStaleSoulseekBrowses()
+  .then((n) => n && console.log(`[slskd] failed ${n} browse snapshot(s) interrupted by restart`))
+  .catch((err) => console.error('[slskd] could not reset stale browse snapshots', err));
+
 // ── Command handlers ──
 
 type ReplyFn = (msg: SidecarEvent) => void;

src/servers/sidecar/slskd/officer.ts

@@ -1,4 +1,14 @@
-import { getSoulseekFavorites, addSoulseekFavorite, removeSoulseekFavorite } from 'officerdb';
+import {
+  getSoulseekFavorites,
+  addSoulseekFavorite,
+  removeSoulseekFavorite,
+  getSoulseekBrowseSnapshots,
+  getSoulseekBrowseSnapshot,
+  getSoulseekBrowseDirs,
+  getSoulseekBrowseDirFiles,
+  deleteSoulseekBrowse,
+} from 'officerdb';
+import { startBrowse } from './browse';
 
 // Officer-owned Soulseek routes — everything slskd itself has no concept of. These are served HERE, by
 // the sidecar, not forwarded upstream: the platform API stays a pure auth-and-forward proxy forever, and
@@ -7,9 +17,16 @@ import { getSoulseekFavorites, addSoulseekFavorite, removeSoulseekFavorite } fro
 //
 // ─────────────────────────────────────────────────────────────────────────────────────────────────
 // CONTRACT (reached from the browser as /api/slskd/_officer/…)
-//   GET    /_officer/favorites                 → string[]  favourited peer usernames, alphabetical
-//   POST   /_officer/favorites { username }    → { ok: true }  add (idempotent)
-//   DELETE /_officer/favorites?username=<u>    → { ok: true }  remove (no-op if absent)
+//   GET    /_officer/favorites                       → string[]  favourited peers, alphabetical
+//   POST   /_officer/favorites { username }          → { ok: true }  add (idempotent)
+//   DELETE /_officer/favorites?username=<u>          → { ok: true }  remove (no-op if absent)
+//
+//   GET    /_officer/browse                          → Snapshot[]  every cached share tree
+//   GET    /_officer/browse/<u>                      → Snapshot | null  one peer's cache state
+//   POST   /_officer/browse/<u>                      → 202 { started }  kick off a background fetch
+//   DELETE /_officer/browse/<u>                      → { ok: true }  drop the cache
+//   GET    /_officer/browse/<u>/dirs?q=&limit=&offset= → { dirs, total }  a page of folders, no files
+//   GET    /_officer/browse/<u>/dirs/<id>/files      → BrowsedFile[]  one folder's files
 // ─────────────────────────────────────────────────────────────────────────────────────────────────
 
 // The authenticated user id arrives in X-Officer-User, injected by the platform proxy after auth. We
@@ -30,30 +47,106 @@ const cleanUsername = (v: unknown): string | null => {
   return CONTROL_CHARS.test(u) ? null : u;
 };
 
-/** Handles a `/_officer/*` request, or returns null if the path isn't one of ours. */
-export async function handleOfficerRoute(req: Request, url: URL): Promise<Response | null> {
-  const path = url.pathname;
-  if (path !== '/_officer/favorites') return null;
+const DIR_PAGE_MAX = 500;
+const DIR_PAGE_DEFAULT = 100;
+const clampInt = (raw: string | null, fallback: number, min: number, max: number): number => {
+  const n = Number(raw);
+  if (!Number.isFinite(n)) return fallback;
+  return Math.min(max, Math.max(min, Math.trunc(n)));
+};
 
-  const userId = userIdOf(req);
-  if (userId === null) return Response.json({ error: 'missing X-Officer-User' }, { status: 401 });
+const badRequest = (error: string) => Response.json({ error }, { status: 400 });
+const notFound = () => new Response('not found', { status: 404 });
+const methodNotAllowed = () => new Response('method not allowed', { status: 405 });
 
+async function handleFavorites(req: Request, url: URL, userId: number): Promise<Response> {
   if (req.method === 'GET') return Response.json(await getSoulseekFavorites(userId));
 
   if (req.method === 'POST') {
     const body = (await req.json().catch(() => null)) as { username?: unknown } | null;
     const username = cleanUsername(body?.username);
-    if (!username) return Response.json({ error: 'username required' }, { status: 400 });
+    if (!username) return badRequest('username required');
     await addSoulseekFavorite(userId, username);
     return Response.json({ ok: true });
   }
 
   if (req.method === 'DELETE') {
     const username = cleanUsername(url.searchParams.get('username'));
-    if (!username) return Response.json({ error: 'username required' }, { status: 400 });
+    if (!username) return badRequest('username required');
     await removeSoulseekFavorite(userId, username);
     return Response.json({ ok: true });
   }
 
-  return new Response('method not allowed', { status: 405 });
+  return methodNotAllowed();
+}
+
+type BrowseRouteParams = { req: Request; url: URL; userId: number; segments: string[] };
+
+async function handleBrowse({ req, url, userId, segments }: BrowseRouteParams): Promise<Response> {
+  // /_officer/browse — every cached tree, for badging the favourites list in one request.
+  if (segments.length === 1) {
+    if (req.method !== 'GET') return methodNotAllowed();
+    return Response.json(await getSoulseekBrowseSnapshots(userId));
+  }
+
+  const username = cleanUsername(decodeURIComponent(segments[1] ?? ''));
+  if (!username) return badRequest('username required');
+
+  // /_officer/browse/<username>
+  if (segments.length === 2) {
+    if (req.method === 'GET') return Response.json(await getSoulseekBrowseSnapshot(userId, username));
+    if (req.method === 'POST') {
+      // 202: the fetch takes minutes and outlives this request. Poll the GET for status.
+      const started = await startBrowse(userId, username);
+      return Response.json({ started }, { status: 202 });
+    }
+    if (req.method === 'DELETE') {
+      await deleteSoulseekBrowse(userId, username);
+      return Response.json({ ok: true });
+    }
+    return methodNotAllowed();
+  }
+
+  if (segments[2] !== 'dirs') return notFound();
+
+  // /_officer/browse/<username>/dirs — a filtered page of folders, files excluded.
+  if (segments.length === 3) {
+    if (req.method !== 'GET') return methodNotAllowed();
+    const page = await getSoulseekBrowseDirs({
+      userId,
+      username,
+      q: url.searchParams.get('q') ?? undefined,
+      limit: clampInt(url.searchParams.get('limit'), DIR_PAGE_DEFAULT, 1, DIR_PAGE_MAX),
+      offset: clampInt(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER),
+    });
+    return Response.json(page);
+  }
+
+  // /_officer/browse/<username>/dirs/<id>/files
+  if (segments.length === 5 && segments[4] === 'files') {
+    if (req.method !== 'GET') return methodNotAllowed();
+    const dirId = Number(segments[3]);
+    if (!Number.isInteger(dirId) || dirId <= 0) return badRequest('invalid directory id');
+    const files = await getSoulseekBrowseDirFiles(userId, dirId);
+    return files ? Response.json(files) : notFound();
+  }
+
+  return notFound();
+}
+
+/** Handles a `/_officer/*` request, or returns null if the path isn't one of ours. */
+export async function handleOfficerRoute(req: Request, url: URL): Promise<Response | null> {
+  const segments = url.pathname
+    .replace(/^\/_officer\//, '')
+    .split('/')
+    .filter(Boolean);
+  if (!segments.length) return null;
+
+  const userId = userIdOf(req);
+  if (userId === null) return Response.json({ error: 'missing X-Officer-User' }, { status: 401 });
+
+  if (segments[0] === 'favorites' && segments.length === 1) return handleFavorites(req, url, userId);
+  if (segments[0] === 'browse') return handleBrowse({ req, url, userId, segments });
+
+  return null;
 }