soulseek: cache peer share trees server-side

Browsing a peer inline could never work. slskd answers GET /users/{u}/browse with
the entire tree in one blocking response — measured at 59 MB / 18k folders / 284k
files for a single real peer — and it takes minutes because it round-trips to that
peer. The browser was made to wait for that, so navigating away threw the whole
thing out and the panel showed an error more often than a tree.

So the fetch moves into the sidecar and the result into Postgres. Clicking "fetch
shares" returns 202 and the job keeps running without the tab; the UI polls the
snapshot row and reads back pages. Folders are rows and files ride along as jsonb
on their folder, because folders are what you filter and page through while files
are only ever read for the one folder you opened — a row per file would be 284k
rows per peer for no gain.

A failed or in-flight refresh deliberately leaves the previous folders in place: a
peer going offline shouldn't cost you a good cache, so the panel drives off rows
existing rather than off status. Interrupted 'pending' snapshots are failed at
sidecar boot, since the job died with the process and would otherwise spin forever.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

src/databases/officer_db/src/index.ts

@@ -103,6 +103,18 @@ export type {
   Playlist,
 } from './queries/music';
 export { getSoulseekFavorites, addSoulseekFavorite, removeSoulseekFavorite } from './queries/soulseek';
+export {
+  getSoulseekBrowseSnapshots,
+  getSoulseekBrowseSnapshot,
+  startSoulseekBrowse,
+  finishSoulseekBrowse,
+  failSoulseekBrowse,
+  resetStaleSoulseekBrowses,
+  getSoulseekBrowseDirs,
+  getSoulseekBrowseDirFiles,
+  deleteSoulseekBrowse,
+} from './queries/soulseek';
+export type { BrowsedFile, BrowseDirInput, BrowseDirPage, SoulseekBrowseSnapshot } from './queries/soulseek';
 export {
   getVaultTokens,
   setVaultTokens,

src/databases/officer_db/src/queries/soulseek.ts

@@ -1,6 +1,6 @@
-import { eq, and, asc } from 'drizzle-orm';
+import { eq, and, asc, ilike, sql } from 'drizzle-orm';
 import { db } from '../db';
-import { soulseekFavorites } from '../schema';
+import { soulseekFavorites, soulseekBrowseSnapshots, soulseekBrowseDirs } from '../schema';
 
 /** A user's favourited Soulseek peers, alphabetical (the order the UI lists them in). */
 export async function getSoulseekFavorites(userId: number): Promise<string[]> {
@@ -23,3 +23,188 @@ export async function removeSoulseekFavorite(userId: number, username: string):
     .delete(soulseekFavorites)
     .where(and(eq(soulseekFavorites.userId, userId), eq(soulseekFavorites.username, username)));
 }
+
+// ── Cached share trees ──
+
+/** A file inside a cached folder, slimmed at ingest (see the schema note on dropped attributes). */
+export type BrowsedFile = { name: string; size: number; extension: string };
+/** One folder to ingest: its slskd name, plus its files. Counts are derived here, not trusted. */
+export type BrowseDirInput = { name: string; files: BrowsedFile[] };
+
+export type SoulseekBrowseSnapshot = {
+  username: string;
+  status: string;
+  error: string | null;
+  directoryCount: number;
+  fileCount: number;
+  totalSize: number;
+  startedAt: Date;
+  completedAt: Date | null;
+};
+
+const snapshotCols = {
+  username: soulseekBrowseSnapshots.username,
+  status: soulseekBrowseSnapshots.status,
+  error: soulseekBrowseSnapshots.error,
+  directoryCount: soulseekBrowseSnapshots.directoryCount,
+  fileCount: soulseekBrowseSnapshots.fileCount,
+  totalSize: soulseekBrowseSnapshots.totalSize,
+  startedAt: soulseekBrowseSnapshots.startedAt,
+  completedAt: soulseekBrowseSnapshots.completedAt,
+};
+
+/** Every cached share tree for a user, so the UI can badge each favourite in one request. */
+export async function getSoulseekBrowseSnapshots(userId: number): Promise<SoulseekBrowseSnapshot[]> {
+  return db
+    .select(snapshotCols)
+    .from(soulseekBrowseSnapshots)
+    .where(eq(soulseekBrowseSnapshots.userId, userId))
+    .orderBy(asc(soulseekBrowseSnapshots.username));
+}
+
+/** One peer's snapshot, or null if never fetched. */
+export async function getSoulseekBrowseSnapshot(
+  userId: number,
+  username: string,
+): Promise<SoulseekBrowseSnapshot | null> {
+  const [row] = await db
+    .select(snapshotCols)
+    .from(soulseekBrowseSnapshots)
+    .where(and(eq(soulseekBrowseSnapshots.userId, userId), eq(soulseekBrowseSnapshots.username, username)))
+    .limit(1);
+  return row ?? null;
+}
+
+/**
+ * Mark a peer's tree as being fetched and return the snapshot id to ingest into. Deliberately leaves the
+ * existing folder rows and counts alone: they stay readable while the refresh runs, and survive it failing.
+ */
+export async function startSoulseekBrowse(userId: number, username: string): Promise<number> {
+  const [row] = await db
+    .insert(soulseekBrowseSnapshots)
+    .values({ userId, username, status: 'pending' })
+    .onConflictDoUpdate({
+      target: [soulseekBrowseSnapshots.userId, soulseekBrowseSnapshots.username],
+      set: { status: 'pending', error: null, startedAt: new Date(), completedAt: null },
+    })
+    .returning({ id: soulseekBrowseSnapshots.id });
+  if (!row) throw new Error('failed to create browse snapshot');
+  return row.id;
+}
+
+// Each row carries a whole folder's file list, so keep batches small enough that no single INSERT gets
+// absurd. 18k folders is ~36 round trips at this size.
+const INGEST_BATCH = 500;
+
+/** Swap in a freshly fetched tree: replace the folder rows and mark the snapshot ready, atomically. */
+export async function finishSoulseekBrowse(snapshotId: number, dirs: BrowseDirInput[]): Promise<void> {
+  const rows = dirs.map((d) => ({
+    snapshotId,
+    name: d.name,
+    fileCount: d.files.length,
+    totalSize: d.files.reduce((n, f) => n + (f.size || 0), 0),
+    files: d.files,
+  }));
+  await db.transaction(async (tx) => {
+    await tx.delete(soulseekBrowseDirs).where(eq(soulseekBrowseDirs.snapshotId, snapshotId));
+    for (let i = 0; i < rows.length; i += INGEST_BATCH) {
+      await tx.insert(soulseekBrowseDirs).values(rows.slice(i, i + INGEST_BATCH));
+    }
+    await tx
+      .update(soulseekBrowseSnapshots)
+      .set({
+        status: 'ready',
+        error: null,
+        completedAt: new Date(),
+        directoryCount: rows.length,
+        fileCount: rows.reduce((n, r) => n + r.fileCount, 0),
+        totalSize: rows.reduce((n, r) => n + r.totalSize, 0),
+      })
+      .where(eq(soulseekBrowseSnapshots.id, snapshotId));
+  });
+}
+
+/** Record a failed fetch. Counts and folder rows are left as they were — a stale cache beats none. */
+export async function failSoulseekBrowse(snapshotId: number, error: string): Promise<void> {
+  await db
+    .update(soulseekBrowseSnapshots)
+    .set({ status: 'failed', error: error.slice(0, 1000), completedAt: new Date() })
+    .where(eq(soulseekBrowseSnapshots.id, snapshotId));
+}
+
+/**
+ * Fail every snapshot still marked pending. Called on sidecar boot: an in-flight fetch dies with the
+ * process, so without this a crash mid-browse leaves a row spinning in the UI forever.
+ */
+export async function resetStaleSoulseekBrowses(): Promise<number> {
+  const rows = await db
+    .update(soulseekBrowseSnapshots)
+    .set({ status: 'failed', error: 'interrupted by a sidecar restart', completedAt: new Date() })
+    .where(eq(soulseekBrowseSnapshots.status, 'pending'))
+    .returning({ id: soulseekBrowseSnapshots.id });
+  return rows.length;
+}
+
+export type BrowseDirPage = {
+  dirs: { id: number; name: string; fileCount: number; totalSize: number }[];
+  total: number;
+};
+type BrowseDirPageParams = { userId: number; username: string; q?: string; limit: number; offset: number };
+
+/**
+ * A page of a peer's cached folders, name-filtered. Files are excluded — that's the whole point of
+ * paging here, and they're fetched per folder on expand.
+ */
+export async function getSoulseekBrowseDirs(params: BrowseDirPageParams): Promise<BrowseDirPage> {
+  const { userId, username, q, limit, offset } = params;
+  const [snap] = await db
+    .select({ id: soulseekBrowseSnapshots.id })
+    .from(soulseekBrowseSnapshots)
+    .where(and(eq(soulseekBrowseSnapshots.userId, userId), eq(soulseekBrowseSnapshots.username, username)))
+    .limit(1);
+  if (!snap) return { dirs: [], total: 0 };
+
+  const term = q?.trim();
+  // ILIKE with a leading wildcard can't use the btree index, but 18k rows per snapshot is a trivial scan.
+  const where = term
+    ? and(eq(soulseekBrowseDirs.snapshotId, snap.id), ilike(soulseekBrowseDirs.name, `%${term}%`))
+    : eq(soulseekBrowseDirs.snapshotId, snap.id);
+
+  const [dirs, [count]] = await Promise.all([
+    db
+      .select({
+        id: soulseekBrowseDirs.id,
+        name: soulseekBrowseDirs.name,
+        fileCount: soulseekBrowseDirs.fileCount,
+        totalSize: soulseekBrowseDirs.totalSize,
+      })
+      .from(soulseekBrowseDirs)
+      .where(where)
+      .orderBy(asc(soulseekBrowseDirs.name))
+      .limit(limit)
+      .offset(offset),
+    db
+      .select({ total: sql<number>`count(*)::int` })
+      .from(soulseekBrowseDirs)
+      .where(where),
+  ]);
+  return { dirs, total: count?.total ?? 0 };
+}
+
+/** One cached folder's files. Scoped by user so a guessed dir id can't read another account's cache. */
+export async function getSoulseekBrowseDirFiles(userId: number, dirId: number): Promise<BrowsedFile[] | null> {
+  const [row] = await db
+    .select({ files: soulseekBrowseDirs.files })
+    .from(soulseekBrowseDirs)
+    .innerJoin(soulseekBrowseSnapshots, eq(soulseekBrowseDirs.snapshotId, soulseekBrowseSnapshots.id))
+    .where(and(eq(soulseekBrowseDirs.id, dirId), eq(soulseekBrowseSnapshots.userId, userId)))
+    .limit(1);
+  return row ? (row.files as BrowsedFile[]) : null;
+}
+
+/** Drop a peer's cache entirely (snapshot + folders, via the FK cascade). */
+export async function deleteSoulseekBrowse(userId: number, username: string): Promise<void> {
+  await db
+    .delete(soulseekBrowseSnapshots)
+    .where(and(eq(soulseekBrowseSnapshots.userId, userId), eq(soulseekBrowseSnapshots.username, username)));
+}

src/databases/officer_db/src/schema/soulseek.ts

@@ -1,4 +1,15 @@
-import { pgTable, serial, integer, text, timestamp, unique } from 'drizzle-orm/pg-core';
+import {
+  pgTable,
+  serial,
+  integer,
+  bigint,
+  text,
+  timestamp,
+  unique,
+  index,
+  jsonb,
+  foreignKey,
+} from 'drizzle-orm/pg-core';
 import { users } from './auth';
 
 // Soulseek state that Officer owns because slskd has none. slskd exposes no favourites/buddy-list API
@@ -22,3 +33,63 @@ export const soulseekFavorites = pgTable(
   // Also the lookup index — (user_id, username) covers "all of a user's favourites" by leftmost prefix.
   (t) => [unique('uq_soulseek_favorites_user_username').on(t.userId, t.username)],
 );
+
+// ── Cached share trees ──
+//
+// A peer's share tree comes from slskd as ONE blocking response with every file of every folder inlined:
+// measured at 59 MB / 18k folders / 284k files for a single real peer. Sending that to a browser is
+// hopeless, and the browse is lost the moment the tab navigates away. So the sidecar fetches it in the
+// background and caches it here, and the UI reads paginated slices instead.
+//
+// One snapshot row per (user, peer) — refreshes replace it rather than accumulating history.
+
+export const soulseekBrowseSnapshots = pgTable(
+  'soulseek_browse_snapshots',
+  {
+    id: serial('id').primaryKey(),
+    userId: integer('user_id')
+      .notNull()
+      .references(() => users.id, { onDelete: 'cascade' }),
+    username: text('username').notNull(),
+    // 'pending' while the sidecar is fetching, then 'ready' or 'failed'. A failed refresh keeps the
+    // previous rows, so a peer going offline never destroys a good cache.
+    status: text('status').notNull().default('pending'),
+    error: text('error'),
+    directoryCount: integer('directory_count').notNull().default(0),
+    fileCount: integer('file_count').notNull().default(0),
+    // Shares run to terabytes, well past int4.
+    totalSize: bigint('total_size', { mode: 'number' }).notNull().default(0),
+    startedAt: timestamp('started_at', { withTimezone: true }).notNull().defaultNow(),
+    completedAt: timestamp('completed_at', { withTimezone: true }),
+  },
+  (t) => [unique('uq_soulseek_browse_snapshots_user_username').on(t.userId, t.username)],
+);
+
+// One row per FOLDER, with that folder's files inlined as jsonb — not a row per file, which would be
+// 284k rows per peer. Folders are what you page and filter through; files are only ever read for the
+// folder you opened, so they ride along in the row.
+export const soulseekBrowseDirs = pgTable(
+  'soulseek_browse_dirs',
+  {
+    id: serial('id').primaryKey(),
+    snapshotId: integer('snapshot_id').notNull(),
+    name: text('name').notNull(),
+    fileCount: integer('file_count').notNull().default(0),
+    totalSize: bigint('total_size', { mode: 'number' }).notNull().default(0),
+    // BrowsedFile[] — slimmed at ingest to {name,size,extension}: slskd's browse response carries
+    // per-file `attributes` (bitrate/duration) that real peers leave empty, and dropping them plus the
+    // wire noise took the sample from 59 MB to 19 MB.
+    files: jsonb('files').notNull().default([]),
+  },
+  (t) => [
+    // Both reads are "this snapshot's folders, ordered/filtered by name".
+    index('idx_soulseek_browse_dirs_snapshot_name').on(t.snapshotId, t.name),
+    // Named explicitly: drizzle's derived name for this pair would be 64 chars, one over Postgres's
+    // 63-char identifier cap, so it would be silently truncated and never match the schema again.
+    foreignKey({
+      name: 'soulseek_browse_dirs_snapshot_id_fk',
+      columns: [t.snapshotId],
+      foreignColumns: [soulseekBrowseSnapshots.id],
+    }).onDelete('cascade'),
+  ],
+);