From 1159c0787d5fce5d239d61a1d5702df217fb8514 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Thu, 30 Jul 2026 01:05:22 +0000 Subject: [PATCH] soulseek: render cached shares as a tree, not a path list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit slskd's browse response is flat: every folder is a full backslash-delimited path. Rendered as-is, a filter for "pogues" gave 26 rows that all began with the same 31 characters, and the real hierarchy — which is the only way to tell an artist folder from an album folder — was invisible. The shape is now derived once, at ingest, in the sidecar: buildTree() links each path to its parent, synthesizes any ancestor slskd omitted (measured: exactly one missing across ~30k folders on two real peers, but a single gap would strand a whole subtree), and rolls subtree file counts and sizes up bottom-up. A parent's own files are usually just cover art, so the number worth showing on a collapsed row is the subtree's. Storing the shape rather than recomputing it is what lets the UI open one level at a time. Levels are still paged, because fan-out is brutal — the widest folder measured has 1,181 children. Filtering keeps the tree instead of falling back to a list: the search route returns matches plus every ancestor, and the UI renders that skeleton pre-expanded, so you see where a hit lives. Matching runs against the whole path, so a matched folder implies its descendants match too and a matched subtree arrives complete. The match cap is reported in the payload and shown in the UI rather than passed off as the whole answer. The two existing snapshots were backfilled by scripts/rebuild-soulseek-tree.ts, which runs the same buildTree + finishSoulseekBrowse the ingest path runs — no second implementation to drift, and no peer contact needed. Kept for the next time the tree shape changes. Co-Authored-By: Claude Opus 4.8 --- scripts/rebuild-soulseek-tree.ts | 50 ++++ src/databases/officer_db/src/index.ts | 13 +- .../officer_db/src/queries/soulseek.ts | 190 +++++++++--- .../officer_db/src/schema/soulseek.ts | 21 +- src/servers/sidecar/slskd/browse.ts | 82 +++++- src/servers/sidecar/slskd/officer.ts | 43 ++- .../src/apps/Soulseek/SharesBrowser.tsx | 274 ++++++++++++------ .../officerdev/src/apps/Soulseek/shared.ts | 25 +- .../src/apps/Soulseek/useSoulseekBrowse.ts | 40 ++- 9 files changed, 582 insertions(+), 156 deletions(-) create mode 100644 scripts/rebuild-soulseek-tree.ts diff --git a/scripts/rebuild-soulseek-tree.ts b/scripts/rebuild-soulseek-tree.ts new file mode 100644 index 00000000..c3f2b9a7 --- /dev/null +++ b/scripts/rebuild-soulseek-tree.ts @@ -0,0 +1,50 @@ +// Re-derive the stored tree shape for every cached share snapshot, from the folder rows already in +// Postgres — no peer contact, so it works with everyone offline and is deterministic. +// +// Written for the backfill when the tree columns were added, but kept because it's the answer to any +// future change in tree shape: it runs the SAME buildTree + finishSoulseekBrowse the sidecar's ingest +// runs, so there's no second implementation to drift. +// +// bun scripts/rebuild-soulseek-tree.ts + +import type { BrowsedFile } from 'officerdb'; +import { eq, asc } from 'drizzle-orm'; +import { db, finishSoulseekBrowse } from 'officerdb'; +import { soulseekBrowseSnapshots, soulseekBrowseDirs } from 'officerdb/schema'; +import { buildTree } from '../src/servers/sidecar/slskd/browse'; + +const snapshots = await db + .select({ + id: soulseekBrowseSnapshots.id, + userId: soulseekBrowseSnapshots.userId, + username: soulseekBrowseSnapshots.username, + status: soulseekBrowseSnapshots.status, + }) + .from(soulseekBrowseSnapshots) + .orderBy(asc(soulseekBrowseSnapshots.id)); + +console.log(`${snapshots.length} snapshot(s) to rebuild`); + +for (const snap of snapshots) { + const started = Date.now(); + const rows = await db + .select({ name: soulseekBrowseDirs.name, files: soulseekBrowseDirs.files }) + .from(soulseekBrowseDirs) + .where(eq(soulseekBrowseDirs.snapshotId, snap.id)); + + if (!rows.length) { + console.log(` #${snap.id} ${snap.username}: no folder rows, skipped`); + continue; + } + + const tree = buildTree(rows.map((r) => ({ name: r.name, files: r.files as BrowsedFile[] }))); + await finishSoulseekBrowse(snap.id, tree); + const roots = tree.filter((t) => t.parentPath === null); + console.log( + ` #${snap.id} ${snap.username}: ${rows.length} -> ${tree.length} rows ` + + `(+${tree.length - rows.length} synthesized), ${roots.length} root(s), ` + + `${roots.reduce((n, r) => n + r.subtreeFileCount, 0)} files, ${Date.now() - started}ms`, + ); +} + +process.exit(0); diff --git a/src/databases/officer_db/src/index.ts b/src/databases/officer_db/src/index.ts index ef80c24f..53b4b852 100644 --- a/src/databases/officer_db/src/index.ts +++ b/src/databases/officer_db/src/index.ts @@ -110,11 +110,20 @@ export { finishSoulseekBrowse, failSoulseekBrowse, resetStaleSoulseekBrowses, - getSoulseekBrowseDirs, + getSoulseekBrowseLevel, + searchSoulseekBrowseTree, getSoulseekBrowseDirFiles, deleteSoulseekBrowse, } from './queries/soulseek'; -export type { BrowsedFile, BrowseDirInput, BrowseDirPage, SoulseekBrowseSnapshot } from './queries/soulseek'; +export type { + BrowsedFile, + BrowseDirInput, + BrowseDirRow, + BrowseTreeNode, + BrowseLevel, + BrowseTreeSearch, + SoulseekBrowseSnapshot, +} from './queries/soulseek'; export { getVaultTokens, setVaultTokens, diff --git a/src/databases/officer_db/src/queries/soulseek.ts b/src/databases/officer_db/src/queries/soulseek.ts index 5b28afbe..60f12ae4 100644 --- a/src/databases/officer_db/src/queries/soulseek.ts +++ b/src/databases/officer_db/src/queries/soulseek.ts @@ -1,4 +1,4 @@ -import { eq, and, asc, ilike, sql } from 'drizzle-orm'; +import { eq, and, asc, isNull, inArray, sql } from 'drizzle-orm'; import { db } from '../db'; import { soulseekFavorites, soulseekBrowseSnapshots, soulseekBrowseDirs } from '../schema'; @@ -28,9 +28,26 @@ export async function removeSoulseekFavorite(userId: number, username: string): /** A file inside a cached folder, slimmed at ingest (see the schema note on dropped attributes). */ export type BrowsedFile = { name: string; size: number; extension: string }; -/** One folder to ingest: its slskd name, plus its files. Counts are derived here, not trusted. */ +/** One folder as slskd gives it: a full backslash-delimited path, plus its files. */ export type BrowseDirInput = { name: string; files: BrowsedFile[] }; +/** + * One folder as it's stored — a tree node. slskd's flat paths are linked up into this shape by the + * sidecar's `buildTree` before they get here, which is why every count is already on the row. + */ +export type BrowseDirRow = { + name: string; + parentPath: string | null; + depth: number; + label: string; + childCount: number; + fileCount: number; + totalSize: number; + subtreeFileCount: number; + subtreeSize: number; + files: BrowsedFile[]; +}; + export type SoulseekBrowseSnapshot = { username: string; status: string; @@ -96,20 +113,17 @@ export async function startSoulseekBrowse(userId: number, username: string): Pro // absurd. 18k folders is ~36 round trips at this size. const INGEST_BATCH = 500; -/** Swap in a freshly fetched tree: replace the folder rows and mark the snapshot ready, atomically. */ -export async function finishSoulseekBrowse(snapshotId: number, dirs: BrowseDirInput[]): Promise { - const rows = dirs.map((d) => ({ - snapshotId, - name: d.name, - fileCount: d.files.length, - totalSize: d.files.reduce((n, f) => n + (f.size || 0), 0), - files: d.files, - })); +/** Swap in a freshly built tree: replace the folder rows and mark the snapshot ready, atomically. */ +export async function finishSoulseekBrowse(snapshotId: number, tree: BrowseDirRow[]): Promise { + const rows = tree.map((r) => ({ ...r, snapshotId })); await db.transaction(async (tx) => { await tx.delete(soulseekBrowseDirs).where(eq(soulseekBrowseDirs.snapshotId, snapshotId)); for (let i = 0; i < rows.length; i += INGEST_BATCH) { await tx.insert(soulseekBrowseDirs).values(rows.slice(i, i + INGEST_BATCH)); } + // Summed over the roots, not over every row: the subtree rollups already include descendants, so + // adding them all up would count each file once per level above it. + const roots = rows.filter((r) => r.parentPath === null); await tx .update(soulseekBrowseSnapshots) .set({ @@ -117,8 +131,8 @@ export async function finishSoulseekBrowse(snapshotId: number, dirs: BrowseDirIn error: null, completedAt: new Date(), directoryCount: rows.length, - fileCount: rows.reduce((n, r) => n + r.fileCount, 0), - totalSize: rows.reduce((n, r) => n + r.totalSize, 0), + fileCount: roots.reduce((n, r) => n + r.subtreeFileCount, 0), + totalSize: roots.reduce((n, r) => n + r.subtreeSize, 0), }) .where(eq(soulseekBrowseSnapshots.id, snapshotId)); }); @@ -145,11 +159,75 @@ export async function resetStaleSoulseekBrowses(): Promise { return rows.length; } -export type BrowseDirPage = { - dirs: { id: number; name: string; fileCount: number; totalSize: number }[]; - total: number; +/** A tree row as the UI renders it. `files` stays out — those load per folder, on expand. */ +export type BrowseTreeNode = { + id: number; + name: string; + parentPath: string | null; + label: string; + depth: number; + childCount: number; + fileCount: number; + totalSize: number; + subtreeFileCount: number; + subtreeSize: number; }; -type BrowseDirPageParams = { userId: number; username: string; q?: string; limit: number; offset: number }; + +const treeCols = { + id: soulseekBrowseDirs.id, + name: soulseekBrowseDirs.name, + parentPath: soulseekBrowseDirs.parentPath, + label: soulseekBrowseDirs.label, + depth: soulseekBrowseDirs.depth, + childCount: soulseekBrowseDirs.childCount, + fileCount: soulseekBrowseDirs.fileCount, + totalSize: soulseekBrowseDirs.totalSize, + subtreeFileCount: soulseekBrowseDirs.subtreeFileCount, + subtreeSize: soulseekBrowseDirs.subtreeSize, +}; + +/** The snapshot a request is scoped to, or null if this peer was never fetched. */ +async function snapshotIdOf(userId: number, username: string): Promise { + const [snap] = await db + .select({ id: soulseekBrowseSnapshots.id }) + .from(soulseekBrowseSnapshots) + .where(and(eq(soulseekBrowseSnapshots.userId, userId), eq(soulseekBrowseSnapshots.username, username))) + .limit(1); + return snap?.id ?? null; +} + +export type BrowseLevel = { nodes: BrowseTreeNode[]; total: number }; +type BrowseLevelParams = { userId: number; username: string; parent?: string; limit: number; offset: number }; + +/** + * One level of the tree: the children of `parent`, or the roots when it's absent. Paged, because fan-out + * is not gentle — the widest folder measured on a real peer has 1,181 children. + */ +export async function getSoulseekBrowseLevel(params: BrowseLevelParams): Promise { + const { userId, username, parent, limit, offset } = params; + const snapshotId = await snapshotIdOf(userId, username); + if (snapshotId === null) return { nodes: [], total: 0 }; + + const where = and( + eq(soulseekBrowseDirs.snapshotId, snapshotId), + parent ? eq(soulseekBrowseDirs.parentPath, parent) : isNull(soulseekBrowseDirs.parentPath), + ); + + const [nodes, [count]] = await Promise.all([ + db + .select(treeCols) + .from(soulseekBrowseDirs) + .where(where) + .orderBy(asc(soulseekBrowseDirs.name)) + .limit(limit) + .offset(offset), + db + .select({ total: sql`count(*)::int` }) + .from(soulseekBrowseDirs) + .where(where), + ]); + return { nodes, total: count?.total ?? 0 }; +} // These names come off strangers' filesystems, so they're littered with typographic punctuation that no // keyboard produces: curly apostrophes, en dashes, and U+2010 HYPHEN, which is pixel-identical to '-'. @@ -177,45 +255,67 @@ const likePattern = (term: string): string => `%${foldPunctuation(term).replace( /** The column, folded the same way `likePattern` folds the term. */ const foldedName = sql`replace(translate(${soulseekBrowseDirs.name}, ${FOLD_FROM}, ${FOLD_TO}), ${ELLIPSIS}, '...')`; +export type BrowseTreeSearch = { + /** Matches AND every ancestor of a match, so the UI has a whole tree to render, not orphans. */ + nodes: BrowseTreeNode[]; + /** `name`s of the rows that actually matched — the rest of `nodes` are there to hold them up. */ + matched: string[]; + /** Matches in the snapshot, which exceeds `matched.length` when the cap kicked in. */ + total: number; + truncated: boolean; +}; +type BrowseTreeSearchParams = { userId: number; username: string; q: string; limit: number }; + +const SEP = '\\'; + +/** Every path above `name`, longest first: 'a\b\c' → ['a\b', 'a']. */ +function ancestorsOf(name: string): string[] { + const out: string[] = []; + for (let cut = name.lastIndexOf(SEP); cut > 0; cut = name.lastIndexOf(SEP, cut - 1)) { + out.push(name.slice(0, cut)); + } + return out; +} + /** - * A page of a peer's cached folders, name-filtered. Files are excluded — that's the whole point of - * paging here, and they're fetched per folder on expand. + * Filtered view of the tree. Filtering never flattens it: matches come back with their full ancestor + * chains, which is what lets the UI open straight down to a hit and keep the shape visible around it. + * + * Ancestors are resolved by string prefix rather than a recursive CTE — the parent's path IS a prefix of + * the child's, so one indexed `name in (…)` lookup does what a walk would take `depth` round trips to do. */ -export async function getSoulseekBrowseDirs(params: BrowseDirPageParams): Promise { - const { userId, username, q, limit, offset } = params; - const [snap] = await db - .select({ id: soulseekBrowseSnapshots.id }) - .from(soulseekBrowseSnapshots) - .where(and(eq(soulseekBrowseSnapshots.userId, userId), eq(soulseekBrowseSnapshots.username, username))) - .limit(1); - if (!snap) return { dirs: [], total: 0 }; +export async function searchSoulseekBrowseTree(params: BrowseTreeSearchParams): Promise { + const { userId, username, q, limit } = params; + const snapshotId = await snapshotIdOf(userId, username); + const term = q.trim(); + if (snapshotId === null || !term) return { nodes: [], matched: [], total: 0, truncated: false }; - const term = q?.trim(); // ILIKE with a leading wildcard can't use the btree index, so this was already a scan of the snapshot's // ~18k rows — which is why folding the column here costs nothing on top. - const where = term - ? and(eq(soulseekBrowseDirs.snapshotId, snap.id), sql`${foldedName} ilike ${likePattern(term)}`) - : eq(soulseekBrowseDirs.snapshotId, snap.id); + const where = and(eq(soulseekBrowseDirs.snapshotId, snapshotId), sql`${foldedName} ilike ${likePattern(term)}`); - const [dirs, [count]] = await Promise.all([ - db - .select({ - id: soulseekBrowseDirs.id, - name: soulseekBrowseDirs.name, - fileCount: soulseekBrowseDirs.fileCount, - totalSize: soulseekBrowseDirs.totalSize, - }) - .from(soulseekBrowseDirs) - .where(where) - .orderBy(asc(soulseekBrowseDirs.name)) - .limit(limit) - .offset(offset), + const [matches, [count]] = await Promise.all([ + db.select(treeCols).from(soulseekBrowseDirs).where(where).orderBy(asc(soulseekBrowseDirs.name)).limit(limit), db .select({ total: sql`count(*)::int` }) .from(soulseekBrowseDirs) .where(where), ]); - return { dirs, total: count?.total ?? 0 }; + + const have = new Set(matches.map((m) => m.name)); + const wanted = new Set(); + for (const m of matches) for (const a of ancestorsOf(m.name)) if (!have.has(a)) wanted.add(a); + + const ancestors = wanted.size + ? await db + .select(treeCols) + .from(soulseekBrowseDirs) + .where(and(eq(soulseekBrowseDirs.snapshotId, snapshotId), inArray(soulseekBrowseDirs.name, [...wanted]))) + : []; + + const nodes = [...matches, ...ancestors].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + const total = count?.total ?? 0; + return { nodes, matched: matches.map((m) => m.name), total, truncated: total > matches.length }; } /** One cached folder's files. Scoped by user so a guessed dir id can't read another account's cache. */ diff --git a/src/databases/officer_db/src/schema/soulseek.ts b/src/databases/officer_db/src/schema/soulseek.ts index 2e6e081c..614aa81e 100644 --- a/src/databases/officer_db/src/schema/soulseek.ts +++ b/src/databases/officer_db/src/schema/soulseek.ts @@ -68,22 +68,41 @@ export const soulseekBrowseSnapshots = pgTable( // One row per FOLDER, with that folder's files inlined as jsonb — not a row per file, which would be // 284k rows per peer. Folders are what you page and filter through; files are only ever read for the // folder you opened, so they ride along in the row. +// +// slskd hands back a flat list of backslash-delimited paths, so the tree is derived once at ingest and +// stored here rather than recomputed per request: `parentPath` links a row to its parent and the subtree +// rollups are precomputed, which is what lets the UI open one level at a time. Measured on real peers, +// the tree is already complete in slskd's response — exactly one intermediate folder was missing across +// 30k rows — but ingest synthesizes any that are absent so a level can never dead-end. export const soulseekBrowseDirs = pgTable( 'soulseek_browse_dirs', { id: serial('id').primaryKey(), snapshotId: integer('snapshot_id').notNull(), name: text('name').notNull(), + // null for a root. Its parent's `name`, i.e. `name` minus the last backslash segment. + parentPath: text('parent_path'), + // Segment count, 1-based, so the UI can indent without walking back up the chain. + depth: integer('depth').notNull().default(1), + // Last path segment — what a tree row actually displays. + label: text('label').notNull().default(''), + childCount: integer('child_count').notNull().default(0), fileCount: integer('file_count').notNull().default(0), totalSize: bigint('total_size', { mode: 'number' }).notNull().default(0), + // This folder AND everything under it. A parent's own file count is usually cover art; the number + // worth showing on a collapsed row is the whole subtree's. + subtreeFileCount: integer('subtree_file_count').notNull().default(0), + subtreeSize: bigint('subtree_size', { mode: 'number' }).notNull().default(0), // BrowsedFile[] — slimmed at ingest to {name,size,extension}: slskd's browse response carries // per-file `attributes` (bitrate/duration) that real peers leave empty, and dropping them plus the // wire noise took the sample from 59 MB to 19 MB. files: jsonb('files').notNull().default([]), }, (t) => [ - // Both reads are "this snapshot's folders, ordered/filtered by name". + // Filtered search and the name lookups behind ancestor expansion. index('idx_soulseek_browse_dirs_snapshot_name').on(t.snapshotId, t.name), + // The tree's hot path: one level, ordered by name. Covers "the roots" too, via a null parent_path. + index('idx_soulseek_browse_dirs_snapshot_parent').on(t.snapshotId, t.parentPath, t.name), // Named explicitly: drizzle's derived name for this pair would be 64 chars, one over Postgres's // 63-char identifier cap, so it would be silently truncated and never match the schema again. foreignKey({ diff --git a/src/servers/sidecar/slskd/browse.ts b/src/servers/sidecar/slskd/browse.ts index 6b7504c5..0b249ef3 100644 --- a/src/servers/sidecar/slskd/browse.ts +++ b/src/servers/sidecar/slskd/browse.ts @@ -1,4 +1,4 @@ -import type { BrowseDirInput, BrowsedFile } from 'officerdb'; +import type { BrowseDirInput, BrowseDirRow, BrowsedFile } from 'officerdb'; import { startSoulseekBrowse, finishSoulseekBrowse, failSoulseekBrowse } from 'officerdb'; import { getSlskdBase, getSlskdApiKey } from './upstream'; @@ -56,6 +56,77 @@ function slim(payload: SlskdBrowsePayload): BrowseDirInput[] { }); } +const SEP = '\\'; + +/** Split a slskd path into the parent path and the last segment, ignoring any trailing separator. */ +function splitPath(name: string): { parentPath: string | null; label: string; depth: number } { + const trimmed = name.endsWith(SEP) ? name.slice(0, -1) : name; + const cut = trimmed.lastIndexOf(SEP); + const depth = trimmed.split(SEP).filter(Boolean).length || 1; + // `cut > 0` not `>= 0`: a leading separator would otherwise make the parent an empty string, which is + // neither null nor a real row, and the level query would never find it. + return { parentPath: cut > 0 ? trimmed.slice(0, cut) : null, label: trimmed.slice(cut + 1), depth }; +} + +/** + * Turn slskd's flat list of paths into a linked tree, once, here — so the UI can open one level at a + * time instead of the platform recomputing the shape on every request. + * + * Two things the response can't be trusted on. Ancestors: measured across two real peers (~30k folders) + * exactly one intermediate folder was missing, so they're nearly always present, but a single gap would + * make a whole subtree unreachable — hence `ensure` synthesizes any absent one as an empty folder. + * Rollups: a parent's own file count is usually just cover art, so the number worth showing on a + * collapsed row is the subtree's, and that's summed bottom-up here rather than per request. + */ +export function buildTree(dirs: BrowseDirInput[]): BrowseDirRow[] { + const byName = new Map(); + + const ensure = (name: string): BrowseDirRow => { + const existing = byName.get(name); + if (existing) return existing; + const { parentPath, label, depth } = splitPath(name); + const row: BrowseDirRow = { + name, + parentPath, + label, + depth, + childCount: 0, + fileCount: 0, + totalSize: 0, + subtreeFileCount: 0, + subtreeSize: 0, + files: [], + }; + // Set before recursing so a cycle-shaped path can't loop, and so the parent's own ancestors follow. + byName.set(name, row); + if (parentPath) ensure(parentPath); + return row; + }; + + for (const dir of dirs) { + const row = ensure(dir.name); + // Concatenated rather than assigned: a path repeated in the response would otherwise lose files. + row.files = row.files.length ? row.files.concat(dir.files) : dir.files; + row.fileCount = row.files.length; + row.totalSize = row.files.reduce((n, f) => n + (f.size || 0), 0); + } + + const rows = [...byName.values()]; + for (const row of rows) { + row.subtreeFileCount = row.fileCount; + row.subtreeSize = row.totalSize; + } + // Deepest first, so every child is final before its parent absorbs it. + for (const row of [...rows].sort((a, b) => b.depth - a.depth)) { + const parent = row.parentPath ? byName.get(row.parentPath) : undefined; + if (!parent) continue; + parent.childCount += 1; + parent.subtreeFileCount += row.subtreeFileCount; + parent.subtreeSize += row.subtreeSize; + } + return rows; +} + async function run(userId: number, username: string, snapshotId: number): Promise { const base = getSlskdBase(); if (!base) { @@ -78,9 +149,14 @@ async function run(userId: number, username: string, snapshotId: number): Promis } const dirs = slim((await res.json()) as SlskdBrowsePayload); - await finishSoulseekBrowse(snapshotId, dirs); + const tree = buildTree(dirs); + await finishSoulseekBrowse(snapshotId, tree); const files = dirs.reduce((n, d) => n + d.files.length, 0); - console.log(`[slskd] browse ${username} -> ${dirs.length} dirs / ${files} files in ${Date.now() - started}ms`); + // Against distinct names, not the array length, so a repeated path doesn't read as a negative. + const synthesized = tree.length - new Set(dirs.map((d) => d.name)).size; + console.log( + `[slskd] browse ${username} -> ${tree.length} dirs (${synthesized} synthesized) / ${files} files in ${Date.now() - started}ms`, + ); } catch (err) { const message = err instanceof Error ? err.message : String(err); console.error(`[slskd] browse ${username} failed after ${Date.now() - started}ms: ${message}`); diff --git a/src/servers/sidecar/slskd/officer.ts b/src/servers/sidecar/slskd/officer.ts index e07994fa..1d60fb9d 100644 --- a/src/servers/sidecar/slskd/officer.ts +++ b/src/servers/sidecar/slskd/officer.ts @@ -4,7 +4,8 @@ import { removeSoulseekFavorite, getSoulseekBrowseSnapshots, getSoulseekBrowseSnapshot, - getSoulseekBrowseDirs, + getSoulseekBrowseLevel, + searchSoulseekBrowseTree, getSoulseekBrowseDirFiles, deleteSoulseekBrowse, } from 'officerdb'; @@ -25,7 +26,8 @@ import { startBrowse } from './browse'; // GET /_officer/browse/ → Snapshot | null one peer's cache state // POST /_officer/browse/ → 202 { started } kick off a background fetch // DELETE /_officer/browse/ → { ok: true } drop the cache -// GET /_officer/browse//dirs?q=&limit=&offset= → { dirs, total } a page of folders, no files +// GET /_officer/browse//tree?parent=&limit=&offset= → { nodes, total } one level (roots if no parent) +// GET /_officer/browse//tree/search?q=&limit= → { nodes, matched, total, truncated } // GET /_officer/browse//dirs//files → BrowsedFile[] one folder's files // ───────────────────────────────────────────────────────────────────────────────────────────────── @@ -47,8 +49,14 @@ const cleanUsername = (v: unknown): string | null => { return CONTROL_CHARS.test(u) ? null : u; }; -const DIR_PAGE_MAX = 500; -const DIR_PAGE_DEFAULT = 100; +// A level, not a page: once you're inside a folder there's nothing to leaf through, so the UI asks for +// the whole level when it's wide. The cap is set above the widest fan-out measured on a real peer (1,181). +const DIR_PAGE_MAX = 2000; +const DIR_PAGE_DEFAULT = 200; +// A match cap, not a page: every match drags its ancestors along, so the response is up to ~depth times +// this. Truncation is reported in the payload rather than passed off as the whole answer. +const SEARCH_MATCH_MAX = 1000; +const SEARCH_MATCH_DEFAULT = 300; const clampInt = (raw: string | null, fallback: number, min: number, max: number): number => { const n = Number(raw); if (!Number.isFinite(n)) return fallback; @@ -107,21 +115,34 @@ async function handleBrowse({ req, url, userId, segments }: BrowseRouteParams): return methodNotAllowed(); } - if (segments[2] !== 'dirs') return notFound(); - - // /_officer/browse//dirs — a filtered page of folders, files excluded. - if (segments.length === 3) { + // /_officer/browse//tree — one level of folders, paged. No `parent` means the roots. + if (segments[2] === 'tree' && segments.length === 3) { if (req.method !== 'GET') return methodNotAllowed(); - const page = await getSoulseekBrowseDirs({ + const level = await getSoulseekBrowseLevel({ userId, username, - q: url.searchParams.get('q') ?? undefined, + parent: url.searchParams.get('parent') ?? undefined, limit: clampInt(url.searchParams.get('limit'), DIR_PAGE_DEFAULT, 1, DIR_PAGE_MAX), offset: clampInt(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER), }); - return Response.json(page); + return Response.json(level); } + // /_officer/browse//tree/search — matches plus their ancestors, as one small tree. Kept off + // the level route because the shape differs: this one is a skeleton spanning depths, not a page. + if (segments[2] === 'tree' && segments.length === 4 && segments[3] === 'search') { + if (req.method !== 'GET') return methodNotAllowed(); + const result = await searchSoulseekBrowseTree({ + userId, + username, + q: url.searchParams.get('q') ?? '', + limit: clampInt(url.searchParams.get('limit'), SEARCH_MATCH_DEFAULT, 1, SEARCH_MATCH_MAX), + }); + return Response.json(result); + } + + if (segments[2] !== 'dirs') return notFound(); + // /_officer/browse//dirs//files if (segments.length === 5 && segments[4] === 'files') { if (req.method !== 'GET') return methodNotAllowed(); diff --git a/src/workspaces/officerdev/src/apps/Soulseek/SharesBrowser.tsx b/src/workspaces/officerdev/src/apps/Soulseek/SharesBrowser.tsx index 4092ab1a..212f8f74 100644 --- a/src/workspaces/officerdev/src/apps/Soulseek/SharesBrowser.tsx +++ b/src/workspaces/officerdev/src/apps/Soulseek/SharesBrowser.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect } from 'react'; +import { useState, useEffect, useMemo } from 'react'; import { FolderOpen, Folder, @@ -12,16 +12,34 @@ import { Trash2, FileAudio, } from 'lucide-react'; -import { useSoulseekBrowseSnapshots, useSoulseekBrowseDirs, useSoulseekBrowseFiles } from './useSoulseekBrowse'; -import { formatSize, formatWhen, type SoulseekBrowseSnapshot } from './shared'; +import { + useSoulseekBrowseSnapshots, + useSoulseekBrowseLevel, + useSoulseekBrowseSearch, + useSoulseekBrowseFiles, +} from './useSoulseekBrowse'; +import { formatSize, formatWhen, type SoulseekBrowseNode, type SoulseekBrowseSnapshot } from './shared'; -// A peer's shared folders, read from Officer's cache rather than browsed live. +// A peer's shared folders, read from Officer's cache rather than browsed live, and rendered as a TREE. // // The live slskd call returns the entire tree in one blocking response (59 MB / 18k folders / 284k files // for a real peer) and takes minutes, because it round-trips to that peer. So nothing here triggers it -// inline: you ask the sidecar to fetch, it keeps going without the tab, and this reads back pages. +// inline: you ask the sidecar to fetch, it keeps going without the tab, and this reads back levels. +// +// slskd's response is flat — every folder is a full backslash-delimited path — which made the old list +// unreadable: 26 rows all beginning with the same 31 characters. The sidecar links the paths into a tree +// at ingest, so a row here shows one segment and one level loads at a time. +// +// Filtering never flattens that. The search route returns matches WITH their ancestors, and this renders +// that skeleton pre-expanded: you see where a hit lives, not just that it exists. Because matching runs +// against the whole path, a folder matching implies all its descendants match too — so a matched subtree +// arrives complete. -const PAGE_SIZE = 100; +// One level's page size. Fan-out is not gentle: the widest folder measured on a real peer has 1,181 +// children, so a level can ask for more (up to the route's cap) rather than paging with Previous/Next — +// there's nothing to page through once you're inside one folder. +const LEVEL_LIMIT = 200; +const LEVEL_LIMIT_MAX = 2000; /** Debounce the filter box so a keystroke doesn't become a request. */ function useDebounced(value: T, ms: number): T { @@ -57,32 +75,41 @@ export const BrowseStateChip = ({ snapshot }: { snapshot: SoulseekBrowseSnapshot ); }; +/** The filtered tree, indexed for rendering: children by parent path, plus which rows actually matched. */ +type Filtered = { childrenOf: Map; matched: Set }; + export const SharesBrowser = ({ username }: { username: string }) => { const { snapshotOf, fetchShares, dropShares } = useSoulseekBrowseSnapshots(); const [filter, setFilter] = useState(''); - const [page, setPage] = useState(0); - const [openDir, setOpenDir] = useState(null); const q = useDebounced(filter.trim(), 250); const snapshot = snapshotOf(username); - // Folder rows survive a failed or in-flight refresh, so drive the list off the data existing rather + // Folder rows survive a failed or in-flight refresh, so drive the tree off the data existing rather // than off the status — a stale tree is still worth browsing while a new one is being fetched. const hasCache = !!snapshot && snapshot.directoryCount > 0; - const { data, isFetching } = useSoulseekBrowseDirs({ username, q, page, pageSize: PAGE_SIZE, enabled: hasCache }); + const roots = useSoulseekBrowseLevel({ username, parent: null, limit: LEVEL_LIMIT, enabled: hasCache && !q }); + const search = useSoulseekBrowseSearch({ username, q, enabled: hasCache }); - // A new peer or a new filter invalidates the current page number and any expanded folder. - useEffect(() => { - setPage(0); - setOpenDir(null); - }, [username, q]); useEffect(() => { setFilter(''); }, [username]); - const total = data?.total ?? 0; - const pages = Math.max(1, Math.ceil(total / PAGE_SIZE)); + const filtered = useMemo(() => { + if (!q || !search.data) return null; + const childrenOf = new Map(); + for (const node of search.data.nodes) { + if (!node.parentPath) continue; + const siblings = childrenOf.get(node.parentPath); + if (siblings) siblings.push(node); + else childrenOf.set(node.parentPath, [node]); + } + return { childrenOf, matched: new Set(search.data.matched) }; + }, [q, search.data]); + + const topNodes = q ? (search.data?.nodes.filter((n) => !n.parentPath) ?? []) : (roots.data?.nodes ?? []); const pending = snapshot?.status === 'pending'; + const busy = q ? search.isFetching : roots.isFetching; return (
@@ -160,68 +187,40 @@ export const SharesBrowser = ({ username }: { username: string }) => { value={filter} onChange={(ev) => setFilter(ev.target.value)} placeholder="Filter folders…" - className="h-7 min-w-0 flex-1 bg-transparent text-sm text-zinc-100 placeholder:text-zinc-600 outline-none" + className="h-7 min-w-0 flex-1 bg-transparent text-sm text-zinc-100 outline-none placeholder:text-zinc-600" /> - {isFetching && } - {total.toLocaleString()} matching + {busy && } + {q && ( + + {(search.data?.total ?? 0).toLocaleString()} matching + + )}
- {total === 0 ? ( -

No folders match “{q}”.

- ) : ( -
- {data?.dirs.map((dir) => ( -
- - {openDir === dir.id && } -
- ))} + {/* Never a silent cap: a truncated search still renders a valid tree, just not all of it. */} + {search.data?.truncated && q && ( +
+ Showing the first {search.data.matched.length.toLocaleString()} of {search.data.total.toLocaleString()}{' '} + matches — narrow the filter to see the rest.
)} - {pages > 1 && ( -
- - {(page * PAGE_SIZE + 1).toLocaleString()}–{Math.min(total, (page + 1) * PAGE_SIZE).toLocaleString()}{' '} - of {total.toLocaleString()} - -
- - - {page + 1} / {pages.toLocaleString()} - - -
+ {q && search.data && search.data.total === 0 ? ( +

No folders match “{q}”.

+ ) : ( + // Keyed by the term so switching between browsing and a filtered view remounts the tree: + // every row's open state is local, and reusing instances across modes would strand it. +
+ {topNodes.map((node) => ( + + ))}
)} @@ -237,26 +236,137 @@ export const SharesBrowser = ({ username }: { username: string }) => { ); }; -type DirFilesProps = { username: string; dirId: number }; +type TreeNodeProps = { + username: string; + node: SoulseekBrowseNode; + /** Non-null in filter mode: children come from this set, not from a request, and rows start open. */ + filtered: Filtered | null; + indent: number; + initialOpen: boolean; +}; -const DirFiles = ({ username, dirId }: DirFilesProps) => { +const TreeNode = ({ username, node, filtered, indent, initialOpen }: TreeNodeProps) => { + const [open, setOpen] = useState(initialOpen); + const [limit, setLimit] = useState(LEVEL_LIMIT); + + const expandable = node.childCount > 0 || node.fileCount > 0; + const level = useSoulseekBrowseLevel({ + username, + parent: node.name, + limit, + enabled: !filtered && open && node.childCount > 0, + }); + + const children = filtered ? (filtered.childrenOf.get(node.name) ?? []) : (level.data?.nodes ?? []); + const shown = filtered ? children.length : (level.data?.total ?? 0); + const isMatch = filtered?.matched.has(node.name) ?? false; + + return ( +
+ + + {open && ( + <> + {!filtered && level.isPending && node.childCount > 0 && ( +

+ Loading folders… +

+ )} + {children.map((child) => ( + + ))} + {/* Wide levels load in one go rather than paging — there's nothing to leaf through inside a folder. */} + {!filtered && shown > children.length && children.length > 0 && ( + + )} + {filtered && node.childCount > children.length && ( +

+ {(node.childCount - children.length).toLocaleString()} more folders here don't match the filter. +

+ )} + {node.fileCount > 0 && } + + )} +
+ ); +}; + +type DirFilesProps = { username: string; dirId: number; indent: number }; + +const DirFiles = ({ username, dirId, indent }: DirFilesProps) => { const { data, isPending, error } = useSoulseekBrowseFiles(username, dirId); + const pad = { paddingLeft: 12 + indent * 14 }; if (isPending) return ( -
+

Loading files… -

+

); - if (error) return

Could not load this folder.

; - if (!data?.length) return

Empty folder.

; + if (error) + return ( +

+ Could not load this folder. +

+ ); + if (!data?.length) return null; return ( -
+
{data.map((file) => ( -
+
+ - + {file.name} {formatSize(file.size)} diff --git a/src/workspaces/officerdev/src/apps/Soulseek/shared.ts b/src/workspaces/officerdev/src/apps/Soulseek/shared.ts index a4742e37..1e982bd2 100644 --- a/src/workspaces/officerdev/src/apps/Soulseek/shared.ts +++ b/src/workspaces/officerdev/src/apps/Soulseek/shared.ts @@ -202,8 +202,29 @@ export type SoulseekBrowseSnapshot = { startedAt: string; completedAt: string | null; }; -export type SoulseekBrowseDir = { id: number; name: string; fileCount: number; totalSize: number }; -export type SoulseekBrowseDirPage = { dirs: SoulseekBrowseDir[]; total: number }; +// A node of the stored tree. The sidecar links slskd's flat backslash paths up at ingest, so `label` is +// already just the last segment and the subtree rollups are already summed — this side never derives +// either, which is the point: a collapsed row can show what's underneath it without fetching it. +export type SoulseekBrowseNode = { + id: number; + name: string; + parentPath: string | null; + label: string; + depth: number; + childCount: number; + fileCount: number; + totalSize: number; + subtreeFileCount: number; + subtreeSize: number; +}; +export type SoulseekBrowseLevel = { nodes: SoulseekBrowseNode[]; total: number }; +/** A filtered tree: the matches plus every ancestor holding them up, so the shape survives filtering. */ +export type SoulseekBrowseSearch = { + nodes: SoulseekBrowseNode[]; + matched: string[]; + total: number; + truncated: boolean; +}; // Browse carries no bitrate/duration (real peers send empty `attributes`), so unlike a search result // there's no quality metadata to show — just name, size, extension. export type SoulseekBrowsedFile = { name: string; size: number; extension: string }; diff --git a/src/workspaces/officerdev/src/apps/Soulseek/useSoulseekBrowse.ts b/src/workspaces/officerdev/src/apps/Soulseek/useSoulseekBrowse.ts index 38e8f11c..90a0f4a7 100644 --- a/src/workspaces/officerdev/src/apps/Soulseek/useSoulseekBrowse.ts +++ b/src/workspaces/officerdev/src/apps/Soulseek/useSoulseekBrowse.ts @@ -1,6 +1,6 @@ import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import { useClient } from 'hooks/useClient'; -import type { SoulseekBrowseSnapshot, SoulseekBrowseDirPage, SoulseekBrowsedFile } from './shared'; +import type { SoulseekBrowseSnapshot, SoulseekBrowseLevel, SoulseekBrowseSearch, SoulseekBrowsedFile } from './shared'; // Client for the sidecar's cached share trees. The expensive work happens server-side and outlives the // tab, so this hook only ever starts a fetch and polls for its status — it never pulls a whole tree. @@ -72,23 +72,43 @@ export function useSoulseekBrowseSnapshots() { }; } -type DirsParams = { username: string | null; q: string; page: number; pageSize: number; enabled?: boolean }; +type LevelParams = { username: string | null; parent: string | null; limit: number; enabled?: boolean }; -/** One page of a peer's cached folders. Keeps the previous page visible while the next loads. */ -export function useSoulseekBrowseDirs({ username, q, page, pageSize, enabled = true }: DirsParams) { +/** + * One level of a peer's tree — the children of `parent`, or the roots when it's null. Cached per level, + * so collapsing and reopening a folder is free, and expanding a shallow node never touches its subtree. + */ +export function useSoulseekBrowseLevel({ username, parent, limit, enabled = true }: LevelParams) { const { get } = useClient(); - const offset = page * pageSize; return useQuery({ - queryKey: ['soulseek', 'browse', username, 'dirs', q, offset, pageSize], + queryKey: ['soulseek', 'browse', username, 'level', parent, limit], queryFn: () => { - const params = new URLSearchParams({ limit: String(pageSize), offset: String(offset) }); - if (q) params.set('q', q); - return get(`/slskd/_officer/browse/${encodeURIComponent(username ?? '')}/dirs?${params}`); + const params = new URLSearchParams({ limit: String(limit) }); + if (parent) params.set('parent', parent); + return get(`/slskd/_officer/browse/${encodeURIComponent(username ?? '')}/tree?${params}`); }, enabled: enabled && !!username, + staleTime: 5 * 60_000, + }); +} + +type SearchParams = { username: string | null; q: string; enabled?: boolean }; + +/** The filtered tree: matches with their ancestors, in one request, ready to render pre-expanded. */ +export function useSoulseekBrowseSearch({ username, q, enabled = true }: SearchParams) { + const { get } = useClient(); + + return useQuery({ + queryKey: ['soulseek', 'browse', username, 'search', q], + queryFn: () => + get( + `/slskd/_officer/browse/${encodeURIComponent(username ?? '')}/tree/search?q=${encodeURIComponent(q)}`, + ), + enabled: enabled && !!username && !!q, + // The previous result stays on screen while the next one loads, so the tree doesn't blink per keystroke. placeholderData: (prev) => prev, - staleTime: 30_000, + staleTime: 60_000, }); }