diff --git a/scripts/rebuild-soulseek-tree.ts b/scripts/rebuild-soulseek-tree.ts new file mode 100644 index 00000000..c3f2b9a7 --- /dev/null +++ b/scripts/rebuild-soulseek-tree.ts @@ -0,0 +1,50 @@ +// Re-derive the stored tree shape for every cached share snapshot, from the folder rows already in +// Postgres — no peer contact, so it works with everyone offline and is deterministic. +// +// Written for the backfill when the tree columns were added, but kept because it's the answer to any +// future change in tree shape: it runs the SAME buildTree + finishSoulseekBrowse the sidecar's ingest +// runs, so there's no second implementation to drift. +// +// bun scripts/rebuild-soulseek-tree.ts + +import type { BrowsedFile } from 'officerdb'; +import { eq, asc } from 'drizzle-orm'; +import { db, finishSoulseekBrowse } from 'officerdb'; +import { soulseekBrowseSnapshots, soulseekBrowseDirs } from 'officerdb/schema'; +import { buildTree } from '../src/servers/sidecar/slskd/browse'; + +const snapshots = await db + .select({ + id: soulseekBrowseSnapshots.id, + userId: soulseekBrowseSnapshots.userId, + username: soulseekBrowseSnapshots.username, + status: soulseekBrowseSnapshots.status, + }) + .from(soulseekBrowseSnapshots) + .orderBy(asc(soulseekBrowseSnapshots.id)); + +console.log(`${snapshots.length} snapshot(s) to rebuild`); + +for (const snap of snapshots) { + const started = Date.now(); + const rows = await db + .select({ name: soulseekBrowseDirs.name, files: soulseekBrowseDirs.files }) + .from(soulseekBrowseDirs) + .where(eq(soulseekBrowseDirs.snapshotId, snap.id)); + + if (!rows.length) { + console.log(` #${snap.id} ${snap.username}: no folder rows, skipped`); + continue; + } + + const tree = buildTree(rows.map((r) => ({ name: r.name, files: r.files as BrowsedFile[] }))); + await finishSoulseekBrowse(snap.id, tree); + const roots = tree.filter((t) => t.parentPath === null); + console.log( + ` #${snap.id} ${snap.username}: ${rows.length} -> ${tree.length} rows ` + + `(+${tree.length - rows.length} synthesized), ${roots.length} root(s), ` + + `${roots.reduce((n, r) => n + r.subtreeFileCount, 0)} files, ${Date.now() - started}ms`, + ); +} + +process.exit(0); diff --git a/src/databases/officer_db/src/index.ts b/src/databases/officer_db/src/index.ts index ef80c24f..53b4b852 100644 --- a/src/databases/officer_db/src/index.ts +++ b/src/databases/officer_db/src/index.ts @@ -110,11 +110,20 @@ export { finishSoulseekBrowse, failSoulseekBrowse, resetStaleSoulseekBrowses, - getSoulseekBrowseDirs, + getSoulseekBrowseLevel, + searchSoulseekBrowseTree, getSoulseekBrowseDirFiles, deleteSoulseekBrowse, } from './queries/soulseek'; -export type { BrowsedFile, BrowseDirInput, BrowseDirPage, SoulseekBrowseSnapshot } from './queries/soulseek'; +export type { + BrowsedFile, + BrowseDirInput, + BrowseDirRow, + BrowseTreeNode, + BrowseLevel, + BrowseTreeSearch, + SoulseekBrowseSnapshot, +} from './queries/soulseek'; export { getVaultTokens, setVaultTokens, diff --git a/src/databases/officer_db/src/queries/soulseek.ts b/src/databases/officer_db/src/queries/soulseek.ts index 5b28afbe..60f12ae4 100644 --- a/src/databases/officer_db/src/queries/soulseek.ts +++ b/src/databases/officer_db/src/queries/soulseek.ts @@ -1,4 +1,4 @@ -import { eq, and, asc, ilike, sql } from 'drizzle-orm'; +import { eq, and, asc, isNull, inArray, sql } from 'drizzle-orm'; import { db } from '../db'; import { soulseekFavorites, soulseekBrowseSnapshots, soulseekBrowseDirs } from '../schema'; @@ -28,9 +28,26 @@ export async function removeSoulseekFavorite(userId: number, username: string): /** A file inside a cached folder, slimmed at ingest (see the schema note on dropped attributes). */ export type BrowsedFile = { name: string; size: number; extension: string }; -/** One folder to ingest: its slskd name, plus its files. Counts are derived here, not trusted. */ +/** One folder as slskd gives it: a full backslash-delimited path, plus its files. */ export type BrowseDirInput = { name: string; files: BrowsedFile[] }; +/** + * One folder as it's stored — a tree node. slskd's flat paths are linked up into this shape by the + * sidecar's `buildTree` before they get here, which is why every count is already on the row. + */ +export type BrowseDirRow = { + name: string; + parentPath: string | null; + depth: number; + label: string; + childCount: number; + fileCount: number; + totalSize: number; + subtreeFileCount: number; + subtreeSize: number; + files: BrowsedFile[]; +}; + export type SoulseekBrowseSnapshot = { username: string; status: string; @@ -96,20 +113,17 @@ export async function startSoulseekBrowse(userId: number, username: string): Pro // absurd. 18k folders is ~36 round trips at this size. const INGEST_BATCH = 500; -/** Swap in a freshly fetched tree: replace the folder rows and mark the snapshot ready, atomically. */ -export async function finishSoulseekBrowse(snapshotId: number, dirs: BrowseDirInput[]): Promise { - const rows = dirs.map((d) => ({ - snapshotId, - name: d.name, - fileCount: d.files.length, - totalSize: d.files.reduce((n, f) => n + (f.size || 0), 0), - files: d.files, - })); +/** Swap in a freshly built tree: replace the folder rows and mark the snapshot ready, atomically. */ +export async function finishSoulseekBrowse(snapshotId: number, tree: BrowseDirRow[]): Promise { + const rows = tree.map((r) => ({ ...r, snapshotId })); await db.transaction(async (tx) => { await tx.delete(soulseekBrowseDirs).where(eq(soulseekBrowseDirs.snapshotId, snapshotId)); for (let i = 0; i < rows.length; i += INGEST_BATCH) { await tx.insert(soulseekBrowseDirs).values(rows.slice(i, i + INGEST_BATCH)); } + // Summed over the roots, not over every row: the subtree rollups already include descendants, so + // adding them all up would count each file once per level above it. + const roots = rows.filter((r) => r.parentPath === null); await tx .update(soulseekBrowseSnapshots) .set({ @@ -117,8 +131,8 @@ export async function finishSoulseekBrowse(snapshotId: number, dirs: BrowseDirIn error: null, completedAt: new Date(), directoryCount: rows.length, - fileCount: rows.reduce((n, r) => n + r.fileCount, 0), - totalSize: rows.reduce((n, r) => n + r.totalSize, 0), + fileCount: roots.reduce((n, r) => n + r.subtreeFileCount, 0), + totalSize: roots.reduce((n, r) => n + r.subtreeSize, 0), }) .where(eq(soulseekBrowseSnapshots.id, snapshotId)); }); @@ -145,11 +159,75 @@ export async function resetStaleSoulseekBrowses(): Promise { return rows.length; } -export type BrowseDirPage = { - dirs: { id: number; name: string; fileCount: number; totalSize: number }[]; - total: number; +/** A tree row as the UI renders it. `files` stays out — those load per folder, on expand. */ +export type BrowseTreeNode = { + id: number; + name: string; + parentPath: string | null; + label: string; + depth: number; + childCount: number; + fileCount: number; + totalSize: number; + subtreeFileCount: number; + subtreeSize: number; }; -type BrowseDirPageParams = { userId: number; username: string; q?: string; limit: number; offset: number }; + +const treeCols = { + id: soulseekBrowseDirs.id, + name: soulseekBrowseDirs.name, + parentPath: soulseekBrowseDirs.parentPath, + label: soulseekBrowseDirs.label, + depth: soulseekBrowseDirs.depth, + childCount: soulseekBrowseDirs.childCount, + fileCount: soulseekBrowseDirs.fileCount, + totalSize: soulseekBrowseDirs.totalSize, + subtreeFileCount: soulseekBrowseDirs.subtreeFileCount, + subtreeSize: soulseekBrowseDirs.subtreeSize, +}; + +/** The snapshot a request is scoped to, or null if this peer was never fetched. */ +async function snapshotIdOf(userId: number, username: string): Promise { + const [snap] = await db + .select({ id: soulseekBrowseSnapshots.id }) + .from(soulseekBrowseSnapshots) + .where(and(eq(soulseekBrowseSnapshots.userId, userId), eq(soulseekBrowseSnapshots.username, username))) + .limit(1); + return snap?.id ?? null; +} + +export type BrowseLevel = { nodes: BrowseTreeNode[]; total: number }; +type BrowseLevelParams = { userId: number; username: string; parent?: string; limit: number; offset: number }; + +/** + * One level of the tree: the children of `parent`, or the roots when it's absent. Paged, because fan-out + * is not gentle — the widest folder measured on a real peer has 1,181 children. + */ +export async function getSoulseekBrowseLevel(params: BrowseLevelParams): Promise { + const { userId, username, parent, limit, offset } = params; + const snapshotId = await snapshotIdOf(userId, username); + if (snapshotId === null) return { nodes: [], total: 0 }; + + const where = and( + eq(soulseekBrowseDirs.snapshotId, snapshotId), + parent ? eq(soulseekBrowseDirs.parentPath, parent) : isNull(soulseekBrowseDirs.parentPath), + ); + + const [nodes, [count]] = await Promise.all([ + db + .select(treeCols) + .from(soulseekBrowseDirs) + .where(where) + .orderBy(asc(soulseekBrowseDirs.name)) + .limit(limit) + .offset(offset), + db + .select({ total: sql`count(*)::int` }) + .from(soulseekBrowseDirs) + .where(where), + ]); + return { nodes, total: count?.total ?? 0 }; +} // These names come off strangers' filesystems, so they're littered with typographic punctuation that no // keyboard produces: curly apostrophes, en dashes, and U+2010 HYPHEN, which is pixel-identical to '-'. @@ -177,45 +255,67 @@ const likePattern = (term: string): string => `%${foldPunctuation(term).replace( /** The column, folded the same way `likePattern` folds the term. */ const foldedName = sql`replace(translate(${soulseekBrowseDirs.name}, ${FOLD_FROM}, ${FOLD_TO}), ${ELLIPSIS}, '...')`; +export type BrowseTreeSearch = { + /** Matches AND every ancestor of a match, so the UI has a whole tree to render, not orphans. */ + nodes: BrowseTreeNode[]; + /** `name`s of the rows that actually matched — the rest of `nodes` are there to hold them up. */ + matched: string[]; + /** Matches in the snapshot, which exceeds `matched.length` when the cap kicked in. */ + total: number; + truncated: boolean; +}; +type BrowseTreeSearchParams = { userId: number; username: string; q: string; limit: number }; + +const SEP = '\\'; + +/** Every path above `name`, longest first: 'a\b\c' → ['a\b', 'a']. */ +function ancestorsOf(name: string): string[] { + const out: string[] = []; + for (let cut = name.lastIndexOf(SEP); cut > 0; cut = name.lastIndexOf(SEP, cut - 1)) { + out.push(name.slice(0, cut)); + } + return out; +} + /** - * A page of a peer's cached folders, name-filtered. Files are excluded — that's the whole point of - * paging here, and they're fetched per folder on expand. + * Filtered view of the tree. Filtering never flattens it: matches come back with their full ancestor + * chains, which is what lets the UI open straight down to a hit and keep the shape visible around it. + * + * Ancestors are resolved by string prefix rather than a recursive CTE — the parent's path IS a prefix of + * the child's, so one indexed `name in (…)` lookup does what a walk would take `depth` round trips to do. */ -export async function getSoulseekBrowseDirs(params: BrowseDirPageParams): Promise { - const { userId, username, q, limit, offset } = params; - const [snap] = await db - .select({ id: soulseekBrowseSnapshots.id }) - .from(soulseekBrowseSnapshots) - .where(and(eq(soulseekBrowseSnapshots.userId, userId), eq(soulseekBrowseSnapshots.username, username))) - .limit(1); - if (!snap) return { dirs: [], total: 0 }; +export async function searchSoulseekBrowseTree(params: BrowseTreeSearchParams): Promise { + const { userId, username, q, limit } = params; + const snapshotId = await snapshotIdOf(userId, username); + const term = q.trim(); + if (snapshotId === null || !term) return { nodes: [], matched: [], total: 0, truncated: false }; - const term = q?.trim(); // ILIKE with a leading wildcard can't use the btree index, so this was already a scan of the snapshot's // ~18k rows — which is why folding the column here costs nothing on top. - const where = term - ? and(eq(soulseekBrowseDirs.snapshotId, snap.id), sql`${foldedName} ilike ${likePattern(term)}`) - : eq(soulseekBrowseDirs.snapshotId, snap.id); + const where = and(eq(soulseekBrowseDirs.snapshotId, snapshotId), sql`${foldedName} ilike ${likePattern(term)}`); - const [dirs, [count]] = await Promise.all([ - db - .select({ - id: soulseekBrowseDirs.id, - name: soulseekBrowseDirs.name, - fileCount: soulseekBrowseDirs.fileCount, - totalSize: soulseekBrowseDirs.totalSize, - }) - .from(soulseekBrowseDirs) - .where(where) - .orderBy(asc(soulseekBrowseDirs.name)) - .limit(limit) - .offset(offset), + const [matches, [count]] = await Promise.all([ + db.select(treeCols).from(soulseekBrowseDirs).where(where).orderBy(asc(soulseekBrowseDirs.name)).limit(limit), db .select({ total: sql`count(*)::int` }) .from(soulseekBrowseDirs) .where(where), ]); - return { dirs, total: count?.total ?? 0 }; + + const have = new Set(matches.map((m) => m.name)); + const wanted = new Set(); + for (const m of matches) for (const a of ancestorsOf(m.name)) if (!have.has(a)) wanted.add(a); + + const ancestors = wanted.size + ? await db + .select(treeCols) + .from(soulseekBrowseDirs) + .where(and(eq(soulseekBrowseDirs.snapshotId, snapshotId), inArray(soulseekBrowseDirs.name, [...wanted]))) + : []; + + const nodes = [...matches, ...ancestors].sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + const total = count?.total ?? 0; + return { nodes, matched: matches.map((m) => m.name), total, truncated: total > matches.length }; } /** One cached folder's files. Scoped by user so a guessed dir id can't read another account's cache. */ diff --git a/src/databases/officer_db/src/schema/soulseek.ts b/src/databases/officer_db/src/schema/soulseek.ts index 2e6e081c..614aa81e 100644 --- a/src/databases/officer_db/src/schema/soulseek.ts +++ b/src/databases/officer_db/src/schema/soulseek.ts @@ -68,22 +68,41 @@ export const soulseekBrowseSnapshots = pgTable( // One row per FOLDER, with that folder's files inlined as jsonb — not a row per file, which would be // 284k rows per peer. Folders are what you page and filter through; files are only ever read for the // folder you opened, so they ride along in the row. +// +// slskd hands back a flat list of backslash-delimited paths, so the tree is derived once at ingest and +// stored here rather than recomputed per request: `parentPath` links a row to its parent and the subtree +// rollups are precomputed, which is what lets the UI open one level at a time. Measured on real peers, +// the tree is already complete in slskd's response — exactly one intermediate folder was missing across +// 30k rows — but ingest synthesizes any that are absent so a level can never dead-end. export const soulseekBrowseDirs = pgTable( 'soulseek_browse_dirs', { id: serial('id').primaryKey(), snapshotId: integer('snapshot_id').notNull(), name: text('name').notNull(), + // null for a root. Its parent's `name`, i.e. `name` minus the last backslash segment. + parentPath: text('parent_path'), + // Segment count, 1-based, so the UI can indent without walking back up the chain. + depth: integer('depth').notNull().default(1), + // Last path segment — what a tree row actually displays. + label: text('label').notNull().default(''), + childCount: integer('child_count').notNull().default(0), fileCount: integer('file_count').notNull().default(0), totalSize: bigint('total_size', { mode: 'number' }).notNull().default(0), + // This folder AND everything under it. A parent's own file count is usually cover art; the number + // worth showing on a collapsed row is the whole subtree's. + subtreeFileCount: integer('subtree_file_count').notNull().default(0), + subtreeSize: bigint('subtree_size', { mode: 'number' }).notNull().default(0), // BrowsedFile[] — slimmed at ingest to {name,size,extension}: slskd's browse response carries // per-file `attributes` (bitrate/duration) that real peers leave empty, and dropping them plus the // wire noise took the sample from 59 MB to 19 MB. files: jsonb('files').notNull().default([]), }, (t) => [ - // Both reads are "this snapshot's folders, ordered/filtered by name". + // Filtered search and the name lookups behind ancestor expansion. index('idx_soulseek_browse_dirs_snapshot_name').on(t.snapshotId, t.name), + // The tree's hot path: one level, ordered by name. Covers "the roots" too, via a null parent_path. + index('idx_soulseek_browse_dirs_snapshot_parent').on(t.snapshotId, t.parentPath, t.name), // Named explicitly: drizzle's derived name for this pair would be 64 chars, one over Postgres's // 63-char identifier cap, so it would be silently truncated and never match the schema again. foreignKey({ diff --git a/src/servers/sidecar/slskd/browse.ts b/src/servers/sidecar/slskd/browse.ts index 6b7504c5..0b249ef3 100644 --- a/src/servers/sidecar/slskd/browse.ts +++ b/src/servers/sidecar/slskd/browse.ts @@ -1,4 +1,4 @@ -import type { BrowseDirInput, BrowsedFile } from 'officerdb'; +import type { BrowseDirInput, BrowseDirRow, BrowsedFile } from 'officerdb'; import { startSoulseekBrowse, finishSoulseekBrowse, failSoulseekBrowse } from 'officerdb'; import { getSlskdBase, getSlskdApiKey } from './upstream'; @@ -56,6 +56,77 @@ function slim(payload: SlskdBrowsePayload): BrowseDirInput[] { }); } +const SEP = '\\'; + +/** Split a slskd path into the parent path and the last segment, ignoring any trailing separator. */ +function splitPath(name: string): { parentPath: string | null; label: string; depth: number } { + const trimmed = name.endsWith(SEP) ? name.slice(0, -1) : name; + const cut = trimmed.lastIndexOf(SEP); + const depth = trimmed.split(SEP).filter(Boolean).length || 1; + // `cut > 0` not `>= 0`: a leading separator would otherwise make the parent an empty string, which is + // neither null nor a real row, and the level query would never find it. + return { parentPath: cut > 0 ? trimmed.slice(0, cut) : null, label: trimmed.slice(cut + 1), depth }; +} + +/** + * Turn slskd's flat list of paths into a linked tree, once, here — so the UI can open one level at a + * time instead of the platform recomputing the shape on every request. + * + * Two things the response can't be trusted on. Ancestors: measured across two real peers (~30k folders) + * exactly one intermediate folder was missing, so they're nearly always present, but a single gap would + * make a whole subtree unreachable — hence `ensure` synthesizes any absent one as an empty folder. + * Rollups: a parent's own file count is usually just cover art, so the number worth showing on a + * collapsed row is the subtree's, and that's summed bottom-up here rather than per request. + */ +export function buildTree(dirs: BrowseDirInput[]): BrowseDirRow[] { + const byName = new Map(); + + const ensure = (name: string): BrowseDirRow => { + const existing = byName.get(name); + if (existing) return existing; + const { parentPath, label, depth } = splitPath(name); + const row: BrowseDirRow = { + name, + parentPath, + label, + depth, + childCount: 0, + fileCount: 0, + totalSize: 0, + subtreeFileCount: 0, + subtreeSize: 0, + files: [], + }; + // Set before recursing so a cycle-shaped path can't loop, and so the parent's own ancestors follow. + byName.set(name, row); + if (parentPath) ensure(parentPath); + return row; + }; + + for (const dir of dirs) { + const row = ensure(dir.name); + // Concatenated rather than assigned: a path repeated in the response would otherwise lose files. + row.files = row.files.length ? row.files.concat(dir.files) : dir.files; + row.fileCount = row.files.length; + row.totalSize = row.files.reduce((n, f) => n + (f.size || 0), 0); + } + + const rows = [...byName.values()]; + for (const row of rows) { + row.subtreeFileCount = row.fileCount; + row.subtreeSize = row.totalSize; + } + // Deepest first, so every child is final before its parent absorbs it. + for (const row of [...rows].sort((a, b) => b.depth - a.depth)) { + const parent = row.parentPath ? byName.get(row.parentPath) : undefined; + if (!parent) continue; + parent.childCount += 1; + parent.subtreeFileCount += row.subtreeFileCount; + parent.subtreeSize += row.subtreeSize; + } + return rows; +} + async function run(userId: number, username: string, snapshotId: number): Promise { const base = getSlskdBase(); if (!base) { @@ -78,9 +149,14 @@ async function run(userId: number, username: string, snapshotId: number): Promis } const dirs = slim((await res.json()) as SlskdBrowsePayload); - await finishSoulseekBrowse(snapshotId, dirs); + const tree = buildTree(dirs); + await finishSoulseekBrowse(snapshotId, tree); const files = dirs.reduce((n, d) => n + d.files.length, 0); - console.log(`[slskd] browse ${username} -> ${dirs.length} dirs / ${files} files in ${Date.now() - started}ms`); + // Against distinct names, not the array length, so a repeated path doesn't read as a negative. + const synthesized = tree.length - new Set(dirs.map((d) => d.name)).size; + console.log( + `[slskd] browse ${username} -> ${tree.length} dirs (${synthesized} synthesized) / ${files} files in ${Date.now() - started}ms`, + ); } catch (err) { const message = err instanceof Error ? err.message : String(err); console.error(`[slskd] browse ${username} failed after ${Date.now() - started}ms: ${message}`); diff --git a/src/servers/sidecar/slskd/officer.ts b/src/servers/sidecar/slskd/officer.ts index e07994fa..1d60fb9d 100644 --- a/src/servers/sidecar/slskd/officer.ts +++ b/src/servers/sidecar/slskd/officer.ts @@ -4,7 +4,8 @@ import { removeSoulseekFavorite, getSoulseekBrowseSnapshots, getSoulseekBrowseSnapshot, - getSoulseekBrowseDirs, + getSoulseekBrowseLevel, + searchSoulseekBrowseTree, getSoulseekBrowseDirFiles, deleteSoulseekBrowse, } from 'officerdb'; @@ -25,7 +26,8 @@ import { startBrowse } from './browse'; // GET /_officer/browse/ → Snapshot | null one peer's cache state // POST /_officer/browse/ → 202 { started } kick off a background fetch // DELETE /_officer/browse/ → { ok: true } drop the cache -// GET /_officer/browse//dirs?q=&limit=&offset= → { dirs, total } a page of folders, no files +// GET /_officer/browse//tree?parent=&limit=&offset= → { nodes, total } one level (roots if no parent) +// GET /_officer/browse//tree/search?q=&limit= → { nodes, matched, total, truncated } // GET /_officer/browse//dirs//files → BrowsedFile[] one folder's files // ───────────────────────────────────────────────────────────────────────────────────────────────── @@ -47,8 +49,14 @@ const cleanUsername = (v: unknown): string | null => { return CONTROL_CHARS.test(u) ? null : u; }; -const DIR_PAGE_MAX = 500; -const DIR_PAGE_DEFAULT = 100; +// A level, not a page: once you're inside a folder there's nothing to leaf through, so the UI asks for +// the whole level when it's wide. The cap is set above the widest fan-out measured on a real peer (1,181). +const DIR_PAGE_MAX = 2000; +const DIR_PAGE_DEFAULT = 200; +// A match cap, not a page: every match drags its ancestors along, so the response is up to ~depth times +// this. Truncation is reported in the payload rather than passed off as the whole answer. +const SEARCH_MATCH_MAX = 1000; +const SEARCH_MATCH_DEFAULT = 300; const clampInt = (raw: string | null, fallback: number, min: number, max: number): number => { const n = Number(raw); if (!Number.isFinite(n)) return fallback; @@ -107,21 +115,34 @@ async function handleBrowse({ req, url, userId, segments }: BrowseRouteParams): return methodNotAllowed(); } - if (segments[2] !== 'dirs') return notFound(); - - // /_officer/browse//dirs — a filtered page of folders, files excluded. - if (segments.length === 3) { + // /_officer/browse//tree — one level of folders, paged. No `parent` means the roots. + if (segments[2] === 'tree' && segments.length === 3) { if (req.method !== 'GET') return methodNotAllowed(); - const page = await getSoulseekBrowseDirs({ + const level = await getSoulseekBrowseLevel({ userId, username, - q: url.searchParams.get('q') ?? undefined, + parent: url.searchParams.get('parent') ?? undefined, limit: clampInt(url.searchParams.get('limit'), DIR_PAGE_DEFAULT, 1, DIR_PAGE_MAX), offset: clampInt(url.searchParams.get('offset'), 0, 0, Number.MAX_SAFE_INTEGER), }); - return Response.json(page); + return Response.json(level); } + // /_officer/browse//tree/search — matches plus their ancestors, as one small tree. Kept off + // the level route because the shape differs: this one is a skeleton spanning depths, not a page. + if (segments[2] === 'tree' && segments.length === 4 && segments[3] === 'search') { + if (req.method !== 'GET') return methodNotAllowed(); + const result = await searchSoulseekBrowseTree({ + userId, + username, + q: url.searchParams.get('q') ?? '', + limit: clampInt(url.searchParams.get('limit'), SEARCH_MATCH_DEFAULT, 1, SEARCH_MATCH_MAX), + }); + return Response.json(result); + } + + if (segments[2] !== 'dirs') return notFound(); + // /_officer/browse//dirs//files if (segments.length === 5 && segments[4] === 'files') { if (req.method !== 'GET') return methodNotAllowed(); diff --git a/src/workspaces/officerdev/src/apps/Soulseek/SharesBrowser.tsx b/src/workspaces/officerdev/src/apps/Soulseek/SharesBrowser.tsx index 4092ab1a..212f8f74 100644 --- a/src/workspaces/officerdev/src/apps/Soulseek/SharesBrowser.tsx +++ b/src/workspaces/officerdev/src/apps/Soulseek/SharesBrowser.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect } from 'react'; +import { useState, useEffect, useMemo } from 'react'; import { FolderOpen, Folder, @@ -12,16 +12,34 @@ import { Trash2, FileAudio, } from 'lucide-react'; -import { useSoulseekBrowseSnapshots, useSoulseekBrowseDirs, useSoulseekBrowseFiles } from './useSoulseekBrowse'; -import { formatSize, formatWhen, type SoulseekBrowseSnapshot } from './shared'; +import { + useSoulseekBrowseSnapshots, + useSoulseekBrowseLevel, + useSoulseekBrowseSearch, + useSoulseekBrowseFiles, +} from './useSoulseekBrowse'; +import { formatSize, formatWhen, type SoulseekBrowseNode, type SoulseekBrowseSnapshot } from './shared'; -// A peer's shared folders, read from Officer's cache rather than browsed live. +// A peer's shared folders, read from Officer's cache rather than browsed live, and rendered as a TREE. // // The live slskd call returns the entire tree in one blocking response (59 MB / 18k folders / 284k files // for a real peer) and takes minutes, because it round-trips to that peer. So nothing here triggers it -// inline: you ask the sidecar to fetch, it keeps going without the tab, and this reads back pages. +// inline: you ask the sidecar to fetch, it keeps going without the tab, and this reads back levels. +// +// slskd's response is flat — every folder is a full backslash-delimited path — which made the old list +// unreadable: 26 rows all beginning with the same 31 characters. The sidecar links the paths into a tree +// at ingest, so a row here shows one segment and one level loads at a time. +// +// Filtering never flattens that. The search route returns matches WITH their ancestors, and this renders +// that skeleton pre-expanded: you see where a hit lives, not just that it exists. Because matching runs +// against the whole path, a folder matching implies all its descendants match too — so a matched subtree +// arrives complete. -const PAGE_SIZE = 100; +// One level's page size. Fan-out is not gentle: the widest folder measured on a real peer has 1,181 +// children, so a level can ask for more (up to the route's cap) rather than paging with Previous/Next — +// there's nothing to page through once you're inside one folder. +const LEVEL_LIMIT = 200; +const LEVEL_LIMIT_MAX = 2000; /** Debounce the filter box so a keystroke doesn't become a request. */ function useDebounced(value: T, ms: number): T { @@ -57,32 +75,41 @@ export const BrowseStateChip = ({ snapshot }: { snapshot: SoulseekBrowseSnapshot ); }; +/** The filtered tree, indexed for rendering: children by parent path, plus which rows actually matched. */ +type Filtered = { childrenOf: Map; matched: Set }; + export const SharesBrowser = ({ username }: { username: string }) => { const { snapshotOf, fetchShares, dropShares } = useSoulseekBrowseSnapshots(); const [filter, setFilter] = useState(''); - const [page, setPage] = useState(0); - const [openDir, setOpenDir] = useState(null); const q = useDebounced(filter.trim(), 250); const snapshot = snapshotOf(username); - // Folder rows survive a failed or in-flight refresh, so drive the list off the data existing rather + // Folder rows survive a failed or in-flight refresh, so drive the tree off the data existing rather // than off the status — a stale tree is still worth browsing while a new one is being fetched. const hasCache = !!snapshot && snapshot.directoryCount > 0; - const { data, isFetching } = useSoulseekBrowseDirs({ username, q, page, pageSize: PAGE_SIZE, enabled: hasCache }); + const roots = useSoulseekBrowseLevel({ username, parent: null, limit: LEVEL_LIMIT, enabled: hasCache && !q }); + const search = useSoulseekBrowseSearch({ username, q, enabled: hasCache }); - // A new peer or a new filter invalidates the current page number and any expanded folder. - useEffect(() => { - setPage(0); - setOpenDir(null); - }, [username, q]); useEffect(() => { setFilter(''); }, [username]); - const total = data?.total ?? 0; - const pages = Math.max(1, Math.ceil(total / PAGE_SIZE)); + const filtered = useMemo(() => { + if (!q || !search.data) return null; + const childrenOf = new Map(); + for (const node of search.data.nodes) { + if (!node.parentPath) continue; + const siblings = childrenOf.get(node.parentPath); + if (siblings) siblings.push(node); + else childrenOf.set(node.parentPath, [node]); + } + return { childrenOf, matched: new Set(search.data.matched) }; + }, [q, search.data]); + + const topNodes = q ? (search.data?.nodes.filter((n) => !n.parentPath) ?? []) : (roots.data?.nodes ?? []); const pending = snapshot?.status === 'pending'; + const busy = q ? search.isFetching : roots.isFetching; return (
@@ -160,68 +187,40 @@ export const SharesBrowser = ({ username }: { username: string }) => { value={filter} onChange={(ev) => setFilter(ev.target.value)} placeholder="Filter folders…" - className="h-7 min-w-0 flex-1 bg-transparent text-sm text-zinc-100 placeholder:text-zinc-600 outline-none" + className="h-7 min-w-0 flex-1 bg-transparent text-sm text-zinc-100 outline-none placeholder:text-zinc-600" /> - {isFetching && } - {total.toLocaleString()} matching + {busy && } + {q && ( + + {(search.data?.total ?? 0).toLocaleString()} matching + + )}
- {total === 0 ? ( -

No folders match “{q}”.

- ) : ( -
- {data?.dirs.map((dir) => ( -
- - {openDir === dir.id && } -
- ))} + {/* Never a silent cap: a truncated search still renders a valid tree, just not all of it. */} + {search.data?.truncated && q && ( +
+ Showing the first {search.data.matched.length.toLocaleString()} of {search.data.total.toLocaleString()}{' '} + matches — narrow the filter to see the rest.
)} - {pages > 1 && ( -
- - {(page * PAGE_SIZE + 1).toLocaleString()}–{Math.min(total, (page + 1) * PAGE_SIZE).toLocaleString()}{' '} - of {total.toLocaleString()} - -
- - - {page + 1} / {pages.toLocaleString()} - - -
+ {q && search.data && search.data.total === 0 ? ( +

No folders match “{q}”.

+ ) : ( + // Keyed by the term so switching between browsing and a filtered view remounts the tree: + // every row's open state is local, and reusing instances across modes would strand it. +
+ {topNodes.map((node) => ( + + ))}
)} @@ -237,26 +236,137 @@ export const SharesBrowser = ({ username }: { username: string }) => { ); }; -type DirFilesProps = { username: string; dirId: number }; +type TreeNodeProps = { + username: string; + node: SoulseekBrowseNode; + /** Non-null in filter mode: children come from this set, not from a request, and rows start open. */ + filtered: Filtered | null; + indent: number; + initialOpen: boolean; +}; -const DirFiles = ({ username, dirId }: DirFilesProps) => { +const TreeNode = ({ username, node, filtered, indent, initialOpen }: TreeNodeProps) => { + const [open, setOpen] = useState(initialOpen); + const [limit, setLimit] = useState(LEVEL_LIMIT); + + const expandable = node.childCount > 0 || node.fileCount > 0; + const level = useSoulseekBrowseLevel({ + username, + parent: node.name, + limit, + enabled: !filtered && open && node.childCount > 0, + }); + + const children = filtered ? (filtered.childrenOf.get(node.name) ?? []) : (level.data?.nodes ?? []); + const shown = filtered ? children.length : (level.data?.total ?? 0); + const isMatch = filtered?.matched.has(node.name) ?? false; + + return ( +
+ + + {open && ( + <> + {!filtered && level.isPending && node.childCount > 0 && ( +

+ Loading folders… +

+ )} + {children.map((child) => ( + + ))} + {/* Wide levels load in one go rather than paging — there's nothing to leaf through inside a folder. */} + {!filtered && shown > children.length && children.length > 0 && ( + + )} + {filtered && node.childCount > children.length && ( +

+ {(node.childCount - children.length).toLocaleString()} more folders here don't match the filter. +

+ )} + {node.fileCount > 0 && } + + )} +
+ ); +}; + +type DirFilesProps = { username: string; dirId: number; indent: number }; + +const DirFiles = ({ username, dirId, indent }: DirFilesProps) => { const { data, isPending, error } = useSoulseekBrowseFiles(username, dirId); + const pad = { paddingLeft: 12 + indent * 14 }; if (isPending) return ( -
+

Loading files… -

+

); - if (error) return

Could not load this folder.

; - if (!data?.length) return

Empty folder.

; + if (error) + return ( +

+ Could not load this folder. +

+ ); + if (!data?.length) return null; return ( -
+
{data.map((file) => ( -
+
+ - + {file.name} {formatSize(file.size)} diff --git a/src/workspaces/officerdev/src/apps/Soulseek/shared.ts b/src/workspaces/officerdev/src/apps/Soulseek/shared.ts index a4742e37..1e982bd2 100644 --- a/src/workspaces/officerdev/src/apps/Soulseek/shared.ts +++ b/src/workspaces/officerdev/src/apps/Soulseek/shared.ts @@ -202,8 +202,29 @@ export type SoulseekBrowseSnapshot = { startedAt: string; completedAt: string | null; }; -export type SoulseekBrowseDir = { id: number; name: string; fileCount: number; totalSize: number }; -export type SoulseekBrowseDirPage = { dirs: SoulseekBrowseDir[]; total: number }; +// A node of the stored tree. The sidecar links slskd's flat backslash paths up at ingest, so `label` is +// already just the last segment and the subtree rollups are already summed — this side never derives +// either, which is the point: a collapsed row can show what's underneath it without fetching it. +export type SoulseekBrowseNode = { + id: number; + name: string; + parentPath: string | null; + label: string; + depth: number; + childCount: number; + fileCount: number; + totalSize: number; + subtreeFileCount: number; + subtreeSize: number; +}; +export type SoulseekBrowseLevel = { nodes: SoulseekBrowseNode[]; total: number }; +/** A filtered tree: the matches plus every ancestor holding them up, so the shape survives filtering. */ +export type SoulseekBrowseSearch = { + nodes: SoulseekBrowseNode[]; + matched: string[]; + total: number; + truncated: boolean; +}; // Browse carries no bitrate/duration (real peers send empty `attributes`), so unlike a search result // there's no quality metadata to show — just name, size, extension. export type SoulseekBrowsedFile = { name: string; size: number; extension: string }; diff --git a/src/workspaces/officerdev/src/apps/Soulseek/useSoulseekBrowse.ts b/src/workspaces/officerdev/src/apps/Soulseek/useSoulseekBrowse.ts index 38e8f11c..90a0f4a7 100644 --- a/src/workspaces/officerdev/src/apps/Soulseek/useSoulseekBrowse.ts +++ b/src/workspaces/officerdev/src/apps/Soulseek/useSoulseekBrowse.ts @@ -1,6 +1,6 @@ import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query'; import { useClient } from 'hooks/useClient'; -import type { SoulseekBrowseSnapshot, SoulseekBrowseDirPage, SoulseekBrowsedFile } from './shared'; +import type { SoulseekBrowseSnapshot, SoulseekBrowseLevel, SoulseekBrowseSearch, SoulseekBrowsedFile } from './shared'; // Client for the sidecar's cached share trees. The expensive work happens server-side and outlives the // tab, so this hook only ever starts a fetch and polls for its status — it never pulls a whole tree. @@ -72,23 +72,43 @@ export function useSoulseekBrowseSnapshots() { }; } -type DirsParams = { username: string | null; q: string; page: number; pageSize: number; enabled?: boolean }; +type LevelParams = { username: string | null; parent: string | null; limit: number; enabled?: boolean }; -/** One page of a peer's cached folders. Keeps the previous page visible while the next loads. */ -export function useSoulseekBrowseDirs({ username, q, page, pageSize, enabled = true }: DirsParams) { +/** + * One level of a peer's tree — the children of `parent`, or the roots when it's null. Cached per level, + * so collapsing and reopening a folder is free, and expanding a shallow node never touches its subtree. + */ +export function useSoulseekBrowseLevel({ username, parent, limit, enabled = true }: LevelParams) { const { get } = useClient(); - const offset = page * pageSize; return useQuery({ - queryKey: ['soulseek', 'browse', username, 'dirs', q, offset, pageSize], + queryKey: ['soulseek', 'browse', username, 'level', parent, limit], queryFn: () => { - const params = new URLSearchParams({ limit: String(pageSize), offset: String(offset) }); - if (q) params.set('q', q); - return get(`/slskd/_officer/browse/${encodeURIComponent(username ?? '')}/dirs?${params}`); + const params = new URLSearchParams({ limit: String(limit) }); + if (parent) params.set('parent', parent); + return get(`/slskd/_officer/browse/${encodeURIComponent(username ?? '')}/tree?${params}`); }, enabled: enabled && !!username, + staleTime: 5 * 60_000, + }); +} + +type SearchParams = { username: string | null; q: string; enabled?: boolean }; + +/** The filtered tree: matches with their ancestors, in one request, ready to render pre-expanded. */ +export function useSoulseekBrowseSearch({ username, q, enabled = true }: SearchParams) { + const { get } = useClient(); + + return useQuery({ + queryKey: ['soulseek', 'browse', username, 'search', q], + queryFn: () => + get( + `/slskd/_officer/browse/${encodeURIComponent(username ?? '')}/tree/search?q=${encodeURIComponent(q)}`, + ), + enabled: enabled && !!username && !!q, + // The previous result stays on screen while the next one loads, so the tree doesn't blink per keystroke. placeholderData: (prev) => prev, - staleTime: 30_000, + staleTime: 60_000, }); }