#!/usr/bin/env bun import { existsSync, readFileSync } from 'node:fs'; import { basename, join, resolve } from 'node:path'; import { upsertMetadata, upsertPlugin } from '../src/databases/marketplace_db/src/plugins/queries'; import { discoverPlugins } from '../src/servers/plugins/discover'; // Fill the marketplace database from the plugin repositories. // // ── What this reads, and why it is not the marketplace's own opinion ── // // Every field except "where to clone from" comes out of the plugin's `manifest.ts` and its README. The // marketplace stores a COPY so the website can render without cloning three repositories per page load; // the manifest stays the source of truth, and the platform re-reads it after cloning. A row here that // disagrees with the repository is a stale cache, not a second opinion — which is why re-running this is // always safe and never merges: it overwrites. // // ── Reading the manifest by importing it ── // // A manifest is TypeScript, not JSON, so it is imported and evaluated rather than parsed. That is code // execution, and it is fine for OUR repositories and not fine as a general mechanism. // // `[blocker for third-party submissions]` A stranger's plugin cannot be imported by the machine that // serves the catalogue. Whoever builds submissions has to either evaluate manifests in a sandbox or make // authors publish a `manifest.json` alongside. Recorded rather than solved: nobody can submit yet. // // ── Local clones, for now ── // // It syncs from `plugins/` on this machine rather than by cloning each repo into a temp directory. That // keeps tonight's scope to "get real data into the database" — and it means the machine running the sync // must have the plugins checked out. See `docs/marketplace.md` → "Sync" for the version that clones. type Sync = { appName: string; developer: string; repoUrl: string }; /** * A clone URL the whole internet can use. * * `plugins/example` and `plugins/offscale` have `git@gitea.officer.dev:plugins/.git` remotes, * because whoever set them up had a key on that host. Publishing that to a catalogue would hand every * installer a URL that only works on the machine it was published from — the install fails at * authentication, which reads as "the plugin is broken" rather than "the catalogue is wrong". */ export function toHttpsRemote(remote: string): string | null { const trimmed = remote.trim(); if (!trimmed) return null; if (trimmed.startsWith('https://')) return trimmed; // git@host:path/to/repo.git → https://host/path/to/repo.git const scp = trimmed.match(/^[^@]+@([^:]+):(.+)$/); if (scp) return `https://${scp[1]}/${scp[2]}`; // ssh://git@host/path.git → https://host/path.git if (trimmed.startsWith('ssh://')) { try { const url = new URL(trimmed); return `https://${url.host}${url.pathname}`; } catch { return null; } } return null; } const git = (dir: string, args: string[]): string | null => { const result = Bun.spawnSync(['git', '-C', dir, ...args], { stdout: 'pipe', stderr: 'ignore' }); if (result.exitCode !== 0) return null; return new TextDecoder().decode(result.stdout).trim() || null; }; const PLUGINS_DIR = resolve(import.meta.dir, '..', 'plugins'); async function syncOne(dir: string): Promise { const appName = basename(dir); const manifestPath = join(dir, 'manifest.ts'); if (!existsSync(manifestPath)) { console.warn(`[marketplace-sync] ${appName}: no manifest.ts — skipped`); return null; } const remote = git(dir, ['remote', 'get-url', 'origin']); const repoUrl = remote ? toHttpsRemote(remote) : null; if (!repoUrl) { console.warn(`[marketplace-sync] ${appName}: no usable https remote (${remote ?? 'none'}) — skipped`); return null; } const { manifest } = (await import(manifestPath)) as { manifest: Record }; if (!manifest) { console.warn(`[marketplace-sync] ${appName}: manifest.ts exports no \`manifest\` — skipped`); return null; } // What the plugin SHIPS, read from the directory rather than declared — the same rule discovery // follows, so a catalogue entry cannot claim a sidecar the repository does not have. const has = { api: existsSync(join(dir, 'api', 'router.ts')), schema: existsSync(join(dir, 'db', 'schema.ts')), sidecar: existsSync(join(dir, 'sidecar', 'index.ts')) || existsSync(join(dir, 'sidecar', 'index.mjs')), web: existsSync(join(dir, 'web')), }; const readmePath = ['README.md', 'readme.md'].map((n) => join(dir, n)).find((p) => existsSync(p)); const readme = readmePath ? readFileSync(readmePath, 'utf-8') : null; const developer = String(manifest.publisher ?? 'unknown'); const pluginId = await upsertPlugin({ appName, developer, repoUrl }); await upsertMetadata(pluginId, { version: String(manifest.version ?? '0.0.0'), platform: String(manifest.platform ?? '>=1.0.0'), label: String(manifest.label ?? appName), summary: String(manifest.summary ?? ''), color: (manifest.color as string) ?? null, tile: (manifest.tile as string) ?? null, // Artwork lives in the repo, so the URL is built from the remote rather than stored by hand. Only // claimed when the file is actually there — a broken image is worse than a glyph. iconUrl: existsSync(join(dir, 'assets', 'icon.png')) ? rawUrl(repoUrl, 'assets/icon.png') : null, icon: (manifest.icon as string) ?? null, permissions: manifest.permissions ?? [], osDependencies: manifest.osDependencies ?? null, has, readme, sourceCommit: git(dir, ['rev-parse', 'HEAD']), }); console.log( `[marketplace-sync] ${appName} ← ${repoUrl}${readme ? '' : ' (no README)'}${has.web ? '' : ' (no web/)'}`, ); return { appName, developer, repoUrl }; } /** * A raw-file URL for a Gitea repository. * * `HEAD`, not a branch name. This said `branch/main` and the comment here warned that "a repo whose * default branch is `master` gets a 404" — which then happened: four plugin repositories were created * on `master` and every one of them served a broken image in the marketplace on any server that reads * the remote catalogue. A warning in a comment is not a guard. * * Gitea resolves `HEAD` to whatever the default branch is, so the question stops being asked. Verified * against both spellings. * * `[open]` Still Gitea-shaped. A GitHub-hosted plugin needs the other form, and that wants the forge to * be part of the registry rather than inferred from the URL. */ function rawUrl(repoUrl: string, path: string): string { return `${repoUrl.replace(/\.git$/, '')}/raw/HEAD/${path}`; } const dirs = existsSync(PLUGINS_DIR) ? [...new Bun.Glob('*/').scanSync({ cwd: PLUGINS_DIR, onlyFiles: false })] // Not a plugin: the shared install target for plugin npm packages, which lives here because bun // resolves node_modules up the tree. `discoverPlugins` skips it for the same reason. .filter((d) => d.replace(/\/$/, '') !== 'node_modules') .map((d) => join(PLUGINS_DIR, d)) : []; if (dirs.length === 0) { console.log(`[marketplace-sync] no plugin directories under ${PLUGINS_DIR} — nothing to sync`); process.exit(0); } // Discovery is not used to FIND them — a plugin that fails discovery is exactly the kind of thing a // catalogue should still be able to describe — but its broken list is worth surfacing here. const { broken } = await discoverPlugins().catch(() => ({ broken: [] as { appName: string; error: string }[] })); for (const { appName, error } of broken) console.warn(`[marketplace-sync] ${appName} is broken on disk: ${error}`); const synced: Sync[] = []; for (const dir of dirs) { const result = await syncOne(dir).catch((err) => { console.error(`[marketplace-sync] ${basename(dir)} failed: ${err instanceof Error ? err.message : err}`); return null; }); if (result) synced.push(result); } console.log(`[marketplace-sync] ${synced.length}/${dirs.length} plugin(s) in the catalogue`); process.exit(0);