platform/scripts/marketplace-sync.ts

#!/usr/bin/env bun
import { existsSync, readFileSync } from 'node:fs';
import { basename, join, resolve } from 'node:path';
import { upsertMetadata, upsertPlugin } from '../src/databases/marketplace_db/src/plugins/queries';
import { discoverPlugins } from '../src/servers/plugins/discover';

// Fill the marketplace database from the plugin repositories.
//
// ── What this reads, and why it is not the marketplace's own opinion ──
//
// Every field except "where to clone from" comes out of the plugin's `manifest.ts` and its README. The
// marketplace stores a COPY so the website can render without cloning three repositories per page load;
// the manifest stays the source of truth, and the platform re-reads it after cloning. A row here that
// disagrees with the repository is a stale cache, not a second opinion — which is why re-running this is
// always safe and never merges: it overwrites.
//
// ── Reading the manifest by importing it ──
//
// A manifest is TypeScript, not JSON, so it is imported and evaluated rather than parsed. That is code
// execution, and it is fine for OUR repositories and not fine as a general mechanism.
//
// `[blocker for third-party submissions]` A stranger's plugin cannot be imported by the machine that
// serves the catalogue. Whoever builds submissions has to either evaluate manifests in a sandbox or make
// authors publish a `manifest.json` alongside. Recorded rather than solved: nobody can submit yet.
//
// ── Local clones, for now ──
//
// It syncs from `plugins/` on this machine rather than by cloning each repo into a temp directory. That
// keeps tonight's scope to "get real data into the database" — and it means the machine running the sync
// must have the plugins checked out. See `docs/marketplace.md` → "Sync" for the version that clones.

type Sync = { appName: string; developer: string; repoUrl: string };

/**
 * A clone URL the whole internet can use.
 *
 * `plugins/example` and `plugins/offscale` have `git@gitea.officer.dev:plugins/<name>.git` remotes,
 * because whoever set them up had a key on that host. Publishing that to a catalogue would hand every
 * installer a URL that only works on the machine it was published from — the install fails at
 * authentication, which reads as "the plugin is broken" rather than "the catalogue is wrong".
 */
export function toHttpsRemote(remote: string): string | null {
  const trimmed = remote.trim();
  if (!trimmed) return null;
  if (trimmed.startsWith('https://')) return trimmed;

  // git@host:path/to/repo.git  →  https://host/path/to/repo.git
  const scp = trimmed.match(/^[^@]+@([^:]+):(.+)$/);
  if (scp) return `https://${scp[1]}/${scp[2]}`;

  // ssh://git@host/path.git → https://host/path.git
  if (trimmed.startsWith('ssh://')) {
    try {
      const url = new URL(trimmed);
      return `https://${url.host}${url.pathname}`;
    } catch {
      return null;
    }
  }
  return null;
}

const git = (dir: string, args: string[]): string | null => {
  const result = Bun.spawnSync(['git', '-C', dir, ...args], { stdout: 'pipe', stderr: 'ignore' });
  if (result.exitCode !== 0) return null;
  return new TextDecoder().decode(result.stdout).trim() || null;
};

const PLUGINS_DIR = resolve(import.meta.dir, '..', 'plugins');

async function syncOne(dir: string): Promise<Sync | null> {
  const appName = basename(dir);
  const manifestPath = join(dir, 'manifest.ts');
  if (!existsSync(manifestPath)) {
    console.warn(`[marketplace-sync] ${appName}: no manifest.ts — skipped`);
    return null;
  }

  const remote = git(dir, ['remote', 'get-url', 'origin']);
  const repoUrl = remote ? toHttpsRemote(remote) : null;
  if (!repoUrl) {
    console.warn(`[marketplace-sync] ${appName}: no usable https remote (${remote ?? 'none'}) — skipped`);
    return null;
  }

  const { manifest } = (await import(manifestPath)) as { manifest: Record<string, unknown> };
  if (!manifest) {
    console.warn(`[marketplace-sync] ${appName}: manifest.ts exports no \`manifest\` — skipped`);
    return null;
  }

  // What the plugin SHIPS, read from the directory rather than declared — the same rule discovery
  // follows, so a catalogue entry cannot claim a sidecar the repository does not have.
  const has = {
    api: existsSync(join(dir, 'api', 'router.ts')),
    schema: existsSync(join(dir, 'db', 'schema.ts')),
    sidecar: existsSync(join(dir, 'sidecar', 'index.ts')) || existsSync(join(dir, 'sidecar', 'index.mjs')),
    web: existsSync(join(dir, 'web')),
  };

  const readmePath = ['README.md', 'readme.md'].map((n) => join(dir, n)).find((p) => existsSync(p));
  const readme = readmePath ? readFileSync(readmePath, 'utf-8') : null;

  const developer = String(manifest.publisher ?? 'unknown');
  const pluginId = await upsertPlugin({ appName, developer, repoUrl });

  await upsertMetadata(pluginId, {
    version: String(manifest.version ?? '0.0.0'),
    platform: String(manifest.platform ?? '>=1.0.0'),
    label: String(manifest.label ?? appName),
    summary: String(manifest.summary ?? ''),
    // Taken verbatim, NOT validated here. `manifestProblems` already refuses a tag outside the
    // vocabulary at discovery, so anything reaching this point is either valid or belongs to a plugin
    // the platform would not load anyway — and a sync that silently dropped a tag would be a catalogue
    // quietly disagreeing with the repository it claims to mirror.
    tags: Array.isArray(manifest.tags) ? manifest.tags : [],
    color: (manifest.color as string) ?? null,
    tile: (manifest.tile as string) ?? null,
    // Artwork lives in the repo, so the URL is built from the remote rather than stored by hand. Only
    // claimed when the file is actually there — a broken image is worse than a glyph.
    iconUrl: existsSync(join(dir, 'assets', 'icon.png')) ? rawUrl(repoUrl, 'assets/icon.png') : null,
    icon: (manifest.icon as string) ?? null,
    permissions: manifest.permissions ?? [],
    osDependencies: manifest.osDependencies ?? null,
    has,
    readme,
    sourceCommit: git(dir, ['rev-parse', 'HEAD']),
  });

  console.log(
    `[marketplace-sync] ${appName} ← ${repoUrl}${readme ? '' : ' (no README)'}${has.web ? '' : ' (no web/)'}`,
  );
  return { appName, developer, repoUrl };
}

/**
 * A raw-file URL for a Gitea repository.
 *
 * `HEAD`, not a branch name. This said `branch/main` and the comment here warned that "a repo whose
 * default branch is `master` gets a 404" — which then happened: four plugin repositories were created
 * on `master` and every one of them served a broken image in the marketplace on any server that reads
 * the remote catalogue. A warning in a comment is not a guard.
 *
 * Gitea resolves `HEAD` to whatever the default branch is, so the question stops being asked. Verified
 * against both spellings.
 *
 * `[open]` Still Gitea-shaped. A GitHub-hosted plugin needs the other form, and that wants the forge to
 * be part of the registry rather than inferred from the URL.
 */
function rawUrl(repoUrl: string, path: string): string {
  return `${repoUrl.replace(/\.git$/, '')}/raw/HEAD/${path}`;
}

const dirs = existsSync(PLUGINS_DIR)
  ? [...new Bun.Glob('*/').scanSync({ cwd: PLUGINS_DIR, onlyFiles: false })]
      // Not a plugin: the shared install target for plugin npm packages, which lives here because bun
      // resolves node_modules up the tree. `discoverPlugins` skips it for the same reason.
      .filter((d) => d.replace(/\/$/, '') !== 'node_modules')
      .map((d) => join(PLUGINS_DIR, d))
  : [];

if (dirs.length === 0) {
  console.log(`[marketplace-sync] no plugin directories under ${PLUGINS_DIR} — nothing to sync`);
  process.exit(0);
}

// Discovery is not used to FIND them — a plugin that fails discovery is exactly the kind of thing a
// catalogue should still be able to describe — but its broken list is worth surfacing here.
const { broken } = await discoverPlugins().catch(() => ({ broken: [] as { appName: string; error: string }[] }));
for (const { appName, error } of broken) console.warn(`[marketplace-sync] ${appName} is broken on disk: ${error}`);

const synced: Sync[] = [];
for (const dir of dirs) {
  const result = await syncOne(dir).catch((err) => {
    console.error(`[marketplace-sync] ${basename(dir)} failed: ${err instanceof Error ? err.message : err}`);
    return null;
  });
  if (result) synced.push(result);
}

console.log(`[marketplace-sync] ${synced.length}/${dirs.length} plugin(s) in the catalogue`);
process.exit(0);