diff --git a/scripts/gen-ecosystem.ts b/scripts/gen-ecosystem.ts index dee2eeab..6cbb3458 100644 --- a/scripts/gen-ecosystem.ts +++ b/scripts/gen-ecosystem.ts @@ -32,7 +32,7 @@ import { CATALOGUE } from '../src/servers/app-store/catalogue'; import { PLATFORM_DIR } from '../src/servers/data-path'; import { discoverPlugins } from '../src/servers/plugins/discover'; import { pluginPm2Entry, type Pm2App } from '../src/servers/plugins/ecosystem'; -import { listPluginInstalls, listSidecarInstalls } from 'officerdb'; +import { listPluginInstalls, listPluginPorts, listSidecarInstalls } from 'officerdb'; const done = (message: string): never => { console.log(`[gen:ecosystem] ${message}`); @@ -92,7 +92,24 @@ const DISABLED = new Set( async function pluginEntries(): Promise { const [installs, { plugins }] = await Promise.all([listPluginInstalls(), discoverPlugins()]); const enabled = new Set(installs.filter((i) => i.enabled).map((i) => i.appName)); - return plugins.filter((p) => enabled.has(p.appName)).flatMap((p) => pluginPm2Entry(p) ?? []); + /* + Ports come from the database, not from a fresh allocation. + + This script rebuilds the whole file from what is installed, and it must produce the SAME entries the + installer produced — allocating here would hand a plugin a different number than the one already + written into somebody's nginx configuration. + */ + const allocations = await listPluginPorts(); + const byPlugin = new Map>(); + for (const row of allocations) { + const current = byPlugin.get(row.appName) ?? {}; + current[row.name] = row.port; + byPlugin.set(row.appName, current); + } + + return plugins + .filter((p) => enabled.has(p.appName)) + .flatMap((p) => pluginPm2Entry(p, byPlugin.get(p.appName)) ?? []); } /** Installed AND enabled. Unlike a plugin, every catalogue entry has a `process` — none are filtered out. */ diff --git a/src/databases/officer_db/src/plugins/queries.ts b/src/databases/officer_db/src/plugins/queries.ts index e0db2b3c..9a59e520 100644 --- a/src/databases/officer_db/src/plugins/queries.ts +++ b/src/databases/officer_db/src/plugins/queries.ts @@ -1,6 +1,6 @@ -import { eq } from 'drizzle-orm'; +import { and, eq } from 'drizzle-orm'; import { db } from '../db'; -import { pluginInstalls } from './schema'; +import { pluginInstalls, pluginPorts } from './schema'; export type PluginInstall = typeof pluginInstalls.$inferSelect; @@ -47,3 +47,110 @@ export async function removePluginInstall(appName: string): Promise { const rows = await db.delete(pluginInstalls).where(eq(pluginInstalls.appName, appName)).returning(); return rows.length > 0; } + +// ── Ports the platform hands out and remembers ──────────────────────────────────────────────────── + +export type PluginPort = typeof pluginPorts.$inferSelect; + +/** Every allocation, for the settings screen and for anyone asking what is spoken for. */ +export async function listPluginPorts(): Promise { + return db.select().from(pluginPorts).orderBy(pluginPorts.port); +} + +/** What this plugin has been given, as `{ name: port }`. Empty when it has asked for nothing. */ +export async function getPluginPorts(appName: string): Promise> { + const rows = await db.select().from(pluginPorts).where(eq(pluginPorts.appName, appName)); + return Object.fromEntries(rows.map((r) => [r.name, r.port])); +} + +/** + * Find a port nothing is using, by asking the kernel and then checking our own book. + * + * Binding `port: 0` is the only honest way to know a port is free — a list of what we have handed out + * says nothing about what else on this machine is listening, and a hardcoded range says nothing about + * either. So the kernel picks, we let go, and the caller binds it a moment later. + * + * ── The gap between letting go and binding ── + * + * Real, and unavoidable without holding the socket open across a process boundary. Something else can + * take the port in between. That is why allocation is not the last word: a sidecar that cannot bind its + * remembered port must say so loudly rather than quietly choosing another, because the number is written + * down in configuration this database cannot see. + */ +/** Can we bind it right now? The only honest test, and true only for this instant. */ +async function isFree(port: number): Promise { + try { + const probe = Bun.serve({ port, hostname: '127.0.0.1', fetch: () => new Response('') }); + probe.stop(true); + return true; + } catch { + return false; + } +} + +async function freePort(taken: Set): Promise { + for (let attempt = 0; attempt < 20; attempt++) { + const probe = Bun.serve({ port: 0, hostname: '127.0.0.1', fetch: () => new Response('') }); + const port = probe.port; + probe.stop(true); + if (port && !taken.has(port)) return port; + } + throw new Error('could not find a free port after 20 attempts'); +} + +/** + * The port this plugin was given for `name`, allocating one the first time. + * + * Idempotent and stable: the same plugin and name always gets the same number back, which is the entire + * point — see the note on `pluginPorts`. Re-installing, upgrading and restarting all return what was + * given the first time. + */ +export async function allocatePluginPort(appName: string, name: string, preferred?: number): Promise { + const [existing] = await db + .select() + .from(pluginPorts) + .where(and(eq(pluginPorts.appName, appName), eq(pluginPorts.name, name))); + if (existing) return existing.port; + + /* + Racing another install is possible and is handled by the unique index rather than by a lock: two + installs running at once can be handed the same number by two different kernel probes, and exactly + one of the inserts will win. The loser tries again with a fresh number rather than failing, because + "somebody else installed a plugin at the same moment" is not a reason to refuse. + */ + for (let attempt = 0; attempt < 5; attempt++) { + const taken = new Set((await db.select().from(pluginPorts)).map((r) => r.port)); + /* + The wish first, and only on the first attempt. + + A preferred port that is already spoken for, or that something else on this machine is listening + on, is not an error — it is a plugin arriving somewhere its ideal address is taken, which is + ordinary. It falls back to any free one rather than failing, because an install that refuses over + a preference would make `preferred` a claim. + */ + const wish = attempt === 0 && preferred && !taken.has(preferred) && (await isFree(preferred)) ? preferred : null; + const port = wish ?? (await freePort(taken)); + try { + const [row] = await db.insert(pluginPorts).values({ appName, name, port }).returning(); + return row!.port; + } catch { + const [now] = await db + .select() + .from(pluginPorts) + .where(and(eq(pluginPorts.appName, appName), eq(pluginPorts.name, name))); + if (now) return now.port; // the other writer was allocating the same thing + } + } + throw new Error(`could not allocate a port for ${appName}/${name}`); +} + +/** + * Give up a plugin's ports. NOT called on uninstall — see the note in schema.ts. + * + * The number lives in configuration this database cannot see, so handing it to something else because a + * plugin was briefly removed points that configuration at the wrong service. Only `purge` does this. + */ +export async function releasePluginPorts(appName: string): Promise { + const rows = await db.delete(pluginPorts).where(eq(pluginPorts.appName, appName)).returning(); + return rows.length; +} diff --git a/src/databases/officer_db/src/plugins/schema.ts b/src/databases/officer_db/src/plugins/schema.ts index 5a34c752..205b0a5a 100644 --- a/src/databases/officer_db/src/plugins/schema.ts +++ b/src/databases/officer_db/src/plugins/schema.ts @@ -1,4 +1,4 @@ -import { pgTable, serial, text, boolean, timestamp, uniqueIndex } from 'drizzle-orm/pg-core'; +import { pgTable, serial, text, integer, boolean, timestamp, uniqueIndex } from 'drizzle-orm/pg-core'; // Which plugins are installed on this machine, and whether they should be mounted. // @@ -56,3 +56,57 @@ export const pluginInstalls = pgTable( uniqueIndex('uq_plugin_installs_app_name').on(t.appName), ], ); + +/** + * Ports the platform has given out to plugins, and remembers. + * + * ── Why the platform allocates these when it allocates nothing else ── + * + * A sidecar's own port is not allocated: it binds `port: 0`, the kernel picks one, and it announces the + * number over the registration socket. That is right for what it is — a private detail between two + * processes, on loopback, re-chosen every restart, and nothing outside Officer ever needs to know it. + * + * A port something ELSE connects to is a different thing entirely. A plugin serving a site has to be + * reachable from nginx, which means the number must be stable across restarts and must be knowable by a + * human writing a proxy configuration. Ephemeral fails both. + * + * Doing it here rather than in each plugin's own settings buys the one thing per-plugin config cannot: + * **two plugins cannot be given the same number**, and the platform knows which are spoken for. That is + * `uq_plugin_ports_port` below, and it is the whole argument for this table. + * + * ── Kept when a plugin is uninstalled ── + * + * Deliberate. The number is written down somewhere this database cannot see — an nginx proxy host, a + * firewall rule, somebody's notes — so handing it to a different plugin because this one was briefly + * removed would point that configuration at the wrong service. Reinstalling gets the same port back. + * `purge` is the deliberate act that gives it up. + */ +export const pluginPorts = pgTable( + 'plugin_ports', + { + id: serial('id').primaryKey(), + /** The plugin's app name — its directory, as everywhere else. */ + appName: text('app_name').notNull(), + /** + * Which of the plugin's ports this is, as the manifest named it. + * + * A plugin may want more than one — a site and a preview of it — so the number alone is not an + * identity. Reaches the sidecar as `PORT_` in its environment, uppercased. + */ + name: text('name').notNull(), + port: integer('port').notNull(), + allocatedAt: timestamp('allocated_at', { withTimezone: true }).notNull().defaultNow(), + }, + (t) => [ + // One allocation per named port per plugin. + uniqueIndex('uq_plugin_ports_app_name_name').on(t.appName, t.name), + /* + No two plugins on the same number, said structurally. + + This is the reason the table exists rather than a convention people follow. A collision here is not + a duplicate row, it is two services silently fighting over a listener, where the loser fails to bind + at startup and the winner is whichever happened to start first. + */ + uniqueIndex('uq_plugin_ports_port').on(t.port), + ], +); diff --git a/src/servers/plugins/ecosystem.ts b/src/servers/plugins/ecosystem.ts index 468aae0d..577e0a3a 100644 --- a/src/servers/plugins/ecosystem.ts +++ b/src/servers/plugins/ecosystem.ts @@ -30,8 +30,22 @@ import type { DiscoveredPlugin } from './manifest'; const ECOSYSTEM_PATH = join(PLATFORM_DIR, 'ecosystem.config.cjs'); -/** One PM2 app entry. Matches what `services.sh` emits for the core processes. */ -export type Pm2App = { name: string; script: string; args?: string; cwd: string; watch: boolean }; +/** + * One PM2 app entry. Matches what `services.sh` emits for the core processes. + * + * `env` carries the ports the platform allocated for this plugin — `PORT_`. It is written into the + * ecosystem file rather than sent over the registration socket because a socket answer arrives after the + * process has already had to decide what to bind, and pm2 is the only thing that can tell a process + * something before it starts. + */ +export type Pm2App = { + name: string; + script: string; + args?: string; + cwd: string; + watch: boolean; + env?: Record; +}; /** * The header to write when there is no file to take one from. @@ -81,12 +95,24 @@ function writeEcosystemApps(apps: Pm2App[]): void { const marker = existing.indexOf('module.exports'); const header = marker > 0 ? existing.slice(0, marker) : FALLBACK_HEADER; - const lines = apps.map( - (app) => + const lines = apps.map((app) => { + /* + Values are emitted as JSON rather than quoted by hand. A port is a number today, but this is the + seam where a path or a URL arrives eventually, and one apostrophe in a value would produce a file + PM2 cannot parse — which takes every process on the machine down, not just this plugin's. + */ + const env = + app.env && Object.keys(app.env).length + ? `, env: { ${Object.entries(app.env) + .map(([k, v]) => `${JSON.stringify(k)}: ${JSON.stringify(v)}`) + .join(', ')} }` + : ''; + return ( ` { name: '${app.name}', script: '${app.script}'` + (app.args ? `, args: '${app.args}'` : '') + - `, cwd: '${app.cwd}', watch: ${app.watch} },`, - ); + `, cwd: '${app.cwd}', watch: ${app.watch}${env} },` + ); + }); writeFileSync(ECOSYSTEM_PATH, `${header}module.exports = {\n apps: [\n${lines.join('\n')}\n ],\n};\n`); } @@ -98,7 +124,7 @@ function writeEcosystemApps(apps: Pm2App[]): void { * the database says is installed and enabled. Neither should duplicate the other's idea of what a * plugin's entry looks like. */ -export function pluginPm2Entry(plugin: DiscoveredPlugin): Pm2App | null { +export function pluginPm2Entry(plugin: DiscoveredPlugin, ports?: Record): Pm2App | null { if (!plugin.sidecar) return null; const name = pluginProcessName(plugin.appName); @@ -108,9 +134,16 @@ export function pluginPm2Entry(plugin: DiscoveredPlugin): Pm2App | null { ? plugin.sidecar.script.slice(PLATFORM_DIR.length + 1) : plugin.sidecar.script; + // `PORT_`, uppercased. Absent entirely when a plugin asked for none, which is nearly all of them. + const env = Object.fromEntries( + Object.entries(ports ?? {}).map(([name, port]) => [`PORT_${name.toUpperCase().replace(/-/g, '_')}`, String(port)]), + ); + + const base = { name, cwd: PLATFORM_DIR, watch: false, ...(Object.keys(env).length ? { env } : {}) }; + return plugin.sidecar.runtime === 'node' - ? { name, script: 'node', args: script, cwd: PLATFORM_DIR, watch: false } - : { name, script: 'bun', args: `run ${script}`, cwd: PLATFORM_DIR, watch: false }; + ? { ...base, script: 'node', args: script } + : { ...base, script: 'bun', args: `run ${script}` }; } /** @@ -119,8 +152,8 @@ export function pluginPm2Entry(plugin: DiscoveredPlugin): Pm2App | null { * Idempotent, because re-installing is how a plugin is upgraded and the script path may have moved. * Returns false when the plugin has no sidecar at all, which is not a failure — most plugins won't. */ -export function addPluginToEcosystem(plugin: DiscoveredPlugin): boolean { - const entry = pluginPm2Entry(plugin); +export function addPluginToEcosystem(plugin: DiscoveredPlugin, ports?: Record): boolean { + const entry = pluginPm2Entry(plugin, ports); if (!entry) return false; const apps = readEcosystemApps().filter((a) => a.name !== entry.name); diff --git a/src/servers/plugins/install.ts b/src/servers/plugins/install.ts index 8dcfe333..e604f974 100644 --- a/src/servers/plugins/install.ts +++ b/src/servers/plugins/install.ts @@ -15,7 +15,7 @@ import { deleteProcess, processStatus, startProcess, stopProcess } from '../app- import { addPluginToEcosystem, pluginProcessName, removePluginFromEcosystem } from './ecosystem'; import { discoverPlugins } from './discover'; import { fetchPlugin } from './fetch'; -import { revokeGrantsForPermissions } from 'officerdb'; +import { allocatePluginPort, releasePluginPorts, revokeGrantsForPermissions } from 'officerdb'; import { grantPluginToAllRoles } from './grant-on-install'; import { publishAssets, unpublishAssets } from '../app-store/assets'; import { refreshPluginMounts, snapshotPlugins } from './mount'; @@ -453,7 +453,29 @@ export async function installPlugin( ); if (hasSidecar(plugin)) { - addPluginToEcosystem(plugin); + /* + Ports the manifest asked the platform to own, allocated before the entry is written because the + entry carries them. + + Idempotent: a plugin that already has an allocation gets the same number back, so re-installing + and upgrading do not move a port somebody has written into an nginx proxy host. See + `pluginPorts` in officerdb for why the platform allocates these when it allocates nothing else. + */ + const ports: Record = {}; + for (const declared of plugin.manifest.extraPorts ?? []) { + ports[declared.name] = await allocatePluginPort(appName, declared.name, declared.preferred); + } + if (Object.keys(ports).length) { + await step( + steps, + onStep, + `ports: ${Object.entries(ports) + .map(([name, port]) => `${name}=${port}`) + .join(', ')}`, + ); + } + + addPluginToEcosystem(plugin, ports); await step(steps, onStep, `ecosystem: ${pluginProcessName(appName)} added`); const started = await startProcess(pluginProcessName(appName), PLATFORM_DIR); @@ -609,6 +631,17 @@ export async function purgePlugin( } } + /* + Ports go here and NOT on uninstall, which is the deliberate asymmetry. + + An allocated number is written down where this database cannot see it — an nginx proxy host, a + firewall rule, somebody's notes. Uninstalling is reversible and often temporary, so giving the number + up there would let a later plugin take it and point that configuration at the wrong service. Purge is + the act that says the plugin is gone for good, and that is when the number goes back in the pool. + */ + const released = await releasePluginPorts(appName); + if (released) await step(steps, onStep, `ports released: ${released}`); + await rm(plugin.dir, { recursive: true, force: true }); await step(steps, onStep, `deleted ${plugin.dir}`); diff --git a/src/servers/plugins/manifest.ts b/src/servers/plugins/manifest.ts index 95913df1..b233e601 100644 --- a/src/servers/plugins/manifest.ts +++ b/src/servers/plugins/manifest.ts @@ -405,6 +405,52 @@ export type PluginManifest = { */ protocolHeaders?: string[]; + /** + * Ports this plugin needs the PLATFORM to allocate and remember. + * + * ── Not the same thing as a sidecar's own port ── + * + * A sidecar already gets a port, and nothing here changes that: it binds `port: 0`, the kernel picks + * one, and it announces the number over the registration socket. That port is a private detail between + * two processes — loopback, re-chosen on every restart, and nothing outside Officer ever needs it. + * + * This is for a port something ELSE connects to. A plugin serving a website has to be reachable from + * nginx, which means the number must survive a restart and must be knowable by whoever writes the proxy + * configuration. Ephemeral fails both, which is why the first site plugin had to hardcode 3011 and hope. + * + * ── What the platform guarantees, and what it does not ── + * + * Guaranteed: the same plugin and name always get the same number back, and no two plugins are ever + * given the same one — that second part is a unique index, not a convention, because a collision is two + * services fighting over a listener rather than a duplicate row. + * + * NOT guaranteed: that the port is still free when the sidecar starts. Something else on the machine + * can take it while Officer is down. A sidecar that cannot bind its remembered port must fail loudly + * rather than quietly choose another, because the number is written down where this platform cannot + * see it. + * + * ── How it reaches the sidecar ── + * + * As `PORT_` in its environment, uppercased, written into the pm2 entry at install. The sidecar + * reads it rather than being told over the socket, because a socket answer would arrive after the + * process has already had to decide what to bind. + * + * ```ts + * extraPorts: [{ name: 'site' }] // any free port, remembered + * extraPorts: [{ name: 'site', preferred: 3011 }] // that one if it is free, otherwise any + * ``` + * + * `preferred` is a WISH, not a claim. It exists because a plugin often arrives after the thing that + * connects to it — an nginx proxy host already pointing at 3011, a firewall rule already written — and + * moving that configuration is work the platform can spare somebody. If the number is already given to + * another plugin, or is not free on this machine, the plugin gets a different one and no error: a + * preference that could fail the install would be a claim wearing a softer word. + * + * Binding is still the sidecar's business, including which interface. A port that exists to be reached + * from a container cannot be loopback, and the platform has no way to know which a plugin means. + */ + extraPorts?: { name: string; preferred?: number }[]; + /** * Host binaries this plugin needs. Absent means it is self-sufficient, which offscale and example are. * @@ -957,6 +1003,31 @@ export function manifestProblems(appName: string, manifest: Partial(); + for (const [i, entry] of manifest.extraPorts.entries()) { + const name = (entry as { name?: unknown } | null)?.name; + if (typeof name !== 'string' || !/^[a-z][a-z0-9-]{0,30}$/.test(name)) { + problems.push(`extraPorts[${i}].name must be a lowercase name, e.g. "site"`); + continue; + } + // The name becomes an environment variable, so two that differ only by case would collide there + // while looking distinct here. + if (seen.has(name)) problems.push(`extraPorts[${i}] repeats the name "${name}"`); + seen.add(name); + + const preferred = (entry as { preferred?: unknown }).preferred; + if (preferred !== undefined) { + // Above 1024 because nothing here runs as root, and below 65536 because that is the range. + const ok = typeof preferred === 'number' && Number.isInteger(preferred) && preferred > 1024 && preferred < 65536; + if (!ok) problems.push(`extraPorts[${i}].preferred must be a port number between 1025 and 65535`); + } + } + } + } if (manifest.protocolHeaders !== undefined) { if (!Array.isArray(manifest.protocolHeaders)) { problems.push('protocolHeaders must be an array of header names');