fofos-admin/api/servers.ts

import { join } from 'node:path';
import { db } from 'officerdb/db';
import { eq } from 'drizzle-orm';
import { fofosAdminSettings } from '../db/schema';
import { randomBytes } from 'node:crypto';
import { encryptSecret, decryptSecret } from 'officerdb/crypto';
import { readDatabases } from './databases';
import { closeDevPort, openDevPort } from './firewall';
import { sidecarPort } from '@@/sidecar-registry';
import { getOwnerUser } from 'officerdb';

/**
 * The site's DEVELOPMENT server, under pm2.
 *
 * ── Production is not here, and that is the point ──
 *
 * The site's own build script says the public site is a directory of files behind nginx, not a Bun
 * process. So production is a build and a web server with no application in it — see `static-site.ts`.
 * This module is development only: `bun --hot`, which is exactly the case pm2 is right for.
 *
 * The two names remain, because a caller asking about `production` should get a real answer rather than
 * a 400, and because the panel still shows two rows. `production` here reports the pm2 process that no
 * longer exists — which is correct, and is why `startServer` refuses it.
 *
 * ── pm2 for BOTH, including development ──
 *
 * A `bun dev` started from a terminal dies with the terminal, and nobody can tell whether it is running
 * without finding the window it was started in. Under pm2 both are the same kind of thing: a named
 * process with a status, a log and an uptime, startable from here and inspectable from anywhere. The
 * cost is that development no longer prints to a console somebody is watching, which is what
 * `pm2 logs` is for.
 *
 * ── This plugin owns EXACTLY two process names, and touches nothing else ──
 *
 * `fofos-dev` and `fofos-prod`. Every command here is built from that pair and from nothing a caller
 * supplies, so there is no request that can stop, restart or delete another process on the machine.
 * That matters more than usual: this runs on a host where Officer's own processes and other people's
 * production services share one pm2 daemon, and a plugin that could name any of them would be a way to
 * take them down through a web form.
 *
 * ── Absolute paths for pm2 and bun ──
 *
 * The platform's PATH is whatever pm2 was started with, which is not a login shell's. `website.ts`
 * already learned this about bun; the same applies to pm2 itself.
 */

const PM2 = '/usr/local/bin/pm2';

export type ServerName = 'development' | 'production';

/** The one place these names exist. Nothing else in this module may build one. */
const PROCESS = { development: 'fofos-dev', production: 'fofos-prod' } as const;

export type ServerStatus = {
  name: ServerName;
  /** The pm2 process name, so somebody can go and look at it with `pm2 logs`. */
  process: string;
  port: number | null;
  /** `online`, `stopped`, `errored` — pm2's own word, or null when it has never been started. */
  status: string | null;
  uptimeMs: number | null;
  restarts: number | null;
  /** How the site would be reached, once it is up. */
  url: string | null;
  /**
   * Does the port actually answer?
   *
   * Asked separately from pm2's status because the two disagree, and the disagreement is the whole
   * point. `bun --hot` survives a crash at startup: the process stays alive, pm2 reports `online`, and
   * nothing is listening. That is exactly what development did — reported up, served nothing, and the
   * panel repeated it confidently.
   *
   * Null when there is no port to try.
   */
  reachable: boolean | null;
};

const readRow = async () => (await db.select().from(fofosAdminSettings).limit(1))[0] ?? null;

const PURPOSE = 'fofos-admin';

/**
 * The site's boot secrets, made once and kept.
 *
 * The site refuses to serve a request without them, and neither is a decision worth putting in front of
 * anybody: they are long random strings, and asking a person to invent one is asking for a worse one.
 * Regenerating on every start would be worse still — it would invalidate every session and every token
 * anybody had written down.
 */
export async function siteSecrets(): Promise<{ apiToken: string; sessionSecret: string }> {
  const row = await readRow();
  const existing = {
    apiToken: row?.siteApiToken ? decryptSecret(PURPOSE, row.siteApiToken) : null,
    sessionSecret: row?.siteSessionSecret ? decryptSecret(PURPOSE, row.siteSessionSecret) : null,
  };
  if (existing.apiToken && existing.sessionSecret) {
    return { apiToken: existing.apiToken, sessionSecret: existing.sessionSecret };
  }

  // 64 hex characters. The site requires at least 32 and there is no reason to be near the limit.
  const apiToken = existing.apiToken ?? randomBytes(32).toString('hex');
  const sessionSecret = existing.sessionSecret ?? randomBytes(32).toString('hex');

  const values = {
    siteApiToken: encryptSecret(PURPOSE, apiToken),
    siteSessionSecret: encryptSecret(PURPOSE, sessionSecret),
    updatedAt: new Date(),
  };
  if (row) await db.update(fofosAdminSettings).set(values).where(eq(fofosAdminSettings.id, row.id));
  else await db.insert(fofosAdminSettings).values(values);

  return { apiToken, sessionSecret };
}

const run = async (
  argv: string[],
  timeoutMs = 60_000,
  env?: Record<string, string>,
): Promise<{ ok: boolean; output: string }> => {
  /*
    pm2 hands the environment it was INVOKED with to the process it starts, so the way to give the site
    its port and its database is to put them in this command's own environment. There is no flag for
    it — `--env` names a block in an ecosystem file, not a variable.
  */
  const proc = Bun.spawn(argv, {
    stdout: 'pipe',
    stderr: 'pipe',
    stdin: 'ignore',
    ...(env ? { env: { ...process.env, ...env } } : {}),
  });
  const timer = setTimeout(() => proc.kill(), timeoutMs);
  try {
    const [stdout, stderr, code] = await Promise.all([
      new Response(proc.stdout).text(),
      new Response(proc.stderr).text(),
      proc.exited,
    ]);
    return { ok: code === 0, output: `${stdout}${stderr}`.trim() };
  } finally {
    clearTimeout(timer);
  }
};

/**
 * Persist pm2's process list, so a reboot brings back what is actually running.
 *
 * Without this, pm2 resurrects the list as it was the last time somebody saved — which, after starting
 * a server here, means the machine comes back up without the site and with no indication that anything
 * was meant to be there. The systemd unit is enabled on this host, so the resurrect definitely happens;
 * the only question is what it restores.
 *
 * Run after every start AND every stop, because the dump has to match reality in both directions: a
 * server stopped here and left in the dump comes back on its own at the next reboot, which is the same
 * bug pointing the other way.
 *
 * A failure to save does not fail the action. The process is already started or stopped by then, and
 * reporting "could not start" about something that did start is worse than a line saying the list was
 * not persisted.
 */
async function persist(): Promise<string> {
  const saved = await run([PM2, 'save'], 30_000);
  return saved.ok ? 'pm2 list saved' : `pm2 save failed: ${saved.output}`;
}

export function checkPort(value: unknown, field: string): number {
  const port = Number(value);
  if (!Number.isInteger(port) || port < 1024 || port > 65535) {
    throw new Error(`The ${field} port must be a whole number between 1024 and 65535.`);
  }
  return port;
}

/** What pm2 says about our two, and nothing about anybody else's. */
export async function readServers(): Promise<ServerStatus[]> {
  const row = await readRow();
  const ports: Record<ServerName, number | null> = {
    development: row?.devPort ?? null,
    production: row?.prodPort ?? null,
  };

  const listed = await run([PM2, 'jlist'], 20_000);
  let processes: Array<{ name?: string; pm2_env?: { status?: string; pm_uptime?: number; restart_time?: number } }> = [];
  if (listed.ok) {
    try {
      processes = JSON.parse(listed.output);
    } catch {
      // pm2 printed something that is not JSON — a daemon starting, usually. Treated as "no answer"
      // rather than as "not running", because those are different and only one of them is alarming.
      processes = [];
    }
  }

  return Promise.all(
    (['development', 'production'] as const).map(async (name) => {
      const found = processes.find((entry) => entry.name === PROCESS[name]);
      const port = ports[name];
      return {
        name,
        process: PROCESS[name],
        port,
        status: found?.pm2_env?.status ?? null,
        uptimeMs:
          found?.pm2_env?.pm_uptime && found.pm2_env.status === 'online' ? Date.now() - found.pm2_env.pm_uptime : null,
        restarts: found?.pm2_env?.restart_time ?? null,
        url: port ? `http://127.0.0.1:${port}` : null,
        reachable: port === null ? null : await answers(port),
      };
    }),
  );
}

export async function savePorts(dev: unknown, prod: unknown): Promise<ServerStatus[]> {
  const devPort = checkPort(dev, 'development');
  const prodPort = checkPort(prod, 'production');
  if (devPort === prodPort) throw new Error('The two servers cannot share a port.');

  const row = await readRow();

  /*
    A changed port leaves the old rule behind, and this is the only place that knows both values.

    Closing it on STOP would be wrong — a stopped development server is usually about to start again,
    and a rule for a port nothing listens on grants nothing. Leaving it on every change is how a
    firewall accumulates holes nobody remembers opening.
  */
  if (row?.devPort && row.devPort !== devPort) await closeDevPort(row.devPort);

  if (row) {
    await db
      .update(fofosAdminSettings)
      .set({ devPort, prodPort, updatedAt: new Date() })
      .where(eq(fofosAdminSettings.id, row.id));
  } else {
    await db.insert(fofosAdminSettings).values({ devPort, prodPort });
  }
  return readServers();
}

/**
 * Start one of them.
 *
 * `pm2 delete` first, then `start`. Not `restart`: restart on a process that has never existed fails,
 * and restart on one that exists reuses the env it was started with — so a port or database changed
 * here would be saved, reported, and not actually in effect until somebody worked out why.
 *
 * The environment is assembled here rather than left to the checkout's `.env`, because the plugin is
 * what knows the ports and the database names. `src/env-file.ts` reads files first and the environment
 * last, so anything written down in the checkout still wins — this fills what is absent, it does not
 * override what somebody chose.
 */
export async function startServer(which: ServerName): Promise<{ ok: boolean; output: string }> {
  /*
    Production has no process to start.

    Refused here rather than in the route, so there is one place that knows it — a check in the panel
    alone would be a rule the API did not have, and the API is what an agent reaches.
  */
  if (which === 'production') {
    throw new Error('Production is a static build served by nginx, not a process. Build it instead.');
  }

  const row = await readRow();
  if (!row?.rootPath) throw new Error('No root location is set.');

  const port = row.devPort;
  if (!port) throw new Error(`No ${which} port is set.`);

  const websiteDir = join(row.rootPath, 'website');
  const databases = await readDatabases();
  const target = databases.development;
  if (!target.name || !target.exists) throw new Error(`The ${which} database does not exist yet.`);

  const raw = process.env.OFFICERDEV_POSTGRES_URL ?? process.env.POSTGRES_URL;
  if (!raw) throw new Error('The platform has no Postgres connection configured.');
  const url = new URL(raw);
  url.pathname = `/${target.name}`;

  /*
    Everything the site refuses to boot without.

    It has no `.env` in the checkout, and it should not need one: every value here is either known to
    this plugin already (the port, the media directory, the database) or a random secret nobody should
    be asked to invent. `src/env-file.ts` reads files first and the environment last, so a checkout that
    DOES write one still wins — this fills absence, it does not override a choice.
  */
  const secrets = await siteSecrets();

  const env: Record<string, string> = {
    BUN_PORT: String(port),
    FOFOS_API_TOKEN: secrets.apiToken,
    ADMIN_SESSION_SECRET: secrets.sessionSecret,
    // Loopback. Production warns about it and serves anyway; a real domain belongs in the checkout's
    // own env file, where it is a deployment fact rather than something this panel invented.
    PUBLIC_URL: `http://127.0.0.1:${port}`,
    API_URL: `http://127.0.0.1:${port}`,
    FOFOS_MEDIA_DIR: join(row.rootPath, 'media'),
    // Both names, because the site's guard compares them and one alone leaves the other falling back
    // to whatever the repository shipped with. Same reasoning as the schema push.
    ...(databases.production.name ? { FOFOS_DB_PRODUCTION: databases.production.name } : {}),
    ...(databases.development.name ? { FOFOS_DB_DEVELOPMENT: databases.development.name } : {}),
    // Only development reaches here — production is refused above — so this is always the dev URL.
    POSTGRES_URL_DEV: url.toString(),
  };

  /*
    How the development server reaches av-club — the same two values the BUILD gets, for the same
    reason and by the same route.

    The site crops each placement to the exact shape and size its slot renders at, rather than shipping
    a whole photograph and asking CSS to hide most of it. Given only to the build, that would make
    development the one place the old behaviour survived: the crop you approved in the admin, the
    picture dev showed you, and the file that actually shipped would be three different things. That is
    precisely the class of bug this whole change exists to remove, so dev runs the same path.

    The port is EPHEMERAL by design — av-club binds whatever is free and reports it — so this is read
    at start rather than written into a file. A restart of av-club therefore leaves dev pointing at a
    port that has moved, and the fix is to restart dev from the panel, which is where its environment
    is assembled anyway. Worth knowing rather than worth engineering around: `startServer` deletes and
    re-creates the process precisely so a changed port is picked up.
  */
  const avClubPort = sidecarPort('av-club');
  // av-club resolves every path against a user's own media directory, so it has to be told who this is
  // working for. The owner, because that is whose root location this plugin administers.
  const owner = await getOwnerUser();
  if (avClubPort && owner?.email) {
    env.AV_CLUB_URL = `http://127.0.0.1:${avClubPort}`;
    env.AV_CLUB_USER = owner.email;
  }

  /*
    Open the firewall to the proxy BEFORE starting, so the port is reachable the moment it binds.

    NPM runs in a container and crosses the docker bridge to reach host services; ufw allows that subnet
    to named ports only. A development server on a port nobody opened answers `curl 127.0.0.1` and is
    invisible to the proxy — which presents as a browser timing out with a valid certificate, a correct
    proxy host and a healthy process, and is written down only in nginx's own error log.

    Never fatal. A rule that could not be added is a line in the output; the server still starts.
  */
  const firewall = await openDevPort(port);

  // Removed first so a stale process cannot survive with the old environment. Failure is expected and
  // ignored: the usual case is that it was never there.
  await run([PM2, 'delete', PROCESS[which]], 20_000);

  const started = await run(
    [
      PM2,
      'start',
      process.execPath,
      '--name',
      PROCESS[which],
      '--cwd',
      websiteDir,
      '--',
      'run',
      'dev',
    ],
    120_000,
    env,
  );

  return {
    ok: started.ok,
    output: [`$ pm2 start ${PROCESS[which]} (${which}, port ${port})`, firewall, started.output, await persist()]
      .filter(Boolean)
      .join('\n'),
  };
}

/** Stop one. `delete`, not `stop`, so a stopped entry does not linger in the list looking like a fault. */
export async function stopServer(which: ServerName): Promise<{ ok: boolean; output: string }> {
  const result = await run([PM2, 'delete', PROCESS[which]], 30_000);
  return {
    ok: result.ok,
    output: [`$ pm2 delete ${PROCESS[which]}`, result.output, await persist()].filter(Boolean).join('\n'),
  };
}

/**
 * Does anything answer on this port?
 *
 * ANY response counts, including a 404 or a 500: the question is whether something is listening, not
 * whether it is happy. A short timeout, because a port nobody holds refuses immediately and one that is
 * firewalled would otherwise hang this request.
 */
async function answers(port: number): Promise<boolean> {
  try {
    await fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(1500), redirect: 'manual' });
    return true;
  } catch {
    return false;
  }
}