fofos-admin/api/backups.ts

import { mkdir, readdir, stat, unlink } from 'node:fs/promises';
import { isAbsolute, join } from 'node:path';
import { CronJob, CronTime, validateCronExpression } from 'cron';
import { db } from 'officerdb/db';
import { eq } from 'drizzle-orm';
import { OFFICER_ROOT } from '@@/data-path';
import { fofosAdminSettings } from '../db/schema';
import { readDatabases } from './databases';

/**
 * Backups of the site's databases: when, where, where else, and how many to keep.
 *
 * ── Local is staging; the SSH destination is the backup ──
 *
 * A dump sitting on the machine it came from is a second copy on one disk, lost to the same failure
 * that would make anybody want it. So a run writes locally and then copies off, and both halves are
 * pruned to the same count.
 *
 * ── Cron for the schedule, a due-check for the durability ──
 *
 * `cron` fires at an HOUR, which is what a backup wants — an interval counted from whenever somebody
 * pressed save drifts across the clock until it is running mid-afternoon.
 *
 * What cron does NOT do is run a job it was down for. A daily backup at three on a machine that
 * restarts overnight is skipped silently, and nothing looks wrong: the schedule is still there, the
 * next fire is still in the future, and a month later there is one file. So the schedule is half of
 * this. On every start, and every fifteen minutes, the last run is compared against the schedule's own
 * period and a backup that should already have happened happens now. That is why `lastBackupAt` is in
 * the database — it is the only thing that knows a run was missed.
 *
 * ── SSH authenticates with the owner's key, never a password ──
 *
 * Nobody is at a terminal when this runs, so every ssh and scp goes out with `BatchMode=yes`: a key
 * that has lost access then FAILS rather than blocking forever on a prompt nothing will answer. It is
 * also why no credential is stored here — there is none to store, and none to leak.
 */

/** How often to look for a run that was missed while this machine was down. */
const CATCHUP_TICK_MS = 15 * 60 * 1000;

/** Long enough for a slow link, short enough that a wedged transfer is not forever. */
const SSH_TIMEOUT_MS = 5 * 60 * 1000;

/** Never wait on a prompt: there is nobody to answer it. */
const SSH_OPTIONS = ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', '-o', 'StrictHostKeyChecking=accept-new'];

/**
 * Where dumps go when nobody has said otherwise.
 *
 * Beside the Postgres container's own directory, under the officerdev root — derived from
 * `OFFICER_ROOT` rather than written down, so it follows the machine instead of the install.
 */
export const defaultBackupDir = (): string => join(OFFICER_ROOT, 'dockers', 'postgres', 'dumps');

export type BackupFile = { file: string; bytes: number; at: string };

export type BackupSettings = {
  cron: string | null;
  dir: string;
  /** False when the directory is only the computed default — nobody has chosen it. */
  dirIsSet: boolean;
  sshHost: string | null;
  sshPath: string | null;
  keep: number | null;
  lastBackupAt: string | null;
  /** From the schedule, so the panel never has to read cron. */
  nextRunAt: string | null;
  recent: BackupFile[];
};

const readRow = async () => (await db.select().from(fofosAdminSettings).limit(1))[0] ?? null;

/** Two successive fires — the schedule's period, which is what "a run was missed" is measured against. */
function periodMs(expression: string): number | null {
  try {
    const time = new CronTime(expression);
    const first = time.getNextDateFrom(new Date());
    const second = time.getNextDateFrom(first.toJSDate());
    return +second.toJSDate() - +first.toJSDate();
  } catch {
    return null;
  }
}

export function checkCron(expression: string): string {
  const trimmed = expression.trim();
  if (!trimmed) throw new Error('A schedule is required.');
  if (!validateCronExpression(trimmed).valid) throw new Error(`"${trimmed}" is not a schedule cron understands.`);
  return trimmed;
}

export function checkDir(path: string): string {
  const trimmed = path.trim();
  if (!trimmed) throw new Error('A location is required.');
  if (!isAbsolute(trimmed)) throw new Error('The location must be an absolute path.');
  return trimmed.replace(/\/+$/, '') || '/';
}

/**
 * `user@host`, or a Host from ssh_config.
 *
 * Narrow on purpose. This value is handed to `ssh` as an argument, and while an argv array means no
 * shell parses it here, a string beginning with `-` would be read by ssh as an OPTION — which is a
 * different and more interesting problem than a bad hostname.
 */
export function checkHost(host: string): string {
  const trimmed = host.trim();
  if (!trimmed) throw new Error('An SSH destination is required.');
  if (!/^[A-Za-z0-9][A-Za-z0-9._@-]*$/.test(trimmed)) {
    throw new Error(`"${trimmed}" is not a host this will connect to. Use user@host, or a Host from your ssh config.`);
  }
  return trimmed;
}

/**
 * An absolute path on the far side.
 *
 * This one IS interpreted by the remote shell — `ssh host mkdir -p /path` runs through it — so the
 * characters a shell would act on are refused rather than escaped. Refusing is checkable by reading it;
 * escaping is checkable by being right about every shell.
 */
export function checkRemotePath(path: string): string {
  const trimmed = path.trim();
  if (!trimmed) throw new Error('A remote directory is required.');
  if (!trimmed.startsWith('/')) throw new Error('The remote directory must be an absolute path.');
  if (/["'`$;&|<>()\\\s]/.test(trimmed)) {
    throw new Error('The remote directory may not contain spaces, quotes or shell characters.');
  }
  return trimmed.replace(/\/+$/, '') || '/';
}

const run = async (argv: string[], timeoutMs = SSH_TIMEOUT_MS): Promise<{ ok: boolean; output: string }> => {
  const proc = Bun.spawn(argv, { stdout: 'pipe', stderr: 'pipe', stdin: 'ignore' });
  const timer = setTimeout(() => proc.kill(), timeoutMs);
  try {
    const [stdout, stderr, code] = await Promise.all([
      new Response(proc.stdout).text(),
      new Response(proc.stderr).text(),
      proc.exited,
    ]);
    return { ok: code === 0, output: `${stdout}${stderr}`.trim() };
  } finally {
    clearTimeout(timer);
  }
};

export async function readBackups(): Promise<BackupSettings> {
  const row = await readRow();
  const cron = row?.backupCron ?? null;
  const dir = row?.backupDir ?? defaultBackupDir();

  let nextRunAt: string | null = null;
  if (cron) {
    try {
      nextRunAt = new CronTime(cron).getNextDateFrom(new Date()).toISO();
    } catch {
      // A stored expression that no longer parses. Reported as no next run rather than thrown: the
      // panel has to render in order to be the place somebody fixes it.
    }
  }

  return {
    cron,
    dir,
    dirIsSet: Boolean(row?.backupDir),
    sshHost: row?.backupSshHost ?? null,
    sshPath: row?.backupSshPath ?? null,
    keep: row?.backupKeep ?? null,
    lastBackupAt: row?.lastBackupAt?.toISOString() ?? null,
    nextRunAt,
    recent: await listDumps(dir),
  };
}

async function listDumps(dir: string): Promise<BackupFile[]> {
  const names = await readdir(dir).catch(() => []);
  const found: BackupFile[] = [];
  for (const name of names) {
    if (!name.endsWith('.sql')) continue;
    const info = await stat(join(dir, name)).catch(() => null);
    if (info) found.push({ file: name, bytes: info.size, at: info.mtime.toISOString() });
  }
  return found.sort((a, b) => b.file.localeCompare(a.file));
}

/**
 * Prove the destination before trusting it with anything.
 *
 * Two steps, reported separately, because they fail for different reasons and the fix is different:
 * the connection is keys and networking, the directory is permissions on the far side.
 */
export async function testDestination(host: string, path: string): Promise<{ ok: boolean; output: string }> {
  const target = checkHost(host);
  const remote = checkRemotePath(path);

  const connect = await run(['ssh', ...SSH_OPTIONS, target, 'true'], 30_000);
  if (!connect.ok) {
    return { ok: false, output: `$ ssh ${target}\n${connect.output || 'Could not connect.'}` };
  }

  const make = await run(['ssh', ...SSH_OPTIONS, target, `mkdir -p ${remote}`], 30_000);
  if (!make.ok) {
    return { ok: false, output: `$ ssh ${target} mkdir -p ${remote}\n${make.output || 'Could not create it.'}` };
  }

  return { ok: true, output: `Connected to ${target}\n${remote} is ready.` };
}

export async function saveBackups(input: {
  cron: string;
  dir: string;
  sshHost: string;
  sshPath: string;
  keep: number | null;
}): Promise<BackupSettings> {
  const cron = checkCron(input.cron);
  const dir = checkDir(input.dir);
  const sshHost = checkHost(input.sshHost);
  const sshPath = checkRemotePath(input.sshPath);
  if (input.keep !== null && (!Number.isInteger(input.keep) || input.keep < 1)) {
    throw new Error('Keep must be a whole number of dumps, or blank for all of them.');
  }

  // Made now rather than at the first run, so a location that cannot be created is a message beside the
  // form instead of a failure at three in the morning.
  await mkdir(dir, { recursive: true });

  const row = await readRow();
  const values = { backupCron: cron, backupDir: dir, backupSshHost: sshHost, backupSshPath: sshPath, backupKeep: input.keep };
  if (row) {
    await db.update(fofosAdminSettings).set({ ...values, updatedAt: new Date() }).where(eq(fofosAdminSettings.id, row.id));
  } else {
    await db.insert(fofosAdminSettings).values(values);
  }

  await schedule();
  return readBackups();
}

/**
 * Dump every named database, copy each off, then prune both ends.
 *
 * `pg_dump` per database rather than `pg_dumpall`: the two here are the site's, and a cluster dump
 * would also carry Officer's own, which is not this plugin's to copy to somebody else's machine.
 */
export async function runBackup(): Promise<{ ok: boolean; files: string[]; output: string }> {
  const row = await readRow();
  const dir = row?.backupDir ?? defaultBackupDir();
  await mkdir(dir, { recursive: true });

  const raw = process.env.OFFICERDEV_POSTGRES_URL ?? process.env.POSTGRES_URL;
  if (!raw) throw new Error('The platform has no Postgres connection configured.');

  const status = await readDatabases();
  const targets = [status.production, status.development].filter((entry) => entry.name && entry.exists);
  if (!targets.length) throw new Error('Neither database exists yet.');

  const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..+$/, '').replace('T', '-');
  const lines: string[] = [];
  const files: string[] = [];
  let ok = true;

  for (const target of targets) {
    const url = new URL(raw);
    url.pathname = `/${target.name}`;
    const name = `${target.name}-${stamp}.sql`;
    const out = join(dir, name);

    const dumped = await run(['pg_dump', url.toString(), '-f', out]);
    if (!dumped.ok) {
      ok = false;
      lines.push(`${target.name}: pg_dump failed — ${dumped.output}`);
      continue;
    }

    const info = await stat(out).catch(() => null);
    files.push(name);
    lines.push(`${name}  ${info?.size ?? 0} bytes`);

    if (row?.backupSshHost && row.backupSshPath) {
      const sent = await run(['scp', ...SSH_OPTIONS, out, `${row.backupSshHost}:${row.backupSshPath}/`]);
      if (sent.ok) lines.push(`  -> ${row.backupSshHost}:${row.backupSshPath}/${name}`);
      else {
        ok = false;
        lines.push(`  -> COPY FAILED: ${sent.output}`);
      }
    }
  }

  if (row?.backupKeep) lines.push(...(await prune(row.backupKeep, dir, row.backupSshHost, row.backupSshPath)));

  /*
    Recorded even when something failed.

    Otherwise one undumpable database makes every catch-up tick think a run is overdue, and the panel
    spends the week retrying at fifteen-minute intervals. The failure is in the output, where somebody
    will read it; the schedule is not the place to express it.
  */
  if (row) {
    await db
      .update(fofosAdminSettings)
      .set({ lastBackupAt: new Date(), updatedAt: new Date() })
      .where(eq(fofosAdminSettings.id, row.id));
  }

  return { ok, files, output: lines.join('\n') };
}

/**
 * Keep the newest `keep` dumps PER DATABASE, in both places.
 *
 * Per database, because two databases sharing one budget means a busy one evicts the other's history —
 * and the filenames already carry the database they came from, so the grouping is free.
 *
 * Sorted by NAME, not mtime. The timestamp is in the filename and is sortable; mtime is not the same
 * thing after a file has been copied, touched, or restored from somewhere else.
 */
async function prune(keep: number, dir: string, host: string | null, path: string | null): Promise<string[]> {
  const lines: string[] = [];
  const group = (names: string[]) => {
    const byDatabase = new Map<string, string[]>();
    for (const name of names) {
      const database = name.replace(/-\d{8}-\d{6}\.sql$/, '');
      if (database === name) continue;
      byDatabase.set(database, [...(byDatabase.get(database) ?? []), name]);
    }
    return byDatabase;
  };

  const local = (await listDumps(dir)).map((entry) => entry.file);
  for (const [, names] of group(local)) {
    for (const name of names.sort().reverse().slice(keep)) {
      await unlink(join(dir, name)).catch(() => {});
      lines.push(`pruned ${name}`);
    }
  }

  if (host && path) {
    // `ls` rather than `find`: it is on every machine, and the answer is one flat directory.
    const listed = await run(['ssh', ...SSH_OPTIONS, host, `ls -1 ${path}`], 30_000);
    if (listed.ok) {
      const remote = listed.output.split('\n').map((line) => line.trim()).filter((line) => line.endsWith('.sql'));
      const doomed: string[] = [];
      for (const [, names] of group(remote)) doomed.push(...names.sort().reverse().slice(keep));
      if (doomed.length) {
        // One call, and each name is one this module wrote — the pattern above only matches its own.
        const removed = await run(['ssh', ...SSH_OPTIONS, host, `cd ${path} && rm -f ${doomed.join(' ')}`], 60_000);
        lines.push(removed.ok ? `pruned ${doomed.length} on ${host}` : `remote prune failed: ${removed.output}`);
      }
    }
  }

  return lines;
}

let job: CronJob | null = null;
let catchup: ReturnType<typeof setInterval> | null = null;

/** Is a scheduled run already overdue, because the machine was off when it should have happened? */
async function runIfMissed(): Promise<void> {
  const row = await readRow();
  if (!row?.backupCron) return;

  const period = periodMs(row.backupCron);
  if (!period) return;

  const last = row.lastBackupAt ? +row.lastBackupAt : 0;
  if (Date.now() - last < period) return;

  await runBackup().catch(() => {
    // Swallowed: this runs on a timer with nobody watching, and the panel reports the state from disk.
  });
}

/**
 * Start, restart or stop the timer to match what is stored.
 *
 * Always stops what was running first. Two saves leaving two jobs is a bug that surfaces as backups
 * arriving in pairs, weeks later, with nothing connecting it to a settings form.
 */
export async function schedule(): Promise<void> {
  job?.stop();
  job = null;
  if (catchup) clearInterval(catchup);
  catchup = null;

  const row = await readRow();
  if (!row?.backupCron) return;

  try {
    job = new CronJob(row.backupCron, () => void runBackup().catch(() => {}), null, true);
  } catch {
    // A stored expression that will not build. Left unscheduled rather than crashing the router import,
    // which would take the whole plugin down over a bad string in one column.
    job = null;
  }

  catchup = setInterval(() => void runIfMissed(), CATCHUP_TICK_MS);
  // And once now, because the restart after a missed run is the most likely moment to discover it.
  void runIfMissed();
}