import { copyFile, mkdir, readdir, stat, unlink } from 'node:fs/promises'; import { isAbsolute, join } from 'node:path'; import { CronJob, CronTime, validateCronExpression } from 'cron'; import { db } from 'officerdb/db'; import { eq } from 'drizzle-orm'; import { OFFICER_ROOT } from '@@/data-path'; import { fofosAdminSettings } from '../db/schema'; import { readDatabases } from './databases'; import { messagesFile } from './messages'; /** * Backups of the site's databases: when, where, where else, and how many to keep. * * ── Local is staging; the SSH destination is the backup ── * * A dump sitting on the machine it came from is a second copy on one disk, lost to the same failure * that would make anybody want it. So a run writes locally and then copies off, and both halves are * pruned to the same count. * * ── Cron for the schedule, a due-check for the durability ── * * `cron` fires at an HOUR, which is what a backup wants — an interval counted from whenever somebody * pressed save drifts across the clock until it is running mid-afternoon. * * What cron does NOT do is run a job it was down for. A daily backup at three on a machine that * restarts overnight is skipped silently, and nothing looks wrong: the schedule is still there, the * next fire is still in the future, and a month later there is one file. So the schedule is half of * this. On every start, and every fifteen minutes, the last run is compared against the schedule's own * period and a backup that should already have happened happens now. That is why `lastBackupAt` is in * the database — it is the only thing that knows a run was missed. * * ── SSH authenticates with the owner's key, never a password ── * * Nobody is at a terminal when this runs, so every ssh and scp goes out with `BatchMode=yes`: a key * that has lost access then FAILS rather than blocking forever on a prompt nothing will answer. It is * also why no credential is stored here — there is none to store, and none to leak. */ /** * "Back up when something is published", rather than at a time. * * Stored in the same column as a cron expression because it answers the same question — WHEN — and a * second column would mean every read had to decide which of the two was authoritative. * * It is the right answer for this site and probably for most: nothing changes until somebody publishes, * so a nightly backup spends most of its life copying a database that is identical to yesterday's, and * the one moment worth capturing is the one it does not cover. */ export const ON_PUBLISH = 'on-publish'; /** How often to look for a run that was missed while this machine was down. */ const CATCHUP_TICK_MS = 15 * 60 * 1000; /** Long enough for a slow link, short enough that a wedged transfer is not forever. */ const SSH_TIMEOUT_MS = 5 * 60 * 1000; /** Never wait on a prompt: there is nobody to answer it. */ const SSH_OPTIONS = ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', '-o', 'StrictHostKeyChecking=accept-new']; /** * Where dumps go when nobody has said otherwise. * * Beside the Postgres container's own directory, under the officerdev root — derived from * `OFFICER_ROOT` rather than written down, so it follows the machine instead of the install. */ export const defaultBackupDir = (): string => join(OFFICER_ROOT, 'dockers', 'postgres', 'dumps'); export type BackupFile = { file: string; bytes: number; at: string }; export type BackupSettings = { cron: string | null; dir: string; /** False when the directory is only the computed default — nobody has chosen it. */ dirIsSet: boolean; sshHost: string | null; sshPath: string | null; keep: number | null; lastBackupAt: string | null; /** From the schedule, so the panel never has to read cron. */ nextRunAt: string | null; recent: BackupFile[]; }; const readRow = async () => (await db.select().from(fofosAdminSettings).limit(1))[0] ?? null; /** Two successive fires — the schedule's period, which is what "a run was missed" is measured against. */ function periodMs(expression: string): number | null { try { const time = new CronTime(expression); const first = time.getNextDateFrom(new Date()); const second = time.getNextDateFrom(first.toJSDate()); return +second.toJSDate() - +first.toJSDate(); } catch { return null; } } export function checkCron(expression: string): string { const trimmed = expression.trim(); if (!trimmed) throw new Error('A schedule is required.'); if (trimmed === ON_PUBLISH) return trimmed; if (!validateCronExpression(trimmed).valid) throw new Error(`"${trimmed}" is not a schedule cron understands.`); return trimmed; } export function checkDir(path: string): string { const trimmed = path.trim(); if (!trimmed) throw new Error('A location is required.'); if (!isAbsolute(trimmed)) throw new Error('The location must be an absolute path.'); return trimmed.replace(/\/+$/, '') || '/'; } /** * `user@host`, or a Host from ssh_config. * * Narrow on purpose. This value is handed to `ssh` as an argument, and while an argv array means no * shell parses it here, a string beginning with `-` would be read by ssh as an OPTION — which is a * different and more interesting problem than a bad hostname. */ export function checkHost(host: string): string { const trimmed = host.trim(); if (!trimmed) throw new Error('An SSH destination is required.'); if (!/^[A-Za-z0-9][A-Za-z0-9._@-]*$/.test(trimmed)) { throw new Error(`"${trimmed}" is not a host this will connect to. Use user@host, or a Host from your ssh config.`); } return trimmed; } /** * A directory on the far side, absolute or relative. * * RELATIVE IS NORMAL, and requiring a leading slash was simply wrong. `scp file host:db-backups/fofos/` * lands under the remote account's home, which is what every ssh tool does and what anybody typing a * destination expects — the home directory is the one place they are certain to be able to write. * * What is still refused: the characters a shell would act on. This value IS interpreted remotely — * `ssh host mkdir -p ` runs through a shell — so they are rejected rather than escaped. Refusing * is checkable by reading this function; escaping is checkable only by being right about every shell. */ export function checkRemotePath(path: string): string { const trimmed = path.trim(); if (!trimmed) throw new Error('A remote directory is required.'); if (/["'`$;&|<>()\\\s]/.test(trimmed)) { throw new Error('The remote directory may not contain spaces, quotes or shell characters.'); } return trimmed.replace(/\/+$/, '') || '/'; } const run = async (argv: string[], timeoutMs = SSH_TIMEOUT_MS): Promise<{ ok: boolean; output: string }> => { const proc = Bun.spawn(argv, { stdout: 'pipe', stderr: 'pipe', stdin: 'ignore' }); const timer = setTimeout(() => proc.kill(), timeoutMs); try { const [stdout, stderr, code] = await Promise.all([ new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited, ]); return { ok: code === 0, output: `${stdout}${stderr}`.trim() }; } finally { clearTimeout(timer); } }; export async function readBackups(): Promise { const row = await readRow(); const cron = row?.backupCron ?? null; const dir = row?.backupDir ?? defaultBackupDir(); let nextRunAt: string | null = null; // Nothing to predict when the trigger is an event: the next run is whenever somebody publishes. if (cron && cron !== ON_PUBLISH) { try { nextRunAt = new CronTime(cron).getNextDateFrom(new Date()).toISO(); } catch { // A stored expression that no longer parses. Reported as no next run rather than thrown: the // panel has to render in order to be the place somebody fixes it. } } return { cron, dir, dirIsSet: Boolean(row?.backupDir), sshHost: row?.backupSshHost ?? null, sshPath: row?.backupSshPath ?? null, keep: row?.backupKeep ?? null, lastBackupAt: row?.lastBackupAt?.toISOString() ?? null, nextRunAt, recent: await listDumps(dir), }; } async function listDumps(dir: string): Promise { const names = await readdir(dir).catch(() => []); const found: BackupFile[] = []; for (const name of names) { if (!name.endsWith('.sql')) continue; const info = await stat(join(dir, name)).catch(() => null); if (info) found.push({ file: name, bytes: info.size, at: info.mtime.toISOString() }); } return found.sort((a, b) => b.file.localeCompare(a.file)); } /** * Prove the destination before trusting it with anything. * * Two steps, reported separately, because they fail for different reasons and the fix is different: * the connection is keys and networking, the directory is permissions on the far side. */ export async function testDestination(host: string, path: string): Promise<{ ok: boolean; output: string }> { const target = checkHost(host); const remote = checkRemotePath(path); const connect = await run(['ssh', ...SSH_OPTIONS, target, 'true'], 30_000); if (!connect.ok) { return { ok: false, output: `$ ssh ${target}\n${connect.output || 'Could not connect.'}` }; } const make = await run(['ssh', ...SSH_OPTIONS, target, `mkdir -p ${remote}`], 30_000); if (!make.ok) { return { ok: false, output: `$ ssh ${target} mkdir -p ${remote}\n${make.output || 'Could not create it.'}` }; } return { ok: true, output: `Connected to ${target}\n${remote} is ready.` }; } export async function saveBackups(input: { cron: string; dir: string; sshHost: string; sshPath: string; keep: number | null; }): Promise { const cron = checkCron(input.cron); const dir = checkDir(input.dir); const sshHost = checkHost(input.sshHost); const sshPath = checkRemotePath(input.sshPath); if (input.keep !== null && (!Number.isInteger(input.keep) || input.keep < 1)) { throw new Error('Keep must be a whole number of dumps, or blank for all of them.'); } // Made now rather than at the first run, so a location that cannot be created is a message beside the // form instead of a failure at three in the morning. await mkdir(dir, { recursive: true }); const row = await readRow(); const values = { backupCron: cron, backupDir: dir, backupSshHost: sshHost, backupSshPath: sshPath, backupKeep: input.keep }; if (row) { await db.update(fofosAdminSettings).set({ ...values, updatedAt: new Date() }).where(eq(fofosAdminSettings.id, row.id)); } else { await db.insert(fofosAdminSettings).values(values); } await schedule(); return readBackups(); } /** * Dump every named database, copy each off, then prune both ends. * * `pg_dump` per database rather than `pg_dumpall`: the two here are the site's, and a cluster dump * would also carry Officer's own, which is not this plugin's to copy to somebody else's machine. */ export async function runBackup(): Promise<{ ok: boolean; files: string[]; output: string }> { const row = await readRow(); const dir = row?.backupDir ?? defaultBackupDir(); await mkdir(dir, { recursive: true }); const raw = process.env.OFFICERDEV_POSTGRES_URL ?? process.env.POSTGRES_URL; if (!raw) throw new Error('The platform has no Postgres connection configured.'); const status = await readDatabases(); const targets = [status.production, status.development].filter((entry) => entry.name && entry.exists); if (!targets.length) throw new Error('Neither database exists yet.'); const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..+$/, '').replace('T', '-'); const lines: string[] = []; const files: string[] = []; let ok = true; for (const target of targets) { const url = new URL(raw); url.pathname = `/${target.name}`; const name = `${target.name}-${stamp}.sql`; const out = join(dir, name); const dumped = await run(['pg_dump', url.toString(), '-f', out]); if (!dumped.ok) { ok = false; lines.push(`${target.name}: pg_dump failed — ${dumped.output}`); continue; } const info = await stat(out).catch(() => null); files.push(name); lines.push(`${name} ${info?.size ?? 0} bytes`); if (row?.backupSshHost && row.backupSshPath) { const sent = await run(['scp', ...SSH_OPTIONS, out, `${row.backupSshHost}:${row.backupSshPath}/`]); if (sent.ok) lines.push(` -> ${row.backupSshHost}:${row.backupSshPath}/${name}`); else { ok = false; lines.push(` -> COPY FAILED: ${sent.output}`); } } } /* The mailbox, timestamped, alongside the dumps. The watcher keeps a CURRENT copy at each destination and that is what protects a message minutes after it arrives. This is the history: without it, restoring last week's site would come with this morning's inbox and no way back to the one that matches. */ if (row?.rootPath) { const mailbox = messagesFile(row.rootPath); if (await stat(mailbox).catch(() => null)) { const copy = join(dir, `messages-${stamp}.db`); await copyFile(mailbox, copy).catch(() => {}); lines.push(`messages-${stamp}.db`); if (row.backupSshHost && row.backupSshPath) { const sent = await run(['scp', ...SSH_OPTIONS, copy, `${row.backupSshHost}:${row.backupSshPath}/`]); if (!sent.ok) { ok = false; lines.push(` -> COPY FAILED: ${sent.output}`); } } } } if (row?.backupKeep) lines.push(...(await prune(row.backupKeep, dir, row.backupSshHost, row.backupSshPath))); /* Recorded even when something failed. Otherwise one undumpable database makes every catch-up tick think a run is overdue, and the panel spends the week retrying at fifteen-minute intervals. The failure is in the output, where somebody will read it; the schedule is not the place to express it. */ if (row) { await db .update(fofosAdminSettings) .set({ lastBackupAt: new Date(), updatedAt: new Date() }) .where(eq(fofosAdminSettings.id, row.id)); } return { ok, files, output: lines.join('\n') }; } /** * Keep the newest `keep` dumps PER DATABASE, in both places. * * Per database, because two databases sharing one budget means a busy one evicts the other's history — * and the filenames already carry the database they came from, so the grouping is free. * * Sorted by NAME, not mtime. The timestamp is in the filename and is sortable; mtime is not the same * thing after a file has been copied, touched, or restored from somewhere else. */ async function prune(keep: number, dir: string, host: string | null, path: string | null): Promise { const lines: string[] = []; const group = (names: string[]) => { const byDatabase = new Map(); for (const name of names) { const database = name.replace(/-\d{8}-\d{6}\.sql$/, ''); if (database === name) continue; byDatabase.set(database, [...(byDatabase.get(database) ?? []), name]); } return byDatabase; }; const local = (await listDumps(dir)).map((entry) => entry.file); for (const [, names] of group(local)) { for (const name of names.sort().reverse().slice(keep)) { await unlink(join(dir, name)).catch(() => {}); lines.push(`pruned ${name}`); } } if (host && path) { // `ls` rather than `find`: it is on every machine, and the answer is one flat directory. const listed = await run(['ssh', ...SSH_OPTIONS, host, `ls -1 ${path}`], 30_000); if (listed.ok) { const remote = listed.output.split('\n').map((line) => line.trim()).filter((line) => line.endsWith('.sql')); const doomed: string[] = []; for (const [, names] of group(remote)) doomed.push(...names.sort().reverse().slice(keep)); if (doomed.length) { // One call, and each name is one this module wrote — the pattern above only matches its own. const removed = await run(['ssh', ...SSH_OPTIONS, host, `cd ${path} && rm -f ${doomed.join(' ')}`], 60_000); lines.push(removed.ok ? `pruned ${doomed.length} on ${host}` : `remote prune failed: ${removed.output}`); } } } return lines; } /** * Load a dump into one of the two databases. * * ── This DESTROYS what is there, and says so ── * * A pg_dump carries `CREATE TABLE`, so loading one into a database that already has the schema fails * on every object rather than replacing it. The only honest restore is therefore to empty the target * first — `DROP SCHEMA public CASCADE` — and that is not a detail to bury. The caller must pass * `confirm: true`, which exists purely so that no code path can reach this by accident. * * ── The file is chosen from the listing, not named freely ── * * `file` is matched against what `listDumps` actually found in the backup directory. So there is no * path to traverse out of and no way to point `psql` at a file this module did not write: an unknown * name is refused before anything is opened, rather than sanitised and hoped about. * * ── psql, not pg_restore ── * * These are plain-SQL dumps, which is what `pg_dump` writes without `-F c`. pg_restore does not read * them. */ export async function restoreBackup(input: { file: string; target: 'production' | 'development'; confirm: boolean; }): Promise<{ ok: boolean; output: string }> { if (!input.confirm) throw new Error('A restore replaces everything in the target database.'); const row = await readRow(); const dir = row?.backupDir ?? defaultBackupDir(); const available = await listDumps(dir); const chosen = available.find((entry) => entry.file === input.file); if (!chosen) throw new Error(`"${input.file}" is not one of the dumps in ${dir}.`); const status = await readDatabases(); const name = input.target === 'production' ? status.production.name : status.development.name; if (!name) throw new Error(`No ${input.target} database is named.`); if (!status.available.includes(name)) throw new Error(`"${name}" does not exist yet.`); const raw = process.env.OFFICERDEV_POSTGRES_URL ?? process.env.POSTGRES_URL; if (!raw) throw new Error('The platform has no Postgres connection configured.'); const url = new URL(raw); url.pathname = `/${name}`; const lines = [`$ restore ${chosen.file} -> ${name}`]; /* Emptied through `psql -c`, not by dropping and recreating the DATABASE. Dropping the database would need a connection to another one, would fail while anything is attached, and would lose whatever was granted on it. Replacing the schema does the same job to the thing that actually matters — every table, view and sequence in `public`. */ const cleared = await run(['psql', url.toString(), '-v', 'ON_ERROR_STOP=1', '-c', 'DROP SCHEMA public CASCADE; CREATE SCHEMA public;']); if (!cleared.ok) return { ok: false, output: `${lines.join('\n')}\nCould not empty ${name}: ${cleared.output}` }; lines.push(`emptied ${name}`); const loaded = await run(['psql', url.toString(), '-v', 'ON_ERROR_STOP=1', '-f', join(dir, chosen.file)]); lines.push(loaded.output || (loaded.ok ? 'loaded' : 'failed')); return { ok: loaded.ok, output: lines.join('\n') }; } let job: CronJob | null = null; let catchup: ReturnType | null = null; /** Is a scheduled run already overdue, because the machine was off when it should have happened? */ async function runIfMissed(): Promise { const row = await readRow(); if (!row?.backupCron || row.backupCron === ON_PUBLISH) return; const period = periodMs(row.backupCron); if (!period) return; const last = row.lastBackupAt ? +row.lastBackupAt : 0; if (Date.now() - last < period) return; await runBackup().catch(() => { // Swallowed: this runs on a timer with nobody watching, and the panel reports the state from disk. }); } /** * Start, restart or stop the timer to match what is stored. * * Always stops what was running first. Two saves leaving two jobs is a bug that surfaces as backups * arriving in pairs, weeks later, with nothing connecting it to a settings form. */ export async function schedule(): Promise { job?.stop(); job = null; if (catchup) clearInterval(catchup); catchup = null; const row = await readRow(); if (!row?.backupCron) return; /* Nothing to schedule, and nothing to catch up on. A missed run is a run whose TIME passed while the machine was down. An event has no time, so there is nothing to have missed — and a catch-up tick here would take a backup every fifteen minutes of a site nobody had touched. */ if (row.backupCron === ON_PUBLISH) return; try { job = new CronJob(row.backupCron, () => void runBackup().catch(() => {}), null, true); } catch { // A stored expression that will not build. Left unscheduled rather than crashing the router import, // which would take the whole plugin down over a bad string in one column. job = null; } catchup = setInterval(() => void runIfMissed(), CATCHUP_TICK_MS); // And once now, because the restart after a missed run is the most likely moment to discover it. void runIfMissed(); } /** * Take a backup because something was just published. * * Does nothing unless that is the configured trigger, so the publish path can call it unconditionally * and never has to know what the backup settings say. Failures are swallowed for the same reason the * timer swallows them: a backup that could not be taken must not turn into a publish that appears to * have failed, when the thing the person asked for actually worked. */ export async function backupAfterPublish(): Promise { const row = await readRow(); if (row?.backupCron !== ON_PUBLISH) return; await runBackup().catch(() => {}); }