import { mkdir, readdir, stat, unlink } from 'node:fs/promises'; import { isAbsolute, join } from 'node:path'; import { CronJob, CronTime, validateCronExpression } from 'cron'; import { db } from 'officerdb/db'; import { eq } from 'drizzle-orm'; import { OFFICER_ROOT } from '@@/data-path'; import { fofosAdminSettings } from '../db/schema'; import { readDatabases } from './databases'; /** * Backups of the site's databases: when, where, where else, and how many to keep. * * ── Local is staging; the SSH destination is the backup ── * * A dump sitting on the machine it came from is a second copy on one disk, lost to the same failure * that would make anybody want it. So a run writes locally and then copies off, and both halves are * pruned to the same count. * * ── Cron for the schedule, a due-check for the durability ── * * `cron` fires at an HOUR, which is what a backup wants — an interval counted from whenever somebody * pressed save drifts across the clock until it is running mid-afternoon. * * What cron does NOT do is run a job it was down for. A daily backup at three on a machine that * restarts overnight is skipped silently, and nothing looks wrong: the schedule is still there, the * next fire is still in the future, and a month later there is one file. So the schedule is half of * this. On every start, and every fifteen minutes, the last run is compared against the schedule's own * period and a backup that should already have happened happens now. That is why `lastBackupAt` is in * the database — it is the only thing that knows a run was missed. * * ── SSH authenticates with the owner's key, never a password ── * * Nobody is at a terminal when this runs, so every ssh and scp goes out with `BatchMode=yes`: a key * that has lost access then FAILS rather than blocking forever on a prompt nothing will answer. It is * also why no credential is stored here — there is none to store, and none to leak. */ /** How often to look for a run that was missed while this machine was down. */ const CATCHUP_TICK_MS = 15 * 60 * 1000; /** Long enough for a slow link, short enough that a wedged transfer is not forever. */ const SSH_TIMEOUT_MS = 5 * 60 * 1000; /** Never wait on a prompt: there is nobody to answer it. */ const SSH_OPTIONS = ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=10', '-o', 'StrictHostKeyChecking=accept-new']; /** * Where dumps go when nobody has said otherwise. * * Beside the Postgres container's own directory, under the officerdev root — derived from * `OFFICER_ROOT` rather than written down, so it follows the machine instead of the install. */ export const defaultBackupDir = (): string => join(OFFICER_ROOT, 'dockers', 'postgres', 'dumps'); export type BackupFile = { file: string; bytes: number; at: string }; export type BackupSettings = { cron: string | null; dir: string; /** False when the directory is only the computed default — nobody has chosen it. */ dirIsSet: boolean; sshHost: string | null; sshPath: string | null; keep: number | null; lastBackupAt: string | null; /** From the schedule, so the panel never has to read cron. */ nextRunAt: string | null; recent: BackupFile[]; }; const readRow = async () => (await db.select().from(fofosAdminSettings).limit(1))[0] ?? null; /** Two successive fires — the schedule's period, which is what "a run was missed" is measured against. */ function periodMs(expression: string): number | null { try { const time = new CronTime(expression); const first = time.getNextDateFrom(new Date()); const second = time.getNextDateFrom(first.toJSDate()); return +second.toJSDate() - +first.toJSDate(); } catch { return null; } } export function checkCron(expression: string): string { const trimmed = expression.trim(); if (!trimmed) throw new Error('A schedule is required.'); if (!validateCronExpression(trimmed).valid) throw new Error(`"${trimmed}" is not a schedule cron understands.`); return trimmed; } export function checkDir(path: string): string { const trimmed = path.trim(); if (!trimmed) throw new Error('A location is required.'); if (!isAbsolute(trimmed)) throw new Error('The location must be an absolute path.'); return trimmed.replace(/\/+$/, '') || '/'; } /** * `user@host`, or a Host from ssh_config. * * Narrow on purpose. This value is handed to `ssh` as an argument, and while an argv array means no * shell parses it here, a string beginning with `-` would be read by ssh as an OPTION — which is a * different and more interesting problem than a bad hostname. */ export function checkHost(host: string): string { const trimmed = host.trim(); if (!trimmed) throw new Error('An SSH destination is required.'); if (!/^[A-Za-z0-9][A-Za-z0-9._@-]*$/.test(trimmed)) { throw new Error(`"${trimmed}" is not a host this will connect to. Use user@host, or a Host from your ssh config.`); } return trimmed; } /** * A directory on the far side, absolute or relative. * * RELATIVE IS NORMAL, and requiring a leading slash was simply wrong. `scp file host:db-backups/fofos/` * lands under the remote account's home, which is what every ssh tool does and what anybody typing a * destination expects — the home directory is the one place they are certain to be able to write. * * What is still refused: the characters a shell would act on. This value IS interpreted remotely — * `ssh host mkdir -p ` runs through a shell — so they are rejected rather than escaped. Refusing * is checkable by reading this function; escaping is checkable only by being right about every shell. */ export function checkRemotePath(path: string): string { const trimmed = path.trim(); if (!trimmed) throw new Error('A remote directory is required.'); if (/["'`$;&|<>()\\\s]/.test(trimmed)) { throw new Error('The remote directory may not contain spaces, quotes or shell characters.'); } return trimmed.replace(/\/+$/, '') || '/'; } const run = async (argv: string[], timeoutMs = SSH_TIMEOUT_MS): Promise<{ ok: boolean; output: string }> => { const proc = Bun.spawn(argv, { stdout: 'pipe', stderr: 'pipe', stdin: 'ignore' }); const timer = setTimeout(() => proc.kill(), timeoutMs); try { const [stdout, stderr, code] = await Promise.all([ new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited, ]); return { ok: code === 0, output: `${stdout}${stderr}`.trim() }; } finally { clearTimeout(timer); } }; export async function readBackups(): Promise { const row = await readRow(); const cron = row?.backupCron ?? null; const dir = row?.backupDir ?? defaultBackupDir(); let nextRunAt: string | null = null; if (cron) { try { nextRunAt = new CronTime(cron).getNextDateFrom(new Date()).toISO(); } catch { // A stored expression that no longer parses. Reported as no next run rather than thrown: the // panel has to render in order to be the place somebody fixes it. } } return { cron, dir, dirIsSet: Boolean(row?.backupDir), sshHost: row?.backupSshHost ?? null, sshPath: row?.backupSshPath ?? null, keep: row?.backupKeep ?? null, lastBackupAt: row?.lastBackupAt?.toISOString() ?? null, nextRunAt, recent: await listDumps(dir), }; } async function listDumps(dir: string): Promise { const names = await readdir(dir).catch(() => []); const found: BackupFile[] = []; for (const name of names) { if (!name.endsWith('.sql')) continue; const info = await stat(join(dir, name)).catch(() => null); if (info) found.push({ file: name, bytes: info.size, at: info.mtime.toISOString() }); } return found.sort((a, b) => b.file.localeCompare(a.file)); } /** * Prove the destination before trusting it with anything. * * Two steps, reported separately, because they fail for different reasons and the fix is different: * the connection is keys and networking, the directory is permissions on the far side. */ export async function testDestination(host: string, path: string): Promise<{ ok: boolean; output: string }> { const target = checkHost(host); const remote = checkRemotePath(path); const connect = await run(['ssh', ...SSH_OPTIONS, target, 'true'], 30_000); if (!connect.ok) { return { ok: false, output: `$ ssh ${target}\n${connect.output || 'Could not connect.'}` }; } const make = await run(['ssh', ...SSH_OPTIONS, target, `mkdir -p ${remote}`], 30_000); if (!make.ok) { return { ok: false, output: `$ ssh ${target} mkdir -p ${remote}\n${make.output || 'Could not create it.'}` }; } return { ok: true, output: `Connected to ${target}\n${remote} is ready.` }; } export async function saveBackups(input: { cron: string; dir: string; sshHost: string; sshPath: string; keep: number | null; }): Promise { const cron = checkCron(input.cron); const dir = checkDir(input.dir); const sshHost = checkHost(input.sshHost); const sshPath = checkRemotePath(input.sshPath); if (input.keep !== null && (!Number.isInteger(input.keep) || input.keep < 1)) { throw new Error('Keep must be a whole number of dumps, or blank for all of them.'); } // Made now rather than at the first run, so a location that cannot be created is a message beside the // form instead of a failure at three in the morning. await mkdir(dir, { recursive: true }); const row = await readRow(); const values = { backupCron: cron, backupDir: dir, backupSshHost: sshHost, backupSshPath: sshPath, backupKeep: input.keep }; if (row) { await db.update(fofosAdminSettings).set({ ...values, updatedAt: new Date() }).where(eq(fofosAdminSettings.id, row.id)); } else { await db.insert(fofosAdminSettings).values(values); } await schedule(); return readBackups(); } /** * Dump every named database, copy each off, then prune both ends. * * `pg_dump` per database rather than `pg_dumpall`: the two here are the site's, and a cluster dump * would also carry Officer's own, which is not this plugin's to copy to somebody else's machine. */ export async function runBackup(): Promise<{ ok: boolean; files: string[]; output: string }> { const row = await readRow(); const dir = row?.backupDir ?? defaultBackupDir(); await mkdir(dir, { recursive: true }); const raw = process.env.OFFICERDEV_POSTGRES_URL ?? process.env.POSTGRES_URL; if (!raw) throw new Error('The platform has no Postgres connection configured.'); const status = await readDatabases(); const targets = [status.production, status.development].filter((entry) => entry.name && entry.exists); if (!targets.length) throw new Error('Neither database exists yet.'); const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..+$/, '').replace('T', '-'); const lines: string[] = []; const files: string[] = []; let ok = true; for (const target of targets) { const url = new URL(raw); url.pathname = `/${target.name}`; const name = `${target.name}-${stamp}.sql`; const out = join(dir, name); const dumped = await run(['pg_dump', url.toString(), '-f', out]); if (!dumped.ok) { ok = false; lines.push(`${target.name}: pg_dump failed — ${dumped.output}`); continue; } const info = await stat(out).catch(() => null); files.push(name); lines.push(`${name} ${info?.size ?? 0} bytes`); if (row?.backupSshHost && row.backupSshPath) { const sent = await run(['scp', ...SSH_OPTIONS, out, `${row.backupSshHost}:${row.backupSshPath}/`]); if (sent.ok) lines.push(` -> ${row.backupSshHost}:${row.backupSshPath}/${name}`); else { ok = false; lines.push(` -> COPY FAILED: ${sent.output}`); } } } if (row?.backupKeep) lines.push(...(await prune(row.backupKeep, dir, row.backupSshHost, row.backupSshPath))); /* Recorded even when something failed. Otherwise one undumpable database makes every catch-up tick think a run is overdue, and the panel spends the week retrying at fifteen-minute intervals. The failure is in the output, where somebody will read it; the schedule is not the place to express it. */ if (row) { await db .update(fofosAdminSettings) .set({ lastBackupAt: new Date(), updatedAt: new Date() }) .where(eq(fofosAdminSettings.id, row.id)); } return { ok, files, output: lines.join('\n') }; } /** * Keep the newest `keep` dumps PER DATABASE, in both places. * * Per database, because two databases sharing one budget means a busy one evicts the other's history — * and the filenames already carry the database they came from, so the grouping is free. * * Sorted by NAME, not mtime. The timestamp is in the filename and is sortable; mtime is not the same * thing after a file has been copied, touched, or restored from somewhere else. */ async function prune(keep: number, dir: string, host: string | null, path: string | null): Promise { const lines: string[] = []; const group = (names: string[]) => { const byDatabase = new Map(); for (const name of names) { const database = name.replace(/-\d{8}-\d{6}\.sql$/, ''); if (database === name) continue; byDatabase.set(database, [...(byDatabase.get(database) ?? []), name]); } return byDatabase; }; const local = (await listDumps(dir)).map((entry) => entry.file); for (const [, names] of group(local)) { for (const name of names.sort().reverse().slice(keep)) { await unlink(join(dir, name)).catch(() => {}); lines.push(`pruned ${name}`); } } if (host && path) { // `ls` rather than `find`: it is on every machine, and the answer is one flat directory. const listed = await run(['ssh', ...SSH_OPTIONS, host, `ls -1 ${path}`], 30_000); if (listed.ok) { const remote = listed.output.split('\n').map((line) => line.trim()).filter((line) => line.endsWith('.sql')); const doomed: string[] = []; for (const [, names] of group(remote)) doomed.push(...names.sort().reverse().slice(keep)); if (doomed.length) { // One call, and each name is one this module wrote — the pattern above only matches its own. const removed = await run(['ssh', ...SSH_OPTIONS, host, `cd ${path} && rm -f ${doomed.join(' ')}`], 60_000); lines.push(removed.ok ? `pruned ${doomed.length} on ${host}` : `remote prune failed: ${removed.output}`); } } } return lines; } let job: CronJob | null = null; let catchup: ReturnType | null = null; /** Is a scheduled run already overdue, because the machine was off when it should have happened? */ async function runIfMissed(): Promise { const row = await readRow(); if (!row?.backupCron) return; const period = periodMs(row.backupCron); if (!period) return; const last = row.lastBackupAt ? +row.lastBackupAt : 0; if (Date.now() - last < period) return; await runBackup().catch(() => { // Swallowed: this runs on a timer with nobody watching, and the panel reports the state from disk. }); } /** * Start, restart or stop the timer to match what is stored. * * Always stops what was running first. Two saves leaving two jobs is a bug that surfaces as backups * arriving in pairs, weeks later, with nothing connecting it to a settings form. */ export async function schedule(): Promise { job?.stop(); job = null; if (catchup) clearInterval(catchup); catchup = null; const row = await readRow(); if (!row?.backupCron) return; try { job = new CronJob(row.backupCron, () => void runBackup().catch(() => {}), null, true); } catch { // A stored expression that will not build. Left unscheduled rather than crashing the router import, // which would take the whole plugin down over a bad string in one column. job = null; } catchup = setInterval(() => void runIfMissed(), CATCHUP_TICK_MS); // And once now, because the restart after a missed run is the most likely moment to discover it. void runIfMissed(); }