diff --git a/api/backups.ts b/api/backups.ts new file mode 100644 index 0000000..3203f77 --- /dev/null +++ b/api/backups.ts @@ -0,0 +1,243 @@ +import { mkdir, readdir, stat } from 'node:fs/promises'; +import { isAbsolute, join } from 'node:path'; +import { CronJob, CronTime, validateCronExpression } from 'cron'; +import { db } from 'officerdb/db'; +import { eq } from 'drizzle-orm'; +import { fofosAdminSettings } from '../db/schema'; +import { readDatabases } from './databases'; + +/** + * Backups of the site's databases: when, where, and the thing that actually runs them. + * + * ── Cron for the schedule, a due-check for the durability ── + * + * `cron` fires at an HOUR, which is what a backup wants — an interval counted from whenever somebody + * pressed save drifts across the clock until it is running mid-afternoon. + * + * What cron does NOT do is run a job it was down for. A daily backup at three on a machine that + * restarts overnight is skipped silently, and nothing looks wrong: the schedule is still there, the + * next fire is still in the future, and a month later there is one backup. So the schedule is only half + * of this. On every start, and on every fire, the last run is compared against the schedule's own + * period, and a backup that should already have happened happens now. + * + * That is why `lastBackupAt` is in the database rather than in memory. It is the only thing that knows + * a run was missed. + * + * ── One job, replaced rather than accumulated ── + * + * The handle is module-level and stopped before a new one is made. Without that, saving the schedule + * twice leaves two jobs running and the third save leaves three — a bug that shows up as backups + * arriving in pairs, long after anybody would connect it to a settings form. + */ + +/** How often to look for a missed run when nothing is scheduled to fire. */ +const CATCHUP_TICK_MS = 15 * 60 * 1000; + +export type BackupSettings = { + cron: string | null; + dir: string | null; + lastBackupAt: string | null; + /** From the schedule, so the panel does not have to know how to read cron. */ + nextRunAt: string | null; + /** Newest first. What is actually on disk, rather than what should be. */ + recent: { file: string; bytes: number; at: string }[]; +}; + +const readRow = async () => (await db.select().from(fofosAdminSettings).limit(1))[0] ?? null; + +/** Two successive fires, which is the schedule's period — what "a run was missed" is measured against. */ +function periodMs(expression: string): number | null { + try { + const time = new CronTime(expression); + const first = time.getNextDateFrom(new Date()); + const second = time.getNextDateFrom(first.toJSDate()); + return +second.toJSDate() - +first.toJSDate(); + } catch { + return null; + } +} + +export function checkCron(expression: string): string { + const trimmed = expression.trim(); + if (!trimmed) throw new Error('A schedule is required.'); + const result = validateCronExpression(trimmed); + if (!result.valid) throw new Error(`"${trimmed}" is not a schedule cron understands.`); + return trimmed; +} + +export function checkDir(path: string): string { + const trimmed = path.trim(); + if (!trimmed) throw new Error('A location is required.'); + if (!isAbsolute(trimmed)) throw new Error('The location must be an absolute path.'); + return trimmed.replace(/\/+$/, ''); +} + +export async function readBackups(): Promise { + const row = await readRow(); + const cron = row?.backupCron ?? null; + const dir = row?.backupDir ?? null; + + let nextRunAt: string | null = null; + if (cron) { + try { + nextRunAt = new CronTime(cron).getNextDateFrom(new Date()).toISO(); + } catch { + // A stored expression that no longer parses. Reported as no next run rather than thrown: the + // panel has to be able to render the settings in order to fix them. + } + } + + return { + cron, + dir, + lastBackupAt: row?.lastBackupAt?.toISOString() ?? null, + nextRunAt, + recent: dir ? await listBackups(dir) : [], + }; +} + +async function listBackups(dir: string): Promise<{ file: string; bytes: number; at: string }[]> { + const names = await readdir(dir).catch(() => []); + const found = []; + for (const name of names) { + if (!name.endsWith('.sql')) continue; + const info = await stat(join(dir, name)).catch(() => null); + if (info) found.push({ file: name, bytes: info.size, at: info.mtime.toISOString() }); + } + return found.sort((a, b) => b.at.localeCompare(a.at)).slice(0, 10); +} + +export async function saveBackups(expression: string, directory: string): Promise { + const cron = checkCron(expression); + const dir = checkDir(directory); + + // Made now rather than at the first run, so a location that cannot be created is a message beside the + // form instead of a failure at three in the morning. + await mkdir(dir, { recursive: true }); + + const row = await readRow(); + if (row) { + await db + .update(fofosAdminSettings) + .set({ backupCron: cron, backupDir: dir, updatedAt: new Date() }) + .where(eq(fofosAdminSettings.id, row.id)); + } else { + await db.insert(fofosAdminSettings).values({ backupCron: cron, backupDir: dir }); + } + + await schedule(); + return readBackups(); +} + +/** + * Dump every named database, once. + * + * `pg_dump` per database rather than `pg_dumpall`: the two databases here are the site's, and a cluster + * dump would also carry Officer's own, which is not this plugin's to copy around. + */ +export async function runBackup(): Promise<{ ok: boolean; files: string[]; output: string }> { + const row = await readRow(); + const dir = row?.backupDir; + if (!dir) throw new Error('No backup location is set.'); + await mkdir(dir, { recursive: true }); + + const raw = process.env.OFFICERDEV_POSTGRES_URL ?? process.env.POSTGRES_URL; + if (!raw) throw new Error('The platform has no Postgres connection configured.'); + + const status = await readDatabases(); + const targets = [status.production, status.development].filter((entry) => entry.name && entry.exists); + if (!targets.length) throw new Error('Neither database exists yet.'); + + // Sortable, and the same shape as the file this plugin's first manual dump used. + const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..+$/, '').replace('T', '-'); + + const files: string[] = []; + const lines: string[] = []; + let ok = true; + + for (const target of targets) { + const url = new URL(raw); + url.pathname = `/${target.name}`; + const out = join(dir, `${target.name}-${stamp}.sql`); + + const proc = Bun.spawn(['pg_dump', url.toString(), '-f', out], { + stdout: 'pipe', + stderr: 'pipe', + stdin: 'ignore', + }); + const [stderr, code] = await Promise.all([new Response(proc.stderr).text(), proc.exited]); + + if (code === 0) { + files.push(`${target.name}-${stamp}.sql`); + const info = await stat(out).catch(() => null); + lines.push(`${target.name} -> ${files.at(-1)} (${info?.size ?? 0} bytes)`); + } else { + ok = false; + lines.push(`${target.name} FAILED: ${stderr.trim()}`); + } + } + + /* + Recorded even when one of them failed. + + Otherwise a database that cannot be dumped makes every subsequent tick think a run is overdue, and + the panel spends the rest of the week trying and failing at fifteen-minute intervals. The failure is + in the output where somebody will see it; the schedule is not the place to express it. + */ + if (row) { + await db + .update(fofosAdminSettings) + .set({ lastBackupAt: new Date(), updatedAt: new Date() }) + .where(eq(fofosAdminSettings.id, row.id)); + } + + return { ok, files, output: lines.join('\n') }; +} + +let job: CronJob | null = null; +let catchup: ReturnType | null = null; + +/** Is a scheduled run already overdue — because the machine was off when it should have happened? */ +async function runIfMissed(): Promise { + const row = await readRow(); + if (!row?.backupCron || !row.backupDir) return; + + const period = periodMs(row.backupCron); + if (!period) return; + + const last = row.lastBackupAt ? +row.lastBackupAt : 0; + if (Date.now() - last < period) return; + + await runBackup().catch(() => { + // Swallowed: this runs on a timer with nobody watching, and the panel reports the state from disk. + }); +} + +/** + * Start, restart, or stop the timer to match what is stored. + * + * Called on save and once at startup. Idempotent, and it always stops what was running first — two + * saves leaving two jobs is a bug that surfaces as backups arriving in pairs, weeks later, with nothing + * connecting it to a settings form. + */ +export async function schedule(): Promise { + job?.stop(); + job = null; + if (catchup) clearInterval(catchup); + catchup = null; + + const row = await readRow(); + if (!row?.backupCron || !row.backupDir) return; + + try { + job = new CronJob(row.backupCron, () => void runBackup().catch(() => {}), null, true); + } catch { + // A stored expression that will not build. Left unscheduled rather than crashing the router import, + // which would take the whole plugin down over a bad string in one column. + job = null; + } + + catchup = setInterval(() => void runIfMissed(), CATCHUP_TICK_MS); + // And once now, because the most likely moment to discover a missed run is the restart after it. + void runIfMissed(); +} diff --git a/api/router.ts b/api/router.ts index a8cd0c6..cabc57e 100644 --- a/api/router.ts +++ b/api/router.ts @@ -9,6 +9,7 @@ import { readSync, startSync } from './sync'; import { readRuns } from './ledger'; import { reconcile } from './derive'; import { createDatabases, pushSchema, readDatabases } from './databases'; +import { readBackups, runBackup, saveBackups, schedule } from './backups'; import { join } from 'node:path'; import { fofosRequest, FofosError, probe, type FofosResponse } from './fofos-client'; @@ -320,6 +321,32 @@ router.post('/_config/databases/create', async (ctx) => { } }); +/** + * Backups — the schedule, the location, and one on demand. + * + * `POST /run` exists so somebody can prove the location and the credentials work without waiting until + * three in the morning to find out they do not. + */ +router.get('/_config/backups', async (ctx) => ctx.json(await readBackups())); + +router.put('/_config/backups', async (ctx) => { + const { cron, dir } = (ctx.get('body') ?? {}) as { cron?: string; dir?: string }; + try { + return ctx.json(await saveBackups(cron ?? '', dir ?? '')); + } catch (error) { + return ctx.json({ error: (error as Error).message }, 400); + } +}); + +router.post('/_config/backups/run', async (ctx) => { + try { + const result = await runBackup(); + return ctx.json({ ...result, settings: await readBackups() }, result.ok ? 200 : 400); + } catch (error) { + return ctx.json({ error: (error as Error).message }, 400); + } +}); + router.get('/_config/summary', async (ctx) => { const user = ctx.get('user'); const settings = await readSettings(user.email); @@ -418,3 +445,13 @@ router.all('/*', async (ctx) => { return unreachable(ctx, error); } }); + +/* + Start the backup timer when this router is imported, which is once, at boot. + + Not awaited: the server must not wait on a database read to finish mounting a router, and a schedule + that starts a moment late costs nothing. Failures are swallowed for the same reason the timer swallows + them — a bad expression in one column must not stop the plugin from loading, because the panel is + where somebody would go to fix it. +*/ +void schedule().catch(() => {}); diff --git a/db/schema.ts b/db/schema.ts index 68f3b7c..6fe30d8 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -102,6 +102,30 @@ export const fofosAdminSettings = pgTable('fofos_admin_settings', { dbProduction: text('db_production'), dbDevelopment: text('db_development'), + /** + * Backups: when, and where they land. + * + * A cron expression, because a backup wants to happen at an HOUR — "every day at three" — and an + * interval measured from whenever somebody last pressed save drifts across the clock until it is + * running at two in the afternoon. The panel offers the handful of schedules anybody wants, and the + * expression is validated on the way in so a malformed one cannot be stored and then fail at a time + * nobody is watching. + * + * Null in either means backups are not set up. Both are needed, so one without the other is not a + * half-configured state to interpret — it is off. + */ + backupCron: text('backup_cron'), + backupDir: text('backup_dir'), + + /** + * When the last one finished, which is what makes the schedule survive a restart. + * + * The timer does not measure elapsed time from when the process started; it asks whether a backup is + * DUE, against this. A daily backup on a machine that restarts every few hours would otherwise never + * fire once — the interval would be reset before it ever elapsed, and nothing would look wrong. + */ + lastBackupAt: timestamp('last_backup_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(), }); diff --git a/web/BackupsRow.tsx b/web/BackupsRow.tsx new file mode 100644 index 0000000..cb9b5e9 --- /dev/null +++ b/web/BackupsRow.tsx @@ -0,0 +1,242 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { Check, ChevronDown, DatabaseBackup, Loader2, PlayCircle } from 'lucide-react'; +import { useClient } from 'hooks/useClient'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { DirectoryPicker } from 'officerdev'; +import { readableError } from './errors'; +import { Console } from './Console'; + +/** + * Backups: when they run, where they land, and what is actually there. + * + * ── A schedule somebody picks, not a cron expression somebody writes ── + * + * The field underneath IS a cron expression, and it is editable — but the four choices above it are + * every schedule anybody setting up a small site has ever wanted. Making people compose "0 3 * * *" + * from memory is how a backup ends up running every minute of the third hour. + * + * ── "Back up now" is not a convenience ── + * + * It is how you find out the location is writable and pg_dump can reach the databases, WITHOUT waiting + * until three in the morning to discover it cannot. A scheduled job that has never once been run by + * hand is a guess. + * + * ── What is on disk, not what should be ── + * + * The list is a `readdir` of the location. A schedule that has been silently failing looks exactly like + * a working one until you count the files, so the files are what is shown. + */ + +type Backup = { file: string; bytes: number; at: string }; + +type Settings = { + cron: string | null; + dir: string | null; + lastBackupAt: string | null; + nextRunAt: string | null; + recent: Backup[]; +}; + +/** Every schedule anybody has actually wanted, and their expressions. */ +const CHOICES = [ + { label: 'Hourly', cron: '0 * * * *' }, + { label: 'Daily, 3am', cron: '0 3 * * *' }, + { label: 'Weekly, Sun 3am', cron: '0 3 * * 0' }, + { label: 'Monthly, 1st 3am', cron: '0 3 1 * *' }, +] as const; + +const size = (bytes: number) => + bytes < 1024 ? `${bytes} B` : bytes < 1024 * 1024 ? `${Math.round(bytes / 1024)} KB` : `${(bytes / 1048576).toFixed(1)} MB`; + +const when = (iso: string | null) => (iso ? new Date(iso).toLocaleString(undefined, { dateStyle: 'medium', timeStyle: 'short' }) : null); + +export const BackupsRow = () => { + const client = useClient(); + const clientRef = useRef(client); + clientRef.current = client; + + const [settings, setSettings] = useState(null); + const [open, setOpen] = useState(false); + const [cron, setCron] = useState('0 3 * * *'); + const [dir, setDir] = useState(''); + const [picking, setPicking] = useState(false); + const [busy, setBusy] = useState<'save' | 'run' | null>(null); + const [error, setError] = useState(null); + const [output, setOutput] = useState(null); + const [failed, setFailed] = useState(false); + + const load = useCallback(async () => { + try { + const current = await clientRef.current.get('/fofos-admin/_config/backups'); + setSettings(current); + setCron((typed) => (current.cron ?? typed)); + setDir((typed) => typed || (current.dir ?? '')); + } catch { + setSettings(null); + } + }, []); + + useEffect(() => { + void load(); + }, [load]); + + const save = async () => { + setBusy('save'); + setError(null); + try { + setSettings(await clientRef.current.put('/fofos-admin/_config/backups', { cron, dir })); + setOpen(false); + } catch (failure) { + setError(readableError(failure)); + } finally { + setBusy(null); + } + }; + + const run = async () => { + setBusy('run'); + setError(null); + setFailed(false); + setOutput(null); + try { + const result = await clientRef.current.post<{ ok: boolean; output: string; settings: Settings }>( + '/fofos-admin/_config/backups/run', + {}, + ); + setSettings(result.settings); + setOutput(result.output); + setFailed(!result.ok); + } catch (failure) { + setOutput(readableError(failure)); + setFailed(true); + void load(); + } finally { + setBusy(null); + } + }; + + if (!settings) return null; + + const configured = Boolean(settings.cron && settings.dir); + const chosen = CHOICES.find((choice) => choice.cron === settings.cron); + + return ( +
+
+
+

Backups

+

+ {configured ? (chosen?.label ?? settings.cron) : 'Not set up'} + {configured && settings.dir ? · {settings.dir} : null} +

+
+ + + {settings.recent.length ? `${settings.recent.length} on disk` : configured ? 'none yet' : ''} + + +
+ {configured && ( + + )} + +
+
+ + {/* Only what has actually happened. A schedule with no runs behind it says so. */} + {configured && !open && ( +

+ {settings.lastBackupAt ? `Last ${when(settings.lastBackupAt)}` : 'Never run'} + {settings.nextRunAt ? ` · next ${when(settings.nextRunAt)}` : ''} +

+ )} + + {open && ( +
+
+

When

+
+ {CHOICES.map((choice) => ( + + ))} +
+ {/* The expression stays visible and editable. The buttons write into it; they do not hide it. */} + setCron(event.target.value)} + className="w-full font-mono text-xs" + autoComplete="off" + spellCheck={false} + /> +
+ +
+

Where

+
+ setDir(event.target.value)} + placeholder="/home/you/backups" + className="w-full font-mono text-xs" + autoComplete="off" + spellCheck={false} + /> + +
+ {picking && ( + setPicking(false)} + onPick={(absolutePath) => { + setDir(absolutePath); + setPicking(false); + }} + /> + )} +
+ + {error &&

{error}

} + +
+ + +
+
+ )} + + + + {settings.recent.length > 0 && !open && ( +
    + {settings.recent.slice(0, 5).map((backup) => ( +
  • + {backup.file} + {size(backup.bytes)} +
    • + ))} +
+ )} +
+ ); +}; diff --git a/web/DatabaseBlock.tsx b/web/DatabaseBlock.tsx index 280627e..7e535f9 100644 --- a/web/DatabaseBlock.tsx +++ b/web/DatabaseBlock.tsx @@ -1,11 +1,12 @@ import { useCallback, useEffect, useRef, useState } from 'react'; -import { ArrowUpFromLine, Check, CircleCheck, CircleDashed, Database, DatabaseBackup, Loader2, Plus } from 'lucide-react'; +import { ArrowUpFromLine, Check, CircleCheck, CircleDashed, Database, Loader2, Plus } from 'lucide-react'; import { useClient } from 'hooks/useClient'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; import { readableError } from './errors'; import { Block } from './Block'; import { Console } from './Console'; +import { BackupsRow } from './BackupsRow'; import { useFlash } from './useFlash'; /** @@ -185,9 +186,6 @@ export const DatabaseBlock = () => { ); const row = (label: string, target: 'production' | 'development', state: Named) => { - // "Schema pushed" is simply: something is in it. See the note on `tables`. - const pushed = (state.tables ?? 0) > 0; - return (
@@ -196,7 +194,7 @@ export const DatabaseBlock = () => {
- {state.tables === null ? '' : pushed ? `${state.tables} tables` : 'empty'} + {state.tables === null ? '' : state.tables > 0 ? `${state.tables} tables` : 'empty'}
@@ -215,19 +213,6 @@ export const DatabaseBlock = () => { Push schema - {/* Nothing to restore INTO until there are tables to restore into — a data-only load against - an empty database fails on every insert, and that failure would read as a bad backup. - Still disabled beyond that: what a backup IS here has not been decided, and a button that - guesses at the source of a restore is not a thing to put near a client's data. */} -
); @@ -255,6 +240,9 @@ export const DatabaseBlock = () => { {error &&

{error}

} + {/* Below the two databases, because it is about both of them rather than either. */} + + {/* The common case is both. Doing them one at a time is still there for when it is not. */}