Restore goes. What it would read from was never decided, and a disabled button with a tooltip explaining it is not built is worse than no button. In its place, backups. `cron` was already a dependency and used nowhere, so the schedule is a real cron expression — a backup wants to happen at an HOUR, and an interval counted from whenever somebody pressed save drifts across the clock until it runs mid-afternoon. Cron alone is not enough, and this is the part worth writing down: it does not run a job it was down for. A daily backup at three on a machine that restarts overnight is skipped silently — the schedule is still there, the next fire is still in the future, and a month later there is one file. So `lastBackupAt` is in the database, and both the startup and a fifteen-minute tick compare it against the schedule's own period, computed from two successive fires. A backup that should already have happened happens now. The job handle is module-level and stopped before a new one is made. Two saves leaving two jobs is a bug that surfaces as backups arriving in pairs, weeks later, with nothing connecting it to a settings form. Back up now exists because a scheduled job nobody has ever run by hand is a guess — it is how you learn the location is writable and pg_dump can reach the databases without waiting until three to find out it cannot. The list underneath is a readdir of the location rather than a record of what should be there: a schedule that has been failing quietly looks exactly like a working one until you count the files. pg_dump per database, not pg_dumpall — a cluster dump would also carry Officer's own, which is not this plugin's to copy around. And a failed run still records the timestamp, or one undumpable database would have the catch-up tick retrying every fifteen minutes for the rest of the week. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
244 lines
9.0 KiB
TypeScript
244 lines
9.0 KiB
TypeScript
import { mkdir, readdir, stat } from 'node:fs/promises';
|
|
import { isAbsolute, join } from 'node:path';
|
|
import { CronJob, CronTime, validateCronExpression } from 'cron';
|
|
import { db } from 'officerdb/db';
|
|
import { eq } from 'drizzle-orm';
|
|
import { fofosAdminSettings } from '../db/schema';
|
|
import { readDatabases } from './databases';
|
|
|
|
/**
|
|
* Backups of the site's databases: when, where, and the thing that actually runs them.
|
|
*
|
|
* ── Cron for the schedule, a due-check for the durability ──
|
|
*
|
|
* `cron` fires at an HOUR, which is what a backup wants — an interval counted from whenever somebody
|
|
* pressed save drifts across the clock until it is running mid-afternoon.
|
|
*
|
|
* What cron does NOT do is run a job it was down for. A daily backup at three on a machine that
|
|
* restarts overnight is skipped silently, and nothing looks wrong: the schedule is still there, the
|
|
* next fire is still in the future, and a month later there is one backup. So the schedule is only half
|
|
* of this. On every start, and on every fire, the last run is compared against the schedule's own
|
|
* period, and a backup that should already have happened happens now.
|
|
*
|
|
* That is why `lastBackupAt` is in the database rather than in memory. It is the only thing that knows
|
|
* a run was missed.
|
|
*
|
|
* ── One job, replaced rather than accumulated ──
|
|
*
|
|
* The handle is module-level and stopped before a new one is made. Without that, saving the schedule
|
|
* twice leaves two jobs running and the third save leaves three — a bug that shows up as backups
|
|
* arriving in pairs, long after anybody would connect it to a settings form.
|
|
*/
|
|
|
|
/** How often to look for a missed run when nothing is scheduled to fire. */
|
|
const CATCHUP_TICK_MS = 15 * 60 * 1000;
|
|
|
|
export type BackupSettings = {
|
|
cron: string | null;
|
|
dir: string | null;
|
|
lastBackupAt: string | null;
|
|
/** From the schedule, so the panel does not have to know how to read cron. */
|
|
nextRunAt: string | null;
|
|
/** Newest first. What is actually on disk, rather than what should be. */
|
|
recent: { file: string; bytes: number; at: string }[];
|
|
};
|
|
|
|
const readRow = async () => (await db.select().from(fofosAdminSettings).limit(1))[0] ?? null;
|
|
|
|
/** Two successive fires, which is the schedule's period — what "a run was missed" is measured against. */
|
|
function periodMs(expression: string): number | null {
|
|
try {
|
|
const time = new CronTime(expression);
|
|
const first = time.getNextDateFrom(new Date());
|
|
const second = time.getNextDateFrom(first.toJSDate());
|
|
return +second.toJSDate() - +first.toJSDate();
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export function checkCron(expression: string): string {
|
|
const trimmed = expression.trim();
|
|
if (!trimmed) throw new Error('A schedule is required.');
|
|
const result = validateCronExpression(trimmed);
|
|
if (!result.valid) throw new Error(`"${trimmed}" is not a schedule cron understands.`);
|
|
return trimmed;
|
|
}
|
|
|
|
export function checkDir(path: string): string {
|
|
const trimmed = path.trim();
|
|
if (!trimmed) throw new Error('A location is required.');
|
|
if (!isAbsolute(trimmed)) throw new Error('The location must be an absolute path.');
|
|
return trimmed.replace(/\/+$/, '');
|
|
}
|
|
|
|
export async function readBackups(): Promise<BackupSettings> {
|
|
const row = await readRow();
|
|
const cron = row?.backupCron ?? null;
|
|
const dir = row?.backupDir ?? null;
|
|
|
|
let nextRunAt: string | null = null;
|
|
if (cron) {
|
|
try {
|
|
nextRunAt = new CronTime(cron).getNextDateFrom(new Date()).toISO();
|
|
} catch {
|
|
// A stored expression that no longer parses. Reported as no next run rather than thrown: the
|
|
// panel has to be able to render the settings in order to fix them.
|
|
}
|
|
}
|
|
|
|
return {
|
|
cron,
|
|
dir,
|
|
lastBackupAt: row?.lastBackupAt?.toISOString() ?? null,
|
|
nextRunAt,
|
|
recent: dir ? await listBackups(dir) : [],
|
|
};
|
|
}
|
|
|
|
async function listBackups(dir: string): Promise<{ file: string; bytes: number; at: string }[]> {
|
|
const names = await readdir(dir).catch(() => []);
|
|
const found = [];
|
|
for (const name of names) {
|
|
if (!name.endsWith('.sql')) continue;
|
|
const info = await stat(join(dir, name)).catch(() => null);
|
|
if (info) found.push({ file: name, bytes: info.size, at: info.mtime.toISOString() });
|
|
}
|
|
return found.sort((a, b) => b.at.localeCompare(a.at)).slice(0, 10);
|
|
}
|
|
|
|
export async function saveBackups(expression: string, directory: string): Promise<BackupSettings> {
|
|
const cron = checkCron(expression);
|
|
const dir = checkDir(directory);
|
|
|
|
// Made now rather than at the first run, so a location that cannot be created is a message beside the
|
|
// form instead of a failure at three in the morning.
|
|
await mkdir(dir, { recursive: true });
|
|
|
|
const row = await readRow();
|
|
if (row) {
|
|
await db
|
|
.update(fofosAdminSettings)
|
|
.set({ backupCron: cron, backupDir: dir, updatedAt: new Date() })
|
|
.where(eq(fofosAdminSettings.id, row.id));
|
|
} else {
|
|
await db.insert(fofosAdminSettings).values({ backupCron: cron, backupDir: dir });
|
|
}
|
|
|
|
await schedule();
|
|
return readBackups();
|
|
}
|
|
|
|
/**
|
|
* Dump every named database, once.
|
|
*
|
|
* `pg_dump` per database rather than `pg_dumpall`: the two databases here are the site's, and a cluster
|
|
* dump would also carry Officer's own, which is not this plugin's to copy around.
|
|
*/
|
|
export async function runBackup(): Promise<{ ok: boolean; files: string[]; output: string }> {
|
|
const row = await readRow();
|
|
const dir = row?.backupDir;
|
|
if (!dir) throw new Error('No backup location is set.');
|
|
await mkdir(dir, { recursive: true });
|
|
|
|
const raw = process.env.OFFICERDEV_POSTGRES_URL ?? process.env.POSTGRES_URL;
|
|
if (!raw) throw new Error('The platform has no Postgres connection configured.');
|
|
|
|
const status = await readDatabases();
|
|
const targets = [status.production, status.development].filter((entry) => entry.name && entry.exists);
|
|
if (!targets.length) throw new Error('Neither database exists yet.');
|
|
|
|
// Sortable, and the same shape as the file this plugin's first manual dump used.
|
|
const stamp = new Date().toISOString().replace(/[-:]/g, '').replace(/\..+$/, '').replace('T', '-');
|
|
|
|
const files: string[] = [];
|
|
const lines: string[] = [];
|
|
let ok = true;
|
|
|
|
for (const target of targets) {
|
|
const url = new URL(raw);
|
|
url.pathname = `/${target.name}`;
|
|
const out = join(dir, `${target.name}-${stamp}.sql`);
|
|
|
|
const proc = Bun.spawn(['pg_dump', url.toString(), '-f', out], {
|
|
stdout: 'pipe',
|
|
stderr: 'pipe',
|
|
stdin: 'ignore',
|
|
});
|
|
const [stderr, code] = await Promise.all([new Response(proc.stderr).text(), proc.exited]);
|
|
|
|
if (code === 0) {
|
|
files.push(`${target.name}-${stamp}.sql`);
|
|
const info = await stat(out).catch(() => null);
|
|
lines.push(`${target.name} -> ${files.at(-1)} (${info?.size ?? 0} bytes)`);
|
|
} else {
|
|
ok = false;
|
|
lines.push(`${target.name} FAILED: ${stderr.trim()}`);
|
|
}
|
|
}
|
|
|
|
/*
|
|
Recorded even when one of them failed.
|
|
|
|
Otherwise a database that cannot be dumped makes every subsequent tick think a run is overdue, and
|
|
the panel spends the rest of the week trying and failing at fifteen-minute intervals. The failure is
|
|
in the output where somebody will see it; the schedule is not the place to express it.
|
|
*/
|
|
if (row) {
|
|
await db
|
|
.update(fofosAdminSettings)
|
|
.set({ lastBackupAt: new Date(), updatedAt: new Date() })
|
|
.where(eq(fofosAdminSettings.id, row.id));
|
|
}
|
|
|
|
return { ok, files, output: lines.join('\n') };
|
|
}
|
|
|
|
let job: CronJob | null = null;
|
|
let catchup: ReturnType<typeof setInterval> | null = null;
|
|
|
|
/** Is a scheduled run already overdue — because the machine was off when it should have happened? */
|
|
async function runIfMissed(): Promise<void> {
|
|
const row = await readRow();
|
|
if (!row?.backupCron || !row.backupDir) return;
|
|
|
|
const period = periodMs(row.backupCron);
|
|
if (!period) return;
|
|
|
|
const last = row.lastBackupAt ? +row.lastBackupAt : 0;
|
|
if (Date.now() - last < period) return;
|
|
|
|
await runBackup().catch(() => {
|
|
// Swallowed: this runs on a timer with nobody watching, and the panel reports the state from disk.
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Start, restart, or stop the timer to match what is stored.
|
|
*
|
|
* Called on save and once at startup. Idempotent, and it always stops what was running first — two
|
|
* saves leaving two jobs is a bug that surfaces as backups arriving in pairs, weeks later, with nothing
|
|
* connecting it to a settings form.
|
|
*/
|
|
export async function schedule(): Promise<void> {
|
|
job?.stop();
|
|
job = null;
|
|
if (catchup) clearInterval(catchup);
|
|
catchup = null;
|
|
|
|
const row = await readRow();
|
|
if (!row?.backupCron || !row.backupDir) return;
|
|
|
|
try {
|
|
job = new CronJob(row.backupCron, () => void runBackup().catch(() => {}), null, true);
|
|
} catch {
|
|
// A stored expression that will not build. Left unscheduled rather than crashing the router import,
|
|
// which would take the whole plugin down over a bad string in one column.
|
|
job = null;
|
|
}
|
|
|
|
catchup = setInterval(() => void runIfMissed(), CATCHUP_TICK_MS);
|
|
// And once now, because the most likely moment to discover a missed run is the restart after it.
|
|
void runIfMissed();
|
|
}
|