From fb234ede5df0ee9770892525421b095ddeffa243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Sat, 29 Aug 2026 13:27:36 +0100 Subject: [PATCH] the contact inbox is a mailbox, and it is copied the moment it changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Messages are the one thing that happens between publishes. A backup triggered by publishing would never contain them, and a RESTORE would destroy every message received since the dump being restored — turning a routine recovery into losing the only copy of something a stranger sent, with nothing about the operation looking like it had done that. So they live in their own SQLite file, in the shape Officer's email plugin already uses. Not a schema that fits a contact form: this IS mail — a from, a subject, a date and a body wearing a form — and storing it as a mailbox means the mail client can open it, and anybody's agent already knows how to read it without being told. The form's own fields become labelled lines in text_body, where a reader and a grep both find them; inventing columns would be the moment it stopped being a mailbox. This side watches the file and copies it within seconds of a write, to the local backup directory and to the machine off this one. The alternative is a window: a message at ten past nine and a disk that fails at twenty past is a message nobody sees, while the sender believes they contacted you. Two details are the whole reason this is not a `cp`. The WAL is checkpointed first — the site writes in WAL mode, so a new message lives in messages.db-wal and not yet in messages.db, and copying the main file alone yields a database that is intact, readable, and missing exactly the message that triggered the copy. And the DIRECTORY is watched rather than the file, because SQLite replaces files during a checkpoint and a watch on a replaced inode goes quiet for good — a watcher that silently stops is worse than none, since the panel would go on implying the inbox is protected. The copy is current, not history; the ordinary backup runs take a timestamped one alongside the dumps, so a restored site can have the inbox that matches it. Co-Authored-By: Claude Opus 5 --- api/backups.ts | 26 ++++++++- api/messages.ts | 136 ++++++++++++++++++++++++++++++++++++++++++++++++ api/router.ts | 10 ++++ 3 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 api/messages.ts diff --git a/api/backups.ts b/api/backups.ts index c98148d..44b5a2a 100644 --- a/api/backups.ts +++ b/api/backups.ts @@ -1,4 +1,4 @@ -import { mkdir, readdir, stat, unlink } from 'node:fs/promises'; +import { copyFile, mkdir, readdir, stat, unlink } from 'node:fs/promises'; import { isAbsolute, join } from 'node:path'; import { CronJob, CronTime, validateCronExpression } from 'cron'; import { db } from 'officerdb/db'; @@ -6,6 +6,7 @@ import { eq } from 'drizzle-orm'; import { OFFICER_ROOT } from '@@/data-path'; import { fofosAdminSettings } from '../db/schema'; import { readDatabases } from './databases'; +import { messagesFile } from './messages'; /** * Backups of the site's databases: when, where, where else, and how many to keep. @@ -304,6 +305,29 @@ export async function runBackup(): Promise<{ ok: boolean; files: string[]; outpu } } + /* + The mailbox, timestamped, alongside the dumps. + + The watcher keeps a CURRENT copy at each destination and that is what protects a message minutes + after it arrives. This is the history: without it, restoring last week's site would come with this + morning's inbox and no way back to the one that matches. + */ + if (row?.rootPath) { + const mailbox = messagesFile(row.rootPath); + if (await stat(mailbox).catch(() => null)) { + const copy = join(dir, `messages-${stamp}.db`); + await copyFile(mailbox, copy).catch(() => {}); + lines.push(`messages-${stamp}.db`); + if (row.backupSshHost && row.backupSshPath) { + const sent = await run(['scp', ...SSH_OPTIONS, copy, `${row.backupSshHost}:${row.backupSshPath}/`]); + if (!sent.ok) { + ok = false; + lines.push(` -> COPY FAILED: ${sent.output}`); + } + } + } + } + if (row?.backupKeep) lines.push(...(await prune(row.backupKeep, dir, row.backupSshHost, row.backupSshPath))); /* diff --git a/api/messages.ts b/api/messages.ts new file mode 100644 index 0000000..245adad --- /dev/null +++ b/api/messages.ts @@ -0,0 +1,136 @@ +import { watch, type FSWatcher } from 'node:fs'; +import { copyFile, mkdir, stat } from 'node:fs/promises'; +import { join } from 'node:path'; +import { Database } from 'bun:sqlite'; +import { db } from 'officerdb/db'; +import { fofosAdminSettings } from '../db/schema'; +import { defaultBackupDir } from './backups'; + +/** + * The contact inbox, and keeping a copy of it somewhere else. + * + * ── Why this is not part of the database backup ── + * + * Messages arrive whenever a visitor sends one — which, for a site that only changes when somebody + * publishes, is the one thing happening between backups. So they live in their own SQLite file rather + * than in Postgres, and this copies that file the moment it changes rather than on a schedule. + * + * The alternative is a window: a message that arrives at ten past nine and a disk that fails at twenty + * past is a message nobody ever sees, and the sender believes they contacted you. + * + * ── The copy is CURRENT, not a history ── + * + * One `messages.db` at each destination, overwritten. History comes from the ordinary backup runs, + * which take a timestamped copy alongside the dumps. Writing a new timestamped file here instead would + * mean one file per message, which is a directory nobody can read and a rotation that evicts real + * dumps to make room. + * + * ── WAL has to be checkpointed first, and this is the whole reason this is not a `cp` ── + * + * The site opens the mailbox in WAL mode, so a recently written message lives in `messages.db-wal` and + * NOT yet in `messages.db`. Copying the main file alone gets a database that is intact, readable, and + * missing exactly the message that triggered the copy. `wal_checkpoint(TRUNCATE)` folds the log in + * first, so what is copied is everything. + */ + +/** Long enough that a burst of writes is one copy; short enough that a message is protected in seconds. */ +const SETTLE_MS = 3000; + +const FILE = 'messages.db'; + +export const messagesDir = (root: string): string => join(root, 'messages'); +export const messagesFile = (root: string): string => join(messagesDir(root), FILE); + +/** + * Fold the write-ahead log into the main file. + * + * Opened read-write, briefly, because a checkpoint is a write — and skipped without complaint if the + * file is not there or is busy, since the next change will bring us back. + */ +function checkpoint(path: string): void { + try { + const handle = new Database(path); + handle.exec('PRAGMA wal_checkpoint(TRUNCATE);'); + handle.close(); + } catch { + // Locked, or not a database yet. The copy still happens; it is simply one message behind, and the + // next write corrects it. + } +} + +/** Copy the mailbox to the local backup directory and, if one is set, to the machine off this one. */ +export async function copyMailbox(): Promise<{ ok: boolean; output: string }> { + const row = (await db.select().from(fofosAdminSettings).limit(1))[0] ?? null; + const root = row?.rootPath; + if (!root) return { ok: false, output: 'No root location is set.' }; + + const source = messagesFile(root); + if (!(await stat(source).catch(() => null))) return { ok: false, output: 'No mailbox yet.' }; + + checkpoint(source); + + const dir = row?.backupDir ?? defaultBackupDir(); + await mkdir(dir, { recursive: true }); + await copyFile(source, join(dir, FILE)); + + const lines = [`${FILE} -> ${dir}`]; + let ok = true; + + if (row?.backupSshHost && row.backupSshPath) { + const proc = Bun.spawn( + [ + 'scp', + '-o', + 'BatchMode=yes', + '-o', + 'ConnectTimeout=10', + '-o', + 'StrictHostKeyChecking=accept-new', + source, + `${row.backupSshHost}:${row.backupSshPath}/${FILE}`, + ], + { stdout: 'pipe', stderr: 'pipe', stdin: 'ignore' }, + ); + const [stderr, code] = await Promise.all([new Response(proc.stderr).text(), proc.exited]); + ok = code === 0; + lines.push(ok ? `${FILE} -> ${row.backupSshHost}:${row.backupSshPath}` : `copy off failed: ${stderr.trim()}`); + } + + return { ok, output: lines.join('\n') }; +} + +let watcher: FSWatcher | null = null; +let settle: ReturnType | null = null; + +/** + * Watch the mailbox and copy it whenever it changes. + * + * The DIRECTORY is watched, not the file. SQLite replaces and recreates files during a checkpoint, and + * a watch on an inode that has been replaced goes quiet for good — a watcher that silently stops is + * worse than no watcher, because the panel would go on implying the inbox is protected. + */ +export async function watchMailbox(): Promise { + watcher?.close(); + watcher = null; + if (settle) clearTimeout(settle); + settle = null; + + const row = (await db.select().from(fofosAdminSettings).limit(1))[0] ?? null; + if (!row?.rootPath) return; + + const dir = messagesDir(row.rootPath); + await mkdir(dir, { recursive: true }); + + try { + watcher = watch(dir, (_event, name) => { + // `messages.db`, `messages.db-wal`, `messages.db-shm` — any of them means the mailbox moved. + if (name && !name.startsWith(FILE)) return; + if (settle) clearTimeout(settle); + settle = setTimeout(() => void copyMailbox().catch(() => {}), SETTLE_MS); + }); + } catch { + // No directory, no permission. Left unwatched rather than throwing: this runs at import, and the + // plugin failing to load over a missing folder would take the panel that fixes it down too. + watcher = null; + } +} diff --git a/api/router.ts b/api/router.ts index 8fd4df7..696f6c7 100644 --- a/api/router.ts +++ b/api/router.ts @@ -10,6 +10,7 @@ import { readRuns } from './ledger'; import { reconcile } from './derive'; import { createDatabases, pushSchema, readDatabases } from './databases'; import { readBackups, restoreBackup, runBackup, saveBackups, schedule, testDestination } from './backups'; +import { watchMailbox } from './messages'; import { join } from 'node:path'; import { fofosRequest, FofosError, probe, type FofosResponse } from './fofos-client'; @@ -509,3 +510,12 @@ router.all('/*', async (ctx) => { where somebody would go to fix it. */ void schedule().catch(() => {}); + +/* + Watch the contact inbox and copy it whenever it changes. + + Separate from the backup schedule on purpose: messages arrive between publishes, so they are the one + thing a publish-triggered backup would never contain. Not awaited, and failures swallowed, for the + same reason as above. +*/ +void watchMailbox().catch(() => {});