From 8da02e1ef65f2b5fb4d0fb4b6aa96eee9c778123 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Sat, 5 Sep 2026 13:21:47 +0100 Subject: [PATCH] watch the dev server, because pm2 cannot tell if it is serving MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fofos-dev sat for eighteen hours reporting `online`, holding its port open and answering nothing. pm2 was not wrong — the process was alive and the socket was bound. It was `bun --hot` wedged after a day of edits, sitting in ep_poll and never completing a request. dev.fofosfluffybastards.com was down the whole time and nothing said so. servers.ts already knows those two answers disagree; that is why `reachable` is asked separately from pm2's status, and its own comment says so. This asks the same question on a timer. Not an hourly restart. That bounces a healthy server fifty times to catch one wedge, drops whatever anybody was doing each time, and still leaves the site broken for up to an hour where a probe catches it in two minutes. It acts only when pm2 says online AND the port does not answer, which is a lie worth correcting. A server somebody STOPPED is a decision and is left alone. Two strikes a minute apart, because --hot legitimately goes quiet while it rebuilds. Five minutes of cooldown after acting. Three failed restarts and it gives up and says to go read the logs, because a watchdog that restarts forever is a crash loop with a nicer name. State is on its own route rather than folded into /_config/servers, which answers with a bare array that two screens read positionally. Co-Authored-By: Claude Opus 5 (1M context) --- api/dev-watchdog.ts | 147 ++++++++++++++++++++++++++++++++++++++++++++ api/router.ts | 24 ++++++++ 2 files changed, 171 insertions(+) create mode 100644 api/dev-watchdog.ts diff --git a/api/dev-watchdog.ts b/api/dev-watchdog.ts new file mode 100644 index 0000000..2c5b6ca --- /dev/null +++ b/api/dev-watchdog.ts @@ -0,0 +1,147 @@ +import { readServers, startServer, type ServerStatus } from './servers'; + +/** + * The development server, watched — because pm2 cannot tell whether it is serving. + * + * ── The failure this exists for ── + * + * On 2026-09-04 `fofos-dev` sat for eighteen hours reporting `online`, holding its port open, and + * answering nothing. pm2 was not wrong: the process was alive and the socket was bound. It was + * `bun --hot` wedged after a day of edits, sitting in `ep_poll` and never completing a request. The + * site at dev.fofosfluffybastards.com had been down the whole time and nothing said so. + * + * `servers.ts` already knows the two answers disagree — that is why `reachable` is asked separately + * from pm2's status, and its own comment says so. This is that same question, asked on a timer. + * + * ── Why not just restart it every hour ── + * + * Because it bounces a healthy server fifty times to catch one wedge, and every bounce drops whatever + * somebody was doing — an editor mid-save, a build reading the database, an agent halfway through a + * request. It also leaves the site broken for up to an hour, where a probe catches it in two minutes. + * A timer that fires on a clock is a worse instrument than one that fires on the symptom. + * + * ── The boundary that matters ── + * + * It acts ONLY when pm2 says `online` and the port does not answer. That combination is a lie, and + * correcting a lie is the whole remit. A server somebody STOPPED is a decision, and starting it again + * behind their back would make this thing an adversary rather than a watchdog. + */ + +/** Long enough not to be noise, short enough that a wedge is measured in minutes. */ +const EVERY_MS = 60_000; + +/** + * Two strikes before acting. + * + * `bun --hot` legitimately stops answering for a moment while it rebuilds after a file changes, and a + * single missed probe during that window is not a wedge. Two consecutive misses a minute apart is. + */ +const STRIKES = 2; + +/** After acting, leave it alone. A restart takes seconds and the first probes after are meaningless. */ +const COOLDOWN_MS = 5 * 60_000; + +/** + * How many times to try before giving up and saying so. + * + * A watchdog that restarts forever is a crash loop with a nicer name: it turns one broken thing into a + * process that is never up long enough for anybody to read its logs. Three failures in a row means the + * problem is not a wedge and restarting is not the answer. + */ +const GIVE_UP_AFTER = 3; + +export type WatchdogState = { + /** When the last probe ran. The only thing that distinguishes "nothing to do" from "it died". */ + lastCheckedAt: string | null; + /** Consecutive probes that found it online-but-silent. */ + strikes: number; + lastRestartAt: string | null; + /** Restarts that did not fix it, in a row. At GIVE_UP_AFTER this stops trying. */ + consecutiveRestarts: number; + /** Set when it has given up, so the panel and the API can say why nothing is happening. */ + surrendered: boolean; +}; + +const state: WatchdogState = { + lastCheckedAt: null, + strikes: 0, + lastRestartAt: null, + consecutiveRestarts: 0, + surrendered: false, +}; + +/** Read by the servers endpoint, so "is the watchdog alive" is answerable without reading a log. */ +export const watchdogState = (): WatchdogState => ({ ...state }); + +let timer: ReturnType | null = null; + +async function check(): Promise { + state.lastCheckedAt = new Date().toISOString(); + + if (state.surrendered) return; + + // Inside the cooldown: it was just restarted and is probably still starting. + if (state.lastRestartAt && Date.now() - Date.parse(state.lastRestartAt) < COOLDOWN_MS) return; + + const servers = await readServers().catch(() => null); + const dev = servers?.find((one: ServerStatus) => one.name === 'development'); + if (!dev || dev.port === null) return; + + /* + Only the lie. `online` and answering is fine; anything else is somebody's decision. + + `reachable` is null when there is no port to try, which is not a failure — it is a server that has + never been configured, and starting that would be inventing an intention nobody had. + */ + if (dev.status !== 'online' || dev.reachable !== false) { + if (state.strikes) console.log('[fofos-admin] dev server answering again'); + state.strikes = 0; + state.consecutiveRestarts = 0; + return; + } + + state.strikes += 1; + console.log(`[fofos-admin] dev server is online but not answering (${state.strikes}/${STRIKES})`); + if (state.strikes < STRIKES) return; + + state.strikes = 0; + state.lastRestartAt = new Date().toISOString(); + state.consecutiveRestarts += 1; + + console.log('[fofos-admin] restarting the dev server — pm2 said online, the port did not answer'); + + /* + `startServer`, not a bare `pm2 restart`. + + It deletes and starts with the full environment the site needs — the database URLs, the media + directory, the boot secrets. A plain restart reuses whatever pm2 captured last time, which is the + same class of bug as the one this is here to fix: something that looks right and is stale. + */ + const result = await startServer('development').catch((error: unknown) => ({ + ok: false, + output: error instanceof Error ? error.message : String(error), + })); + + if (!result.ok) console.warn('[fofos-admin] the restart failed:', result.output.slice(0, 400)); + + if (state.consecutiveRestarts >= GIVE_UP_AFTER) { + state.surrendered = true; + console.error( + `[fofos-admin] the dev server has not stayed up after ${GIVE_UP_AFTER} restarts — not trying again. ` + + 'Look at `pm2 logs fofos-dev`; this is not a wedge.', + ); + } +} + +/** + * Start watching. Safe to call twice — the second call replaces the first rather than adding one. + * + * Not on a schedule the platform owns, because this is a fact about THIS plugin's server and belongs + * with the code that knows how to start it. + */ +export function watchDevServer(): void { + if (timer) clearInterval(timer); + timer = setInterval(() => void check().catch(() => {}), EVERY_MS); + // `unref` so a timer cannot be the reason the process will not exit. + timer.unref?.(); +} diff --git a/api/router.ts b/api/router.ts index dabe833..54d9c53 100644 --- a/api/router.ts +++ b/api/router.ts @@ -12,6 +12,7 @@ import { listOriginals, originalsDir, reconcile, safeScope } from './derive'; import { createDatabases, pushSchema, readDatabases } from './databases'; import { readBackups, restoreBackup, runBackup, saveBackups, schedule, testDestination } from './backups'; import { watchMailbox } from './messages'; +import { watchDevServer, watchdogState } from './dev-watchdog'; import { watchOriginals } from './originals-watch'; import { readServers, savePorts, siteSecrets, startServer, stopServer } from './servers'; import { buildSite, readStatic, startStatic, stopStatic } from './static-site'; @@ -564,6 +565,19 @@ router.post('/_config/backups/run', async (ctx) => { */ router.get('/_config/servers', async (ctx) => ctx.json(await readServers())); +/* + The watchdog's own state, on its own route. + + NOT folded into `/_config/servers`, which answers with a bare array that SitePreview and + ServersBlock both read positionally. Wrapping it to make room here would have broken two screens to + save one request. + + `lastCheckedAt` is the point of exposing any of this: a timer that has died looks exactly like a + timer with nothing to do, and a stale timestamp is the only thing that tells them apart without + going to read a log. +*/ +router.get('/_config/servers/watchdog', (ctx) => ctx.json(watchdogState())); + router.put('/_config/servers', async (ctx) => { const { development, production } = (ctx.get('body') ?? {}) as { development?: number; production?: number }; try { @@ -800,3 +814,13 @@ void watchMailbox().catch(() => {}); to know about. */ void watchOriginals().catch(() => {}); + +/* + Watch the DEVELOPMENT server, because pm2 cannot tell whether it is serving. + + It reported `online` for eighteen hours while holding its port open and answering nothing — a wedged + `bun --hot`, and the site was down the whole time with nothing to say so. servers.ts already asks + `reachable` separately from pm2's status for exactly this reason; this asks it on a timer and + restarts only when the two disagree. See dev-watchdog.ts. +*/ +watchDevServer();