import { readServers, startServer, type ServerStatus } from './servers'; /** * The development server, watched — because pm2 cannot tell whether it is serving. * * ── The failure this exists for ── * * On 2026-09-04 `fofos-dev` sat for eighteen hours reporting `online`, holding its port open, and * answering nothing. pm2 was not wrong: the process was alive and the socket was bound. It was * `bun --hot` wedged after a day of edits, sitting in `ep_poll` and never completing a request. The * site at dev.fofosfluffybastards.com had been down the whole time and nothing said so. * * `servers.ts` already knows the two answers disagree — that is why `reachable` is asked separately * from pm2's status, and its own comment says so. This is that same question, asked on a timer. * * ── Why not just restart it every hour ── * * Because it bounces a healthy server fifty times to catch one wedge, and every bounce drops whatever * somebody was doing — an editor mid-save, a build reading the database, an agent halfway through a * request. It also leaves the site broken for up to an hour, where a probe catches it in two minutes. * A timer that fires on a clock is a worse instrument than one that fires on the symptom. * * ── The boundary that matters ── * * It acts ONLY when pm2 says `online` and the port does not answer. That combination is a lie, and * correcting a lie is the whole remit. A server somebody STOPPED is a decision, and starting it again * behind their back would make this thing an adversary rather than a watchdog. */ /** Long enough not to be noise, short enough that a wedge is measured in minutes. */ const EVERY_MS = 60_000; /** * Two strikes before acting. * * `bun --hot` legitimately stops answering for a moment while it rebuilds after a file changes, and a * single missed probe during that window is not a wedge. Two consecutive misses a minute apart is. */ const STRIKES = 2; /** After acting, leave it alone. A restart takes seconds and the first probes after are meaningless. */ const COOLDOWN_MS = 5 * 60_000; /** * How many times to try before giving up and saying so. * * A watchdog that restarts forever is a crash loop with a nicer name: it turns one broken thing into a * process that is never up long enough for anybody to read its logs. Three failures in a row means the * problem is not a wedge and restarting is not the answer. */ const GIVE_UP_AFTER = 3; export type WatchdogState = { /** When the last probe ran. The only thing that distinguishes "nothing to do" from "it died". */ lastCheckedAt: string | null; /** Consecutive probes that found it online-but-silent. */ strikes: number; lastRestartAt: string | null; /** Restarts that did not fix it, in a row. At GIVE_UP_AFTER this stops trying. */ consecutiveRestarts: number; /** Set when it has given up, so the panel and the API can say why nothing is happening. */ surrendered: boolean; }; const state: WatchdogState = { lastCheckedAt: null, strikes: 0, lastRestartAt: null, consecutiveRestarts: 0, surrendered: false, }; /** Read by the servers endpoint, so "is the watchdog alive" is answerable without reading a log. */ export const watchdogState = (): WatchdogState => ({ ...state }); let timer: ReturnType | null = null; async function check(): Promise { state.lastCheckedAt = new Date().toISOString(); const servers = await readServers().catch(() => null); const dev = servers?.find((one: ServerStatus) => one.name === 'development'); if (!dev || dev.port === null) return; /* ── Answering again clears everything, INCLUDING having given up ── This branch used to sit below the `surrendered` check, which made surrender permanent: once it stopped trying it also stopped noticing, so a server somebody fixed by hand left a watchdog that would never watch again until the whole platform restarted. Reading the state first and clearing it here is what makes giving up a pause rather than a death. `online` and answering is fine; anything else is somebody's decision — see the note at the top. `reachable` is null when there is no port to try, which is a server nobody has configured. */ if (dev.status !== 'online' || dev.reachable !== false) { if (state.strikes || state.surrendered) console.log('[fofos-admin] dev server answering again'); state.strikes = 0; state.consecutiveRestarts = 0; state.surrendered = false; return; } if (state.surrendered) return; // Inside the cooldown: it was just restarted and is probably still starting. if (state.lastRestartAt && Date.now() - Date.parse(state.lastRestartAt) < COOLDOWN_MS) return; state.strikes += 1; console.log(`[fofos-admin] dev server is online but not answering (${state.strikes}/${STRIKES})`); if (state.strikes < STRIKES) return; state.strikes = 0; state.lastRestartAt = new Date().toISOString(); state.consecutiveRestarts += 1; console.log('[fofos-admin] restarting the dev server — pm2 said online, the port did not answer'); /* `startServer`, not a bare `pm2 restart`. It deletes and starts with the full environment the site needs — the database URLs, the media directory, the boot secrets. A plain restart reuses whatever pm2 captured last time, which is the same class of bug as the one this is here to fix: something that looks right and is stale. */ const result = await startServer('development').catch((error: unknown) => ({ ok: false, output: error instanceof Error ? error.message : String(error), })); /* A failed start is the dangerous outcome, not the loud one. `startServer` DELETES the pm2 entry before starting it. If the start then fails there is no process at all — and `status: null` reads, one tick later, as "a server somebody stopped", which this deliberately leaves alone. So a single bad restart would turn the watchdog off and take the server with it, silently. Counted as a failure so `GIVE_UP_AFTER` still bounds the retries, and said loudly, because a development server that no longer exists is not something to find out about tomorrow. */ if (!result.ok) { console.error( '[fofos-admin] the dev server FAILED to start and no process is left — `pm2 start` was preceded ' + 'by a delete. Start it from the panel, or `pm2 logs`:', result.output.slice(0, 400), ); } if (state.consecutiveRestarts >= GIVE_UP_AFTER) { state.surrendered = true; console.error( `[fofos-admin] the dev server has not stayed up after ${GIVE_UP_AFTER} restarts — not trying again. ` + 'Look at `pm2 logs fofos-dev`; this is not a wedge.', ); } } /** * Start watching. Safe to call twice — the second call replaces the first rather than adding one. * * Not on a schedule the platform owns, because this is a fact about THIS plugin's server and belongs * with the code that knows how to start it. */ export function watchDevServer(): void { if (timer) clearInterval(timer); timer = setInterval(() => void check().catch(() => {}), EVERY_MS); // `unref` so a timer cannot be the reason the process will not exit. timer.unref?.(); }