import { readServers, startServer, type ServerStatus } from './servers'; /** * The development server, watched — because pm2 cannot tell whether it is serving. * * ── The failure this exists for ── * * On 2026-09-04 `fofos-dev` sat for eighteen hours reporting `online`, holding its port open, and * answering nothing. pm2 was not wrong: the process was alive and the socket was bound. It was * `bun --hot` wedged after a day of edits, sitting in `ep_poll` and never completing a request. The * site at dev.fofosfluffybastards.com had been down the whole time and nothing said so. * * `servers.ts` already knows the two answers disagree — that is why `reachable` is asked separately * from pm2's status, and its own comment says so. This is that same question, asked on a timer. * * ── Why not just restart it every hour ── * * Because it bounces a healthy server fifty times to catch one wedge, and every bounce drops whatever * somebody was doing — an editor mid-save, a build reading the database, an agent halfway through a * request. It also leaves the site broken for up to an hour, where a probe catches it in two minutes. * A timer that fires on a clock is a worse instrument than one that fires on the symptom. * * ── The boundary that matters ── * * It acts ONLY when pm2 says `online` and the port does not answer. That combination is a lie, and * correcting a lie is the whole remit. A server somebody STOPPED is a decision, and starting it again * behind their back would make this thing an adversary rather than a watchdog. */ /** Long enough not to be noise, short enough that a wedge is measured in minutes. */ const EVERY_MS = 60_000; /** * Two strikes before acting. * * `bun --hot` legitimately stops answering for a moment while it rebuilds after a file changes, and a * single missed probe during that window is not a wedge. Two consecutive misses a minute apart is. */ const STRIKES = 2; /** After acting, leave it alone. A restart takes seconds and the first probes after are meaningless. */ const COOLDOWN_MS = 5 * 60_000; /** * How many times to try before giving up and saying so. * * A watchdog that restarts forever is a crash loop with a nicer name: it turns one broken thing into a * process that is never up long enough for anybody to read its logs. Three failures in a row means the * problem is not a wedge and restarting is not the answer. */ const GIVE_UP_AFTER = 3; export type WatchdogState = { /** When the last probe ran. The only thing that distinguishes "nothing to do" from "it died". */ lastCheckedAt: string | null; /** Consecutive probes that found it online-but-silent. */ strikes: number; lastRestartAt: string | null; /** Restarts that did not fix it, in a row. At GIVE_UP_AFTER this stops trying. */ consecutiveRestarts: number; /** Set when it has given up, so the panel and the API can say why nothing is happening. */ surrendered: boolean; }; const state: WatchdogState = { lastCheckedAt: null, strikes: 0, lastRestartAt: null, consecutiveRestarts: 0, surrendered: false, }; /** Read by the servers endpoint, so "is the watchdog alive" is answerable without reading a log. */ export const watchdogState = (): WatchdogState => ({ ...state }); let timer: ReturnType | null = null; async function check(): Promise { state.lastCheckedAt = new Date().toISOString(); if (state.surrendered) return; // Inside the cooldown: it was just restarted and is probably still starting. if (state.lastRestartAt && Date.now() - Date.parse(state.lastRestartAt) < COOLDOWN_MS) return; const servers = await readServers().catch(() => null); const dev = servers?.find((one: ServerStatus) => one.name === 'development'); if (!dev || dev.port === null) return; /* Only the lie. `online` and answering is fine; anything else is somebody's decision. `reachable` is null when there is no port to try, which is not a failure — it is a server that has never been configured, and starting that would be inventing an intention nobody had. */ if (dev.status !== 'online' || dev.reachable !== false) { if (state.strikes) console.log('[fofos-admin] dev server answering again'); state.strikes = 0; state.consecutiveRestarts = 0; return; } state.strikes += 1; console.log(`[fofos-admin] dev server is online but not answering (${state.strikes}/${STRIKES})`); if (state.strikes < STRIKES) return; state.strikes = 0; state.lastRestartAt = new Date().toISOString(); state.consecutiveRestarts += 1; console.log('[fofos-admin] restarting the dev server — pm2 said online, the port did not answer'); /* `startServer`, not a bare `pm2 restart`. It deletes and starts with the full environment the site needs — the database URLs, the media directory, the boot secrets. A plain restart reuses whatever pm2 captured last time, which is the same class of bug as the one this is here to fix: something that looks right and is stale. */ const result = await startServer('development').catch((error: unknown) => ({ ok: false, output: error instanceof Error ? error.message : String(error), })); if (!result.ok) console.warn('[fofos-admin] the restart failed:', result.output.slice(0, 400)); if (state.consecutiveRestarts >= GIVE_UP_AFTER) { state.surrendered = true; console.error( `[fofos-admin] the dev server has not stayed up after ${GIVE_UP_AFTER} restarts — not trying again. ` + 'Look at `pm2 logs fofos-dev`; this is not a wedge.', ); } } /** * Start watching. Safe to call twice — the second call replaces the first rather than adding one. * * Not on a schedule the platform owns, because this is a fact about THIS plugin's server and belongs * with the code that knows how to start it. */ export function watchDevServer(): void { if (timer) clearInterval(timer); timer = setInterval(() => void check().catch(() => {}), EVERY_MS); // `unref` so a timer cannot be the reason the process will not exit. timer.unref?.(); }