fofos-admin/api/dev-watchdog.ts

import { readServers, startServer, type ServerStatus } from './servers';

/**
 * The development server, watched — because pm2 cannot tell whether it is serving.
 *
 * ── The failure this exists for ──
 *
 * On 2026-09-04 `fofos-dev` sat for eighteen hours reporting `online`, holding its port open, and
 * answering nothing. pm2 was not wrong: the process was alive and the socket was bound. It was
 * `bun --hot` wedged after a day of edits, sitting in `ep_poll` and never completing a request. The
 * site at dev.fofosfluffybastards.com had been down the whole time and nothing said so.
 *
 * `servers.ts` already knows the two answers disagree — that is why `reachable` is asked separately
 * from pm2's status, and its own comment says so. This is that same question, asked on a timer.
 *
 * ── Why not just restart it every hour ──
 *
 * Because it bounces a healthy server fifty times to catch one wedge, and every bounce drops whatever
 * somebody was doing — an editor mid-save, a build reading the database, an agent halfway through a
 * request. It also leaves the site broken for up to an hour, where a probe catches it in two minutes.
 * A timer that fires on a clock is a worse instrument than one that fires on the symptom.
 *
 * ── The boundary that matters ──
 *
 * It acts ONLY when pm2 says `online` and the port does not answer. That combination is a lie, and
 * correcting a lie is the whole remit. A server somebody STOPPED is a decision, and starting it again
 * behind their back would make this thing an adversary rather than a watchdog.
 */

/** Long enough not to be noise, short enough that a wedge is measured in minutes. */
const EVERY_MS = 60_000;

/**
 * Two strikes before acting.
 *
 * `bun --hot` legitimately stops answering for a moment while it rebuilds after a file changes, and a
 * single missed probe during that window is not a wedge. Two consecutive misses a minute apart is.
 */
const STRIKES = 2;

/** After acting, leave it alone. A restart takes seconds and the first probes after are meaningless. */
const COOLDOWN_MS = 5 * 60_000;

/**
 * How many times to try before giving up and saying so.
 *
 * A watchdog that restarts forever is a crash loop with a nicer name: it turns one broken thing into a
 * process that is never up long enough for anybody to read its logs. Three failures in a row means the
 * problem is not a wedge and restarting is not the answer.
 */
const GIVE_UP_AFTER = 3;

export type WatchdogState = {
  /** When the last probe ran. The only thing that distinguishes "nothing to do" from "it died". */
  lastCheckedAt: string | null;
  /** Consecutive probes that found it online-but-silent. */
  strikes: number;
  lastRestartAt: string | null;
  /** Restarts that did not fix it, in a row. At GIVE_UP_AFTER this stops trying. */
  consecutiveRestarts: number;
  /** Set when it has given up, so the panel and the API can say why nothing is happening. */
  surrendered: boolean;
};

const state: WatchdogState = {
  lastCheckedAt: null,
  strikes: 0,
  lastRestartAt: null,
  consecutiveRestarts: 0,
  surrendered: false,
};

/** Read by the servers endpoint, so "is the watchdog alive" is answerable without reading a log. */
export const watchdogState = (): WatchdogState => ({ ...state });

let timer: ReturnType<typeof setInterval> | null = null;

async function check(): Promise<void> {
  state.lastCheckedAt = new Date().toISOString();

  if (state.surrendered) return;

  // Inside the cooldown: it was just restarted and is probably still starting.
  if (state.lastRestartAt && Date.now() - Date.parse(state.lastRestartAt) < COOLDOWN_MS) return;

  const servers = await readServers().catch(() => null);
  const dev = servers?.find((one: ServerStatus) => one.name === 'development');
  if (!dev || dev.port === null) return;

  /*
    Only the lie. `online` and answering is fine; anything else is somebody's decision.

    `reachable` is null when there is no port to try, which is not a failure — it is a server that has
    never been configured, and starting that would be inventing an intention nobody had.
  */
  if (dev.status !== 'online' || dev.reachable !== false) {
    if (state.strikes) console.log('[fofos-admin] dev server answering again');
    state.strikes = 0;
    state.consecutiveRestarts = 0;
    return;
  }

  state.strikes += 1;
  console.log(`[fofos-admin] dev server is online but not answering (${state.strikes}/${STRIKES})`);
  if (state.strikes < STRIKES) return;

  state.strikes = 0;
  state.lastRestartAt = new Date().toISOString();
  state.consecutiveRestarts += 1;

  console.log('[fofos-admin] restarting the dev server — pm2 said online, the port did not answer');

  /*
    `startServer`, not a bare `pm2 restart`.

    It deletes and starts with the full environment the site needs — the database URLs, the media
    directory, the boot secrets. A plain restart reuses whatever pm2 captured last time, which is the
    same class of bug as the one this is here to fix: something that looks right and is stale.
  */
  const result = await startServer('development').catch((error: unknown) => ({
    ok: false,
    output: error instanceof Error ? error.message : String(error),
  }));

  if (!result.ok) console.warn('[fofos-admin] the restart failed:', result.output.slice(0, 400));

  if (state.consecutiveRestarts >= GIVE_UP_AFTER) {
    state.surrendered = true;
    console.error(
      `[fofos-admin] the dev server has not stayed up after ${GIVE_UP_AFTER} restarts — not trying again. ` +
        'Look at `pm2 logs fofos-dev`; this is not a wedge.',
    );
  }
}

/**
 * Start watching. Safe to call twice — the second call replaces the first rather than adding one.
 *
 * Not on a schedule the platform owns, because this is a fact about THIS plugin's server and belongs
 * with the code that knows how to start it.
 */
export function watchDevServer(): void {
  if (timer) clearInterval(timer);
  timer = setInterval(() => void check().catch(() => {}), EVERY_MS);
  // `unref` so a timer cannot be the reason the process will not exit.
  timer.unref?.();
}