fofos-admin/api/dev-watchdog.ts

import { readServers, startServer, type ServerStatus } from './servers';

/**
 * The development server, watched — because pm2 cannot tell whether it is serving.
 *
 * ── The failure this exists for ──
 *
 * On 2026-09-04 `fofos-dev` sat for eighteen hours reporting `online`, holding its port open, and
 * answering nothing. pm2 was not wrong: the process was alive and the socket was bound. It was
 * `bun --hot` wedged after a day of edits, sitting in `ep_poll` and never completing a request. The
 * site at dev.fofosfluffybastards.com had been down the whole time and nothing said so.
 *
 * `servers.ts` already knows the two answers disagree — that is why `reachable` is asked separately
 * from pm2's status, and its own comment says so. This is that same question, asked on a timer.
 *
 * ── Why not just restart it every hour ──
 *
 * Because it bounces a healthy server fifty times to catch one wedge, and every bounce drops whatever
 * somebody was doing — an editor mid-save, a build reading the database, an agent halfway through a
 * request. It also leaves the site broken for up to an hour, where a probe catches it in two minutes.
 * A timer that fires on a clock is a worse instrument than one that fires on the symptom.
 *
 * ── The boundary that matters ──
 *
 * It acts ONLY when pm2 says `online` and the port does not answer. That combination is a lie, and
 * correcting a lie is the whole remit. A server somebody STOPPED is a decision, and starting it again
 * behind their back would make this thing an adversary rather than a watchdog.
 */

/** Long enough not to be noise, short enough that a wedge is measured in minutes. */
const EVERY_MS = 60_000;

/**
 * Two strikes before acting.
 *
 * `bun --hot` legitimately stops answering for a moment while it rebuilds after a file changes, and a
 * single missed probe during that window is not a wedge. Two consecutive misses a minute apart is.
 */
const STRIKES = 2;

/** After acting, leave it alone. A restart takes seconds and the first probes after are meaningless. */
const COOLDOWN_MS = 5 * 60_000;

/**
 * How many times to try before giving up and saying so.
 *
 * A watchdog that restarts forever is a crash loop with a nicer name: it turns one broken thing into a
 * process that is never up long enough for anybody to read its logs. Three failures in a row means the
 * problem is not a wedge and restarting is not the answer.
 */
const GIVE_UP_AFTER = 3;

export type WatchdogState = {
  /** When the last probe ran. The only thing that distinguishes "nothing to do" from "it died". */
  lastCheckedAt: string | null;
  /** Consecutive probes that found it online-but-silent. */
  strikes: number;
  lastRestartAt: string | null;
  /** Restarts that did not fix it, in a row. At GIVE_UP_AFTER this stops trying. */
  consecutiveRestarts: number;
  /** Set when it has given up, so the panel and the API can say why nothing is happening. */
  surrendered: boolean;
};

const state: WatchdogState = {
  lastCheckedAt: null,
  strikes: 0,
  lastRestartAt: null,
  consecutiveRestarts: 0,
  surrendered: false,
};

/** Read by the servers endpoint, so "is the watchdog alive" is answerable without reading a log. */
export const watchdogState = (): WatchdogState => ({ ...state });

let timer: ReturnType<typeof setInterval> | null = null;

async function check(): Promise<void> {
  state.lastCheckedAt = new Date().toISOString();

  const servers = await readServers().catch(() => null);
  const dev = servers?.find((one: ServerStatus) => one.name === 'development');
  if (!dev || dev.port === null) return;

  /*
    ── Answering again clears everything, INCLUDING having given up ──

    This branch used to sit below the `surrendered` check, which made surrender permanent: once it
    stopped trying it also stopped noticing, so a server somebody fixed by hand left a watchdog that
    would never watch again until the whole platform restarted. Reading the state first and clearing
    it here is what makes giving up a pause rather than a death.

    `online` and answering is fine; anything else is somebody's decision — see the note at the top.
    `reachable` is null when there is no port to try, which is a server nobody has configured.
  */
  if (dev.status !== 'online' || dev.reachable !== false) {
    if (state.strikes || state.surrendered) console.log('[fofos-admin] dev server answering again');
    state.strikes = 0;
    state.consecutiveRestarts = 0;
    state.surrendered = false;
    return;
  }

  if (state.surrendered) return;

  // Inside the cooldown: it was just restarted and is probably still starting.
  if (state.lastRestartAt && Date.now() - Date.parse(state.lastRestartAt) < COOLDOWN_MS) return;

  state.strikes += 1;
  console.log(`[fofos-admin] dev server is online but not answering (${state.strikes}/${STRIKES})`);
  if (state.strikes < STRIKES) return;

  state.strikes = 0;
  state.lastRestartAt = new Date().toISOString();
  state.consecutiveRestarts += 1;

  console.log('[fofos-admin] restarting the dev server — pm2 said online, the port did not answer');

  /*
    `startServer`, not a bare `pm2 restart`.

    It deletes and starts with the full environment the site needs — the database URLs, the media
    directory, the boot secrets. A plain restart reuses whatever pm2 captured last time, which is the
    same class of bug as the one this is here to fix: something that looks right and is stale.
  */
  const result = await startServer('development').catch((error: unknown) => ({
    ok: false,
    output: error instanceof Error ? error.message : String(error),
  }));

  /*
    A failed start is the dangerous outcome, not the loud one.

    `startServer` DELETES the pm2 entry before starting it. If the start then fails there is no
    process at all — and `status: null` reads, one tick later, as "a server somebody stopped", which
    this deliberately leaves alone. So a single bad restart would turn the watchdog off and take the
    server with it, silently.

    Counted as a failure so `GIVE_UP_AFTER` still bounds the retries, and said loudly, because a
    development server that no longer exists is not something to find out about tomorrow.
  */
  if (!result.ok) {
    console.error(
      '[fofos-admin] the dev server FAILED to start and no process is left — `pm2 start` was preceded ' +
        'by a delete. Start it from the panel, or `pm2 logs`:',
      result.output.slice(0, 400),
    );
  }

  if (state.consecutiveRestarts >= GIVE_UP_AFTER) {
    state.surrendered = true;
    console.error(
      `[fofos-admin] the dev server has not stayed up after ${GIVE_UP_AFTER} restarts — not trying again. ` +
        'Look at `pm2 logs fofos-dev`; this is not a wedge.',
    );
  }
}

/**
 * Start watching. Safe to call twice — the second call replaces the first rather than adding one.
 *
 * Not on a schedule the platform owns, because this is a fact about THIS plugin's server and belongs
 * with the code that knows how to start it.
 */
export function watchDevServer(): void {
  if (timer) clearInterval(timer);
  timer = setInterval(() => void check().catch(() => {}), EVERY_MS);
  // `unref` so a timer cannot be the reason the process will not exit.
  timer.unref?.();
}