diff --git a/scripts/setup/officer-setup.sh b/scripts/setup/officer-setup.sh index d0486c6d..946f4303 100755 --- a/scripts/setup/officer-setup.sh +++ b/scripts/setup/officer-setup.sh @@ -657,6 +657,14 @@ if ! skip; then write_ecosystem ok "written — $(ecosystem_file)" + # Starting against a database that is not answering is not fatal — the server + # waits and the agent retries forever — but it makes the Verify section below + # report a failure that is really just a race, and that is the kind of noise + # that teaches people to ignore a red line. + if pg_container_running && ! pg_wait_ready 30; then + warn "Postgres is not answering — starting anyway, but Verify may report failures" + fi + if OUT="$(pm2_start)"; then ok "processes started" pm2_save >/dev/null 2>&1 && ok "process list saved (survives a pm2 restart)" diff --git a/src/databases/officer_db/src/db.ts b/src/databases/officer_db/src/db.ts index 7bf48716..a39fd41f 100644 --- a/src/databases/officer_db/src/db.ts +++ b/src/databases/officer_db/src/db.ts @@ -10,3 +10,47 @@ if (!POSTGRES_URL) { const client = postgres(POSTGRES_URL); export const db = drizzle(client, { schema }); + +/** + * Block until Postgres answers, or give up after `timeoutMs`. + * + * `postgres()` above is LAZY — it opens no socket until the first query — so nothing here fails at + * import time when the database is not up yet. That is the right default and it has a cost: startup + * work that queries fires, fails once, and is swallowed by whatever `.catch()` it was written with. + * + * That is not hypothetical. `server.tsx` runs `initQueue()` and `cleanupOnStartup()` as + * fire-and-forget promises, and the second marks jobs that were interrupted by the last restart and + * promotes the queued backlog. If Postgres is a few seconds behind — which is exactly what happens on + * a reboot, when pm2's resurrect races Docker starting the container — both log a line and do nothing. + * Interrupted jobs then stay marked running forever, because the only thing that would have corrected + * them already ran. + * + * So: wait, rather than try once. Callers that genuinely cannot proceed without the database await + * this first; request handlers do not, since by then it is either up or the request fails honestly. + * + * Bounded, and it resolves false rather than throwing on timeout. An unbounded wait here would hold a + * process open with no way to tell whether it is starting or hung, and the caller is better placed to + * decide what "gave up" means than this function is. + */ +export async function waitForDatabase(timeoutMs = 60_000): Promise { + const started = Date.now(); + let announced = false; + + for (;;) { + try { + await client`select 1`; + if (announced) console.log('[db] Postgres is up'); + return true; + } catch (err) { + if (Date.now() - started >= timeoutMs) { + console.error(`[db] Postgres did not answer within ${Math.round(timeoutMs / 1000)}s:`, err instanceof Error ? err.message : err); + return false; + } + if (!announced) { + console.log('[db] waiting for Postgres…'); + announced = true; + } + await new Promise((r) => setTimeout(r, 1_000)); + } + } +} diff --git a/src/databases/officer_db/src/index.ts b/src/databases/officer_db/src/index.ts index 40d72e4a..c64823b4 100644 --- a/src/databases/officer_db/src/index.ts +++ b/src/databases/officer_db/src/index.ts @@ -21,7 +21,7 @@ // blank line below is the only thing marking the boundary. // The connection, and drizzle-kit's view of the schema. See ./schema.ts for the core/plugin split. -export { db } from './db'; +export { db, waitForDatabase } from './db'; export * as schema from './schema'; // ── Core ───────────────────────────────────────────────────────────────────────────────────────── diff --git a/src/server.tsx b/src/server.tsx index 7957e019..8615ccbf 100644 --- a/src/server.tsx +++ b/src/server.tsx @@ -382,11 +382,31 @@ import { listAllJobs as queueList, readJob as queueGet, } from './servers/queue/init'; -initQueue().catch((err) => console.error('[queue] failed to initialize:', err)); - -// Mark any orphaned pipeline jobs from previous server run import { cleanupOnStartup } from './servers/api/tasks/pipeline-job-manager'; -cleanupOnStartup().catch((err) => console.error('[pipeline-jobs] startup cleanup failed:', err)); +import { waitForDatabase } from 'officerdb'; + +// ── Startup work that needs the database, and waits for it ── +// +// Both of these query Postgres, and both used to be fired as bare promises with a `.catch()` that +// logged. That is fine when the database is up and silently wrong when it is not — which is precisely +// what a reboot looks like, with pm2's resurrect racing Docker starting the Postgres container. +// +// `cleanupOnStartup` is the one that matters: it marks jobs interrupted by the previous shutdown and +// promotes the queued backlog. Fail it once and those jobs stay marked running forever, because the +// only thing that would have corrected them has already run. Nothing retries and nothing complains +// again — the log line scrolls past during boot and the jobs are simply stuck. +// +// Deliberately NOT awaited before serve(): the HTTP listener is already up by here, and holding it +// closed for a minute would turn a database that is thirty seconds late into a reverse proxy serving +// connection-refused instead of a page. Requests that need the database fail honestly in the meantime. +void (async () => { + if (!(await waitForDatabase())) { + console.error('[startup] skipping queue init and pipeline cleanup — the database never answered'); + return; + } + await initQueue().catch((err) => console.error('[queue] failed to initialize:', err)); + await cleanupOnStartup().catch((err) => console.error('[pipeline-jobs] startup cleanup failed:', err)); +})(); // PulseAudio and the `virtual_out` sink used to be set up here, at every boot of a process that has no // audio responsibilities. They belong to the music sidecar, which owns both cliamp halves now