From 5c3ffa7a0681cb3d4b2888bcdb035b3b10887ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Padez?= Date: Sun, 23 Aug 2026 00:20:56 +0000 Subject: [PATCH] the watcher loop lives next to the claim it depends on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit prompts/deploy-commander.md treats the poll loop as something an agent rewrites from scratch on every arming. That is also how the same defect keeps arriving: the loop left at data/deploy-watch.sh gated on "$v.done" — a file written only on SUCCESS — and never called deploy-claim.sh at all. A failing release therefore spawned a fresh paid agent every poll. Measured against that script with a stub agent: 10 spawns in 20 seconds at a 2s poll, which is 30 an hour at its real one, all night. That is exactly the storm deploy-claim.sh exists to make impossible, reintroduced in the one place the claim cannot defend — the caller that decides whether to call it. So the loop moves into the repository beside it. The baseline is the claim, not the success: a release that failed is handled, and handled means a human looks at it. Arming today fires on nothing, because all six existing releases carry a .done and deploy-claim.sh honours one. The claim is taken before the spawn, which means the child has to be told it holds one. It is about to read the procedure, whose claim instruction is addressed to whoever reads it — an obedient child claims for itself, gets exit 1, and correctly refuses the deploy it was started to perform. Co-Authored-By: Claude Opus 5 --- scripts/deploy-watch.sh | 144 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100755 scripts/deploy-watch.sh diff --git a/scripts/deploy-watch.sh b/scripts/deploy-watch.sh new file mode 100755 index 00000000..f3169cab --- /dev/null +++ b/scripts/deploy-watch.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# +# The wait half of the deploy commander: poll for a release tag, and when one appears that has not been +# attempted, spawn a headless agent to deploy it. +# +# WHY THIS IS IN THE REPOSITORY AND NOT WRITTEN FRESH EACH ARMING +# +# prompts/deploy-commander.md says the watcher loop is rewritten from scratch every time one arms, and +# treats that as a given. It is also how the same defect keeps arriving. The loop that was on disk at +# data/deploy-watch.sh from 2026-08-21 gated on "$v.done" — a file that exists only on SUCCESS — and never +# called deploy-claim.sh at all. That is precisely the retry storm the claim script was written to make +# impossible, reintroduced in the one place the claim script cannot reach: the caller that decides whether +# to invoke it. A rule enforced by a script nobody calls is not enforced. +# +# So the loop lives next to the claim it depends on, and changes with it. +# +# THE BASELINE IS THE CLAIM, NOT THE SUCCESS +# +# A watcher needs to know what counts as new. Here that is "no claim directory for this version" — a record +# of what has been HANDLED, not of what has SUCCEEDED. The distinction is the whole point: a release that +# failed is handled. It is looked at by a human, not retried by a loop. +# +# deploy-claim.sh honours a pre-existing .done with no .claim, so releases deployed before it existed are +# already skipped and arming this today fires on none of them. +# +# THE CLAIM IS TAKEN HERE, BEFORE THE SPAWN +# +# An action that claims its own work has claimed nothing if it dies before getting that far. So the loop +# claims, then spawns — which means the child must be TOLD the claim is already held. It is about to read +# prompts/deploy-commander.md, whose claim instruction is addressed to whoever reads it, and an obedient +# child that claims for itself gets exit 1 and correctly refuses to deploy. See PROMPT below. +set -u + +PLATFORM_DIR="${PLATFORM_DIR:-$HOME/officer-dev/platform}" +# Empty in the sidecar's environment, where anything built from it resolves to /data/... and the record +# lands where nothing reads it. A watcher whose baseline is invisible to itself has none. +OFFICER_ROOT="${OFFICER_ROOT:-$HOME/officer-dev}" +DEPLOYS_DIR="${DEPLOYS_DIR:-$OFFICER_ROOT/data/deploys}" +POLL_SECONDS="${POLL_SECONDS:-120}" +export OFFICER_ROOT DEPLOYS_DIR + +CLAIM_SH="$PLATFORM_DIR/scripts/deploy-claim.sh" + +stamp() { date -Iseconds; } + +if [ ! -x "$CLAIM_SH" ]; then + echo "$(stamp) deploy-watch: no claim script at $CLAIM_SH — refusing to arm without one" >&2 + exit 2 +fi + +mkdir -p "$DEPLOYS_DIR" || exit 2 + +echo "$(stamp) deploy-watch: armed. polling $(cd "$PLATFORM_DIR" && git remote get-url origin) every ${POLL_SECONDS}s" +echo "$(stamp) deploy-watch: baseline is $DEPLOYS_DIR — a version with a claim is never fired again" + +while true; do + # origin is the LOCAL gitea over the tailnet. The public hostname arrives through a proxy where every + # connection shares one source IP and SSH is refused for long stretches — which presents as an + # access-rights error and is not one. Eighty minutes of detection latency on 2026-08-21. + raw=$(cd "$PLATFORM_DIR" && timeout 30 git ls-remote --tags origin 'refs/tags/deploy/*' 2>&1) + rc=$? + + if [ $rc -ne 0 ]; then + echo "$(stamp) tick: ls-remote failed (exit $rc) — ${raw%%$'\n'*}" + sleep "$POLL_SECONDS" + continue + fi + + versions=$(printf '%s\n' "$raw" | grep -v '\^{}' | awk '{print $2}' | sed -E 's#.*refs/tags/deploy/##' | sort -u) + + pending="" + for v in $versions; do + [ -n "$v" ] || continue + [ "$("$CLAIM_SH" state "$v" 2>/dev/null)" = unclaimed ] && pending="$pending $v" + done + + # Every tick leaves a line, including the quiet ones: a fresh timestamp is the only thing that tells + # "nothing to deploy" apart from "the loop died". + echo "$(stamp) tick: $(printf '%s\n' "$versions" | grep -c .) tags,${pending:- none} pending" + + for v in $pending; do + if ! "$CLAIM_SH" claim "$v"; then + # Lost a race with another watcher, or it was claimed between the state check and here. Either way + # it is not ours and never becomes ours. + continue + fi + + echo "$(stamp) deploy/$v claimed — spawning deploy agent" + + # The child reads the procedure, which tells ITS reader to claim first. It must not: we hold the claim + # on its behalf, and its own claim would return exit 1 and stop the deploy it was spawned to do. + PROMPT="Read prompts/deploy-commander.md in $PLATFORM_DIR and follow it exactly, for the release deploy/$v. + +The claim for $v is ALREADY HELD on your behalf by the watcher that spawned you. Do NOT run +'scripts/deploy-claim.sh claim $v' — it will refuse, correctly, because the claim is taken, and stopping +there would abandon a deploy you were started to perform. Everything else in that file applies unchanged, +including running 'scripts/deploy-claim.sh done $v' once the rollout has succeeded. + +Roll it out across every host the plan names, skip the host you are running on, and write the report +exactly as the procedure specifies." + + raw_log="$DEPLOYS_DIR/$v.child.jsonl" + + # Nothing links a spawned agent to its parent and a child transcript does not last, so the stream is + # kept here as it arrives. stderr never enters the pipe: the CLI writes warnings there and one landing + # in the stream corrupts the first jq parse. + { + cd "$PLATFORM_DIR" && claude -p "$PROMPT" \ + --dangerously-skip-permissions \ + --output-format=stream-json \ + --verbose + } 2> >(while IFS= read -r l; do echo " ! $l"; done) \ + | tee -a "$raw_log" \ + | jq -r --unbuffered ' + if .type=="system" and .subtype=="init" then + " · child session " + .session_id + elif .type=="assistant" then + (.message.content[]? | + if .type=="tool_use" then + " ⚙ " + .name + ": " + ((.input.description // .input.command // .input.pattern // (.input | tostring))) + elif .type=="text" then + " » " + .text + else empty end) + elif .type=="user" then + (.message.content[]? | select(.type=="tool_result") | + " " + (if (.content|type)=="string" then .content else (.content // [] | map(.text? // "") | join(" ")) end)) + elif .type=="result" then + " = deploy agent " + .subtype + ", " + (.num_turns|tostring) + " turns, " + (.total_cost_usd|tostring) + " usd" + else empty end + ' 2>/dev/null + + if [ -f "$DEPLOYS_DIR/$v.done" ]; then + echo "$(stamp) deploy/$v SUCCEEDED — recorded, transcript at $raw_log" + else + # The claim stays held. This is the loop's most important line and the one that used to say + # "will retry next poll". + echo "$(stamp) deploy/$v FAILED — no .done written. NOT retrying; the claim is held." + echo "$(stamp) transcript: $raw_log" + echo "$(stamp) to allow another attempt, deliberately: rm -rf $DEPLOYS_DIR/$v.claim" + fi + done + + sleep "$POLL_SECONDS" +done