diff --git a/scripts/deploy-watch.sh b/scripts/deploy-watch.sh new file mode 100755 index 00000000..f3169cab --- /dev/null +++ b/scripts/deploy-watch.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# +# The wait half of the deploy commander: poll for a release tag, and when one appears that has not been +# attempted, spawn a headless agent to deploy it. +# +# WHY THIS IS IN THE REPOSITORY AND NOT WRITTEN FRESH EACH ARMING +# +# prompts/deploy-commander.md says the watcher loop is rewritten from scratch every time one arms, and +# treats that as a given. It is also how the same defect keeps arriving. The loop that was on disk at +# data/deploy-watch.sh from 2026-08-21 gated on "$v.done" — a file that exists only on SUCCESS — and never +# called deploy-claim.sh at all. That is precisely the retry storm the claim script was written to make +# impossible, reintroduced in the one place the claim script cannot reach: the caller that decides whether +# to invoke it. A rule enforced by a script nobody calls is not enforced. +# +# So the loop lives next to the claim it depends on, and changes with it. +# +# THE BASELINE IS THE CLAIM, NOT THE SUCCESS +# +# A watcher needs to know what counts as new. Here that is "no claim directory for this version" — a record +# of what has been HANDLED, not of what has SUCCEEDED. The distinction is the whole point: a release that +# failed is handled. It is looked at by a human, not retried by a loop. +# +# deploy-claim.sh honours a pre-existing .done with no .claim, so releases deployed before it existed are +# already skipped and arming this today fires on none of them. +# +# THE CLAIM IS TAKEN HERE, BEFORE THE SPAWN +# +# An action that claims its own work has claimed nothing if it dies before getting that far. So the loop +# claims, then spawns — which means the child must be TOLD the claim is already held. It is about to read +# prompts/deploy-commander.md, whose claim instruction is addressed to whoever reads it, and an obedient +# child that claims for itself gets exit 1 and correctly refuses to deploy. See PROMPT below. +set -u + +PLATFORM_DIR="${PLATFORM_DIR:-$HOME/officer-dev/platform}" +# Empty in the sidecar's environment, where anything built from it resolves to /data/... and the record +# lands where nothing reads it. A watcher whose baseline is invisible to itself has none. +OFFICER_ROOT="${OFFICER_ROOT:-$HOME/officer-dev}" +DEPLOYS_DIR="${DEPLOYS_DIR:-$OFFICER_ROOT/data/deploys}" +POLL_SECONDS="${POLL_SECONDS:-120}" +export OFFICER_ROOT DEPLOYS_DIR + +CLAIM_SH="$PLATFORM_DIR/scripts/deploy-claim.sh" + +stamp() { date -Iseconds; } + +if [ ! -x "$CLAIM_SH" ]; then + echo "$(stamp) deploy-watch: no claim script at $CLAIM_SH — refusing to arm without one" >&2 + exit 2 +fi + +mkdir -p "$DEPLOYS_DIR" || exit 2 + +echo "$(stamp) deploy-watch: armed. polling $(cd "$PLATFORM_DIR" && git remote get-url origin) every ${POLL_SECONDS}s" +echo "$(stamp) deploy-watch: baseline is $DEPLOYS_DIR — a version with a claim is never fired again" + +while true; do + # origin is the LOCAL gitea over the tailnet. The public hostname arrives through a proxy where every + # connection shares one source IP and SSH is refused for long stretches — which presents as an + # access-rights error and is not one. Eighty minutes of detection latency on 2026-08-21. + raw=$(cd "$PLATFORM_DIR" && timeout 30 git ls-remote --tags origin 'refs/tags/deploy/*' 2>&1) + rc=$? + + if [ $rc -ne 0 ]; then + echo "$(stamp) tick: ls-remote failed (exit $rc) — ${raw%%$'\n'*}" + sleep "$POLL_SECONDS" + continue + fi + + versions=$(printf '%s\n' "$raw" | grep -v '\^{}' | awk '{print $2}' | sed -E 's#.*refs/tags/deploy/##' | sort -u) + + pending="" + for v in $versions; do + [ -n "$v" ] || continue + [ "$("$CLAIM_SH" state "$v" 2>/dev/null)" = unclaimed ] && pending="$pending $v" + done + + # Every tick leaves a line, including the quiet ones: a fresh timestamp is the only thing that tells + # "nothing to deploy" apart from "the loop died". + echo "$(stamp) tick: $(printf '%s\n' "$versions" | grep -c .) tags,${pending:- none} pending" + + for v in $pending; do + if ! "$CLAIM_SH" claim "$v"; then + # Lost a race with another watcher, or it was claimed between the state check and here. Either way + # it is not ours and never becomes ours. + continue + fi + + echo "$(stamp) deploy/$v claimed — spawning deploy agent" + + # The child reads the procedure, which tells ITS reader to claim first. It must not: we hold the claim + # on its behalf, and its own claim would return exit 1 and stop the deploy it was spawned to do. + PROMPT="Read prompts/deploy-commander.md in $PLATFORM_DIR and follow it exactly, for the release deploy/$v. + +The claim for $v is ALREADY HELD on your behalf by the watcher that spawned you. Do NOT run +'scripts/deploy-claim.sh claim $v' — it will refuse, correctly, because the claim is taken, and stopping +there would abandon a deploy you were started to perform. Everything else in that file applies unchanged, +including running 'scripts/deploy-claim.sh done $v' once the rollout has succeeded. + +Roll it out across every host the plan names, skip the host you are running on, and write the report +exactly as the procedure specifies." + + raw_log="$DEPLOYS_DIR/$v.child.jsonl" + + # Nothing links a spawned agent to its parent and a child transcript does not last, so the stream is + # kept here as it arrives. stderr never enters the pipe: the CLI writes warnings there and one landing + # in the stream corrupts the first jq parse. + { + cd "$PLATFORM_DIR" && claude -p "$PROMPT" \ + --dangerously-skip-permissions \ + --output-format=stream-json \ + --verbose + } 2> >(while IFS= read -r l; do echo " ! $l"; done) \ + | tee -a "$raw_log" \ + | jq -r --unbuffered ' + if .type=="system" and .subtype=="init" then + " · child session " + .session_id + elif .type=="assistant" then + (.message.content[]? | + if .type=="tool_use" then + " ⚙ " + .name + ": " + ((.input.description // .input.command // .input.pattern // (.input | tostring))) + elif .type=="text" then + " » " + .text + else empty end) + elif .type=="user" then + (.message.content[]? | select(.type=="tool_result") | + " " + (if (.content|type)=="string" then .content else (.content // [] | map(.text? // "") | join(" ")) end)) + elif .type=="result" then + " = deploy agent " + .subtype + ", " + (.num_turns|tostring) + " turns, " + (.total_cost_usd|tostring) + " usd" + else empty end + ' 2>/dev/null + + if [ -f "$DEPLOYS_DIR/$v.done" ]; then + echo "$(stamp) deploy/$v SUCCEEDED — recorded, transcript at $raw_log" + else + # The claim stays held. This is the loop's most important line and the one that used to say + # "will retry next poll". + echo "$(stamp) deploy/$v FAILED — no .done written. NOT retrying; the claim is held." + echo "$(stamp) transcript: $raw_log" + echo "$(stamp) to allow another attempt, deliberately: rm -rf $DEPLOYS_DIR/$v.claim" + fi + done + + sleep "$POLL_SECONDS" +done