Recertify empty-search fix from authenticated push #168
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Agent Guild — machine-operated ship loop (corrective pass 2026-07-22, r3). | |
| # | |
| # r3 root cause (found by LIVE operation, not review): workflow_run events are | |
| # NOT emitted for workflow runs that were themselves dispatched with | |
| # GITHUB_TOKEN — GitHub's recursion guard swallows the completion event, so an | |
| # auto-merge leg hung off `workflow_run` never fires for the ci runs this loop | |
| # dispatches. The r3 design removes that dependency: the push-triggered ship | |
| # run (always a USER event — pushes come from an authenticated session, never | |
| # from GITHUB_TOKEN) drives the ENTIRE loop inline: | |
| # | |
| # push ship/<topic> | |
| # → open (or reuse) the PR to main | |
| # → dispatch the full `ci` workflow on the branch and WAIT for that exact | |
| # run (workflow_dispatch is the documented exception that DOES create | |
| # runs from GITHUB_TOKEN) | |
| # → decide via the UNIT-TESTED live/scripts/ship_decision.py: | |
| # merge head == certified SHA and contains main | |
| # update_and_recertify main advanced: update branch, dispatch ci | |
| # again, wait again — merge only the COMBINED | |
| # state (bounded rounds, never silent) | |
| # refuse_unprotected protection on main is MANDATORY — halt + issue | |
| # refuse_head_mismatch a newer push owns its own run — stop here | |
| # → merge (squash) → check out THE EXACT MERGED SHA → deployment-aware | |
| # release gate from that tree → upload attestation → dispatch registry | |
| # publish whenever the exact local registry version is not yet served | |
| # → red gate: machine-complete recovery — revert branch + PR + ci wait + | |
| # merge + RE-GATE the recovery, all inline. The issue filed is | |
| # telemetry, never the mechanism. A failed recovery halts (no | |
| # oscillation). | |
| # | |
| # No human anywhere in the loop. Autonomous sessions: push `ship/<topic>` and | |
| # stop. Direct pushes to main are refused by branch protection. | |
| name: ship | |
| on: | |
| push: | |
| branches: ["ship/**"] | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| actions: write | |
| issues: write | |
| concurrency: | |
| group: release-gate-production | |
| cancel-in-progress: false | |
| jobs: | |
| ship: | |
| if: "github.event_name == 'push' && !startsWith(github.ref_name, 'ship/revert-')" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: open PR, certify via dispatched ci, merge only the certified combined state | |
| id: merge | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BRANCH: ${{ github.ref_name }} | |
| REPO: ${{ github.repository }} | |
| run: | | |
| set -euo pipefail | |
| echo "merged=false" >> "$GITHUB_OUTPUT" | |
| existing=$(gh pr list --head "$BRANCH" --base main --state open \ | |
| --json number --jq '.[0].number // empty') | |
| if [ -z "$existing" ]; then | |
| title=$(git log -1 --format=%s) | |
| gh pr create --base main --head "$BRANCH" \ | |
| --title "$title" \ | |
| --body "$(printf 'Machine-operated ship branch `%s`.\n\nThe ship workflow dispatches the full `ci` matrix, waits for it, merges only the certified combined state with main, then runs the deployment-aware release gate against the merged SHA — with automatic certified revert on a red gate. See .github/workflows/ship.yml.' "$BRANCH")" | |
| fi | |
| number=$(gh pr list --head "$BRANCH" --base main --state open \ | |
| --json number --jq '.[0].number') | |
| dispatch_and_wait() { | |
| # dispatch ci on $BRANCH and wait for THAT run; echoes the run's | |
| # head SHA on success, fails the function on a red run. | |
| local before rid tries | |
| before=$(gh run list --workflow=ci.yml --branch "$BRANCH" \ | |
| --event workflow_dispatch --limit 1 \ | |
| --json databaseId --jq '.[0].databaseId // 0') | |
| # r4: EVERYTHING except the final SHA goes to stderr — gh's | |
| # informational stdout (a URL, found live in ship #3 attempt 2) | |
| # otherwise contaminates the captured "certified" value. | |
| gh workflow run ci.yml --ref "$BRANCH" >&2 | |
| rid="$before"; tries=0 | |
| while [ "$rid" = "$before" ] && [ $tries -lt 24 ]; do | |
| sleep 5; tries=$((tries+1)) | |
| rid=$(gh run list --workflow=ci.yml --branch "$BRANCH" \ | |
| --event workflow_dispatch --limit 1 \ | |
| --json databaseId --jq '.[0].databaseId // 0') | |
| done | |
| [ "$rid" != "$before" ] || { echo "dispatched ci run never appeared" >&2; return 1; } | |
| echo "waiting on ci run $rid" >&2 | |
| gh run watch "$rid" --exit-status --interval 15 >&2 | |
| gh run view "$rid" --json headSha --jq .headSha | |
| } | |
| for round in 1 2 3; do | |
| echo "=== certification round $round ===" | |
| certified=$(dispatch_and_wait) || { | |
| echo "ci is RED for $BRANCH — nothing merges. Fix and push again."; exit 1; } | |
| # r4: accept ONLY a 40-hex SHA; anything else is stdout noise | |
| certified=$(printf '%s\n' "$certified" | grep -E '^[0-9a-f]{40}$' | tail -n1 || true) | |
| if [ -z "$certified" ]; then | |
| echo "could not extract a certified SHA from the ci wait — refusing to decide on garbage" | |
| exit 1 | |
| fi | |
| git fetch --force origin main "refs/heads/$BRANCH" | |
| head=$(gh pr view "$number" --json headRefOid --jq .headRefOid) | |
| main_sha=$(git rev-parse origin/main) | |
| if git merge-base --is-ancestor "$main_sha" "$head" 2>/dev/null; then | |
| ancestor=true; else ancestor=false; fi | |
| protected=$(gh api "repos/$REPO/branches/main" --jq .protected) | |
| decision=$(python3 live/scripts/ship_decision.py merge \ | |
| --protected "$protected" --pr-head "$head" \ | |
| --certified "$certified" --main-is-ancestor "$ancestor") | |
| action=$(echo "$decision" | jq -r .action) | |
| echo "decision: $decision" | |
| case "$action" in | |
| merge) | |
| if gh pr merge "$number" --squash --delete-branch; then | |
| merged_sha=$(gh pr view "$number" --json mergeCommit --jq .mergeCommit.oid) | |
| echo "merged PR #$number → main @ $merged_sha" | |
| echo "merged=true" >> "$GITHUB_OUTPUT" | |
| echo "merged_sha=$merged_sha" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| echo "merge refused by GitHub (raced) — updating and re-certifying" | |
| gh api -X PUT "repos/$REPO/pulls/$number/update-branch" \ | |
| -f expected_head_sha="$head" || true | |
| continue ;; | |
| update_and_recertify) | |
| if ! gh api -X PUT "repos/$REPO/pulls/$number/update-branch" \ | |
| -f expected_head_sha="$head"; then | |
| gh issue create --title "SHIP BLOCKED: cannot auto-update $BRANCH with main" \ | |
| --body "PR #$number is behind main and the automatic update failed (likely a conflict). The next autonomous session should rebase and push $BRANCH again — the loop takes it from there." | |
| exit 1 | |
| fi | |
| continue ;; | |
| refuse_unprotected) | |
| gh issue list --state open --search "SHIP LOOP HALTED: main unprotected" \ | |
| --json number --jq '.[0].number // empty' | grep -q . || \ | |
| gh issue create --title "SHIP LOOP HALTED: main unprotected" \ | |
| --body "The ship workflow refuses to merge while main has no branch protection. Run live/scripts/protect_main.sh (required PRs + required non-release ci checks + strict up-to-date + enforce_admins). Re-run this workflow run once protection is on." | |
| echo "halting: protection is mandatory"; exit 1 ;; | |
| refuse_head_mismatch) | |
| echo "a newer push owns its own ship run — stopping this one"; exit 0 ;; | |
| *) echo "unknown action $action"; exit 1 ;; | |
| esac | |
| done | |
| gh issue create --title "SHIP BLOCKED: $BRANCH could not certify a combined state in 3 rounds" \ | |
| --body "main kept advancing (or updates kept failing) during certification of PR #$number. Investigate and push the branch again." | |
| exit 1 | |
| - name: check out THE EXACT MERGED SHA (the gate must run from the code production serves) | |
| if: steps.merge.outputs.merged == 'true' | |
| run: | | |
| set -euo pipefail | |
| git fetch --force origin main | |
| git checkout --force "${{ steps.merge.outputs.merged_sha }}" | |
| git rev-parse HEAD | |
| - if: steps.merge.outputs.merged == 'true' | |
| run: pip install -r live/guild/requirements.txt -r live/trustplane/requirements/core.txt | |
| - name: deployment-aware release gate (production must serve the merged SHA) | |
| id: release_gate | |
| if: steps.merge.outputs.merged == 'true' | |
| run: | | |
| python live/scripts/release_gate.py \ | |
| --sha "${{ steps.merge.outputs.merged_sha }}" \ | |
| --timeout 1800 --interval 20 \ | |
| --attestation release_attestation.json | |
| - name: upload release attestation (machine-readable) | |
| if: always() && steps.merge.outputs.merged == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: release-attestation-${{ steps.merge.outputs.merged_sha }} | |
| path: release_attestation.json | |
| if-no-files-found: warn | |
| - name: decide whether the exact MCP Registry version still needs publication | |
| id: registry | |
| if: steps.release_gate.outcome == 'success' | |
| run: | | |
| python live/scripts/registry_publish_needed.py \ | |
| --server-json server.json \ | |
| --server-json registry/x402-payment-safety/server.json \ | |
| --github-output "$GITHUB_OUTPUT" | |
| - name: dispatch registry publish (exact local version is not served) | |
| if: steps.registry.outputs.needed == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh workflow run publish-mcp.yml --ref main | |
| - name: red gate → machine-complete recovery (certified revert, merged and re-gated inline) | |
| if: failure() && steps.release_gate.outcome == 'failure' && steps.merge.outputs.merged == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| BRANCH: ${{ github.ref_name }} | |
| REPO: ${{ github.repository }} | |
| MERGED_SHA: ${{ steps.merge.outputs.merged_sha }} | |
| # r5 (2026-08-31): optional fine-grained PAT (this repo only; | |
| # Contents + Workflows read/write) stored as the SHIP_RECOVERY_TOKEN | |
| # actions secret. A GITHUB_TOKEN push cannot create the push-event | |
| # ci runs that branch protection recognises (recursion guard), so a | |
| # bot-pushed ship/revert-* branch leaves every required context stuck | |
| # at "Expected" even after a GREEN dispatched ci run — observed live | |
| # on PR #168, unblocked that day by a manual user-authenticated | |
| # empty commit. Pushing the revert branch WITH this token makes the | |
| # push a user event: ci starts on push and reports the required | |
| # contexts exactly as it does for a normal ship branch. Absent the | |
| # secret, the old dispatch path still runs and the stall is reported | |
| # with the manual unblock instructions. | |
| RECOVERY_PUSH_TOKEN: ${{ secrets.SHIP_RECOVERY_TOKEN }} | |
| run: | | |
| set -euo pipefail | |
| # The gate records WHY it failed. A revert is only correct when the | |
| # merged code is LIVE and defective; if the deployment never | |
| # arrived, production still serves the previous release and a revert | |
| # would change nothing live while pushing a second build through the | |
| # pipeline that just timed out. | |
| gate_outcome=$(jq -r '.outcome // ""' release_attestation.json 2>/dev/null || echo "") | |
| echo "gate outcome: ${gate_outcome:-unknown}" | |
| decision=$(python3 live/scripts/ship_decision.py recover \ | |
| --failed-branch "$BRANCH" --gate-outcome "$gate_outcome") | |
| action=$(echo "$decision" | jq -r .action) | |
| short=${MERGED_SHA:0:9} | |
| gh issue create \ | |
| --title "RED RELEASE: gate failed for $MERGED_SHA" \ | |
| --body "The ship workflow merged $MERGED_SHA but the deployment-aware release gate did NOT certify production. Recovery decision: $decision. Run: ${{ github.server_url }}/$REPO/actions/runs/${{ github.run_id }}. This issue is telemetry — recovery is the automatic certified revert below (if any)." || true | |
| if [ "$action" != "revert" ]; then | |
| echo "halting: $decision"; exit 0 | |
| fi | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git fetch --force origin main | |
| git checkout -B "ship/revert-$short" origin/main | |
| git revert --no-edit "$MERGED_SHA" | |
| if [ -n "${RECOVERY_PUSH_TOKEN:-}" ]; then | |
| # User-authenticated push: ci.yml (push: ship/**) starts on its | |
| # own and reports the required contexts on this exact head SHA. | |
| # The ship job's own ship/revert-* guard keeps the loop acyclic. | |
| git push --force \ | |
| "https://x-access-token:${RECOVERY_PUSH_TOKEN}@github.com/$REPO.git" \ | |
| "ship/revert-$short" | |
| else | |
| git push --force origin "ship/revert-$short" | |
| fi | |
| gh pr create --base main --head "ship/revert-$short" \ | |
| --title "Automatic certified revert of red release $short" \ | |
| --body "The release gate failed for $MERGED_SHA. This revert is certified, merged and re-gated by the SAME run that detected the failure. No human is required." || true | |
| rnumber=$(gh pr list --head "ship/revert-$short" --base main --state open \ | |
| --json number --jq '.[0].number') | |
| if [ -n "${RECOVERY_PUSH_TOKEN:-}" ]; then | |
| # Wait for the push-event ci run born from the authenticated push. | |
| rid=0; tries=0 | |
| while [ "$rid" = "0" ] && [ $tries -lt 24 ]; do | |
| sleep 5; tries=$((tries+1)) | |
| rid=$(gh run list --workflow=ci.yml --branch "ship/revert-$short" \ | |
| --event push --limit 1 --json databaseId --jq '.[0].databaseId // 0') | |
| done | |
| [ "$rid" != "0" ] || { | |
| echo "push-event ci run never appeared for ship/revert-$short (is SHIP_RECOVERY_TOKEN valid?)" >&2 | |
| exit 1; } | |
| else | |
| before=$(gh run list --workflow=ci.yml --branch "ship/revert-$short" \ | |
| --event workflow_dispatch --limit 1 --json databaseId --jq '.[0].databaseId // 0') | |
| gh workflow run ci.yml --ref "ship/revert-$short" >&2 | |
| rid="$before"; tries=0 | |
| while [ "$rid" = "$before" ] && [ $tries -lt 24 ]; do | |
| sleep 5; tries=$((tries+1)) | |
| rid=$(gh run list --workflow=ci.yml --branch "ship/revert-$short" \ | |
| --event workflow_dispatch --limit 1 --json databaseId --jq '.[0].databaseId // 0') | |
| done | |
| [ "$rid" != "$before" ] || { echo "dispatched ci run never appeared" >&2; exit 1; } | |
| fi | |
| gh run watch "$rid" --exit-status --interval 15 | |
| # CI can be green a few seconds before branch-policy state converges. | |
| # Queue the certified revert under the policy, then wait boundedly | |
| # for GitHub to perform the merge. Never use --admin or bypass the | |
| # protections that the normal ship path just proved. | |
| gh pr merge "$rnumber" --squash --delete-branch --auto | |
| recovery_sha=""; recovery_state="OPEN"; tries=0 | |
| while [ -z "$recovery_sha" ] && [ $tries -lt 120 ]; do | |
| sleep 5; tries=$((tries+1)) | |
| recovery_state=$(gh pr view "$rnumber" --json state --jq .state) | |
| recovery_sha=$(gh pr view "$rnumber" --json mergeCommit \ | |
| --jq '.mergeCommit.oid // empty') | |
| if [ "$recovery_state" = "CLOSED" ] && [ -z "$recovery_sha" ]; then | |
| echo "certified recovery PR closed without merging"; exit 1 | |
| fi | |
| done | |
| [ -n "$recovery_sha" ] || { | |
| gh issue create --title "RECOVERY MERGE STALLED: ship/revert-$short (PR #$rnumber)" \ | |
| --body "The certified revert PR did not merge within 10 minutes. Most likely cause when SHIP_RECOVERY_TOKEN is not configured: the revert branch was pushed with GITHUB_TOKEN, so branch protection never received push-event required contexts (observed on PR #168). Unblock now: from a user-authenticated checkout, \`git commit --allow-empty -m 'ci: attach protected rollback checks' && git push origin ship/revert-$short\`. Fix permanently: add a fine-grained PAT (this repo; Contents + Workflows read/write) as the SHIP_RECOVERY_TOKEN actions secret — the recovery path then pushes revert branches as a user event automatically." || true | |
| echo "certified recovery PR did not merge within 10 minutes"; exit 1; } | |
| echo "recovery merged → main @ $recovery_sha — certifying the RECOVERY" | |
| python live/scripts/release_gate.py \ | |
| --sha "$recovery_sha" --timeout 1800 --interval 20 \ | |
| --attestation recovery_attestation.json || { | |
| gh issue create --title "RECOVERY GATE ALSO RED: $recovery_sha" \ | |
| --body "The certified revert of $MERGED_SHA merged as $recovery_sha but the recovery gate did not certify production either. Halting (no revert-of-revert). Manual-free next step: the next autonomous session pushes a fresh ship/ branch with a fix."; exit 1; } | |
| # A deployment can legitimately arrive after the bounded ship gate. This | |
| # dispatchable path certifies the CURRENT main SHA and finishes a stranded | |
| # registry publication without inventing a new release or bypassing a gate. | |
| recover_deployment: | |
| if: github.event_name == 'workflow_dispatch' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| actions: write | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| fetch-depth: 0 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: identify the exact current main SHA | |
| id: expected | |
| run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" | |
| - run: pip install -r live/guild/requirements.txt -r live/trustplane/requirements/core.txt | |
| - name: certify production against the exact current main SHA | |
| id: release_gate | |
| run: | | |
| python live/scripts/release_gate.py \ | |
| --sha "${{ steps.expected.outputs.sha }}" \ | |
| --timeout 1800 --interval 20 \ | |
| --attestation release_attestation.json | |
| - name: upload recovery release attestation | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: release-recovery-attestation-${{ steps.expected.outputs.sha }} | |
| path: release_attestation.json | |
| if-no-files-found: warn | |
| - name: decide whether the exact MCP Registry version still needs publication | |
| id: registry | |
| if: steps.release_gate.outcome == 'success' | |
| run: | | |
| python live/scripts/registry_publish_needed.py \ | |
| --server-json server.json \ | |
| --server-json registry/x402-payment-safety/server.json \ | |
| --github-output "$GITHUB_OUTPUT" | |
| - name: dispatch registry publish (exact local version is not served) | |
| if: steps.registry.outputs.needed == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh workflow run publish-mcp.yml --ref main |