diff --git a/.github/scripts/build_plan.py b/.github/scripts/build_plan.py index 22ceb38e6b..603aa68b00 100644 --- a/.github/scripts/build_plan.py +++ b/.github/scripts/build_plan.py @@ -13,6 +13,7 @@ log, parse_json_object, resolve_build_selection, + write_step_summary, ) @@ -67,6 +68,22 @@ def main() -> None: + ", ".join(item["coins"]) ) + summary_lines = [ + f"### Build plan ({build_env})", + "", + "| Runner | Coins |", + "| --- | --- |", + ] + for item in runner_matrix: + summary_lines.append(f"| {item['runner']} | {', '.join(item['coins'])} |") + if selection.skipped_prod_only and selection.requested_all: + summary_lines += [ + "", + "Skipped prod-only coins for env=dev: " + + ", ".join(selection.skipped_prod_only), + ] + write_step_summary("\n".join(summary_lines)) + if __name__ == "__main__": main() diff --git a/.github/scripts/deploy_plan.py b/.github/scripts/deploy_plan.py index 8539cf7101..903d05590d 100644 --- a/.github/scripts/deploy_plan.py +++ b/.github/scripts/deploy_plan.py @@ -14,6 +14,7 @@ parse_json_object, require_coin_config, resolve_deploy_selection, + write_step_summary, ) @@ -90,6 +91,20 @@ def main() -> None: log("Selected coins: " + ", ".join(requested)) log("Connectivity regex: " + connectivity_regex) + summary_lines = [ + "### Deploy plan", + "", + "| Coin | Runner |", + "| --- | --- |", + ] + for item in runner_matrix: + summary_lines.append(f"| {item['coin']} | {item['runner']} |") + summary_lines += [ + "", + "E2E test coins: " + ", ".join(unique_test_coins), + ] + write_step_summary("\n".join(summary_lines)) + if __name__ == "__main__": main() diff --git a/.github/scripts/runner.py b/.github/scripts/runner.py index 06de28d61f..21fa350821 100644 --- a/.github/scripts/runner.py +++ b/.github/scripts/runner.py @@ -41,6 +41,15 @@ def log(message: str) -> None: print(f"{LOG_PREFIX} {message}", flush=True) +def write_step_summary(markdown: str) -> None: + # Renders on the workflow run's summary page; a no-op outside Actions. + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + with open(summary_path, "a", encoding="utf-8") as summary: + summary.write(markdown.rstrip("\n") + "\n") + + def parse_json_object(raw: str, description: str) -> dict: try: payload = json.loads(raw) diff --git a/.github/scripts/validate_branch_or_tag.py b/.github/scripts/validate_branch_or_tag.py index 5ae9fe3320..2aa95fef9b 100755 --- a/.github/scripts/validate_branch_or_tag.py +++ b/.github/scripts/validate_branch_or_tag.py @@ -23,8 +23,8 @@ def fail(message: str) -> None: def ref_exists(repo: str, ref: str, kind: str) -> bool: remote = f"https://github.com/{repo}.git" try: - subprocess.run( - ["git", "ls-remote", "--exit-code", f"--{kind}", remote, ref], + result = subprocess.run( + ["git", "ls-remote", "--exit-code", f"--{kind}", remote, "--", ref], check=True, capture_output=True, text=True, @@ -34,7 +34,15 @@ def ref_exists(repo: str, ref: str, kind: str) -> bool: raise AssertionError("unreachable") from exc except subprocess.CalledProcessError: return False - return True + # ls-remote patterns are tail-matching globs, so e.g. 'mas*' or 'master' + # can match a different ref than the one actions/checkout will later use; + # accept only an exact ref-name match. + expected = f"refs/{kind}/{ref}" + for line in result.stdout.splitlines(): + parts = line.split("\t") + if len(parts) == 2 and parts[1] == expected: + return True + return False def validate_branch_or_tag(repo: str, ref: str) -> str: diff --git a/.github/scripts/validate_branch_or_tag_test.py b/.github/scripts/validate_branch_or_tag_test.py index 630115730e..0592566625 100644 --- a/.github/scripts/validate_branch_or_tag_test.py +++ b/.github/scripts/validate_branch_or_tag_test.py @@ -1,7 +1,8 @@ +import subprocess import unittest from unittest.mock import patch -from validate_branch_or_tag import validate_branch_or_tag +from validate_branch_or_tag import ref_exists, validate_branch_or_tag class ValidateBranchOrTagTest(unittest.TestCase): @@ -21,5 +22,34 @@ def test_rejects_missing_ref(self) -> None: validate_branch_or_tag("trezor/blockbook", "missing-ref") +class RefExistsTest(unittest.TestCase): + @staticmethod + def _ls_remote(stdout: str) -> subprocess.CompletedProcess: + return subprocess.CompletedProcess(args=[], returncode=0, stdout=stdout, stderr="") + + def test_accepts_exact_ref_name(self) -> None: + with patch( + "validate_branch_or_tag.subprocess.run", + return_value=self._ls_remote("abc123\trefs/heads/master\n"), + ): + self.assertTrue(ref_exists("trezor/blockbook", "master", "heads")) + + def test_rejects_glob_pattern_matching_another_ref(self) -> None: + with patch( + "validate_branch_or_tag.subprocess.run", + return_value=self._ls_remote("abc123\trefs/heads/master\n"), + ): + self.assertFalse(ref_exists("trezor/blockbook", "mas*", "heads")) + + def test_rejects_tail_match_of_longer_ref(self) -> None: + # ls-remote patterns tail-match, so 'master' would match + # 'feature/master' even when no branch named 'master' exists. + with patch( + "validate_branch_or_tag.subprocess.run", + return_value=self._ls_remote("abc123\trefs/heads/feature/master\n"), + ): + self.assertFalse(ref_exists("trezor/blockbook", "master", "heads")) + + if __name__ == "__main__": unittest.main() diff --git a/.github/scripts/wait_for_sync.py b/.github/scripts/wait_for_sync.py index 41c4770396..c4760efff4 100644 --- a/.github/scripts/wait_for_sync.py +++ b/.github/scripts/wait_for_sync.py @@ -21,6 +21,15 @@ def log(message: str) -> None: print(f"{LOG_PREFIX} {message}", flush=True) +def read_error_body(exc: urllib.error.HTTPError) -> bytes: + # The connection can be reset while reading the error body (e.g. a proxy + # 502); that must degrade to an empty body, not kill the retry loop. + try: + return exc.read() + except Exception: + return b"" + + def parse_requested_coins(raw: str) -> list[str]: text = raw.strip() if not text: @@ -192,7 +201,7 @@ def main() -> None: status, body = fetch_status(base_url, request_timeout, ssl_context) except urllib.error.HTTPError as exc: status = exc.code - body = exc.read() + body = read_error_body(exc) except Exception as exc: last_seen[coin] = f"{base_url}/api/status request failed: {exc}" continue @@ -204,7 +213,7 @@ def main() -> None: status, body = fetch_status(base_url, request_timeout, ssl_context) except urllib.error.HTTPError as exc: status = exc.code - body = exc.read() + body = read_error_body(exc) except Exception as exc: last_seen[coin] = f"{base_url}/api/status request failed: {exc}" continue @@ -224,14 +233,16 @@ def main() -> None: if not pending: break - remaining_seconds = int(max(0, deadline - time.monotonic())) - if remaining_seconds == 0: + # Keep sub-second remainders: int() truncation used to declare a + # timeout with up to a second of budget (and one last poll) unused. + remaining_seconds = deadline - time.monotonic() + if remaining_seconds <= 0: break details = "; ".join( f"{coin}: {last_seen[coin]}" for coin in sorted(pending) ) - log(f"Still waiting for Blockbook sync ({remaining_seconds}s left): {details}") + log(f"Still waiting for Blockbook sync ({remaining_seconds:.0f}s left): {details}") time.sleep(min(poll_seconds, remaining_seconds)) if pending: diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c1e2177174..97a133685e 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,5 +1,10 @@ name: Build / Deploy +# Surface what the run actually does in the run list; without this every +# dispatch shows only the generic workflow name. env context is not +# available in run-name, so the ref fallback is inlined. +run-name: "${{ inputs.mode }} ${{ inputs.coins }} (${{ inputs.mode == 'build' && inputs.env || 'dev' }}) @ ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }}" + on: workflow_dispatch: inputs: @@ -39,19 +44,21 @@ on: permissions: contents: read +env: + RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} + jobs: prepare_build: name: Prepare Build Plan runs-on: ubuntu-latest if: ${{ inputs.mode == 'build' }} - env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} outputs: runner_matrix: ${{ steps.plan.outputs.runner_matrix }} - coins_csv: ${{ steps.plan.outputs.coins_csv }} steps: - name: Checkout workflow code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false - name: Validate branch or tag env: @@ -62,6 +69,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Build build plan id: plan @@ -75,16 +83,15 @@ jobs: name: Prepare Deploy Plan runs-on: ubuntu-latest if: ${{ inputs.mode == 'deploy' }} - env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} outputs: runner_matrix: ${{ steps.plan.outputs.runner_matrix }} connectivity_regex: ${{ steps.plan.outputs.connectivity_regex }} - coins_csv: ${{ steps.plan.outputs.coins_csv }} test_coins_csv: ${{ steps.plan.outputs.test_coins_csv }} steps: - name: Checkout workflow code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false - name: Validate branch or tag env: @@ -95,6 +102,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Build deploy/e2e plan id: plan @@ -104,11 +112,12 @@ jobs: run: python3 ./.github/scripts/deploy_plan.py build: - name: Build (${{ matrix.runner }}) + name: "Build (${{ matrix.runner }}: ${{ matrix.coins_csv }})" needs: prepare_build if: ${{ inputs.mode == 'build' }} - env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} + # Generous ceilings so a hung step can't hold a self-hosted runner for + # the 360-minute default; sized well above normal runtimes. + timeout-minutes: 240 strategy: fail-fast: false matrix: @@ -119,6 +128,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Export repository variables uses: ./.github/actions/export-env-vars @@ -139,8 +149,7 @@ jobs: name: Deploy (${{ matrix.coin }}) needs: prepare_deploy if: ${{ inputs.mode == 'deploy' }} - env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} + timeout-minutes: 45 strategy: fail-fast: false matrix: @@ -151,6 +160,7 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Export repository variables uses: ./.github/actions/export-env-vars @@ -175,23 +185,27 @@ jobs: env: BRANCH_OR_TAG: ${{ env.RESOLVED_BRANCH_OR_TAG }} BB_BUILD_ENV: dev - run: ./contrib/scripts/deploy-bb-and-backend.sh "${{ matrix.coin }}" --force-confnew + DEPLOY_COIN: ${{ matrix.coin }} + run: ./contrib/scripts/deploy-bb-and-backend.sh "$DEPLOY_COIN" --force-confnew wait-for-sync: name: Wait For Sync needs: [prepare_deploy, deploy] - if: ${{ needs.deploy.result == 'success' }} + # deploy is a fail-fast:false matrix: one failed coin must not skip sync + # verification for the coins that did deploy, so run on partial failure + # too. Only 'skipped' (mode=build) and cancellation keep this job off. + if: ${{ !cancelled() && (needs.deploy.result == 'success' || needs.deploy.result == 'failure') }} runs-on: [self-hosted, bb-dev-selfhosted] - timeout-minutes: 31 + timeout-minutes: 6 env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} COINS_INPUT: ${{ needs.prepare_deploy.outputs.test_coins_csv }} - SYNC_TIMEOUT_SECONDS: "1800" + SYNC_TIMEOUT_SECONDS: "300" steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Export repository variables uses: ./.github/actions/export-env-vars @@ -206,16 +220,19 @@ jobs: e2e-tests: name: E2E Tests (post-deploy) needs: [prepare_deploy, deploy, wait-for-sync] - if: ${{ needs.deploy.result == 'success' && needs.wait-for-sync.result == 'success' }} + # Mirror wait-for-sync: partially failed deploys still get e2e coverage + # for the coins that deployed, as long as they reached sync. + if: ${{ !cancelled() && (needs.deploy.result == 'success' || needs.deploy.result == 'failure') && needs.wait-for-sync.result == 'success' }} runs-on: [self-hosted, bb-dev-selfhosted] + timeout-minutes: 60 env: - RESOLVED_BRANCH_OR_TAG: ${{ inputs.branch_or_tag != '' && inputs.branch_or_tag || github.ref_name }} CONNECTIVITY_REGEX: ${{ needs.prepare_deploy.outputs.connectivity_regex }} steps: - name: Checkout code uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: ref: ${{ env.RESOLVED_BRANCH_OR_TAG }} + persist-credentials: false - name: Export repository variables uses: ./.github/actions/export-env-vars diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c86ab1a3da..55d1c8dacb 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -9,10 +9,20 @@ on: permissions: contents: read +# Rapid pushes queue full unit->connectivity->integration chains for +# already-superseded commits on the shared self-hosted pool. Cancel only +# superseded PR runs; pushes to master/develop are never cancelled. +concurrency: + group: testing-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: unit-tests: name: Unit Tests runs-on: [self-hosted, bb-dev-selfhosted] + # Generous ceilings so a hung step can't hold a self-hosted runner for + # the 360-minute default; sized well above normal runtimes. + timeout-minutes: 60 # Defense-in-depth: never run untrusted fork-PR code on the self-hosted # runner. connectivity/integration already carry this guard; without it # here, unit-tests relied solely on the repo "require approval for fork @@ -33,6 +43,7 @@ jobs: connectivity-tests: name: Connectivity Tests runs-on: [self-hosted, bb-dev-selfhosted] + timeout-minutes: 30 needs: unit-tests if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} @@ -58,6 +69,7 @@ jobs: integration-tests: name: Integration Tests (RPC + Sync) runs-on: [self-hosted, bb-dev-selfhosted] + timeout-minutes: 180 needs: connectivity-tests if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} @@ -105,3 +117,8 @@ jobs: - name: Validate backend artifact sources run: python3 ./.github/scripts/validate_backend_artifacts.py + + - name: Run CI pipeline script tests + # Stdlib-only unit tests for the build/deploy plan scripts; without + # this step a regression in .github/scripts ships with green CI. + run: python3 -m unittest discover -s .github/scripts -p '*_test.py'