From 5cde031e77fafc0230388635db1eb0a9b47d89de Mon Sep 17 00:00:00 2001 From: pasta Date: Sat, 29 Aug 2026 16:40:05 +0200 Subject: [PATCH] ci: skip builds for documentation-only changes --- .github/workflows/build.yml | 110 +++++++++++- .github/workflows/classify_ci_changes.py | 163 ++++++++++++++++++ .github/workflows/test_classify_ci_changes.py | 122 +++++++++++++ 3 files changed, 389 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/classify_ci_changes.py create mode 100644 .github/workflows/test_classify_ci_changes.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ce8191dc99ea..bae1f28968bb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,6 +8,7 @@ permissions: actions: read contents: read packages: write + pull-requests: read concurrency: group: | @@ -25,6 +26,7 @@ jobs: runs-on: ${{ vars.RUNNER_CHECK_SKIP || 'ubuntu-24.04-arm' }} outputs: skip: ${{ steps.skip-check.outputs.skip }} + run-build-tests: ${{ steps.classify-changes.outputs.run-build-tests }} runner-amd64: ${{ steps.select-runner.outputs.runner_amd64 }} runner-arm64: ${{ steps.select-runner.outputs.runner_arm64 }} use-blacksmith: ${{ steps.select-runner.outputs.use_blacksmith }} @@ -39,18 +41,108 @@ jobs: run: | if [[ "${{ github.event_name }}" == "push" && "${{ vars.SKIP_ON_PUSH }}" != "" ]]; then echo "Skipping build on push due to SKIP_ON_PUSH environment variable" - echo "skip=true" >> $GITHUB_OUTPUT + echo "skip=true" >> "$GITHUB_OUTPUT" elif [[ "${{ github.event_name }}" == "pull_request_target" && "${{ vars.SKIP_ON_PR }}" != "" ]]; then echo "Skipping build on pull request due to SKIP_ON_PR environment variable" - echo "skip=true" >> $GITHUB_OUTPUT + echo "skip=true" >> "$GITHUB_OUTPUT" else - echo "skip=false" >> $GITHUB_OUTPUT + echo "skip=false" >> "$GITHUB_OUTPUT" fi - name: Checkout code if: ${{ steps.skip-check.outputs.skip == 'false' }} uses: actions/checkout@v6 + - name: Collect changed paths + id: changed-paths + if: ${{ steps.skip-check.outputs.skip == 'false' }} + uses: actions/github-script@v8 + env: + CHANGED_PATHS_FILE: ${{ runner.temp }}/changed-paths.json + with: + script: | + const fs = require('fs'); + let complete = true; + let files = []; + + try { + if (context.eventName === 'pull_request_target') { + files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + per_page: 100, + }); + if (files.length >= 3000) { + complete = false; + core.warning('Pull request file list reached the GitHub API limit'); + } + } else if (context.eventName === 'push') { + const after = context.payload.after; + if (context.ref.startsWith('refs/tags/')) { + complete = false; + } else { + // Comparing with the last successful run preserves coverage when a + // later push cancels an in-flight run before its build matrix finishes. + const previousRuns = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'build.yml', + branch: context.ref.replace('refs/heads/', ''), + event: 'push', + status: 'success', + per_page: 1, + }); + const previousSha = previousRuns.data.workflow_runs[0]?.head_sha; + if (!previousSha) { + complete = false; + } else { + const comparison = await github.rest.repos.compareCommitsWithBasehead({ + owner: context.repo.owner, + repo: context.repo.repo, + basehead: `${previousSha}...${after}`, + }); + files = comparison.data.files || []; + if (files.length >= 300) { + complete = false; + core.warning('Push file list reached the GitHub API limit'); + } + } + } + } else { + complete = false; + } + } catch (error) { + complete = false; + core.warning(`Unable to collect a complete changed path list: ${error.message}`); + } + + const paths = [...new Set(files.flatMap(file => + [file.filename, file.previous_filename].filter(Boolean) + ))].sort(); + fs.writeFileSync(process.env.CHANGED_PATHS_FILE, JSON.stringify(paths)); + core.setOutput('complete', complete.toString()); + core.setOutput('removed-or-renamed', files.some(file => + file.status === 'removed' || file.status === 'renamed' + ).toString()); + + - name: Classify changed paths + id: classify-changes + if: ${{ steps.skip-check.outputs.skip == 'false' }} + env: + CHANGED_PATHS_COMPLETE: ${{ steps.changed-paths.outputs.complete }} + CHANGED_PATHS_FILE: ${{ runner.temp }}/changed-paths.json + CHANGED_PATHS_REMOVED_OR_RENAMED: ${{ steps.changed-paths.outputs.removed-or-renamed }} + run: | + if ! python3 .github/workflows/classify_ci_changes.py \ + --complete "$CHANGED_PATHS_COMPLETE" \ + --removed-or-renamed "$CHANGED_PATHS_REMOVED_OR_RENAMED" \ + "$CHANGED_PATHS_FILE"; then + echo "Path classification failed; running the full build and test matrix" + echo "run-build-tests=true" >> "$GITHUB_OUTPUT" + echo "decision-reason=path classification failed" >> "$GITHUB_OUTPUT" + fi + - name: Select runners id: select-runner if: ${{ steps.skip-check.outputs.skip == 'false' }} @@ -65,7 +157,9 @@ jobs: - name: Get base image digest id: base-image - if: ${{ steps.skip-check.outputs.skip == 'false' }} + if: | + steps.skip-check.outputs.skip == 'false' && + steps.classify-changes.outputs.run-build-tests == 'true' run: | # Fetch the canonical manifest digest for ubuntu:noble so the # depends cache key changes when Canonical pushes base image @@ -86,7 +180,9 @@ jobs: cache-sources: name: Cache depends sources needs: [check-skip] - if: ${{ needs.check-skip.outputs.skip == 'false' }} + if: | + needs.check-skip.outputs.skip == 'false' && + needs.check-skip.outputs.run-build-tests == 'true' uses: ./.github/workflows/cache-depends-sources.yml with: runs-on: ${{ needs.check-skip.outputs['runner-arm64'] }} @@ -94,7 +190,9 @@ jobs: container: name: Build container needs: [check-skip] - if: ${{ needs.check-skip.outputs.skip == 'false' }} + if: | + needs.check-skip.outputs.skip == 'false' && + needs.check-skip.outputs.run-build-tests == 'true' uses: ./.github/workflows/build-container.yml with: context: ./contrib/containers/ci diff --git a/.github/workflows/classify_ci_changes.py b/.github/workflows/classify_ci_changes.py new file mode 100644 index 000000000000..492d196c953f --- /dev/null +++ b/.github/workflows/classify_ci_changes.py @@ -0,0 +1,163 @@ +# Copyright (c) 2026 The Dash Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +import argparse +import json +import os +from pathlib import PurePosixPath +from typing import Dict, List, Optional, Sequence + + +EXCLUDED_EXACT_PATHS = { + "doc/.gitignore", +} +EXCLUDED_PREFIXES = ( + ".github/ISSUE_TEMPLATE/", + ".github/PULL_REQUEST_TEMPLATE/", +) +EXCLUDED_DOC_SUFFIXES = {".1", ".5", ".8", ".png", ".svg", ".txt"} +BUILD_RELEVANT_PATHS = {"COPYING", "doc/README_windows.txt"} + + +def is_excluded_path(path: str) -> bool: + if not path or path in BUILD_RELEVANT_PATHS: + return False + if path in EXCLUDED_EXACT_PATHS or path.startswith(EXCLUDED_PREFIXES): + return True + + suffix = PurePosixPath(path).suffix + if suffix == ".md": + return True + return path.startswith("doc/") and suffix in EXCLUDED_DOC_SUFFIXES + + +def classify_changes( + paths: Sequence[str], complete: bool, removed_or_renamed: bool = False +) -> Dict[str, object]: + unique_paths = sorted(set(paths)) + if not complete: + return { + "run_build_tests": True, + "reason": "changed path list is incomplete", + "paths": unique_paths, + "triggering_paths": [], + } + if removed_or_renamed: + return { + "run_build_tests": True, + "reason": "a changed path was removed or renamed", + "paths": unique_paths, + "triggering_paths": [], + } + if not unique_paths: + return { + "run_build_tests": True, + "reason": "no changed paths were reported", + "paths": unique_paths, + "triggering_paths": [], + } + + triggering_paths = [path for path in unique_paths if not is_excluded_path(path)] + if triggering_paths: + reason = "build-relevant or unclassified paths changed" + else: + reason = "all changed paths are in CI exclusion zones" + return { + "run_build_tests": bool(triggering_paths), + "reason": reason, + "paths": unique_paths, + "triggering_paths": triggering_paths, + } + + +def load_paths(path: str) -> List[str]: + with open(path, "r", encoding="utf-8") as file: + paths = json.load(file) + if not isinstance(paths, list) or not all(isinstance(item, str) for item in paths): + raise ValueError("changed paths must be a JSON array of strings") + return paths + + +def write_github_output(path: Optional[str], result: Dict[str, object]) -> None: + if not path: + return + with open(path, "a", encoding="utf-8") as file: + file.write( + "run-build-tests={}\n".format( + "true" if result["run_build_tests"] else "false" + ) + ) + file.write("decision-reason={}\n".format(result["reason"])) + + +def write_step_summary( + path: Optional[str], result: Dict[str, object], complete: bool +) -> None: + if not path: + return + + triggering_paths = result["triggering_paths"] + with open(path, "a", encoding="utf-8") as file: + file.write("### Build and test path classification\n") + file.write( + "- Changed path list complete: {}\n".format( + "yes" if complete else "no" + ) + ) + file.write("- Changed paths examined: {}\n".format(len(result["paths"]))) + file.write( + "- Full build and test matrix: {}\n".format( + "required" if result["run_build_tests"] else "skipped" + ) + ) + file.write("- Decision: `{}`\n".format(result["reason"])) + if triggering_paths: + file.write("- Triggering paths (up to 20):\n") + for changed_path in triggering_paths[:20]: + formatted_path = json.dumps(changed_path, ensure_ascii=True).replace( + "`", "\\u0060" + ) + file.write( + " - `{}`\n".format(formatted_path) + ) + + +def parse_args(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Classify changed paths for Dash CI.") + parser.add_argument("paths_json", help="JSON file containing changed paths") + parser.add_argument( + "--complete", + choices=("true", "false"), + required=True, + help="Whether the changed path list is known to be complete", + ) + parser.add_argument( + "--removed-or-renamed", + choices=("true", "false"), + required=True, + help="Whether a changed path was removed or renamed", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str]) -> int: + args = parse_args(argv) + complete = args.complete == "true" + result = classify_changes( + load_paths(args.paths_json), complete, args.removed_or_renamed == "true" + ) + write_github_output(os.environ.get("GITHUB_OUTPUT"), result) + write_step_summary(os.environ.get("GITHUB_STEP_SUMMARY"), result, complete) + print( + "run-build-tests={} ({})".format( + "true" if result["run_build_tests"] else "false", result["reason"] + ) + ) + return 0 + + +if __name__ == "__main__": + import sys + + sys.exit(main(sys.argv[1:])) diff --git a/.github/workflows/test_classify_ci_changes.py b/.github/workflows/test_classify_ci_changes.py new file mode 100644 index 000000000000..e8016021fe0d --- /dev/null +++ b/.github/workflows/test_classify_ci_changes.py @@ -0,0 +1,122 @@ +import importlib.util +import json +import pathlib +import tempfile +import unittest + + +MODULE_PATH = pathlib.Path(__file__).with_name("classify_ci_changes.py") +SPEC = importlib.util.spec_from_file_location("classify_ci_changes", MODULE_PATH) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MODULE) + + +class ClassifyCIChangesTest(unittest.TestCase): + def test_markdown_files_are_excluded_at_any_depth(self): + for path in ("README.md", "doc/release-notes-1234.md", "depends/README.md"): + with self.subTest(path=path): + self.assertTrue(MODULE.is_excluded_path(path)) + + def test_static_documentation_files_are_excluded(self): + for path in ( + "doc/assets/diagram.png", + "doc/assets/diagram.svg", + "doc/man/dashd.1", + "doc/notes.txt", + ): + with self.subTest(path=path): + self.assertTrue(MODULE.is_excluded_path(path)) + + def test_documentation_build_inputs_are_not_excluded(self): + for path in ("doc/Doxyfile.in", "doc/man/Makefile.am"): + with self.subTest(path=path): + self.assertFalse(MODULE.is_excluded_path(path)) + + def test_repository_templates_are_excluded(self): + for path in ( + ".github/ISSUE_TEMPLATE/config.yml", + ".github/PULL_REQUEST_TEMPLATE/release.md", + ".github/PULL_REQUEST_TEMPLATE.md", + ): + with self.subTest(path=path): + self.assertTrue(MODULE.is_excluded_path(path)) + + def test_windows_installer_inputs_are_not_excluded(self): + for path in ("COPYING", "doc/README_windows.txt"): + with self.subTest(path=path): + self.assertFalse(MODULE.is_excluded_path(path)) + + def test_build_relevant_and_unknown_paths_require_tests(self): + for path in ( + "src/net.cpp", + "test/functional/p2p_invalid_messages.py", + "depends/packages/boost.mk", + "ci/dash/matrix.sh", + ".github/workflows/build.yml", + "configure.ac", + "new-area/README.txt", + ): + with self.subTest(path=path): + self.assertFalse(MODULE.is_excluded_path(path)) + + def test_only_excluded_paths_skip_builds_and_tests(self): + result = MODULE.classify_changes( + ["README.md", "doc/assets/diagram.svg", ".github/ISSUE_TEMPLATE/config.yml"], + complete=True, + ) + + self.assertFalse(result["run_build_tests"]) + self.assertEqual(result["triggering_paths"], []) + + def test_mixed_changes_require_builds_and_tests(self): + result = MODULE.classify_changes( + ["doc/release-notes-1234.md", "src/net.cpp"], complete=True + ) + + self.assertTrue(result["run_build_tests"]) + self.assertEqual(result["triggering_paths"], ["src/net.cpp"]) + + def test_previous_rename_path_can_require_builds_and_tests(self): + result = MODULE.classify_changes( + ["doc/retired-code.md", "src/retired-code.cpp"], complete=True + ) + + self.assertTrue(result["run_build_tests"]) + self.assertEqual(result["triggering_paths"], ["src/retired-code.cpp"]) + + def test_incomplete_or_empty_path_lists_fail_open(self): + for paths, complete in ((["README.md"], False), ([], True)): + with self.subTest(paths=paths, complete=complete): + result = MODULE.classify_changes(paths, complete) + self.assertTrue(result["run_build_tests"]) + + def test_removed_or_renamed_documentation_fails_open(self): + result = MODULE.classify_changes( + ["doc/man/dashd.1"], complete=True, removed_or_renamed=True + ) + + self.assertTrue(result["run_build_tests"]) + self.assertIn("removed or renamed", result["reason"]) + + def test_load_paths_rejects_malformed_input(self): + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "paths.json" + path.write_text(json.dumps({"path": "README.md"}), encoding="utf-8") + + with self.assertRaisesRegex(ValueError, "JSON array of strings"): + MODULE.load_paths(str(path)) + + def test_step_summary_escapes_markdown_backticks_in_paths(self): + result = MODULE.classify_changes(["src/name` [link](example).cpp"], complete=True) + with tempfile.TemporaryDirectory() as directory: + path = pathlib.Path(directory) / "summary.md" + MODULE.write_step_summary(str(path), result, complete=True) + summary = path.read_text(encoding="utf-8") + + self.assertIn(r"name\u0060 [link](example).cpp", summary) + self.assertNotIn("name` [link](example).cpp", summary) + + +if __name__ == "__main__": + unittest.main()