diff --git a/Makefile b/Makefile index a8702bc385..45c8e03142 100644 --- a/Makefile +++ b/Makefile @@ -286,6 +286,7 @@ help: ## Print this help message hack/ci/pr-removes-fixed-skips.t hack/ci/pr-should-include-tests.t hack/ci/logformatter.t + hack/ci/github_log_summary.t test/system/helpers.t .PHONY: lint diff --git a/hack/ci/github_log_summary.py b/hack/ci/github_log_summary.py index 125530f4ed..0d8af47398 100755 --- a/hack/ci/github_log_summary.py +++ b/hack/ci/github_log_summary.py @@ -104,12 +104,20 @@ def handle_endtag(self, tag): +# logformatter decides how to mark up a log by looking at the log contents, +# not at the test name, and only its ginkgo path emits the "log-failed" class. +# Detect the format the same way, so that ginkgo suites which are not named +# "int-" (the bindings suite, for example) are parsed as ginkgo rather than +# falling through to the bats parser. +GINKGO_MARKER = 'class="log-failed"' + + def filter_html_file(file_path): # Read the HTML content with open(file_path, 'r', encoding='utf-8') as f: html_content = f.read() - if 'int-' in file_path: + if GINKGO_MARKER in html_content: parser = GinkgoLogFilterParser() parser.feed(html_content) return parser.results @@ -119,10 +127,14 @@ def filter_html_file(file_path): return [parser.data] -# Running the filter -matching_elements = filter_html_file(sys.argv[1]) +def main(file_paths): + for file_path in file_paths: + for element in filter_html_file(file_path): + print("```") + print(element) + print("```") -for element in matching_elements: - print(f"```") - print(element) - print("```") + +# Running the filter +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/hack/ci/github_log_summary.t b/hack/ci/github_log_summary.t new file mode 100755 index 0000000000..1667fcabe5 --- /dev/null +++ b/hack/ci/github_log_summary.t @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# +# tests for github_log_summary.py +# + +import importlib.util +import os +import tempfile +import unittest + +TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) + +# The tool is a script, not a module, so load it by path. +spec = importlib.util.spec_from_file_location( + "github_log_summary", os.path.join(TESTS_DIR, "github_log_summary.py") +) +github_log_summary = importlib.util.module_from_spec(spec) +spec.loader.exec_module(github_log_summary) + + +# Trimmed-down versions of what logformatter emits. The ginkgo path wraps +# failures in a "log-failed" span inside the "tt" block; the bats path marks +# up each line with a "bats-*" class. +GINKGO_HTML = """
+[+0298s] [FAILED] podman pod correctly sets up PIDNS +[+0298s] expected exit code 0, got 125 +
+""" + +GINKGO_PASSING_HTML = """
+[+0271s] ok, all tests passed +
+""" + +BATS_HTML = """
+not ok 1 podman run +# expected 0, got 125 +
+""" + + +def summarize(html, name): + """Write html to a file called name, then run it through the filter.""" + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, name) + with open(path, "w", encoding="utf-8") as f: + f.write(html) + return github_log_summary.filter_html_file(path) + + +class TestFormatDetection(unittest.TestCase): + def test_ginkgo_int_suite(self): + """The int suite is ginkgo and is detected as such.""" + out = "".join(summarize(GINKGO_HTML, "int-local-root-fedora.log.html")) + self.assertIn("[FAILED] podman pod correctly sets up PIDNS", out) + self.assertIn("expected exit code 0, got 125", out) + + def test_ginkgo_suite_not_named_int(self): + """Ginkgo suites are detected by content, not by the file name. + + The bindings suite is ginkgo but is not called "int-". Keying off the + name meant its failures were parsed with the bats parser, which found + no bats markup and so reported nothing useful. + """ + out = "".join(summarize(GINKGO_HTML, "bindings-root-fedora.log.html")) + self.assertIn("[FAILED] podman pod correctly sets up PIDNS", out) + self.assertIn("expected exit code 0, got 125", out) + + def test_bats_suite(self): + """The bats parser still handles bats logs.""" + out = "".join(summarize(BATS_HTML, "sys-local-root-fedora.log.html")) + self.assertIn("not ok 1 podman run", out) + self.assertIn("expected 0, got 125", out) + + def test_ginkgo_without_failures(self): + """A ginkgo log with no failures has nothing to report.""" + out = "".join(summarize(GINKGO_PASSING_HTML, "int-local-root-fedora.log.html")) + self.assertEqual(out.strip(), "") + + +if __name__ == "__main__": + unittest.main()