diff --git a/ddtrace/internal/settings/_supported_configurations.py b/ddtrace/internal/settings/_supported_configurations.py index 8722d96e719..122b9098b28 100644 --- a/ddtrace/internal/settings/_supported_configurations.py +++ b/ddtrace/internal/settings/_supported_configurations.py @@ -459,6 +459,8 @@ "DD_TEST_DEBUG", "DD_TEST_MANAGEMENT_ATTEMPT_TO_FIX_RETRIES", "DD_TEST_MANAGEMENT_ENABLED", + "DD_TEST_OPTIMIZATION_DISCOVERY_ENABLED", + "DD_TEST_OPTIMIZATION_DISCOVERY_FILE", "DD_TEST_OPTIMIZATION_ENV_DATA_FILE", "DD_TEST_OPTIMIZATION_MANIFEST_FILE", "DD_TEST_OPTIMIZATION_PAYLOADS_IN_FILES", diff --git a/ddtrace/testing/internal/constants.py b/ddtrace/testing/internal/constants.py index 1f8a40c393c..56d0bac71e8 100644 --- a/ddtrace/testing/internal/constants.py +++ b/ddtrace/testing/internal/constants.py @@ -25,5 +25,9 @@ class ITRSkippingLevel(Enum): DD_TEST_OPTIMIZATION_ENV_DATA_FILE = "DD_TEST_OPTIMIZATION_ENV_DATA_FILE" TEST_UNDECLARED_OUTPUTS_DIR = "TEST_UNDECLARED_OUTPUTS_DIR" +# Test discovery environment variables +DD_TEST_OPTIMIZATION_DISCOVERY_ENABLED = "DD_TEST_OPTIMIZATION_DISCOVERY_ENABLED" +DD_TEST_OPTIMIZATION_DISCOVERY_FILE = "DD_TEST_OPTIMIZATION_DISCOVERY_FILE" + # The only supported .testoptimization manifest version SUPPORTED_MANIFEST_VERSION = 1 diff --git a/ddtrace/testing/internal/pytest/_discovery.py b/ddtrace/testing/internal/pytest/_discovery.py new file mode 100644 index 00000000000..1a42de9fc2d --- /dev/null +++ b/ddtrace/testing/internal/pytest/_discovery.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import json +import logging +from pathlib import Path +import typing as t + +import pytest + +from ddtrace.internal.settings import env +from ddtrace.testing.internal.constants import DD_TEST_OPTIMIZATION_DISCOVERY_ENABLED +from ddtrace.testing.internal.constants import DD_TEST_OPTIMIZATION_DISCOVERY_FILE +from ddtrace.testing.internal.git import get_workspace_path +from ddtrace.testing.internal.pytest.utils import _get_test_parameters_json +from ddtrace.testing.internal.pytest.utils import item_to_test_ref +from ddtrace.testing.internal.utils import asbool + + +log = logging.getLogger(__name__) + +_DEFAULT_OUTPUT_PATH = ".testoptimization/tests-discovery/tests.json" + + +def is_discovery_mode_enabled() -> bool: + return asbool(env.get(DD_TEST_OPTIMIZATION_DISCOVERY_ENABLED)) + + +def _get_output_path() -> Path: + return Path(env.get(DD_TEST_OPTIMIZATION_DISCOVERY_FILE, _DEFAULT_OUTPUT_PATH)) + + +def _is_item_skipped(item: pytest.Item) -> bool: + """Return True if the item will definitely be skipped at execution time. + + Handles pytest.mark.skip (unconditional) and pytest.mark.skipif with + non-string conditions. String conditions are not evaluated (would require + exec in the test module's namespace) so those tests are conservatively + included. + """ + if item.get_closest_marker("skip") is not None: + return True + for marker in item.iter_markers("skipif"): + condition = marker.args[0] if marker.args else marker.kwargs.get("condition") + if condition is None or isinstance(condition, str): + # String conditions require eval in the test module's namespace, which + # we can't do safely at collection time. None conditions arise when + # @pytest.mark.skipif is used with only a reason= kwarg and no positional + # condition — pytest 9 accepts this and skips the test at runtime, but + # does not signal it during collection. + # Conservatively include in both cases rather than risk hiding a test. + continue + if condition: + return True + return False + + +def _get_suite_source_file(item: pytest.Item, workspace_path: t.Optional[Path]) -> str: + item_path = Path(item.path if hasattr(item, "path") else getattr(item, "fspath", "")).absolute() + if workspace_path is not None: + try: + return item_path.relative_to(workspace_path).as_posix() + except ValueError: + pass + return str(item_path) + + +@pytest.hookimpl(tryfirst=True) # type: ignore[misc] +def pytest_collection_finish(session: pytest.Session) -> None: + if not is_discovery_mode_enabled(): + return + + workspace_path: t.Optional[Path] = None + try: + workspace_path = get_workspace_path() + except Exception: + log.debug("Could not determine workspace path for test discovery", exc_info=True) + + output_path = _get_output_path() + output_path.parent.mkdir(parents=True, exist_ok=True) + + with open(output_path, "w", encoding="utf-8") as f: + for item in session.items: + if _is_item_skipped(item): + continue + + test_ref = item_to_test_ref(item) + module = test_ref.suite.module.name + suite = test_ref.suite.name + name = test_ref.name + parameters = _get_test_parameters_json(item) + suite_source_file = _get_suite_source_file(item, workspace_path) + + test_info: dict[str, t.Any] = { + "name": name, + "suite": suite, + "module": module, + "parameters": parameters, + "suiteSourceFile": suite_source_file, + } + f.write(json.dumps(test_info) + "\n") + + log.info("Test discovery complete: wrote tests to %s", output_path) + pytest.exit("Test discovery complete", returncode=0) diff --git a/ddtrace/testing/internal/pytest/plugin.py b/ddtrace/testing/internal/pytest/plugin.py index 6f5d3901737..d09e9c6b21e 100644 --- a/ddtrace/testing/internal/pytest/plugin.py +++ b/ddtrace/testing/internal/pytest/plugin.py @@ -2,10 +2,8 @@ from collections import defaultdict from io import StringIO -import json import logging from pathlib import Path -import re import traceback import typing as t @@ -27,11 +25,13 @@ from ddtrace.testing.internal.logging import catch_and_log_exceptions from ddtrace.testing.internal.logging import setup_logging from ddtrace.testing.internal.offline_mode import get_offline_mode +from ddtrace.testing.internal.pytest._discovery import is_discovery_mode_enabled from ddtrace.testing.internal.pytest.bdd import BddTestOptPlugin from ddtrace.testing.internal.pytest.benchmark import BenchmarkData from ddtrace.testing.internal.pytest.benchmark import get_benchmark_tags_and_metrics from ddtrace.testing.internal.pytest.hookspecs import TestOptHooks from ddtrace.testing.internal.pytest.report_links import print_test_report_links +from ddtrace.testing.internal.pytest.utils import _get_test_parameters_json from ddtrace.testing.internal.pytest.utils import item_to_test_ref from ddtrace.testing.internal.retry_handlers import RetryHandler from ddtrace.testing.internal.session_manager import SessionManager @@ -1074,6 +1074,9 @@ def _is_enabled_early(early_config: pytest.Config, args: list[str]) -> bool: if _is_test_optimization_disabled_by_kill_switch(): return False + if is_discovery_mode_enabled(): + return False + if _is_option_true("no-ddtrace", early_config, args): return False @@ -1138,6 +1141,18 @@ def pytest_configure(config: pytest.Config) -> None: if _is_test_optimization_disabled_by_kill_switch(): return + if is_discovery_mode_enabled(): + # Register hook specs so item_to_test_ref can call the custom name hooks during + # discovery, giving the same module/suite/name resolution as a real test run. + # AIDEV-NOTE: BddTestOptPlugin is not registered here, so pytest-bdd tests will + # fall back to nodeid-based names rather than feature-file names during discovery. + # TODO: register BddTestOptPlugin in discovery mode to support pytest-bdd. + import ddtrace.testing.internal.pytest._discovery as _ddtrace_discovery + + config.pluginmanager.add_hookspecs(TestOptHooks) + config.pluginmanager.register(_ddtrace_discovery, "_ddtrace_discovery") + return + session_manager = config.stash.get(SESSION_MANAGER_STASH_KEY, None) if not session_manager: log.debug("Session manager not initialized (plugin was not enabled)") @@ -1243,38 +1258,10 @@ def _get_user_property(report: pytest.TestReport, user_property: str) -> t.Optio return None -def _get_test_parameters_json(item: pytest.Item) -> t.Optional[str]: - callspec: t.Optional[pytest.python.CallSpec2] = getattr(item, "callspec", None) - - if callspec is None: - return None - - parameters: dict[str, dict[str, str]] = {"arguments": {}, "metadata": {}} - for param_name, param_val in item.callspec.params.items(): - try: - parameters["arguments"][param_name] = _encode_test_parameter(param_val) - except Exception: - parameters["arguments"][param_name] = "Could not encode" - log.warning("Failed to encode %r", param_name, exc_info=True) - - try: - return json.dumps(parameters, sort_keys=True) - except TypeError: - log.warning("Failed to serialize parameters for test %s", item, exc_info=True) - return None - - def _get_test_original_name(item: pytest.Item) -> t.Optional[str]: return getattr(item, "originalname", None) -def _encode_test_parameter(parameter: t.Any) -> str: - param_repr = repr(parameter) - # if the representation includes an id() we'll remove it - # because it isn't constant across executions - return re.sub(r" at 0[xX][0-9a-fA-F]+", "", param_repr) - - def _get_skipif_condition(marker: pytest.Mark) -> t.Any: # DEV: pytest allows the condition to be a string to be evaluated. We currently don't support this. if marker.args: diff --git a/ddtrace/testing/internal/pytest/utils.py b/ddtrace/testing/internal/pytest/utils.py index b745360f3f6..05dbe5c1265 100644 --- a/ddtrace/testing/internal/pytest/utils.py +++ b/ddtrace/testing/internal/pytest/utils.py @@ -1,4 +1,9 @@ +from __future__ import annotations + +import json +import logging import re +import typing as t import pytest @@ -7,6 +12,8 @@ from ddtrace.testing.internal.test_data import TestRef +log = logging.getLogger(__name__) + _NODEID_REGEX = re.compile("^(((?P.*)/)?(?P[^/]*?))::(?P.*?)$") @@ -40,3 +47,31 @@ def item_to_test_ref(item: pytest.Item) -> TestRef: test_ref = TestRef(suite_ref, custom_test or default_test) return test_ref + + +def _encode_test_parameter(parameter: t.Any) -> str: + param_repr = repr(parameter) + # if the representation includes an id() we'll remove it + # because it isn't constant across executions + return re.sub(r" at 0[xX][0-9a-fA-F]+", "", param_repr) + + +def _get_test_parameters_json(item: pytest.Item) -> t.Optional[str]: + callspec: t.Optional[pytest.python.CallSpec2] = getattr(item, "callspec", None) + + if callspec is None: + return None + + parameters: dict[str, dict[str, str]] = {"arguments": {}, "metadata": {}} + for param_name, param_val in item.callspec.params.items(): + try: + parameters["arguments"][param_name] = _encode_test_parameter(param_val) + except Exception: + parameters["arguments"][param_name] = "Could not encode" + log.warning("Failed to encode %r", param_name, exc_info=True) + + try: + return json.dumps(parameters, sort_keys=True) + except TypeError: + log.warning("Failed to serialize parameters for test %s", item, exc_info=True) + return None diff --git a/supported-configurations.json b/supported-configurations.json index 33729e9e0ac..72e8f187771 100644 --- a/supported-configurations.json +++ b/supported-configurations.json @@ -3516,6 +3516,20 @@ "default": null } ], + "DD_TEST_OPTIMIZATION_DISCOVERY_ENABLED": [ + { + "implementation": "A", + "type": "boolean", + "default": "false" + } + ], + "DD_TEST_OPTIMIZATION_DISCOVERY_FILE": [ + { + "implementation": "A", + "type": "string", + "default": null + } + ], "DD_TEST_OPTIMIZATION_ENV_DATA_FILE": [ { "implementation": "A", diff --git a/tests/testing/internal/pytest/test_plugin.py b/tests/testing/internal/pytest/test_plugin.py index 05f046e5bdf..003bec39842 100644 --- a/tests/testing/internal/pytest/test_plugin.py +++ b/tests/testing/internal/pytest/test_plugin.py @@ -16,7 +16,6 @@ from ddtrace.testing.internal.pytest.plugin import SKIPPED_BY_ITR_REASON from ddtrace.testing.internal.pytest.plugin import TestOptPlugin from ddtrace.testing.internal.pytest.plugin import XdistTestOptPlugin -from ddtrace.testing.internal.pytest.plugin import _encode_test_parameter from ddtrace.testing.internal.pytest.plugin import _get_exception_tags from ddtrace.testing.internal.pytest.plugin import _get_module_path_from_item from ddtrace.testing.internal.pytest.plugin import _get_source_lines @@ -25,6 +24,7 @@ from ddtrace.testing.internal.pytest.plugin import _get_test_original_name from ddtrace.testing.internal.pytest.plugin import _get_test_parameters_json from ddtrace.testing.internal.pytest.plugin import _get_user_property +from ddtrace.testing.internal.pytest.utils import _encode_test_parameter from ddtrace.testing.internal.pytest.utils import nodeid_to_names from ddtrace.testing.internal.test_data import TestStatus from ddtrace.testing.internal.test_data import TestTag diff --git a/tests/testing/internal/pytest/test_pytest_discovery.py b/tests/testing/internal/pytest/test_pytest_discovery.py new file mode 100644 index 00000000000..77c00bcd520 --- /dev/null +++ b/tests/testing/internal/pytest/test_pytest_discovery.py @@ -0,0 +1,251 @@ +"""Integration tests for the test discovery mode (DD_TEST_OPTIMIZATION_DISCOVERY_ENABLED).""" + +from __future__ import annotations + +import json +from pathlib import Path +import sys + +from _pytest.pytester import Pytester +import pytest + + +class TestDiscoveryMode: + """Tests for pytest_collection_finish in discovery mode. + + Discovery mode: DD_TEST_OPTIMIZATION_DISCOVERY_ENABLED=1 causes pytest to write + collected tests as JSON Lines to a file and exit without running any tests. + """ + + @pytest.fixture(autouse=True) + def output_file(self, pytester: Pytester, monkeypatch: pytest.MonkeyPatch) -> Path: + """Use a stable output path inside the pytester tmpdir for each test.""" + output = pytester.path / "discovery_output.json" + monkeypatch.setenv("DD_TEST_OPTIMIZATION_DISCOVERY_ENABLED", "1") + monkeypatch.setenv("DD_TEST_OPTIMIZATION_DISCOVERY_FILE", str(output)) + return output + + def _read_discovered(self, output_file: Path) -> list[dict]: + return [json.loads(line) for line in output_file.read_text().splitlines() if line.strip()] + + # ------------------------------------------------------------------ + # Basic behaviour + # ------------------------------------------------------------------ + + def test_writes_json_lines_file(self, pytester: Pytester, output_file: Path) -> None: + """Discovery mode writes one JSON object per line for each collected test.""" + pytester.makepyfile( + test_foo=""" + def test_alpha(): + pass + + def test_beta(): + pass + """ + ) + + result = pytester.inline_run() + + assert result.ret == 0 + tests = self._read_discovered(output_file) + assert len(tests) == 2 + names = {t["name"] for t in tests} + assert names == {"test_alpha", "test_beta"} + + def test_no_items_collected_writes_empty_file(self, pytester: Pytester, output_file: Path) -> None: + """When no tests are collected the output file is created but empty.""" + pytester.makepyfile(test_empty="") + + result = pytester.inline_run() + + assert result.ret == 0 + assert output_file.exists() + assert output_file.read_text(encoding="utf-8") == "" + + def test_does_not_run_tests(self, pytester: Pytester, output_file: Path) -> None: + """Discovery mode exits after collection — test bodies are never executed.""" + pytester.makepyfile( + test_boom=""" + def test_would_fail(): + raise RuntimeError("This should never run") + """ + ) + + result = pytester.inline_run() + + assert result.ret == 0 + + def test_json_fields_are_correct(self, pytester: Pytester, output_file: Path) -> None: + """Each JSON object has name, suite, module, parameters, suiteSourceFile. + + module is the dotted directory path of the test file (matching what the + plugin reports at run time), not the framework name. A root-level file + has an empty module; a nested file carries the dotted path. + """ + # Root-level test: module should be "" + pytester.makepyfile( + test_fields=""" + def test_something(): + pass + """ + ) + # Nested test: module should reflect the subdirectory + pytester.mkdir("subpkg") + pytester.path.joinpath("subpkg", "test_nested.py").write_text("def test_inner(): pass") + + pytester.inline_run() + + tests = {e["suite"]: e for e in self._read_discovered(output_file)} + + root_entry = tests["test_fields.py"] + assert root_entry["name"] == "test_something" + assert root_entry["module"] == "" + assert root_entry["parameters"] is None + assert root_entry["suiteSourceFile"].endswith("test_fields.py") + + nested_entry = tests["test_nested.py"] + assert nested_entry["name"] == "test_inner" + assert nested_entry["module"] == "subpkg" + assert nested_entry["suiteSourceFile"].endswith("test_nested.py") + + def test_exit_code_zero(self, pytester: Pytester, output_file: Path) -> None: + """Discovery mode always exits with code 0.""" + pytester.makepyfile(test_x="def test_pass(): pass") + result = pytester.inline_run() + assert result.ret == 0 + + # ------------------------------------------------------------------ + # Parametrized tests + # ------------------------------------------------------------------ + + def test_parametrized_tests_include_parameters(self, pytester: Pytester, output_file: Path) -> None: + """Parametrized tests have their parameters encoded in the parameters field.""" + pytester.makepyfile( + test_params=""" + import pytest + + @pytest.mark.parametrize("x,y", [(1, 2), (3, 4)]) + def test_add(x, y): + pass + """ + ) + + pytester.inline_run() + + tests = self._read_discovered(output_file) + assert len(tests) == 2 + for entry in tests: + assert entry["parameters"] is not None + params = json.loads(entry["parameters"]) + assert "arguments" in params + assert "x" in params["arguments"] + assert "y" in params["arguments"] + + # ------------------------------------------------------------------ + # Skip filtering + # ------------------------------------------------------------------ + + def test_skip_marked_tests_are_excluded(self, pytester: Pytester, output_file: Path) -> None: + """Tests decorated with @pytest.mark.skip are excluded from the output.""" + pytester.makepyfile( + test_skip=""" + import pytest + + @pytest.mark.skip(reason="not ready") + def test_skipped(): + pass + + def test_included(): + pass + """ + ) + + pytester.inline_run() + + tests = self._read_discovered(output_file) + assert len(tests) == 1 + assert tests[0]["name"] == "test_included" + + def test_skipif_true_tests_are_excluded(self, pytester: Pytester, output_file: Path) -> None: + """Tests with @pytest.mark.skipif(True, ...) are excluded from the output.""" + pytester.makepyfile( + test_skipif=""" + import pytest + + @pytest.mark.skipif(True, reason="always skip") + def test_always_skipped(): + pass + + @pytest.mark.skipif(False, reason="never skip") + def test_never_skipped(): + pass + """ + ) + + pytester.inline_run() + + tests = self._read_discovered(output_file) + assert len(tests) == 1 + assert tests[0]["name"] == "test_never_skipped" + + def test_skipif_with_version_check(self, pytester: Pytester, output_file: Path) -> None: + """Tests with @pytest.mark.skipif(sys.version_info...) are handled correctly.""" + always_true = sys.version_info >= (2, 0) + always_false = sys.version_info >= (99, 0) + + pytester.makepyfile( + test_version=""" + import sys + import pytest + + @pytest.mark.skipif(sys.version_info >= (2, 0), reason="skipped on all Pythons") + def test_skipped_on_all(): + pass + + @pytest.mark.skipif(sys.version_info >= (99, 0), reason="only on Python 99+") + def test_included_on_all(): + pass + """ + ) + + pytester.inline_run() + + tests = self._read_discovered(output_file) + names = {t["name"] for t in tests} + + if always_true: + assert "test_skipped_on_all" not in names + if not always_false: + assert "test_included_on_all" in names + + def test_skipif_string_condition_included(self, pytester: Pytester, output_file: Path) -> None: + """Tests with string-form skipif conditions are conservatively included.""" + pytester.makepyfile( + test_string_skip=""" + import pytest + + @pytest.mark.skipif("True", reason="string condition not evaluated") + def test_with_string_condition(): + pass + """ + ) + + pytester.inline_run() + + tests = self._read_discovered(output_file) + assert len(tests) == 1 + assert tests[0]["name"] == "test_with_string_condition" + + # ------------------------------------------------------------------ + # Interaction with --ddtrace + # ------------------------------------------------------------------ + + def test_discovery_mode_disables_ci_visibility(self, pytester: Pytester, output_file: Path) -> None: + """When discovery mode is active, CI visibility is not initialised even if --ddtrace is passed.""" + pytester.makepyfile(test_ci="def test_one(): pass") + + # Should exit cleanly with no API key / agent configured + result = pytester.inline_run("--ddtrace") + assert result.ret == 0 + tests = self._read_discovered(output_file) + assert len(tests) == 1