diff --git a/CHANGES.rst b/CHANGES.rst index 7960238..f186682 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,7 +4,9 @@ Changelog 16.4 (unreleased) ----------------- -- Nothing changed yet. +- Added the ``pytest_rerunfailures_rerun_policy`` hook: plugins can return a + ``RerunPolicy`` to give a specific failure class its own rerun count / + all-reruns-need-to-pass mode, without changing how other failures are rerun. 16.3 (2026-05-22) diff --git a/README.rst b/README.rst index 30d2669..3974f48 100644 --- a/README.rst +++ b/README.rst @@ -236,6 +236,31 @@ Each retried attempt's traceback is appended to the ``rerun test summary info`` section. The section is emitted automatically when the flag is set, so ``-rR`` is not required. +Per-failure rerun policy (hook) +------------------------------- + +Plugins can give a *specific failure class* its own rerun count and semantics via the +``pytest_rerunfailures_rerun_policy`` hook, without changing how other failures are +rerun. The hook is called once per test, on its first failing report, and returns a +``RerunPolicy`` (or ``None`` for the default behavior): + +.. code-block:: python + + # conftest.py + from pytest_rerunfailures import RerunPolicy + + def pytest_rerunfailures_rerun_policy(item, report, call): + # e.g. give transient provider/infra errors one rerun, pass on recovery — + # while other failures keep the global --reruns / --all-reruns-need-to-pass + # behavior. + exc = call.excinfo.value if call.excinfo else None + if isinstance(exc, ConnectionError): + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) + return None + +``RerunPolicy`` fields left ``None`` keep the plugin's default for that item. The policy +is fixed at the item's first failure and applies to its whole rerun sequence. + Output ------ diff --git a/pyproject.toml b/pyproject.toml index 2800983..f0fe8c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,10 @@ entry-points.pytest11.rerunfailures = "pytest_rerunfailures" [tool.setuptools.dynamic] readme = { file = [ "HEADER.rst", "README.rst", "CHANGES.rst" ] } +[tool.setuptools] +package-dir = { "" = "src" } +py-modules = [ "pytest_rerunfailures", "pytest_rerunfailures_newhooks" ] + [tool.ruff] fix = true lint.select = [ diff --git a/src/pytest_rerunfailures.py b/src/pytest_rerunfailures.py index 127cdc9..3c5a8bc 100644 --- a/src/pytest_rerunfailures.py +++ b/src/pytest_rerunfailures.py @@ -10,6 +10,7 @@ import traceback import warnings from contextlib import suppress +from dataclasses import dataclass import pytest from _pytest.outcomes import fail @@ -46,6 +47,32 @@ def works_with_current_xdist(): RERUNS_DELAY_DESC = "add time (seconds) delay between reruns." +@dataclass(frozen=True) +class RerunPolicy: + """Per-failure rerun policy returned by ``pytest_rerunfailures_rerun_policy``. + + Any field left ``None`` keeps the plugin's default for that item. + + Attributes: + reruns: Rerun count for this failure (overrides ``--reruns`` / the marker). + all_reruns_need_to_pass: Override all-reruns-need-to-pass for this failure. + """ + + reruns: int | None = None + all_reruns_need_to_pass: bool | None = None + + +def pytest_addhooks(pluginmanager): + """Register this plugin's own hookspecs (see ``pytest_rerunfailures_newhooks``). + + The specs live in a dedicated module so ``add_hookspecs`` sees only them, not this + module's own hook implementations. + """ + import pytest_rerunfailures_newhooks + + pluginmanager.add_hookspecs(pytest_rerunfailures_newhooks) + + # command line options def pytest_addoption(parser): group = parser.getgroup( @@ -111,6 +138,16 @@ def pytest_addoption(parser): dest="fail_on_flaky", help="Fail the test run with exit code 7 if a flaky test passes on a rerun.", ) + group._addoption( + "--all-reruns-need-to-pass", + action="store_true", + dest="all_reruns_need_to_pass", + default=False, + help=( + "If enabled, after an initial failure, all reruns must pass " + "for the test to succeed." + ), + ) group._addoption( "--rerun-show-tracebacks", action="store_true", @@ -124,6 +161,11 @@ def pytest_addoption(parser): arg_type = "string" parser.addini("reruns", RERUNS_DESC, type=arg_type) parser.addini("reruns_delay", RERUNS_DELAY_DESC, type=arg_type) + parser.addini( + "all_reruns_need_to_pass", + "If enabled, all reruns must pass after initial failure", + type=arg_type, + ) # making sure the options make sense @@ -205,6 +247,33 @@ def get_reruns_delay(item): return delay +def get_all_reruns_need_to_pass(item): + """Get whether all reruns need to pass from marker, config, or ini.""" + rerun_marker = _get_marker(item) + + # Check marker kwargs first + if rerun_marker is not None and "all_reruns_need_to_pass" in rerun_marker.kwargs: + return rerun_marker.kwargs["all_reruns_need_to_pass"] + + # Check command-line option + all_need_pass = item.session.config.getvalue("all_reruns_need_to_pass") + if all_need_pass is not None: + return all_need_pass + + # Check ini value + try: + all_need_pass = item.session.config.getini("all_reruns_need_to_pass") + if all_need_pass: + # Parse string values like "True", "true", "1", etc. + if isinstance(all_need_pass, str): + return all_need_pass.lower() in ("true", "1", "yes", "on") + return bool(all_need_pass) + except (TypeError, ValueError): + pass + + return False + + def get_reruns_condition(item): rerun_marker = _get_marker(item) @@ -375,9 +444,10 @@ def pytest_configure(config): # add flaky marker config.addinivalue_line( "markers", - "flaky(reruns=1, reruns_delay=0): mark test to re-run up " - "to 'reruns' times. Add a delay of 'reruns_delay' seconds " - "between re-runs.", + "flaky(reruns=1, reruns_delay=0, all_reruns_need_to_pass=False): " + "mark test to re-run up to 'reruns' times. Add a delay of " + "'reruns_delay' seconds between re-runs. If " + "'all_reruns_need_to_pass' is True, all reruns must pass.", ) if config.pluginmanager.hasplugin("xdist") and HAS_PYTEST_HANDLECRASHITEM: @@ -599,6 +669,13 @@ def pytest_runtest_makereport(item, call): item, result, call.excinfo ) + # On the first failure, ask plugins for a per-failure rerun policy so the protocol + # can give this failure class its own rerun count / semantics. + if result.failed and not hasattr(item, "_rerunfailures_policy"): + item._rerunfailures_policy = item.ihook.pytest_rerunfailures_rerun_policy( + item=item, report=result, call=call + ) + def pytest_runtest_protocol(item, nextitem): """ @@ -617,6 +694,7 @@ def pytest_runtest_protocol(item, nextitem): # first item if necessary check_options(item.session.config) delay = get_reruns_delay(item) + all_reruns_need_to_pass = get_all_reruns_need_to_pass(item) parallel = not is_master(item.config) db = item.session.config.failures_db item.execution_count = db.get_test_failures(item.nodeid) @@ -625,31 +703,106 @@ def pytest_runtest_protocol(item, nextitem): if item.execution_count > reruns: return True + # Track rerun results when all reruns need to pass + initial_failure_occurred = False + rerun_results = [] # Track result of each rerun (True=passed, False=failed) + policy_applied = False + need_to_run = True while need_to_run: item.execution_count += 1 item.ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location) reports = runtestprotocol(item, nextitem=nextitem, log=False) + # Apply a per-failure rerun policy (pytest_rerunfailures_rerun_policy) once the + # first attempt has run and makereport has classified its failure. Lets a plugin + # override the rerun count / all-reruns mode for THIS item, without changing how + # other failures are rerun. + if not policy_applied: + policy = getattr(item, "_rerunfailures_policy", None) + if policy is not None: + policy_applied = True + if policy.reruns is not None: + reruns = policy.reruns + db.set_test_reruns(item.nodeid, reruns) + if policy.all_reruns_need_to_pass is not None: + all_reruns_need_to_pass = policy.all_reruns_need_to_pass + for report in reports: # 3 reports: setup, call, teardown report.rerun = item.execution_count - 1 - if _should_not_rerun(item, report, reruns): - # last run or no failure detected, log normally - item.ihook.pytest_runtest_logreport(report=report) + + # Track initial failure for all_reruns_need_to_pass mode + if ( + all_reruns_need_to_pass + and report.when == "call" + and report.failed + and item.execution_count == 1 + ): + initial_failure_occurred = True + + # Track rerun results (after initial failure) - only track call phase + if ( + all_reruns_need_to_pass + and initial_failure_occurred + and item.execution_count > 1 + and report.when == "call" + ): + rerun_results.append( + not report.failed + ) # True if passed, False if failed + + # In all_reruns_need_to_pass mode with initial failure, + # override normal behavior + if all_reruns_need_to_pass and initial_failure_occurred: + # execution_count starts at 1, so: + # - execution_count==1: initial run (failed) + # - execution_count==2..reruns+1: reruns (must run all of them) + is_last_rerun = item.execution_count > reruns + + if is_last_rerun: + # Last run, check if all reruns passed + if any(not r for r in rerun_results): + # At least one rerun failed, mark final outcome as failed + if report.when == "call": + report.outcome = "failed" + # Log the final report + item.ihook.pytest_runtest_logreport(report=report) + else: + # Not the last rerun yet + # Only trigger rerun after processing the call phase + if report.when == "call": + report.outcome = "rerun" + time.sleep(delay) + if not parallel or works_with_current_xdist(): + item.ihook.pytest_runtest_logreport(report=report) + + _remove_cached_results_from_failed_fixtures(item) + _remove_failed_setup_state_from_session(item) + break # trigger rerun + else: + # For setup/teardown, just log normally + item.ihook.pytest_runtest_logreport(report=report) else: - # failure detected and reruns not exhausted, since i < reruns - report.outcome = "rerun" - time.sleep(delay) + # Normal rerun behavior + should_not_rerun = _should_not_rerun(item, report, reruns) - if not parallel or works_with_current_xdist(): - # will rerun test, log intermediate result + if should_not_rerun: + # last run or no failure detected, log normally item.ihook.pytest_runtest_logreport(report=report) + else: + # failure detected and reruns not exhausted + report.outcome = "rerun" + time.sleep(delay) + + if not parallel or works_with_current_xdist(): + # will rerun test, log intermediate result + item.ihook.pytest_runtest_logreport(report=report) - # cleanin item's cashed results from any level of setups - _remove_cached_results_from_failed_fixtures(item) - _remove_failed_setup_state_from_session(item) + # cleanin item's cashed results from any level of setups + _remove_cached_results_from_failed_fixtures(item) + _remove_failed_setup_state_from_session(item) - break # trigger rerun + break # trigger rerun else: need_to_run = False diff --git a/src/pytest_rerunfailures_newhooks.py b/src/pytest_rerunfailures_newhooks.py new file mode 100644 index 0000000..2947d2c --- /dev/null +++ b/src/pytest_rerunfailures_newhooks.py @@ -0,0 +1,30 @@ +"""Hook specifications added by the smith fork of pytest-rerunfailures. + +Kept in a dedicated module so ``PluginManager.add_hookspecs`` registers only these +specs and not the plugin's own ``pytest_*`` hook implementations (pytest treats any +``pytest_``-prefixed function in a namespace as a hookspec). +""" + +import pytest + + +@pytest.hookspec(firstresult=True) +def pytest_rerunfailures_rerun_policy(item, report, call): + """Return a ``RerunPolicy`` for a test's first failure, or ``None`` for default. + + Consulted once per test, on its first failing report (``.when`` may be ``"setup"``, + ``"call"``, or ``"teardown"``). The returned policy is locked to the item's WHOLE rerun + sequence -- it is not recomputed on later attempts, so a differently-classed failure on + a rerun inherits this policy (the rerun count / mode is fixed by the first failure). Lets + a plugin give a specific failure class (e.g. provider-infra errors) its own rerun count / + semantics, without affecting how other failures are rerun. The first non-None result wins. + + Args: + item: The test item that failed. + report: The first failing ``TestReport`` (``.when`` is ``"setup"``, ``"call"``, + or ``"teardown"``). + call: The ``CallInfo`` (``call.excinfo`` carries the exception). + + Returns: + A ``pytest_rerunfailures.RerunPolicy``, or ``None`` for default behavior. + """ diff --git a/tests/test_pytest_rerunfailures.py b/tests/test_pytest_rerunfailures.py index b60716c..da21d22 100644 --- a/tests/test_pytest_rerunfailures.py +++ b/tests/test_pytest_rerunfailures.py @@ -1458,6 +1458,214 @@ def test_1(session_fixture, function_fixture): result.stdout.fnmatch_lines("session teardown") +def test_all_reruns_need_to_pass_disabled_by_default(testdir): + """Test that default behavior is unchanged (flag disabled by default).""" + testdir.makepyfile( + """ + import py + def test_default_behavior(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('Fail on first attempt') + # Pass on second attempt + """ + ) + result = testdir.runpytest("--reruns", "3") + # Should pass because one successful rerun is enough (default behavior) + assert_outcomes(result, passed=1, rerun=1) + + +def test_all_reruns_need_to_pass_all_pass(testdir): + """Test that when all reruns pass, test passes.""" + testdir.makepyfile( + """ + import py + def test_all_pass(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('Fail on first attempt') + # Pass on all subsequent attempts + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Should pass because all 3 reruns pass + assert_outcomes(result, passed=1, rerun=3) + + +def test_all_reruns_need_to_pass_some_fail(testdir): + """Test that when some reruns fail, test fails.""" + testdir.makepyfile( + """ + import py + def test_some_fail(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + # Fail on attempt 0 (initial) and attempt 2 (second rerun) + if count == 0 or count == 2: + raise Exception(f'Fail on attempt {count}') + # Pass on attempts 1 and 3 + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Should fail because rerun 2 fails + assert_outcomes(result, passed=0, failed=1, rerun=3) + + +def test_all_reruns_need_to_pass_all_fail(testdir): + """Test that when all reruns fail, test fails.""" + testdir.makepyfile( + """ + def test_all_fail(): + raise Exception('Always fail') + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Should fail because all reruns fail + assert_outcomes(result, passed=0, failed=1, rerun=3) + + +def test_all_reruns_need_to_pass_marker_override(testdir): + """Test that marker can override command-line flag.""" + testdir.makepyfile( + """ + import pytest + import py + + @pytest.mark.flaky(reruns=3, all_reruns_need_to_pass=True) + def test_marker_override(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('Fail on first attempt') + # Pass on all subsequent attempts + """ + ) + # Not passing --all-reruns-need-to-pass on command line, but marker enables it + result = testdir.runpytest("--verbose") + assert_outcomes(result, passed=1, rerun=3) + + +def test_all_reruns_need_to_pass_marker_can_disable(testdir): + """Test that marker can disable flag even when set on command line.""" + testdir.makepyfile( + """ + import pytest + import py + + @pytest.mark.flaky(reruns=3, all_reruns_need_to_pass=False) + def test_marker_disables(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('Fail on first attempt') + # Pass on second attempt + """ + ) + # Passing --all-reruns-need-to-pass, but marker disables it + result = testdir.runpytest("--all-reruns-need-to-pass", "--verbose") + # Should pass with just one successful rerun (default behavior) + assert_outcomes(result, passed=1, rerun=1) + + +def test_all_reruns_need_to_pass_with_only_rerun(testdir): + """Test interaction with --only-rerun flag.""" + testdir.makepyfile( + """ + import py + def test_with_only_rerun(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise ValueError('Fail on first attempt') + # Pass on subsequent attempts + """ + ) + result = testdir.runpytest( + "--reruns", "3", "--all-reruns-need-to-pass", "--only-rerun", "ValueError" + ) + # Should pass because all reruns pass + assert_outcomes(result, passed=1, rerun=3) + + +def test_all_reruns_need_to_pass_initial_pass(testdir): + """Test that flag has no effect if test passes on first attempt.""" + testdir.makepyfile( + """ + def test_initial_pass(): + pass # Always passes + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Should pass without any reruns + assert_outcomes(result, passed=1, rerun=0) + + +def test_all_reruns_need_to_pass_zero_reruns(testdir): + """Test that flag has no effect with reruns=0.""" + testdir.makepyfile( + """ + def test_zero_reruns(): + raise Exception('Fail') + """ + ) + result = testdir.runpytest("--reruns", "0", "--all-reruns-need-to-pass") + # Should fail without any reruns + assert_outcomes(result, passed=0, failed=1, rerun=0) + + +def test_all_reruns_need_to_pass_setup_failure(testdir): + """Test behavior when setup fails - only call phase is tracked.""" + testdir.makepyfile( + """ + import pytest + import py + + @pytest.fixture + def failing_fixture(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count < 2: + raise Exception(f'Fixture fails on attempt {count}') + return "ok" + + def test_setup_failure(failing_fixture): + pass + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Note: all_reruns_need_to_pass only tracks call phase failures, not setup/teardown + # Setup eventually passes, so test passes + assert_outcomes(result, passed=1, rerun=2) + + +def test_all_reruns_need_to_pass_command_line(testdir): + """Test that command line flag works as expected.""" + testdir.makepyfile( + """ + import py + def test_cli_flag(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('Fail on first attempt') + # Pass on all subsequent attempts + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Should pass because all reruns pass + assert_outcomes(result, passed=1, rerun=3) + + @pytest.mark.parametrize("mark_params", ["", "reruns=1"]) def test_force_reruns(testdir, mark_params): testdir.makepyfile( @@ -1526,3 +1734,246 @@ def test_pass(): result = testdir.runpytest("--reruns-mode", "bogus") assert result.ret != 0 + + +# --- pytest_rerunfailures_rerun_policy: per-failure rerun policy hook --- + + +def test_rerun_policy_none_keeps_global_behavior(testdir): + """A policy hook that returns None does not change the global rerun behavior.""" + testdir.makeconftest( + """ + def pytest_rerunfailures_rerun_policy(item, report, call): + return None + """ + ) + testdir.makepyfile("def test_fail(): raise Exception('boom')") + result = testdir.runpytest("--reruns", "2") + assert_outcomes(result, passed=0, failed=1, rerun=2) + + +def test_rerun_policy_overrides_rerun_count(testdir): + """A policy can cap the rerun count for its failure below the global --reruns.""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=1) + """ + ) + testdir.makepyfile("def test_fail(): raise Exception('boom')") + # Global --reruns is 5, but the policy caps this failure at a single rerun. + result = testdir.runpytest("--reruns", "5") + assert_outcomes(result, passed=0, failed=1, rerun=1) + + +def test_rerun_policy_recovers_on_single_rerun(testdir): + """A failure that recovers on its one rerun passes under all-reruns mode.""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) + """ + ) + testdir.makepyfile( + """ + import py + def test_transient(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('transient failure') + """ + ) + # Global mode is all-reruns-need-to-pass with 3 reruns; the policy overrides it + # to "1 rerun, pass on recovery" for this failure. + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + assert_outcomes(result, passed=1, rerun=1) + + +def test_rerun_policy_overrides_all_reruns_need_to_pass(testdir): + """A policy can disable all-reruns-need-to-pass for its failure (sustained fail).""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) + """ + ) + testdir.makepyfile("def test_fail(): raise Exception('always down')") + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # One rerun (not three), and it fails -> a single failure. + assert_outcomes(result, passed=0, failed=1, rerun=1) + + +def test_rerun_policy_only_applies_to_matching_failures(testdir): + """The policy can target one failure class; others keep default behavior.""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + if call.excinfo is not None and call.excinfo.typename == "InfraError": + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) + return None + """ + ) + testdir.makepyfile( + """ + class InfraError(Exception): + pass + + def test_infra(): + raise InfraError('provider down') + + def test_behavioral(): + raise AssertionError('real bug') + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # test_infra: policy -> 1 rerun; test_behavioral: default -> 3. Both fail. + assert_outcomes(result, passed=0, failed=2, rerun=4) + + +@pytest.mark.skipif(not has_xdist, reason="requires pytest-xdist") +def test_rerun_policy_under_xdist(testdir): + """The policy override + recovery work under xdist (policy per-item).""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) + """ + ) + testdir.makepyfile( + """ + import py + def test_transient(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise Exception('transient failure') + """ + ) + result = testdir.runpytest("-n", "1", "--reruns", "3", "--all-reruns-need-to-pass") + assert_outcomes(result, passed=1, rerun=1) + + +def test_rerun_policy_no_hook_impl_keeps_global_behavior(testdir): + """With NO policy hook implemented at all, global rerun behavior is unchanged. + + Guards the additive contract: suites that never implement the hook must behave + exactly as before (this is the common case). + """ + testdir.makepyfile("def test_fail(): raise Exception('boom')") + result = testdir.runpytest("--reruns", "2") + assert_outcomes(result, passed=0, failed=1, rerun=2) + + +def test_rerun_policy_can_increase_rerun_count(testdir): + """A policy can RAISE the rerun count for its failure above the global --reruns.""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=3) + """ + ) + testdir.makepyfile("def test_fail(): raise Exception('boom')") + # Global --reruns is 1, but the policy raises this failure's count to 3. + result = testdir.runpytest("--reruns", "1") + assert_outcomes(result, passed=0, failed=1, rerun=3) + + +def test_rerun_policy_firstresult_first_non_none_wins(testdir): + """firstresult=True: the first non-None policy wins over a later implementation.""" + testdir.makepyfile( + plugin_first=""" + import pytest + from pytest_rerunfailures import RerunPolicy + @pytest.hookimpl(tryfirst=True) + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=1) + """ + ) + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=5) + """ + ) + testdir.makepyfile("def test_fail(): raise Exception('boom')") + testdir.syspathinsert() # make plugin_first importable for `-p` + # plugin_first is consulted first (tryfirst) and returns non-None, so its reruns=1 + # wins over the conftest's reruns=5 (which would give rerun=5 if it won). + result = testdir.runpytest("-p", "plugin_first", "--reruns", "3") + assert_outcomes(result, passed=0, failed=1, rerun=1) + + +def test_rerun_policy_disables_all_reruns_stops_on_first_pass(testdir): + """Overriding all_reruns_need_to_pass=False takes effect with reruns>=2. + + Distinguishes the two modes (which are indistinguishable at reruns=1): a + fail/pass/fail sequence stops on the first passing rerun under the override + (passed), whereas global all-reruns-need-to-pass would run every rerun and fail + because a later rerun failed. + """ + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + return RerunPolicy(reruns=2, all_reruns_need_to_pass=False) + """ + ) + testdir.makepyfile( + """ + import py + def test_flaky(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count != 1: # fail (0), pass (1), fail (2) + raise Exception('flaky failure') + """ + ) + result = testdir.runpytest("--reruns", "2", "--all-reruns-need-to-pass") + # Override -> normal mode: stops on the first passing rerun. Without the override + # (all-reruns-need-to-pass) it would be passed=0, failed=1, rerun=2. + assert_outcomes(result, passed=1, rerun=1) + + +def test_rerun_policy_locked_at_first_failure(testdir): + """The policy is fixed at the first failure; a differently-classed rerun failure + inherits it and is NOT recomputed.""" + testdir.makeconftest( + """ + from pytest_rerunfailures import RerunPolicy + def pytest_rerunfailures_rerun_policy(item, report, call): + # Only the first failure (InfraError) ever reaches this hook; give it 1 rerun. + if call.excinfo is not None and call.excinfo.typename == "InfraError": + return RerunPolicy(reruns=1, all_reruns_need_to_pass=False) + return RerunPolicy(reruns=3) # would apply if (wrongly) recomputed on the rerun + """ + ) + testdir.makepyfile( + """ + import py + class InfraError(Exception): + pass + + def test_mixed(): + path = py.path.local(__file__).dirpath().ensure('test.res') + count = int(path.read() or 0) + path.write(count + 1) + if count == 0: + raise InfraError('provider blip') # first failure -> policy reruns=1 + raise AssertionError('real bug on the rerun') # different class on the rerun + """ + ) + result = testdir.runpytest("--reruns", "3", "--all-reruns-need-to-pass") + # Locked to the infra policy (1 rerun), NOT recomputed to reruns=3 for the behavioral + # failure that surfaces on the rerun. + assert_outcomes(result, passed=0, failed=1, rerun=1)