diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..fdb0540f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tpl/pytest-doctestplus"] + path = tpl/pytest-doctestplus + url = https://github.com/Erotemic/pytest-doctestplus.git diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a13c1a8..0b92e227 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Version 1.3.3 - Unreleased ### Added +* doctestplus compatibility with `FLOAT_CMP`, `IGNORE_OUTPUT` directives and + `__doctest_skip__`, `__doctest_requires__` module level special vars. * Added `deferred_output_matching` and `optional_want` config knobs, plus CLI flags, to opt into stdlib/doctest-like output semantics without changing the default xdoctest behavior. diff --git a/dev/_compare/demo_enhancements.py b/dev/_compare/demo_enhancements.py index af3f39cb..59a84425 100644 --- a/dev/_compare/demo_enhancements.py +++ b/dev/_compare/demo_enhancements.py @@ -78,20 +78,40 @@ def multiple_eval_for_loops_v2(): """ -def compact_style_code(): +def top_level_async(): """ - This compact style is a bit ugly, but it should still be valid python + xdoctest supports top-level async examples. - Exception: - >>> try: raise Exception # doctest: +ELLIPSIS - ... except Exception: raise - Traceback (most recent call last): - ... - Exception - ... + >>> async def func(): + >>> return 'awaited' + >>> await func() + 'awaited' + """ + + +def prefixed_triple_quotes(): + """ + >>> x = ''' + >>> Prefixing every line with >>> is ok too + >>> even inside a string literal + >>> ''' + >>> print(x.strip()) + Prefixing every line with >>> is ok too + even inside a string literal + """ + pass + +def assert_based_examples_can_ignore_stdout(): + """ + >>> print('debug output that is not part of the test') + >>> value = 1 + 1 + >>> assert value == 2 + """ + + +def block_directives(): + """ + >>> # xdoctest: +SKIP + >>> raise AssertionError('xdoctest skips this whole block') """ - try: - raise Exception # NOQA - except Exception: - pass # NOQA diff --git a/docs/source/conf.py b/docs/source/conf.py index d16543ec..9e106c67 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -111,10 +111,9 @@ # sys.path.insert(0, os.path.abspath('.')) # -- Project information ----------------------------------------------------- +from os.path import dirname, exists, join + import sphinx_rtd_theme -from os.path import exists -from os.path import dirname -from os.path import join def parse_version(fpath): diff --git a/src/xdoctest/__init__.py b/src/xdoctest/__init__.py index d93ffb7d..c06b1d37 100644 --- a/src/xdoctest/__init__.py +++ b/src/xdoctest/__init__.py @@ -332,6 +332,30 @@ def fib(n): doctest_callable, doctest_module, ) +from xdoctest.directive_facade import ( + BLANKLINE_MARKER, + DONT_ACCEPT_BLANKLINE, + ELLIPSIS, + ELLIPSIS_MARKER, + FLOAT_CMP, + IGNORE_EXCEPTION_DETAIL, + IGNORE_OUTPUT, + IGNORE_WHITESPACE, + IGNORE_WANT, + NORMALIZE_REPR, + NORMALIZE_WHITESPACE, + REPORT_CDIFF, + REPORT_NDIFF, + REPORT_UDIFF, + optionflags_to_runtime_state, + register_optionflag, + runtime_state_to_optionflags, +) +from xdoctest.checker_facade import ( + OutputChecker, + register_checker, + resolve_checker, +) __all__ = [ 'DoctestParseError', @@ -343,4 +367,24 @@ def fib(n): 'utils', 'docstr', '__version__', + 'BLANKLINE_MARKER', + 'ELLIPSIS_MARKER', + 'DONT_ACCEPT_BLANKLINE', + 'NORMALIZE_WHITESPACE', + 'ELLIPSIS', + 'IGNORE_EXCEPTION_DETAIL', + 'REPORT_UDIFF', + 'REPORT_CDIFF', + 'REPORT_NDIFF', + 'IGNORE_WANT', + 'IGNORE_OUTPUT', + 'IGNORE_WHITESPACE', + 'FLOAT_CMP', + 'NORMALIZE_REPR', + 'register_optionflag', + 'runtime_state_to_optionflags', + 'optionflags_to_runtime_state', + 'register_checker', + 'resolve_checker', + 'OutputChecker', ] diff --git a/src/xdoctest/checker.py b/src/xdoctest/checker.py index f92b41f2..51e40ae4 100644 --- a/src/xdoctest/checker.py +++ b/src/xdoctest/checker.py @@ -36,6 +36,8 @@ from __future__ import annotations import difflib +import math +import types import re import typing from typing import Dict, Set @@ -51,6 +53,15 @@ TRAILING_WS = re.compile(r'[ \t]*$', re.UNICODE | re.MULTILINE) +_FLOAT_CMP_NUMBER_RE = re.compile( + r'(?P(? bool: + """ + Check output using the currently configured output checker backend. + + Args: + got (str): text produced by the test + want (str): target to match against + runstate (xdoctest.directive.RuntimeState | None): current state + + Returns: + bool: True if got matches want or if the check is disabled + """ + if not want: # nocover + return True + if runstate is None: + runstate = directive.RuntimeState() + from xdoctest import checker_facade + optionflags = checker_facade.runtime_state_to_optionflags(runstate) + output_checker = checker_facade.resolve_current_checker(runstate) + return bool(output_checker.check_output(want, got, optionflags)) + + def _check_match( got: str, want: str, runstate: directive.RuntimeState | dict ) -> bool: @@ -278,6 +315,10 @@ def _check_match( if got == want: return True + if runstate['FLOAT_CMP']: + if _float_cmp_match(got, want, runstate): + return True + if runstate['ELLIPSIS']: if _ellipsis_match(got, want): return True @@ -369,6 +410,178 @@ def _ellipsis_match(got: typing.Any, want: typing.Any) -> bool: return True +def _float_cmp_match( + got: str, want: str, runstate: directive.RuntimeState | dict +) -> bool: + """ + Compare output strings with numeric substrings matched approximately. + + Text outside of numeric substrings must still agree. Ellipsis is handled in + the same token stream so that ``FLOAT_CMP`` composes with ``ELLIPSIS``. + """ + allow_ellipsis = bool(runstate['ELLIPSIS']) + got_tokens = _tokenize_float_cmp_text(got, allow_ellipsis=allow_ellipsis) + want_tokens = _tokenize_float_cmp_text(want, allow_ellipsis=allow_ellipsis) + return _float_cmp_match_tokens(got_tokens, want_tokens) + + +def _tokenize_float_cmp_text( + text: str, allow_ellipsis: bool +) -> list[tuple[str, str]]: + tokens: list[tuple[str, str]] = [] + pos = 0 + for match in _FLOAT_CMP_NUMBER_RE.finditer(text): + start, stop = match.span() + if start > pos: + _append_float_cmp_text_tokens( + tokens, text[pos:start], allow_ellipsis=allow_ellipsis + ) + tokens.append(('number', match.group('number'))) + pos = stop + if pos < len(text): + _append_float_cmp_text_tokens( + tokens, text[pos:], allow_ellipsis=allow_ellipsis + ) + return tokens + + +def _append_float_cmp_text_tokens( + tokens: list[tuple[str, str]], text: str, allow_ellipsis: bool +) -> None: + if not allow_ellipsis or ELLIPSIS_MARKER not in text: + if text: + tokens.append(('text', text)) + return + + parts = re.split(r'(\.\.\.)', text) + for part in parts: + if not part: + continue + if part == ELLIPSIS_MARKER: + tokens.append(('ellipsis', part)) + else: + tokens.append(('text', part)) + + +def _float_cmp_match_tokens( + got_tokens: list[tuple[str, str]], + want_tokens: list[tuple[str, str]], +) -> bool: + """ + Match tokenized output for ``FLOAT_CMP``. + + ``want_tokens`` may contain ``('ellipsis', '...')`` tokens, which act like + a wildcard over zero or more tokens in ``got_tokens``. + + Example: + >>> from xdoctest.checker import _float_cmp_match_tokens + >>> got_tokens = [('text', 'best='), ('number', '0.3333333333')] + >>> want_tokens = [('text', 'best='), ('number', '0.333333')] + >>> _float_cmp_match_tokens(got_tokens, want_tokens) + True + + Example: + >>> from xdoctest.checker import _float_cmp_match_tokens + >>> got_tokens = [('text', 'best='), ('number', '1.0')] + >>> want_tokens = [('text', 'best='), ('number', '1.1')] + >>> _float_cmp_match_tokens(got_tokens, want_tokens) + False + + Example: + >>> from xdoctest.checker import _float_cmp_match_tokens + >>> got_tokens = [ + ... ('text', 'prefix '), + ... ('number', '0.3333333333'), + ... ('text', ' middle '), + ... ('number', '2.0'), + ... ('text', ' suffix'), + ... ] + >>> want_tokens = [ + ... ('text', 'prefix '), + ... ('ellipsis', '...'), + ... ('text', ' suffix'), + ... ] + >>> _float_cmp_match_tokens(got_tokens, want_tokens) + True + + Example: + >>> from xdoctest.checker import _float_cmp_match_tokens + >>> got_tokens = [('text', 'prefix '), ('text', 'suffix')] + >>> want_tokens = [('text', 'prefix '), ('ellipsis', '...')] + >>> _float_cmp_match_tokens(got_tokens, want_tokens) + True + """ + def token_equal(got_token: tuple[str, str], want_token: tuple[str, str]) -> bool: + gkind, gvalue = got_token + wkind, wvalue = want_token + if gkind != wkind: + return False + if gkind == 'text': + return gvalue == wvalue + if gkind == 'ellipsis': + return True + return _float_cmp_numeric_equal(gvalue, wvalue) + + # Collapse consecutive ellipses first. + compact_want: list[tuple[str, str]] = [] + for tok in want_tokens: + if ( + tok[0] == 'ellipsis' + and compact_want + and compact_want[-1][0] == 'ellipsis' + ): + continue + compact_want.append(tok) + want_tokens = compact_want + + gi = 0 + wi = 0 + last_ellipsis_wi = -1 + retry_gi = -1 + + while gi < len(got_tokens): + if wi < len(want_tokens) and want_tokens[wi][0] == 'ellipsis': + last_ellipsis_wi = wi + wi += 1 + retry_gi = gi + elif wi < len(want_tokens) and token_equal(got_tokens[gi], want_tokens[wi]): + gi += 1 + wi += 1 + elif last_ellipsis_wi != -1: + retry_gi += 1 + gi = retry_gi + wi = last_ellipsis_wi + 1 + else: + return False + + while wi < len(want_tokens) and want_tokens[wi][0] == 'ellipsis': + wi += 1 + + return wi == len(want_tokens) + + +def _float_cmp_numeric_equal(got_text: str, want_text: str) -> bool: + got_value = _parse_float_cmp_number(got_text) + want_value = _parse_float_cmp_number(want_text) + + if math.isnan(got_value) or math.isnan(want_value): + return math.isnan(got_value) and math.isnan(want_value) + if math.isinf(got_value) or math.isinf(want_value): + return got_value == want_value + return math.isclose(got_value, want_value, rel_tol=1e-5, abs_tol=1e-8) + + +def _parse_float_cmp_number(text: str) -> float: + lowered = text.lower() + if lowered in {'nan', '+nan', '-nan'}: + return float('nan') + if lowered in {'inf', '+inf', 'infinity', '+infinity'}: + return float('inf') + if lowered in {'-inf', '-infinity'}: + return float('-inf') + return float(text) + + def normalize( got: str, want: str, @@ -534,7 +747,7 @@ def _do_a_fancy_diff( return False - def output_difference( + def _output_difference_xdoctest( self, runstate: directive.RuntimeState | None = None, colored: bool = True, @@ -633,6 +846,29 @@ def output_difference( text = 'Expected nothing\nGot nothing\n' return text + def output_difference( + self, + runstate: directive.RuntimeState | None = None, + colored: bool = True, + ) -> str: + if runstate is None: + runstate = directive.RuntimeState() + + from xdoctest import checker_facade + output_checker = checker_facade.resolve_current_checker(runstate) + if ( + output_checker.__class__.output_difference + is not checker_facade.OutputChecker.output_difference + ): + example = types.SimpleNamespace(want=self.want) + optionflags = checker_facade.runtime_state_to_optionflags(runstate) + return output_checker.output_difference(example, self.got, optionflags) + + return self._output_difference_xdoctest( + runstate=runstate, + colored=colored, + ) + def output_repr_difference( self, runstate: directive.RuntimeState | None = None ) -> str: diff --git a/src/xdoctest/checker_facade.py b/src/xdoctest/checker_facade.py new file mode 100644 index 00000000..85d0d0bf --- /dev/null +++ b/src/xdoctest/checker_facade.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import doctest +import types +import typing + +from xdoctest import checker, directive +from xdoctest.directive_facade import ( + optionflags_to_runtime_state, + register_optionflag, + runtime_state_to_optionflags, +) + +_REGISTERED_CHECKERS: dict[ + str, doctest.OutputChecker | type[doctest.OutputChecker] +] = {} + + +def register_checker( + name: str, + checker_: doctest.OutputChecker | type[doctest.OutputChecker], +) -> None: + _REGISTERED_CHECKERS[name] = checker_ + + + +def resolve_checker(name: str) -> doctest.OutputChecker: + if name not in _REGISTERED_CHECKERS: + raise KeyError( + 'Unknown output checker {!r}. Known checkers are {}'.format( + name, sorted(_REGISTERED_CHECKERS) + ) + ) + checker_ = _REGISTERED_CHECKERS[name] + if isinstance(checker_, doctest.OutputChecker): + return checker_ + return checker_() + + + +def resolve_current_checker( + runstate: directive.RuntimeState | dict | None, +) -> doctest.OutputChecker: + if isinstance(runstate, directive.RuntimeState): + checker_name = runstate.get_output_checker() + elif isinstance(runstate, dict): + checker_name = str(runstate.get('_output_checker', 'xdoctest')) + else: + checker_name = 'xdoctest' + return resolve_checker(checker_name) + + +class OutputChecker(doctest.OutputChecker): + def check_output(self, want: str, got: str, flags: int) -> bool: + runstate = optionflags_to_runtime_state(flags) + return checker._xdoctest_check_output(got, want, runstate) + + def output_difference( + self, + example: typing.Any, + got: str, + flags: int, + ) -> str: + runstate = optionflags_to_runtime_state(flags) + want = getattr(example, 'want', example) + ex = checker.GotWantException('got differs with doctest want', got, want) + return ex._output_difference_xdoctest(runstate=runstate, colored=False) + + +register_checker('xdoctest', OutputChecker) + + +__all__ = [ + 'OutputChecker', + 'register_checker', + 'resolve_checker', + 'resolve_current_checker', + 'register_optionflag', + 'runtime_state_to_optionflags', + 'optionflags_to_runtime_state', +] diff --git a/src/xdoctest/directive.py b/src/xdoctest/directive.py index d98833b3..0558f7d9 100644 --- a/src/xdoctest/directive.py +++ b/src/xdoctest/directive.py @@ -29,6 +29,10 @@ * ``IGNORE_WANT``: False, + * ``IGNORE_OUTPUT``: False, + + * ``FLOAT_CMP``: False, + * ``NORMALIZE_REPR``: True, * ``REPORT_CDIFF``: False, @@ -199,6 +203,8 @@ class RuntimeStateDict(TypedDict, total=False): IGNORE_EXCEPTION_DETAIL: bool NORMALIZE_WHITESPACE: bool IGNORE_WANT: bool + IGNORE_OUTPUT: bool + FLOAT_CMP: bool NORMALIZE_REPR: bool REPORT_CDIFF: bool REPORT_NDIFF: bool @@ -219,6 +225,8 @@ class RuntimeStateDict(TypedDict, total=False): 'IGNORE_EXCEPTION_DETAIL': False, 'NORMALIZE_WHITESPACE': True, 'IGNORE_WANT': False, + 'IGNORE_OUTPUT': False, + 'FLOAT_CMP': False, # 'IGNORE_MEASUREMENTS': False, # TODO: I want this flag to turn on normalization of numbers, # I.E: non-determenistic measurements do not cause doctest failure, but @@ -281,7 +289,9 @@ class RuntimeState(utils.NiceRepr): ASYNC: False, DONT_ACCEPT_BLANKLINE: False, ELLIPSIS: True, + FLOAT_CMP: False, IGNORE_EXCEPTION_DETAIL: False, + IGNORE_OUTPUT: False, IGNORE_WANT: False, IGNORE_WHITESPACE: False, NORMALIZE_REPR: True, @@ -306,6 +316,9 @@ def __init__(self, default_state: RuntimeStateDict | None = None) -> None: if default_state: self._global_state.update(default_state) self._inline_state: dict[str, typing.Any] = {} + self._output_checker = 'xdoctest' + self._output_checker_flags = 0 + self._inline_output_checker_flags = 0 def to_dict(self) -> OrderedDict[str, bool | set[str]]: """ @@ -354,6 +367,30 @@ def __setitem__(self, key: str, value: bool | set[str]) -> None: raise KeyError('Unknown key: {}'.format(key)) cast(Dict[str, Union[bool, Set[str]]], self._global_state)[key] = value + def get_output_checker(self) -> str: + return self._output_checker + + def set_output_checker(self, name: str) -> None: + self._output_checker = name + + def get_output_checker_flags(self) -> int: + return self._output_checker_flags | self._inline_output_checker_flags + + def set_output_checker_flags(self, flags: int) -> None: + self._output_checker_flags = int(flags) + + def add_output_checker_flags(self, flags: int, inline: bool = False) -> None: + if inline: + self._inline_output_checker_flags |= int(flags) + else: + self._output_checker_flags |= int(flags) + + def remove_output_checker_flags(self, flags: int, inline: bool = False) -> None: + if inline: + self._inline_output_checker_flags &= ~int(flags) + else: + self._output_checker_flags &= ~int(flags) + def set_report_style( self, reportchoice: ReportStyle, @@ -393,6 +430,7 @@ def update(self, directives: list[Directive]) -> None: """ # Clear the previous inline state self._inline_state.clear() + self._inline_output_checker_flags = 0 for directive in directives: for effect in directive.effects(): action, key, value = effect @@ -400,6 +438,16 @@ def update(self, directives: list[Directive]) -> None: continue if key not in self._global_state: + from xdoctest import directive_facade + + if directive_facade.is_registered_optionflag(key): + flag = directive_facade.get_optionflag(key) + if action == 'assign': + if value: + self.add_output_checker_flags(flag, inline=bool(directive.inline)) + else: + self.remove_output_checker_flags(flag, inline=bool(directive.inline)) + continue warnings.warn('Unknown state: {}'.format(key)) # Determine if this impacts the local (inline) or global state. @@ -984,12 +1032,14 @@ def parse_directive_optstr( name = name.upper() if name not in COMMANDS: - msg = 'Unknown directive: {!r}'.format(optpart) - warnings.warn(msg) - return None - else: - directive = Directive(name, positive, args, inline) - return directive + from xdoctest import directive_facade + + if not directive_facade.is_registered_optionflag(name): + msg = 'Unknown directive: {!r}'.format(optpart) + warnings.warn(msg) + return None + directive = Directive(name, positive, args, inline) + return directive if __name__ == '__main__': diff --git a/src/xdoctest/directive_facade.py b/src/xdoctest/directive_facade.py new file mode 100644 index 00000000..b62e8d55 --- /dev/null +++ b/src/xdoctest/directive_facade.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import doctest +from typing import Dict + +from xdoctest import directive + +BLANKLINE_MARKER = doctest.BLANKLINE_MARKER +ELLIPSIS_MARKER = doctest.ELLIPSIS_MARKER + +DONT_ACCEPT_BLANKLINE = doctest.DONT_ACCEPT_BLANKLINE +NORMALIZE_WHITESPACE = doctest.NORMALIZE_WHITESPACE +ELLIPSIS = doctest.ELLIPSIS +IGNORE_EXCEPTION_DETAIL = doctest.IGNORE_EXCEPTION_DETAIL +REPORT_UDIFF = doctest.REPORT_UDIFF +REPORT_CDIFF = doctest.REPORT_CDIFF +REPORT_NDIFF = doctest.REPORT_NDIFF + + +class RuntimeFlagFacade: + def __init__(self) -> None: + self._optionflags_by_name: Dict[str, int] = {} + self._optionflag_names_by_value: Dict[int, str] = {} + self._runtime_key_to_optionflag: Dict[str, int] = {} + self._optionflag_to_runtime_key: Dict[int, str] = {} + + def register_builtin_optionflag( + self, + name: str, + flag: int, + runtime_state_key: str | None = None, + ) -> int: + self._optionflags_by_name[name] = flag + self._optionflag_names_by_value[flag] = name + if runtime_state_key is not None: + self._runtime_key_to_optionflag[runtime_state_key] = flag + self._optionflag_to_runtime_key[flag] = runtime_state_key + return flag + + def register_optionflag( + self, + name: str, + runtime_state_key: str | None = None, + ) -> int: + if name in self._optionflags_by_name: + flag = self._optionflags_by_name[name] + if runtime_state_key is not None: + self._runtime_key_to_optionflag[runtime_state_key] = flag + self._optionflag_to_runtime_key[flag] = runtime_state_key + return flag + flag = doctest.register_optionflag(name) + return self.register_builtin_optionflag(name, flag, runtime_state_key) + + def is_registered_optionflag(self, name: str) -> bool: + return name in self._optionflags_by_name + + def get_optionflag(self, name: str) -> int: + return self._optionflags_by_name[name] + + def get_registered_optionflags(self) -> Dict[str, int]: + return dict(self._optionflags_by_name) + + def runtime_state_to_optionflags( + self, + runstate: directive.RuntimeState | dict | None, + ) -> int: + flags = 0 + if isinstance(runstate, directive.RuntimeState): + flags |= runstate.get_output_checker_flags() + lookup = runstate.__getitem__ + elif runstate is None: + runstate = directive.RuntimeState() + flags |= runstate.get_output_checker_flags() + lookup = runstate.__getitem__ + else: + lookup = runstate.__getitem__ + if isinstance(runstate, dict): + flags |= int(runstate.get('_optionflags', 0)) + + for runtime_key, flag in self._runtime_key_to_optionflag.items(): + try: + value = lookup(runtime_key) + except KeyError: + continue + if isinstance(value, bool) and value: + flags |= flag + return flags + + def optionflags_to_runtime_state( + self, + optionflags: int, + default_state: directive.RuntimeStateDict | None = None, + ) -> directive.RuntimeState: + base_state = {} if default_state is None else dict(default_state) + for runtime_key in self._runtime_key_to_optionflag: + if runtime_key == 'REQUIRES': + continue + base_state.setdefault(runtime_key, False) + runstate = directive.RuntimeState(base_state) + runstate.set_output_checker_flags(optionflags) + for flag, runtime_key in self._optionflag_to_runtime_key.items(): + value = runstate[runtime_key] + if isinstance(value, bool) and (optionflags & flag): + runstate[runtime_key] = True + return runstate + + +_RUNTIME_FLAGS = RuntimeFlagFacade() + + +def register_optionflag(name: str, runtime_state_key: str | None = None) -> int: + return _RUNTIME_FLAGS.register_optionflag(name, runtime_state_key) + + +def runtime_state_to_optionflags( + runstate: directive.RuntimeState | dict | None, +) -> int: + return _RUNTIME_FLAGS.runtime_state_to_optionflags(runstate) + + +def optionflags_to_runtime_state( + optionflags: int, + default_state: directive.RuntimeStateDict | None = None, +) -> directive.RuntimeState: + return _RUNTIME_FLAGS.optionflags_to_runtime_state(optionflags, default_state) + + +def is_registered_optionflag(name: str) -> bool: + return _RUNTIME_FLAGS.is_registered_optionflag(name) + + +def get_optionflag(name: str) -> int: + return _RUNTIME_FLAGS.get_optionflag(name) + + +def get_registered_optionflags() -> Dict[str, int]: + return _RUNTIME_FLAGS.get_registered_optionflags() + + +_RUNTIME_FLAGS.register_builtin_optionflag( + 'DONT_ACCEPT_BLANKLINE', DONT_ACCEPT_BLANKLINE, 'DONT_ACCEPT_BLANKLINE' +) +_RUNTIME_FLAGS.register_builtin_optionflag( + 'NORMALIZE_WHITESPACE', NORMALIZE_WHITESPACE, 'NORMALIZE_WHITESPACE' +) +_RUNTIME_FLAGS.register_builtin_optionflag('ELLIPSIS', ELLIPSIS, 'ELLIPSIS') +_RUNTIME_FLAGS.register_builtin_optionflag( + 'IGNORE_EXCEPTION_DETAIL', + IGNORE_EXCEPTION_DETAIL, + 'IGNORE_EXCEPTION_DETAIL', +) +_RUNTIME_FLAGS.register_builtin_optionflag('REPORT_UDIFF', REPORT_UDIFF, 'REPORT_UDIFF') +_RUNTIME_FLAGS.register_builtin_optionflag('REPORT_CDIFF', REPORT_CDIFF, 'REPORT_CDIFF') +_RUNTIME_FLAGS.register_builtin_optionflag('REPORT_NDIFF', REPORT_NDIFF, 'REPORT_NDIFF') +IGNORE_WANT = register_optionflag('IGNORE_WANT', 'IGNORE_WANT') +IGNORE_OUTPUT = register_optionflag('IGNORE_OUTPUT', 'IGNORE_OUTPUT') +IGNORE_WHITESPACE = register_optionflag('IGNORE_WHITESPACE', 'IGNORE_WHITESPACE') +FLOAT_CMP = register_optionflag('FLOAT_CMP', 'FLOAT_CMP') +NORMALIZE_REPR = register_optionflag('NORMALIZE_REPR', 'NORMALIZE_REPR') + + +__all__ = [ + 'BLANKLINE_MARKER', + 'ELLIPSIS_MARKER', + 'DONT_ACCEPT_BLANKLINE', + 'NORMALIZE_WHITESPACE', + 'ELLIPSIS', + 'IGNORE_EXCEPTION_DETAIL', + 'REPORT_UDIFF', + 'REPORT_CDIFF', + 'REPORT_NDIFF', + 'IGNORE_WANT', + 'IGNORE_OUTPUT', + 'IGNORE_WHITESPACE', + 'FLOAT_CMP', + 'NORMALIZE_REPR', + 'register_optionflag', + 'runtime_state_to_optionflags', + 'optionflags_to_runtime_state', + 'is_registered_optionflag', + 'get_optionflag', + 'get_registered_optionflags', +] diff --git a/src/xdoctest/doctest_example.py b/src/xdoctest/doctest_example.py index 74c33d22..53bcf281 100644 --- a/src/xdoctest/doctest_example.py +++ b/src/xdoctest/doctest_example.py @@ -15,10 +15,25 @@ import typing import warnings from collections import OrderedDict +from fnmatch import fnmatch from inspect import CO_COROUTINE from typing import TYPE_CHECKING, Any, Union, cast -from xdoctest import ( +importlib_metadata_compat: types.ModuleType + +try: + from importlib import metadata as importlib_metadata +except ImportError: # nocover + import importlib_metadata as importlib_metadata_compat # type: ignore +else: + importlib_metadata_compat = importlib_metadata + +try: + from packaging.requirements import Requirement +except ImportError: # nocover + Requirement = None # type: ignore + +from xdoctest import ( # NOQA checker, constants, directive, @@ -30,7 +45,8 @@ if TYPE_CHECKING: from xdoctest.doctest_part import DoctestPart -from xdoctest import static_analysis as static + +from xdoctest import static_analysis as static # NOQA __devnotes__ = """ TODO: @@ -57,6 +73,8 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: and sys.stdout.isatty(), 'reportchoice': 'udiff', 'default_runtime_state': {}, + 'output_checker': 'xdoctest', + 'output_checker_flags': 0, 'offset_linenos': False, 'deferred_output_matching': True, 'global_exec': None, @@ -85,11 +103,13 @@ def _populate_from_cli(self, ns): _examp_conf = { 'default_runtime_state': default_runtime_state, 'deferred_output_matching': ns['deferred_output_matching'], + 'output_checker': ns['output_checker'], 'offset_linenos': ns['offset_linenos'], 'colored': ns['colored'], 'reportchoice': ns['reportchoice'], 'global_exec': ns['global_exec'], 'optional_want': ns['optional_want'], + 'output_checker_flags': self['output_checker_flags'], 'supress_import_errors': ns['supress_import_errors'], 'verbose': ns['verbose'], } @@ -162,9 +182,7 @@ def str_lower(x: str) -> str: dest='deferred_output_matching', action='store_false', default=argparse.SUPPRESS, - help=( - 'Disable deferred stdout matching between parts' - ), + help=('Disable deferred stdout matching between parts'), ), ), ( @@ -185,6 +203,15 @@ def str_lower(x: str) -> str: ), ), ), + ( + ['--output-checker'], + dict( + dest='output_checker', + type=str, + default=self['output_checker'], + help='Named output checker backend to use for got/want matching', + ), + ), # used to build default_runtime_state ( ['--options'], @@ -315,6 +342,62 @@ def getvalue(self, key: str, given: typing.Any = None) -> object: return given +def _doctest_requirement_satisfied(requirement_text: str) -> bool: + """ + Return True when a doctestplus-style requirement is satisfied. + + Example: + >>> from xdoctest.doctest_example import _doctest_requirement_satisfied + >>> _doctest_requirement_satisfied('os') + True + >>> _doctest_requirement_satisfied('definitely_missing_package_123456') + False + + Example: + >>> from xdoctest.doctest_example import _doctest_requirement_satisfied + >>> _doctest_requirement_satisfied('packaging>=0') + True + >>> _doctest_requirement_satisfied('packaging>999999') + False + + Example: + >>> from xdoctest.doctest_example import _doctest_requirement_satisfied + >>> _doctest_requirement_satisfied('bad requirement') + Traceback (most recent call last): + ... + ValueError: Invalid __doctest_requires__ requirement: 'bad requirement' + """ + if Requirement is None: + raise ImportError( + 'packaging is required to evaluate __doctest_requires__' + ) + + try: + requirement = Requirement(requirement_text) + except Exception as ex: + raise ValueError( + 'Invalid __doctest_requires__ requirement: {!r}'.format( + requirement_text + ) + ) from ex + + try: + installed_version = importlib_metadata_compat.version(requirement.name) + except Exception: + installed_version = None + + if not requirement.specifier: + if installed_version is not None: + return True + from importlib.util import find_spec + + return find_spec(requirement.name) is not None + + if installed_version is None: + return False + return requirement.specifier.contains(installed_version, prereleases=True) + + class DocTest: """ Holds information necessary to execute and verify a doctest @@ -954,6 +1037,116 @@ def anything_ran(self) -> bool: assert self.logged_stdout is not None return len(self.logged_stdout) > 0 + def _apply_module_doctest_metadata( + self, runstate: directive.RuntimeState + ) -> None: + """ + Apply module-level doctestplus metadata to the current example. + + This is intentionally evaluated after the :class:`DocTest` already + exists and has a callname, so the parser output shape stays unchanged. + """ + skip_spec = None + requires_spec = None + + if self.module is not None: + skip_spec = getattr(self.module, '__doctest_skip__', None) + requires_spec = getattr(self.module, '__doctest_requires__', None) + else: + modpath = self.modpath + if isinstance(modpath, (str, os.PathLike)) and os.path.exists( + modpath + ): + for key in ['__doctest_skip__', '__doctest_requires__']: + try: + value = static.parse_static_value( + key, fpath=os.fspath(modpath) + ) + except NameError: + value = None + except Exception as ex: + raise ValueError( + 'Failed to read {!r} from {!r}: {}'.format( + key, modpath, ex + ) + ) + if key == '__doctest_skip__': + skip_spec = value + else: + requires_spec = value + + if skip_spec is not None and self._matches_doctest_skip(skip_spec): + runstate['SKIP'] = True + return + + if requires_spec is not None and not self._module_requires_satisfied( + requires_spec + ): + runstate['SKIP'] = True + + def _matches_doctest_skip(self, skip_spec: Any) -> bool: + if isinstance(skip_spec, str): + patterns = [skip_spec] + elif isinstance(skip_spec, (list, tuple, set)): + patterns = list(skip_spec) + else: + raise ValueError( + '__doctest_skip__ must be a string or sequence of strings' + ) + + for pattern in patterns: + if not isinstance(pattern, str): + raise ValueError( + '__doctest_skip__ patterns must be strings, got {!r}'.format( + pattern + ) + ) + if pattern == '.': + pattern = '__doc__' + if fnmatch(self.callname, pattern): + return True + return False + + def _module_requires_satisfied(self, requires_spec: Any) -> bool: + if not isinstance(requires_spec, dict): + raise ValueError( + '__doctest_requires__ must be a dictionary of patterns to requirements' + ) + + for key, reqs in requires_spec.items(): + if isinstance(key, str): + patterns = [key] + elif isinstance(key, (list, tuple, set)): + patterns = list(key) + else: + raise ValueError( + '__doctest_requires__ keys must be strings or sequences of strings' + ) + + if not any( + fnmatch(self.callname, pattern if pattern != '.' else '__doc__') + for pattern in patterns + ): + continue + if isinstance(reqs, str): + req_list = [reqs] + elif isinstance(reqs, (list, tuple, set)): + req_list = list(reqs) + else: + raise ValueError( + '__doctest_requires__ values must be strings or sequences of strings' + ) + + for req_text in req_list: + if not isinstance(req_text, str): + raise ValueError( + '__doctest_requires__ requirements must be strings' + ) + if not _doctest_requirement_satisfied(req_text): + return False + + return True + def run( self, verbose: int | None | bool = None, on_error: str | None = None ) -> dict[str, typing.Any]: @@ -1000,6 +1193,10 @@ def run( # Initialize a new runtime state default_state = self.config['default_runtime_state'] runstate = self._runstate = directive.RuntimeState(default_state) + runstate.set_output_checker(self.config.get('output_checker', 'xdoctest')) + runstate.set_output_checker_flags( + int(self.config.get('output_checker_flags', 0)) + ) # setup reporting choice runstate.set_report_style(self.config['reportchoice'].lower()) @@ -1095,6 +1292,15 @@ def run( summary = self._post_run(verbose) return summary + self._apply_module_doctest_metadata(runstate) + + if runstate['SKIP']: + if DEBUG: + print(f'part[{partx}] skipped by module metadata') + self._skipped_parts.append(part) + did_pre_import = True + continue + test_globals, compileflags = self._test_globals() if DEBUG: @@ -1436,17 +1642,24 @@ def _check_or_defer_part_output( stdout for later trailing matching, while parts with a local want are checked immediately. The `deferred_output_matching` knob disables the deferred-trailing behavior, and `optional_want` requires output - producing parts to have a local want unless `IGNORE_WANT` is active. - Any part with `IGNORE_WANT` active is treated as a boundary and does - not contribute output to later matching. + producing parts to have a local want unless `IGNORE_WANT` or + `IGNORE_OUTPUT` is active. Any part with either directive active is + treated as a boundary and does not contribute output to later + matching. """ deferred_output_matching = bool( self.config.getvalue('deferred_output_matching') ) optional_want = bool(self.config.getvalue('optional_want')) + ignore_want = bool(runstate['IGNORE_WANT']) + ignore_output = bool(runstate['IGNORE_OUTPUT']) if part.want is None: - if runstate['IGNORE_WANT']: + if ignore_output: + self._unmatched_stdout = [] + return + + if ignore_want: self._unmatched_stdout = [] return @@ -1478,7 +1691,7 @@ def _check_or_defer_part_output( self._unmatched_stdout.append(got_stdout) else: assert got_stdout is not None - if not runstate['IGNORE_WANT']: + if not ignore_want and not ignore_output: part.check( got_stdout, got_eval, @@ -1486,7 +1699,7 @@ def _check_or_defer_part_output( unmatched=self._unmatched_stdout, ) # Any want-bearing part is a boundary for deferred stdout, even when - # IGNORE_WANT skips local comparison. + # IGNORE_WANT / IGNORE_OUTPUT skip local comparison. self._unmatched_stdout = [] @property diff --git a/tests/test_checker.py b/tests/test_checker.py index 2910688c..9a9bada9 100644 --- a/tests/test_checker.py +++ b/tests/test_checker.py @@ -50,3 +50,45 @@ def test_blankline_not_accept() -> None: got = 'foo\n\nbar' want = 'foo\n\nbar' assert not checker.check_output(got, want, runstate) + + +def test_float_cmp_simple_pass() -> None: + runstate = directive.RuntimeState({'FLOAT_CMP': True}) + got = '0.3333333333333333' + want = '0.333333' + assert checker.check_output(got, want, runstate) + + +def test_float_cmp_simple_mismatch_fail() -> None: + runstate = directive.RuntimeState({'FLOAT_CMP': True}) + got = '1.0' + want = '1.1' + assert not checker.check_output(got, want, runstate) + + +def test_float_cmp_text_with_numbers_pass() -> None: + runstate = directive.RuntimeState({'FLOAT_CMP': True}) + got = 'best=0.3333333333333333 ave=0.6666666666666666' + want = 'best=0.333333 ave=0.666666' + assert checker.check_output(got, want, runstate) + + +def test_float_cmp_ellipsis_composes() -> None: + runstate = directive.RuntimeState({'FLOAT_CMP': True, 'ELLIPSIS': True}) + got = 'prefix best=0.3333333333 s ave=0.6666666666 suffix' + want = 'prefix best=... s ave=0.666666 suffix' + assert checker.check_output(got, want, runstate) + + +def test_float_cmp_does_not_enable_ellipsis_by_itself() -> None: + runstate = directive.RuntimeState({'FLOAT_CMP': True, 'ELLIPSIS': False}) + got = 'prefix best=0.3333333333 suffix' + want = 'prefix best=... suffix' + assert not checker.check_output(got, want, runstate) + + +def test_float_cmp_special_values_and_multiple_numbers() -> None: + runstate = directive.RuntimeState({'FLOAT_CMP': True}) + got = 'value=NaN upper=INF lower=-Infinity pair=1.0,2.0000000001' + want = 'value=nan upper=inf lower=-inf pair=1,2' + assert checker.check_output(got, want, runstate) diff --git a/tests/test_checker_compat.py b/tests/test_checker_compat.py new file mode 100644 index 00000000..c3c4d5c1 --- /dev/null +++ b/tests/test_checker_compat.py @@ -0,0 +1,311 @@ +from __future__ import annotations + +import doctest + +import xdoctest +from xdoctest import checker, doctest_example, utils +from xdoctest import directive_facade + + +def test_register_optionflag_is_stable() -> None: + flag1 = xdoctest.register_optionflag('CUSTOM_STABLE_FLAG') + flag2 = xdoctest.register_optionflag('CUSTOM_STABLE_FLAG') + assert flag1 == flag2 + + +def test_runtime_state_optionflag_roundtrip_for_builtin_flags() -> None: + runstate = xdoctest.optionflags_to_runtime_state( + xdoctest.FLOAT_CMP | xdoctest.ELLIPSIS + ) + flags = xdoctest.runtime_state_to_optionflags(runstate) + assert flags & xdoctest.FLOAT_CMP + assert flags & xdoctest.ELLIPSIS + + +FIX = doctest.register_optionflag('FIX') + + +class DoctestPlusLikeChecker(doctest.OutputChecker): + def check_output(self, want: str, got: str, flags: int) -> bool: + if flags & FIX: + want = want.replace('L', '') + got = got.replace('L', '') + return xdoctest.OutputChecker().check_output(want, got, flags) + + def output_difference(self, example, got: str, flags: int) -> str: + return 'compat-diff: ' + xdoctest.OutputChecker().output_difference( + example, got, flags + ) + + +xdoctest.register_checker('doctestplus_like', DoctestPlusLikeChecker) + + +def test_registered_checker_can_use_doctest_style_optionflags() -> None: + docsrc = utils.codeblock( + ''' + >>> print('10') + 10L + ''' + ) + self = doctest_example.DocTest(docsrc=docsrc) + self.config['output_checker'] = 'doctestplus_like' + self.config['output_checker_flags'] = FIX + result = self.run(verbose=0, on_error='raise') + assert result['passed'] + + +def test_registered_checker_receives_runtime_state_flags() -> None: + seen: list[int] = [] + + class FlagRecorder(doctest.OutputChecker): + def check_output(self, want: str, got: str, flags: int) -> bool: + seen.append(flags) + return xdoctest.OutputChecker().check_output(want, got, flags) + + xdoctest.register_checker('flag_recorder', FlagRecorder) + + docsrc = utils.codeblock( + ''' + >>> print(0.3333333333) # xdoctest: +FLOAT_CMP + 0.333333 + ''' + ) + self = doctest_example.DocTest(docsrc=docsrc) + self.config['output_checker'] = 'flag_recorder' + result = self.run(verbose=0, on_error='raise') + assert result['passed'] + assert seen + assert seen[-1] & directive_facade.FLOAT_CMP + + +def test_registered_checker_output_difference_is_used() -> None: + docsrc = utils.codeblock( + ''' + >>> print('alpha') + beta + ''' + ) + self = doctest_example.DocTest(docsrc=docsrc) + self.config['output_checker'] = 'doctestplus_like' + self.config['output_checker_flags'] = FIX + result = self.run(verbose=0, on_error='return') + assert result['failed'] + text = '\n'.join(self.repr_failure()) + assert 'compat-diff:' in text + + + +def test_registered_optionflag_can_be_set_via_directive() -> None: + seen: list[int] = [] + allow_bytes = xdoctest.register_optionflag('ALLOW_BYTES') + + class BytesFlagRecorder(doctest.OutputChecker): + def check_output(self, want: str, got: str, flags: int) -> bool: + seen.append(flags) + return xdoctest.OutputChecker().check_output(want, got, flags) + + xdoctest.register_checker('bytes_flag_recorder', BytesFlagRecorder) + + docsrc = utils.codeblock( + ''' + >>> print('alpha') # xdoctest: +ALLOW_BYTES + alpha + >>> print('beta') + beta + ''' + ) + self = doctest_example.DocTest(docsrc=docsrc) + self.config['output_checker'] = 'bytes_flag_recorder' + result = self.run(verbose=0, on_error='raise') + assert result['passed'] + assert len(seen) >= 2 + assert seen[0] & allow_bytes + assert not (seen[-1] & allow_bytes) + + +def test_resolve_current_checker_honors_mapping_state() -> None: + class MappingChecker(doctest.OutputChecker): + pass + + xdoctest.register_checker('mapping_checker', MappingChecker) + resolved = xdoctest.checker_facade.resolve_current_checker( + {'_output_checker': 'mapping_checker'} + ) + assert isinstance(resolved, MappingChecker) + + +def test_registered_optionflag_inline_reaches_checker() -> None: + seen: list[int] = [] + fix_inline = xdoctest.register_optionflag('FIX_INLINE') + + class InlineRecorder(doctest.OutputChecker): + def check_output(self, want: str, got: str, flags: int) -> bool: + seen.append(flags) + return xdoctest.OutputChecker().check_output(want, got, flags) + + xdoctest.register_checker('inline_flag_recorder', InlineRecorder) + + docsrc = utils.codeblock( + """ + >>> print('alpha') # xdoctest: +FIX_INLINE + alpha + """ + ) + self = doctest_example.DocTest(docsrc=docsrc) + self.config['output_checker'] = 'inline_flag_recorder' + result = self.run(verbose=0, on_error='raise') + assert result['passed'] + assert seen + assert seen[-1] & fix_inline + + + +def test_registered_optionflag_inline_clears_after_part() -> None: + seen: list[int] = [] + fix_inline_once = xdoctest.register_optionflag('FIX_INLINE_ONCE') + + class InlineOnceRecorder(doctest.OutputChecker): + def check_output(self, want: str, got: str, flags: int) -> bool: + seen.append(flags) + return xdoctest.OutputChecker().check_output(want, got, flags) + + xdoctest.register_checker('inline_once_recorder', InlineOnceRecorder) + + docsrc = utils.codeblock( + """ + >>> print('alpha') # xdoctest: +FIX_INLINE_ONCE + alpha + >>> print('beta') + beta + """ + ) + self = doctest_example.DocTest(docsrc=docsrc) + self.config['output_checker'] = 'inline_once_recorder' + result = self.run(verbose=0, on_error='raise') + assert result['passed'] + assert len(seen) >= 2 + assert seen[0] & fix_inline_once + assert not (seen[1] & fix_inline_once) + + + +def test_registered_optionflag_block_persists_until_disabled() -> None: + seen: list[int] = [] + fix_block = xdoctest.register_optionflag('FIX_BLOCK') + + class BlockRecorder(doctest.OutputChecker): + def check_output(self, want: str, got: str, flags: int) -> bool: + seen.append(flags) + return xdoctest.OutputChecker().check_output(want, got, flags) + + xdoctest.register_checker('block_flag_recorder', BlockRecorder) + + docsrc = utils.codeblock( + """ + >>> # xdoctest: +FIX_BLOCK + >>> print('alpha') + alpha + >>> print('beta') + beta + >>> # xdoctest: -FIX_BLOCK + >>> print('gamma') + gamma + """ + ) + self = doctest_example.DocTest(docsrc=docsrc) + self.config['output_checker'] = 'block_flag_recorder' + result = self.run(verbose=0, on_error='raise') + assert result['passed'] + assert len(seen) >= 3 + assert seen[0] & fix_block + assert seen[1] & fix_block + assert not (seen[2] & fix_block) + + + +def test_registered_optionflag_negative_block_clears_correctly() -> None: + seen: list[int] = [] + fix_toggle = xdoctest.register_optionflag('FIX_TOGGLE') + + class ToggleRecorder(doctest.OutputChecker): + def check_output(self, want: str, got: str, flags: int) -> bool: + seen.append(flags) + return xdoctest.OutputChecker().check_output(want, got, flags) + + xdoctest.register_checker('toggle_flag_recorder', ToggleRecorder) + + docsrc = utils.codeblock( + """ + >>> # xdoctest: +FIX_TOGGLE + >>> print('alpha') + alpha + >>> # xdoctest: -FIX_TOGGLE + >>> print('beta') + beta + >>> # xdoctest: +FIX_TOGGLE + >>> print('gamma') + gamma + """ + ) + self = doctest_example.DocTest(docsrc=docsrc) + self.config['output_checker'] = 'toggle_flag_recorder' + result = self.run(verbose=0, on_error='raise') + assert result['passed'] + assert len(seen) >= 3 + assert seen[0] & fix_toggle + assert not (seen[1] & fix_toggle) + assert seen[2] & fix_toggle + + + +def test_custom_checker_selected_by_name_receives_registered_directive_flags() -> None: + seen: list[int] = [] + fix_named = xdoctest.register_optionflag('FIX_SELECTED_BY_NAME') + + class NamedRecorder(doctest.OutputChecker): + def check_output(self, want: str, got: str, flags: int) -> bool: + seen.append(flags) + return xdoctest.OutputChecker().check_output(want, got, flags) + + xdoctest.register_checker('named_flag_recorder', NamedRecorder) + + docsrc = utils.codeblock( + """ + >>> print('alpha') # xdoctest: +FIX_SELECTED_BY_NAME + alpha + """ + ) + self = doctest_example.DocTest(docsrc=docsrc) + self.config['output_checker'] = 'named_flag_recorder' + result = self.run(verbose=0, on_error='raise') + assert result['passed'] + assert seen + assert seen[-1] & fix_named + + + +def test_end_to_end_registered_checker_flag_works_via_directive() -> None: + fix_end_to_end = xdoctest.register_optionflag('FIX_END_TO_END') + + class FixDirectiveChecker(doctest.OutputChecker): + def check_output(self, want: str, got: str, flags: int) -> bool: + if flags & fix_end_to_end: + want = want.replace('L', '') + got = got.replace('L', '') + return xdoctest.OutputChecker().check_output(want, got, flags) + + xdoctest.register_checker('fix_directive_checker', FixDirectiveChecker) + + docsrc = utils.codeblock( + """ + >>> print('10') # xdoctest: +FIX_END_TO_END + 10L + >>> print('20') + 20 + """ + ) + self = doctest_example.DocTest(docsrc=docsrc) + self.config['output_checker'] = 'fix_directive_checker' + result = self.run(verbose=0, on_error='raise') + assert result['passed'] diff --git a/tests/test_doctest_example.py b/tests/test_doctest_example.py index d55af222..b3f87c01 100644 --- a/tests/test_doctest_example.py +++ b/tests/test_doctest_example.py @@ -2,6 +2,9 @@ import argparse import typing +from pathlib import Path + +import pytest from xdoctest import checker, constants, doctest_example, exceptions, utils @@ -257,16 +260,17 @@ def test_doctestconfig_cli_flags() -> None: parser = argparse.ArgumentParser() config._update_argparse_cli(parser.add_argument) - ns = parser.parse_args( - ['--no-optional-want', '--deferred-output-matching'] - ) + ns = parser.parse_args(['--no-optional-want', '--deferred-output-matching']) assert ns.optional_want is False assert ns.deferred_output_matching is True prefixed = argparse.ArgumentParser() config._update_argparse_cli(prefixed.add_argument, prefix=['xdoctest']) ns2 = prefixed.parse_args( - ['--xdoctest-no-optional-want', '--xdoctest-no-deferred-output-matching'] + [ + '--xdoctest-no-optional-want', + '--xdoctest-no-deferred-output-matching', + ] ) assert ns2.xdoctest_optional_want is False assert ns2.xdoctest_deferred_output_matching is False @@ -323,6 +327,106 @@ def test_optional_want_false_ignored_no_want_part_passes() -> None: assert result['passed'] +def test_ignore_output_wrong_want_passes() -> None: + docsrc = utils.codeblock( + """ + >>> print('foo') # xdoctest: +IGNORE_OUTPUT + bar + """ + ) + self = doctest_example.DocTest(docsrc=docsrc) + result = self.run(verbose=0, on_error='return') + + assert result['passed'] + + +def test_ignore_output_suppresses_optional_want_stdout_failure() -> None: + docsrc = utils.codeblock( + """ + >>> print('foo') # xdoctest: +IGNORE_OUTPUT + """ + ) + self = doctest_example.DocTest(docsrc=docsrc) + self.config['optional_want'] = False + + result = self.run(verbose=0, on_error='return') + + assert result['passed'] + + +def test_ignore_output_suppresses_optional_want_eval_failure() -> None: + docsrc = utils.codeblock( + """ + >>> 1 + 1 # xdoctest: +IGNORE_OUTPUT + """ + ) + self = doctest_example.DocTest(docsrc=docsrc) + self._parse() + assert self._parts is not None + self._parts[0].compile_mode = 'eval' + self.config['optional_want'] = False + + result = self.run(verbose=0, on_error='return') + + assert result['passed'] + + +def test_ignore_output_breaks_deferred_matching() -> None: + docsrc = utils.codeblock( + """ + >>> print('prefix') + + >>> print('ignored') # xdoctest: +IGNORE_OUTPUT + junk + + >>> print('suffix') + prefix + suffix + """ + ) + self = doctest_example.DocTest(docsrc=docsrc) + self._parse() + assert self._parts is not None + assert len(self._parts) == 3 + + result = self.run(verbose=0, on_error='return') + + assert result['failed'] + assert not result['passed'] + + +def test_module_doctest_requires_invalid_value_fails(tmp_path: Path) -> None: + fpath = tmp_path / 'mod_requires_bad.py' + fpath.write_text( + utils.codeblock( + """ + __doctest_requires__ = {'foo': ['bad requirement']} + + def foo(): + ''' + >>> 1 + 1 + ''' + """ + ) + ) + docsrc = utils.codeblock( + """ + >>> 1 + 1 + """ + ) + self = doctest_example.DocTest( + docsrc=docsrc, + modpath=str(fpath), + callname='foo', + fpath=str(fpath), + ) + + with pytest.raises(ValueError): + self.run(verbose=0, on_error='raise') + + def test_deferred_output_matching_false_disables_trailing_match() -> None: docsrc = utils.codeblock( """ @@ -391,6 +495,7 @@ def test_doctest_fails_because_ignore_want_clears_unmatched_stdout_v1() -> None: # The idea here is that we shouldn't be able to match things before an # ignore, because it is cleared by the want. import pytest + docsrc = utils.codeblock( """ >>> print('prefix') @@ -414,6 +519,7 @@ def test_doctest_fails_because_ignore_want_clears_unmatched_stdout_v2() -> None: # Similar idea to the v1 test, but this one would pass if the ignore was # removed. import pytest + docsrc = utils.codeblock( """ >>> print('prefix') diff --git a/tests/test_errors.py b/tests/test_errors.py index 121ad1c2..46cd4ed5 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -212,14 +212,15 @@ def test_optional_want_false_extracts_bad_repr() -> None: A no-want eval output should still report bad reprs through the existing got-repr failure machinery when optional_want is disabled. """ + class MyObj: def __repr__(self) -> str: raise Exception('this repr fails') source = utils.codeblock( - ''' + """ >>> obj - ''' + """ ) self = doctest_example.DocTest(docsrc=source) self._parse() diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 0ed06d84..f3831042 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -427,7 +427,9 @@ def add_one(x): reprec = testdir.inline_run(p, '--xdoctest-modules', *EXTRA_ARGS) reprec.assertoutcome(skipped=1, failed=0, passed=0) - def test_xdoctest_optional_want_addopts(self, testdir: pytest.Testdir) -> None: + def test_xdoctest_optional_want_addopts( + self, testdir: pytest.Testdir + ) -> None: """Test prefixed config knobs in pytest addopts. CommandLine: @@ -1524,6 +1526,89 @@ def test_vacuous_all_skipped(self, testdir, makedoctest) -> None: reprec.assertoutcome(passed=0, skipped=0) +class TestXDoctestModuleMetadata: + def test_doctestplus_skip_metadata(self, testdir: pytest.Testdir) -> None: + testdir.makepyfile( + meta=utils.codeblock( + """ + def _skip_targets(): + return ['skip_me', 'SkipClass.*'] + + __doctest_skip__ = _skip_targets() + + def skip_me(): + ''' + >>> 1 + 2 + ''' + + def keep_me(): + ''' + >>> 1 + 1 + ''' + + class SkipClass: + def method(self): + ''' + >>> 1 + 2 + ''' + """ + ) + ) + reprec = testdir.inline_run('--xdoctest-modules', *EXTRA_ARGS) + reprec.assertoutcome(passed=1, skipped=2) + + def test_doctestplus_requires_metadata( + self, testdir: pytest.Testdir + ) -> None: + testdir.makepyfile( + meta=utils.codeblock( + """ + __doctest_requires__ = { + 'needs_*': ['sys'], + 'needs_version': ['pytest>=1'], + 'needs_missing': ['definitely_missing_package_123456'], + 'needs_unsatisfied': ['pytest>999999'], + } + + def needs_sys(): + ''' + >>> 1 + 1 + ''' + + def needs_version(): + ''' + >>> 1 + 1 + ''' + + def needs_missing(): + ''' + >>> 1 + 1 + ''' + + def needs_unsatisfied(): + ''' + >>> 1 + 1 + ''' + + def keep_me(): + ''' + >>> 1 + 1 + ''' + """ + ) + ) + reprec = testdir.inline_run('--xdoctest-modules', *EXTRA_ARGS) + reprec.assertoutcome(passed=3, skipped=2) + + class TestXDoctestAutoUseFixtures: SCOPES = ['module', 'session', 'class', 'function'] diff --git a/tpl/pytest-doctestplus b/tpl/pytest-doctestplus new file mode 160000 index 00000000..d9048086 --- /dev/null +++ b/tpl/pytest-doctestplus @@ -0,0 +1 @@ +Subproject commit d904808650f150dba3deffa3ff8bf5eb40aa3fe2