diff --git a/cardano_node_tests/utils/logfiles.py b/cardano_node_tests/utils/logfiles.py index 32d48a1f7..3e6542c11 100644 --- a/cardano_node_tests/utils/logfiles.py +++ b/cardano_node_tests/utils/logfiles.py @@ -770,6 +770,12 @@ def find_msgs_in_logs( final line of the live log file is not searched, so a truncated line is never returned for it. An unterminated final line of a rotated log file is searched, as it is the file's final content. + + Raises: + FileNotFoundError: When the log file keeps getting rotated during the search. + Unlike the error-aggregating searches, the failure is propagated - the + caller needs the matching lines, and an empty result would be misread + as "no matches". """ # Compile as BYTES regex for speed; note: \w/\b are ASCII-only in bytes mode. regex_b = _compile_bytes_from_str(pat=regex) @@ -895,7 +901,8 @@ def check_msgs_presence_in_logs( # noqa: C901 timestamp: Passed to `_get_rotated_logs`. Returns: - Error messages for missing entries and for globs that matched no log file. + Error messages for missing entries, for globs that matched no log file, and + for log files that could not be searched. """ def _search( @@ -957,11 +964,21 @@ def _search( for logfile in matching_files: start_seek, inode = seek_offsets.get(logfile) or (0, None) - line_found = _retry_search( - functools.partial( - _search, start_seek=start_seek, inode=inode, logfile=logfile, regex_b=regex_b + try: + line_found = _retry_search( + functools.partial( + _search, + start_seek=start_seek, + inode=inode, + logfile=logfile, + regex_b=regex_b, + ) ) - ) + except FileNotFoundError as err: + # Report the failure instead of raising, so that findings that were + # already collected for other regexes and files are not lost + errors.append(f"Cannot search '{logfile}' for `{regex}`: {err}") + continue if not line_found: errors.append(f"No line matching `{regex}` found in '{logfile}'.") @@ -969,7 +986,7 @@ def _search( @contextlib.contextmanager -def expect_errors(regex_pairs: list[tuple[str, str]], *, worker_id: str) -> tp.Iterator[None]: +def expect_errors(regex_pairs: list[tuple[str, str]], *, worker_id: str) -> tp.Generator[None]: """Make sure the expected errors are present in logs. Context manager. @@ -1008,7 +1025,7 @@ def expect_errors(regex_pairs: list[tuple[str, str]], *, worker_id: str) -> tp.I @contextlib.contextmanager -def expect_messages(regex_pairs: list[tuple[str, str]]) -> tp.Iterator[None]: +def expect_messages(regex_pairs: list[tuple[str, str]]) -> tp.Generator[None]: """Make sure the expected messages are present in logs. Context manager. @@ -1081,18 +1098,23 @@ def _search( ) # Search for errors in the log file - errors.extend( - _retry_search( - functools.partial( - _search, - logfile=logfile, - seek=seek, - timestamp=timestamp, - inode=inode, - errors_ignored=errors_ignored, + try: + errors.extend( + _retry_search( + functools.partial( + _search, + logfile=logfile, + seek=seek, + timestamp=timestamp, + inode=inode, + errors_ignored=errors_ignored, + ) ) ) - ) + except FileNotFoundError as err: + # Report the failure instead of raising, so that errors that were already + # found in other log files are not lost + errors.append((logfile, f"Cannot search the log file: {err}")) return errors @@ -1116,8 +1138,12 @@ def _search() -> list[tuple[pl.Path, str]]: errors_re=ERRORS_RE, ) - # Search for errors in the log file - errors = _retry_search(_search) + # Search for errors in the log file. Report a failure instead of raising, so that + # errors found by the other log searches are not lost. + try: + errors = _retry_search(_search) + except FileNotFoundError as err: + errors = [(logfile, f"Cannot search the log file: {err}")] return errors @@ -1143,8 +1169,12 @@ def _search() -> list[tuple[pl.Path, str]]: errors_re=SUPERVISORD_ERRORS_RE, ) - # Search for errors in the log file - errors = _retry_search(_search) + # Search for errors in the log file. Report a failure instead of raising, so that + # errors found by the other log searches are not lost. + try: + errors = _retry_search(_search) + except FileNotFoundError as err: + errors = [(logfile, f"Cannot search the log file: {err}")] return errors diff --git a/framework_tests/test_logfiles.py b/framework_tests/test_logfiles.py index 5238a2b77..65d1b8d6f 100644 --- a/framework_tests/test_logfiles.py +++ b/framework_tests/test_logfiles.py @@ -1164,3 +1164,76 @@ def test_check_msgs_metacharacters_in_state_dir(tmp_path: pl.Path): timestamp=0.0, ) assert errors == [] + + +@pytest.fixture +def _failing_validate_inode(monkeypatch: pytest.MonkeyPatch): + """Make every log file look permanently rotated and skip the retry sleep.""" + + def _raise(log: logfiles.RotableLog) -> None: + msg = f"Log file {log.logfile} was rotated during search." + raise FileNotFoundError(msg) + + monkeypatch.setattr(logfiles, "_validate_inode", _raise) + monkeypatch.setattr(logfiles.time, "sleep", lambda _seconds: None) + + +@pytest.mark.usefixtures("_failing_validate_inode") +def test_check_msgs_search_failure_reported(tmp_path: pl.Path): + """Check that a failed search is reported instead of raised. + + When the search of a log file keeps failing (e.g. on repeated rotation), the failure + must not discard the results that were already collected for other regexes and files. + """ + logfile = _write_log(state_dir=tmp_path, name="node1.stdout", content="expected msg\n") + + errors = logfiles.check_msgs_presence_in_logs( + regex_pairs=[("*.stdout", "expected msg"), ("*.stdout", "other msg")], + seek_offsets={str(logfile): (0, logfile.stat().st_ino)}, + state_dir=tmp_path, + timestamp=0.0, + ) + assert len(errors) == 2 + assert all("Cannot search" in e for e in errors) + + +@pytest.mark.usefixtures("_failing_validate_inode") +def test_search_cluster_logs_search_failure_reported(cluster_env: cluster_nodes.ClusterEnv): + """Check that a failed log file search is reported as an error entry. + + A raise would abort the whole log check and discard errors that were already found + in other log files. + """ + logfile = _write_log( + state_dir=cluster_env.state_dir, name="node1.stdout", content="error one\n" + ) + + errors = logfiles.search_cluster_logs() + assert len(errors) == 1 + assert errors[0][0] == logfile + assert "Cannot search the log file" in errors[0][1] + + +@pytest.mark.usefixtures("_failing_validate_inode") +def test_search_framework_log_failure_reported(tmp_path: pl.Path, monkeypatch: pytest.MonkeyPatch): + """Check that a failed framework log search is reported as an error entry.""" + logfile = _write_log(state_dir=tmp_path, name="framework.log", content="error one\n") + monkeypatch.setattr(logfiles.framework_log, "get_framework_log_path", lambda: logfile) + + errors = logfiles.search_framework_log() + assert errors == [ + (logfile, f"Cannot search the log file: Log file {logfile} was rotated during search.") + ] + + +@pytest.mark.usefixtures("_failing_validate_inode") +def test_search_supervisord_logs_failure_reported(cluster_env: cluster_nodes.ClusterEnv): + """Check that a failed supervisord log search is reported as an error entry.""" + logfile = _write_log( + state_dir=cluster_env.state_dir, name="supervisord.log", content="FATAL x\n" + ) + + errors = logfiles.search_supervisord_logs() + assert len(errors) == 1 + assert errors[0][0] == logfile + assert "Cannot search the log file" in errors[0][1]