fix(logfiles): report search failures instead of losing findings

cardano_node_tests/utils/logfiles.py

@@ -770,6 +770,12 @@ def find_msgs_in_logs(
         final line of the live log file is not searched, so a truncated line is never
         returned for it. An unterminated final line of a rotated log file is searched,
         as it is the file's final content.
+
+    Raises:
+        FileNotFoundError: When the log file keeps getting rotated during the search.
+            Unlike the error-aggregating searches, the failure is propagated - the
+            caller needs the matching lines, and an empty result would be misread
+            as "no matches".
     """
     # Compile as BYTES regex for speed; note: \w/\b are ASCII-only in bytes mode.
     regex_b = _compile_bytes_from_str(pat=regex)
@@ -895,7 +901,8 @@ def check_msgs_presence_in_logs(  # noqa: C901
         timestamp: Passed to `_get_rotated_logs`.
 
     Returns:
-        Error messages for missing entries and for globs that matched no log file.
+        Error messages for missing entries, for globs that matched no log file, and
+        for log files that could not be searched.
     """
 
     def _search(
@@ -957,19 +964,29 @@ def _search(
 
         for logfile in matching_files:
             start_seek, inode = seek_offsets.get(logfile) or (0, None)
-            line_found = _retry_search(
-                functools.partial(
-                    _search, start_seek=start_seek, inode=inode, logfile=logfile, regex_b=regex_b
+            try:
+                line_found = _retry_search(
+                    functools.partial(
+                        _search,
+                        start_seek=start_seek,
+                        inode=inode,
+                        logfile=logfile,
+                        regex_b=regex_b,
+                    )
                 )
-            )
+            except FileNotFoundError as err:
+                # Report the failure instead of raising, so that findings that were
+                # already collected for other regexes and files are not lost
+                errors.append(f"Cannot search '{logfile}' for `{regex}`: {err}")
+                continue
             if not line_found:
                 errors.append(f"No line matching `{regex}` found in '{logfile}'.")
 
     return errors
 
 
 @contextlib.contextmanager
-def expect_errors(regex_pairs: list[tuple[str, str]], *, worker_id: str) -> tp.Iterator[None]:
+def expect_errors(regex_pairs: list[tuple[str, str]], *, worker_id: str) -> tp.Generator[None]:
     """Make sure the expected errors are present in logs.
 
     Context manager.
@@ -1008,7 +1025,7 @@ def expect_errors(regex_pairs: list[tuple[str, str]], *, worker_id: str) -> tp.I
 
 
 @contextlib.contextmanager
-def expect_messages(regex_pairs: list[tuple[str, str]]) -> tp.Iterator[None]:
+def expect_messages(regex_pairs: list[tuple[str, str]]) -> tp.Generator[None]:
     """Make sure the expected messages are present in logs.
 
     Context manager.
@@ -1081,18 +1098,23 @@ def _search(
             )
 
             # Search for errors in the log file
-            errors.extend(
-                _retry_search(
-                    functools.partial(
-                        _search,
-                        logfile=logfile,
-                        seek=seek,
-                        timestamp=timestamp,
-                        inode=inode,
-                        errors_ignored=errors_ignored,
+            try:
+                errors.extend(
+                    _retry_search(
+                        functools.partial(
+                            _search,
+                            logfile=logfile,
+                            seek=seek,
+                            timestamp=timestamp,
+                            inode=inode,
+                            errors_ignored=errors_ignored,
+                        )
                     )
                 )
-            )
+            except FileNotFoundError as err:
+                # Report the failure instead of raising, so that errors that were already
+                # found in other log files are not lost
+                errors.append((logfile, f"Cannot search the log file: {err}"))
 
     return errors
 
@@ -1116,8 +1138,12 @@ def _search() -> list[tuple[pl.Path, str]]:
             errors_re=ERRORS_RE,
         )
 
-    # Search for errors in the log file
-    errors = _retry_search(_search)
+    # Search for errors in the log file. Report a failure instead of raising, so that
+    # errors found by the other log searches are not lost.
+    try:
+        errors = _retry_search(_search)
+    except FileNotFoundError as err:
+        errors = [(logfile, f"Cannot search the log file: {err}")]
 
     return errors
 
@@ -1143,8 +1169,12 @@ def _search() -> list[tuple[pl.Path, str]]:
                 errors_re=SUPERVISORD_ERRORS_RE,
             )
 
-        # Search for errors in the log file
-        errors = _retry_search(_search)
+        # Search for errors in the log file. Report a failure instead of raising, so that
+        # errors found by the other log searches are not lost.
+        try:
+            errors = _retry_search(_search)
+        except FileNotFoundError as err:
+            errors = [(logfile, f"Cannot search the log file: {err}")]
 
     return errors
 

framework_tests/test_logfiles.py

@@ -1164,3 +1164,76 @@ def test_check_msgs_metacharacters_in_state_dir(tmp_path: pl.Path):
         timestamp=0.0,
     )
     assert errors == []
+
+
+@pytest.fixture
+def _failing_validate_inode(monkeypatch: pytest.MonkeyPatch):
+    """Make every log file look permanently rotated and skip the retry sleep."""
+
+    def _raise(log: logfiles.RotableLog) -> None:
+        msg = f"Log file {log.logfile} was rotated during search."
+        raise FileNotFoundError(msg)
+
+    monkeypatch.setattr(logfiles, "_validate_inode", _raise)
+    monkeypatch.setattr(logfiles.time, "sleep", lambda _seconds: None)
+
+
+@pytest.mark.usefixtures("_failing_validate_inode")
+def test_check_msgs_search_failure_reported(tmp_path: pl.Path):
+    """Check that a failed search is reported instead of raised.
+
+    When the search of a log file keeps failing (e.g. on repeated rotation), the failure
+    must not discard the results that were already collected for other regexes and files.
+    """
+    logfile = _write_log(state_dir=tmp_path, name="node1.stdout", content="expected msg\n")
+
+    errors = logfiles.check_msgs_presence_in_logs(
+        regex_pairs=[("*.stdout", "expected msg"), ("*.stdout", "other msg")],
+        seek_offsets={str(logfile): (0, logfile.stat().st_ino)},
+        state_dir=tmp_path,
+        timestamp=0.0,
+    )
+    assert len(errors) == 2
+    assert all("Cannot search" in e for e in errors)
+
+
+@pytest.mark.usefixtures("_failing_validate_inode")
+def test_search_cluster_logs_search_failure_reported(cluster_env: cluster_nodes.ClusterEnv):
+    """Check that a failed log file search is reported as an error entry.
+
+    A raise would abort the whole log check and discard errors that were already found
+    in other log files.
+    """
+    logfile = _write_log(
+        state_dir=cluster_env.state_dir, name="node1.stdout", content="error one\n"
+    )
+
+    errors = logfiles.search_cluster_logs()
+    assert len(errors) == 1
+    assert errors[0][0] == logfile
+    assert "Cannot search the log file" in errors[0][1]
+
+
+@pytest.mark.usefixtures("_failing_validate_inode")
+def test_search_framework_log_failure_reported(tmp_path: pl.Path, monkeypatch: pytest.MonkeyPatch):
+    """Check that a failed framework log search is reported as an error entry."""
+    logfile = _write_log(state_dir=tmp_path, name="framework.log", content="error one\n")
+    monkeypatch.setattr(logfiles.framework_log, "get_framework_log_path", lambda: logfile)
+
+    errors = logfiles.search_framework_log()
+    assert errors == [
+        (logfile, f"Cannot search the log file: Log file {logfile} was rotated during search.")
+    ]
+
+
+@pytest.mark.usefixtures("_failing_validate_inode")
+def test_search_supervisord_logs_failure_reported(cluster_env: cluster_nodes.ClusterEnv):
+    """Check that a failed supervisord log search is reported as an error entry."""
+    logfile = _write_log(
+        state_dir=cluster_env.state_dir, name="supervisord.log", content="FATAL x\n"
+    )
+
+    errors = logfiles.search_supervisord_logs()
+    assert len(errors) == 1
+    assert errors[0][0] == logfile
+    assert "Cannot search the log file" in errors[0][1]