测试问题修复：🐛 修复 engineV4 终态日志 torch_error / paddle_cuda 重叠问题

engineV4.py

@@ -47,6 +47,10 @@
 os.environ["FLAGS_use_system_allocator"] = "1"
 os.environ["NVIDIA_TF32_OVERRIDE"] = "0"
 
+FATAL_CUDA_EXIT_CODE = 99
+FATAL_OOM_EXIT_CODE = 98
+FATAL_TORCH_EXIT_CODE = 97
+
 VALID_TEST_ARGS = {
     "test_amp",
     "test_backward",
@@ -1283,13 +1287,49 @@ def run_test_case(api_config_str, options):
     try:
         case.test()
     except Exception as err:
-        # if fatal error happens, subprocess need to exit with non-zero status
-        if "CUDA error" in str(err) or "memory corruption" in str(err):
-            os._exit(99)
-        if "CUDA out of memory" in str(err) or "Out of memory error" in str(err):
-            os._exit(98)
-        if "AssertionError" in str(err) or "Tensor-likes are not equal" in str(err):
-            os._exit(1)
+        err_msg = str(err).lower()
+        terminal_log_type = get_terminal_log_type(api_config_str)
+        oom_markers = (
+            "cuda out of memory",
+            "out of memory error",
+            "resourceexhaustederror",
+            "out of memory",
+            "outofmemoryerror",
+            "cannot allocate memory",
+            "std::bad_alloc",
+            "bad allocation",
+            "memoryerror",
+            "cublas_status_alloc_failed",
+        )
+        cuda_markers = (
+            "cuda error",
+            "memory corruption",
+            "illegal memory access",
+            "invalid configuration argument",
+            "invalid resource handle",
+        )
+        exit_code = None
+        if any(marker in err_msg for marker in oom_markers):
+            exit_code = FATAL_OOM_EXIT_CODE
+        elif terminal_log_type == "torch_error" and any(
+            marker in err_msg for marker in cuda_markers
+        ):
+            exit_code = FATAL_TORCH_EXIT_CODE
+        elif any(marker in err_msg for marker in cuda_markers):
+            exit_code = FATAL_CUDA_EXIT_CODE
+        if exit_code is not None:
+            if has_terminal_log(api_config_str):
+                write_checkpoint(api_config_str)
+            try:
+                close_process_files()
+            finally:
+                try:
+                    restore_stdio()
+                finally:
+                    os._exit(exit_code)
+        if has_terminal_log(api_config_str):
+            write_checkpoint(api_config_str)
+            return
         # if not fatal error, subprocess will be alive and report error
         print(f"[test error] {api_config_str}: {err}", flush=True)
         raise
@@ -1959,16 +1999,22 @@ def cleanup_handler(*args):
                         flush=True,
                     )
                 elif msg_type == "crashed":
-                    if exitcode == 99:
+                    if exitcode == FATAL_CUDA_EXIT_CODE:
                         write_to_log("paddle_cuda", config)
                         print(
-                            f"[error] CUDA error for {config}",
+                            f"[paddle_cuda] {config}: worker exited with CUDA error",
                             flush=True,
                         )
-                    elif exitcode == 98:
+                    elif exitcode == FATAL_OOM_EXIT_CODE:
                         write_to_log("oom", config)
                         print(
-                            f"[error] CUDA out of memory for {config}",
+                            f"[oom] {config}: worker exited with OOM",
+                            flush=True,
+                        )
+                    elif exitcode == FATAL_TORCH_EXIT_CODE:
+                        write_to_log("torch_error", config)
+                        print(
+                            f"[torch_error] {config}: worker exited with torch CUDA error",
                             flush=True,
                         )
                     elif (

tester/api_config/log_writer.py

@@ -487,6 +487,39 @@ def aggregate_logs(end=False, cleanup=False):
                     f.writelines(f"{line}\n" for line in sorted(api_configs))
             except Exception as err:
                 print(f"Error writing to {incomplete_file}: {err}", flush=True)
+
+        # === 终态日志互斥性检查 ===
+        config_to_types: dict[str, list[str]] = {}
+        for log_type, prefix in LOG_PREFIXES.items():
+            if log_type == "checkpoint":
+                continue
+            log_file = TEST_LOG_PATH / f"{prefix}.txt"
+            if not log_file.exists():
+                continue
+            try:
+                with log_file.open("r") as f:
+                    for raw_line in f:
+                        line = raw_line.strip()
+                        if line:
+                            config_to_types.setdefault(line, []).append(log_type)
+            except Exception:
+                pass
+
+        duplicates = {config: types for config, types in config_to_types.items() if len(types) > 1}
+        if duplicates:
+            print("\n" + "!" * 50)
+            print("INTEGRITY ERROR: configs found in multiple log types:")
+            for config, types in sorted(duplicates.items())[:20]:
+                print(f"  {config}")
+                print(f"    -> {', '.join(types)}")
+            if len(duplicates) > 20:
+                print(f"  ... and {len(duplicates) - 20} more")
+            print("!" * 50 + "\n")
+            assert not duplicates, (
+                f"Log integrity violation: {len(duplicates)} config(s) appear in "
+                f"multiple terminal log types. This indicates a classification bug."
+            )
+
         return log_counts
 
 
@@ -522,18 +555,18 @@ def print_log_info(all_case, log_counts=None):
     print("\n" + "=" * 50)
     print("Test Case Statistics".center(50))
     print("=" * 50)
-    print(f"{'Pending cases':<30}: {all_case}")
-    print(f"{'Tested cases':<30}: {test_case}")
-    print(f"{'Pass cases':<30}: {pass_case}")
-    print(f"{'Skip cases':<30}: {skip_case}")
-    print(f"{'Paddle issue cases':<30}: {paddle_issue_case}")
-    print(f"{'Test issue cases':<30}: {test_issue_case}")
-    print(f"{'Retest cases':<30}: {retest_case}")
+    print(f"{'Pending cases':<30}: {all_case:>8}")
+    print(f"{'Tested cases':<30}: {test_case:>8}")
+    print(f"{'Pass cases':<30}: {pass_case:>8}")
+    print(f"{'Skip cases':<30}: {skip_case:>8}")
+    print(f"{'Paddle issue cases':<30}: {paddle_issue_case:>8}")
+    print(f"{'Test issue cases':<30}: {test_issue_case:>8}")
+    print(f"{'Retest cases':<30}: {retest_case:>8}")
     if log_counts:
         print("-" * 50)
         print("Log Type Breakdown:")
         for log_type, count in log_counts.items():
-            print(f"  {log_type:<28}: {count}")
+            print(f"  {log_type:<28}: {count:>8}")
     print("=" * 50 + "\n")
 
 

tester/base.py

@@ -234,8 +234,6 @@ def report_runtime_error(self, err, default_log_type, phase="", allow_ignore_pad
             return "pass", False
         if log_type is None:
             log_type = default_log_type
-        elif default_log_type == "torch_error" and log_type != "oom":
-            log_type = "torch_error"
         phase_text = f" phase={phase}" if phase else ""
         print(
             f"[{log_type}]{phase_text} {self.api_config.config}\n{err_msg}",