diff --git a/engineV4.py b/engineV4.py index a98bb167..7ac07a15 100644 --- a/engineV4.py +++ b/engineV4.py @@ -47,6 +47,10 @@ os.environ["FLAGS_use_system_allocator"] = "1" os.environ["NVIDIA_TF32_OVERRIDE"] = "0" +FATAL_CUDA_EXIT_CODE = 99 +FATAL_OOM_EXIT_CODE = 98 +FATAL_TORCH_EXIT_CODE = 97 + VALID_TEST_ARGS = { "test_amp", "test_backward", @@ -1283,13 +1287,49 @@ def run_test_case(api_config_str, options): try: case.test() except Exception as err: - # if fatal error happens, subprocess need to exit with non-zero status - if "CUDA error" in str(err) or "memory corruption" in str(err): - os._exit(99) - if "CUDA out of memory" in str(err) or "Out of memory error" in str(err): - os._exit(98) - if "AssertionError" in str(err) or "Tensor-likes are not equal" in str(err): - os._exit(1) + err_msg = str(err).lower() + terminal_log_type = get_terminal_log_type(api_config_str) + oom_markers = ( + "cuda out of memory", + "out of memory error", + "resourceexhaustederror", + "out of memory", + "outofmemoryerror", + "cannot allocate memory", + "std::bad_alloc", + "bad allocation", + "memoryerror", + "cublas_status_alloc_failed", + ) + cuda_markers = ( + "cuda error", + "memory corruption", + "illegal memory access", + "invalid configuration argument", + "invalid resource handle", + ) + exit_code = None + if any(marker in err_msg for marker in oom_markers): + exit_code = FATAL_OOM_EXIT_CODE + elif terminal_log_type == "torch_error" and any( + marker in err_msg for marker in cuda_markers + ): + exit_code = FATAL_TORCH_EXIT_CODE + elif any(marker in err_msg for marker in cuda_markers): + exit_code = FATAL_CUDA_EXIT_CODE + if exit_code is not None: + if has_terminal_log(api_config_str): + write_checkpoint(api_config_str) + try: + close_process_files() + finally: + try: + restore_stdio() + finally: + os._exit(exit_code) + if has_terminal_log(api_config_str): + write_checkpoint(api_config_str) + return # if not fatal error, subprocess will be alive and report error print(f"[test error] {api_config_str}: {err}", flush=True) raise @@ -1959,16 +1999,22 @@ def cleanup_handler(*args): flush=True, ) elif msg_type == "crashed": - if exitcode == 99: + if exitcode == FATAL_CUDA_EXIT_CODE: write_to_log("paddle_cuda", config) print( - f"[error] CUDA error for {config}", + f"[paddle_cuda] {config}: worker exited with CUDA error", flush=True, ) - elif exitcode == 98: + elif exitcode == FATAL_OOM_EXIT_CODE: write_to_log("oom", config) print( - f"[error] CUDA out of memory for {config}", + f"[oom] {config}: worker exited with OOM", + flush=True, + ) + elif exitcode == FATAL_TORCH_EXIT_CODE: + write_to_log("torch_error", config) + print( + f"[torch_error] {config}: worker exited with torch CUDA error", flush=True, ) elif ( diff --git a/tester/api_config/log_writer.py b/tester/api_config/log_writer.py index 5daf558c..468211cc 100644 --- a/tester/api_config/log_writer.py +++ b/tester/api_config/log_writer.py @@ -487,6 +487,39 @@ def aggregate_logs(end=False, cleanup=False): f.writelines(f"{line}\n" for line in sorted(api_configs)) except Exception as err: print(f"Error writing to {incomplete_file}: {err}", flush=True) + + # === 终态日志互斥性检查 === + config_to_types: dict[str, list[str]] = {} + for log_type, prefix in LOG_PREFIXES.items(): + if log_type == "checkpoint": + continue + log_file = TEST_LOG_PATH / f"{prefix}.txt" + if not log_file.exists(): + continue + try: + with log_file.open("r") as f: + for raw_line in f: + line = raw_line.strip() + if line: + config_to_types.setdefault(line, []).append(log_type) + except Exception: + pass + + duplicates = {config: types for config, types in config_to_types.items() if len(types) > 1} + if duplicates: + print("\n" + "!" * 50) + print("INTEGRITY ERROR: configs found in multiple log types:") + for config, types in sorted(duplicates.items())[:20]: + print(f" {config}") + print(f" -> {', '.join(types)}") + if len(duplicates) > 20: + print(f" ... and {len(duplicates) - 20} more") + print("!" * 50 + "\n") + assert not duplicates, ( + f"Log integrity violation: {len(duplicates)} config(s) appear in " + f"multiple terminal log types. This indicates a classification bug." + ) + return log_counts @@ -522,18 +555,18 @@ def print_log_info(all_case, log_counts=None): print("\n" + "=" * 50) print("Test Case Statistics".center(50)) print("=" * 50) - print(f"{'Pending cases':<30}: {all_case}") - print(f"{'Tested cases':<30}: {test_case}") - print(f"{'Pass cases':<30}: {pass_case}") - print(f"{'Skip cases':<30}: {skip_case}") - print(f"{'Paddle issue cases':<30}: {paddle_issue_case}") - print(f"{'Test issue cases':<30}: {test_issue_case}") - print(f"{'Retest cases':<30}: {retest_case}") + print(f"{'Pending cases':<30}: {all_case:>8}") + print(f"{'Tested cases':<30}: {test_case:>8}") + print(f"{'Pass cases':<30}: {pass_case:>8}") + print(f"{'Skip cases':<30}: {skip_case:>8}") + print(f"{'Paddle issue cases':<30}: {paddle_issue_case:>8}") + print(f"{'Test issue cases':<30}: {test_issue_case:>8}") + print(f"{'Retest cases':<30}: {retest_case:>8}") if log_counts: print("-" * 50) print("Log Type Breakdown:") for log_type, count in log_counts.items(): - print(f" {log_type:<28}: {count}") + print(f" {log_type:<28}: {count:>8}") print("=" * 50 + "\n") diff --git a/tester/base.py b/tester/base.py index 42901b21..1753eeb9 100644 --- a/tester/base.py +++ b/tester/base.py @@ -234,8 +234,6 @@ def report_runtime_error(self, err, default_log_type, phase="", allow_ignore_pad return "pass", False if log_type is None: log_type = default_log_type - elif default_log_type == "torch_error" and log_type != "oom": - log_type = "torch_error" phase_text = f" phase={phase}" if phase else "" print( f"[{log_type}]{phase_text} {self.api_config.config}\n{err_msg}",