Skip to content

测试基建：✨ 补齐 FP8 与 TE MoE 对齐测试能力 #682

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cangtianhuang merged 5 commits into from
Jul 16, 2026
Merged

测试基建：✨ 补齐 FP8 与 TE MoE 对齐测试能力 #682

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7877ab3
⚡️ Fix wrong classify, use report_compare_error
cangtianhuang Jul 16, 2026
ffa1647
🐛 Fix numpy error
cangtianhuang Jul 16, 2026
62a6360
✨ Impl TE moe
cangtianhuang Jul 16, 2026
6520810
🚸 Rename PADDLEAPITEST_IMPL
cangtianhuang Jul 16, 2026
2429f56
Merge remote-tracking branch 'upstream/main'
cangtianhuang Jul 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 1 addition & 13 deletions tester/accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,20 +334,8 @@ def compare_paddle_and_torch(paddle_tensor, torch_tensor, idx=0) -> bool:
paddle_tensor, torch_tensor, atol=self.get_atol(), rtol=self.get_rtol()
)
except Exception as err:
err_str = str(err)
phase = "backward" if self.is_backward else "forward"
if err_str.startswith("[torch_assert_OOM]"):
print(
f"[oom] phase={phase} idx={idx} {self.api_config.config}\n{err_str}",
flush=True,
)
write_to_log("oom", self.api_config.config)
else:
print(
f"[paddle_accuracy] phase={phase} idx={idx} {self.api_config.config}\n{err_str}",
flush=True,
)
write_to_log("paddle_accuracy", self.api_config.config)
self.report_compare_error(err, f"{phase} idx={idx}")
if self.exit_on_error:
raise
return False
Expand Down
56 changes: 4 additions & 52 deletions tester/accuracy_stable.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,19 +453,7 @@ def compare(self, input1, input2, comp):
try:
self.assert_accuracy(input1, input2, comp)
except Exception as err:
err_str = str(err)
if err_str.startswith("[torch_assert_OOM]"):
print(
f"[oom] comp={comp} {self.api_config.config}\n{err_str}",
flush=True,
)
write_to_comp_log(comp, "oom", self.api_config.config)
else:
print(
f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err_str}",
flush=True,
)
write_to_comp_log(comp, "paddle_accuracy", self.api_config.config)
self.report_compare_error(err, f"comp={comp}")
return
else:
print(
Expand Down Expand Up @@ -500,39 +488,15 @@ def compare(self, input1, input2, comp):
try:
self.assert_accuracy(item1, item2, comp, idx)
except Exception as err:
err_str = str(err)
if err_str.startswith("[torch_assert_OOM]"):
print(
f"[oom] comp={comp} {self.api_config.config}\n{err_str}",
flush=True,
)
write_to_comp_log(comp, "oom", self.api_config.config)
else:
print(
f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err_str}",
flush=True,
)
write_to_comp_log(comp, "paddle_accuracy", self.api_config.config)
self.report_compare_error(err, f"comp={comp} idx={idx}")
return
elif not isinstance(item1, (paddle.Tensor, torch.Tensor)) and not isinstance(
item2, (paddle.Tensor, torch.Tensor)
):
try:
self.assert_accuracy(torch.tensor(item1), torch.tensor(item2), comp, idx)
except Exception as err:
err_str = str(err)
if err_str.startswith("[torch_assert_OOM]"):
print(
f"[oom] comp={comp} {self.api_config.config}\n{err_str}",
flush=True,
)
write_to_comp_log(comp, "oom", self.api_config.config)
else:
print(
f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err_str}",
flush=True,
)
write_to_comp_log(comp, "paddle_accuracy", self.api_config.config)
self.report_compare_error(err, f"comp={comp} idx={idx}")
return
else:
print(
Expand All @@ -546,19 +510,7 @@ def compare(self, input1, input2, comp):
try:
self.assert_accuracy(torch.tensor(input1), torch.tensor(input2), comp)
except Exception as err:
err_str = str(err)
if err_str.startswith("[torch_assert_OOM]"):
print(
f"[oom] comp={comp} {self.api_config.config}\n{err_str}",
flush=True,
)
write_to_comp_log(comp, "oom", self.api_config.config)
else:
print(
f"[paddle_accuracy] comp={comp} {self.api_config.config}\n{err_str}",
flush=True,
)
write_to_comp_log(comp, "paddle_accuracy", self.api_config.config)
self.report_compare_error(err, f"comp={comp}")
return

def assert_accuracy(self, tensor1, tensor2, comp, idx=0):
Expand Down
158 changes: 93 additions & 65 deletions tester/api_config/config_analyzer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1738,9 +1738,12 @@ def get_padding_offset(bsz, max_seq_len, seq_lens_this_time):
vocab_size = numpy.random.randint(10, 1000)
if isinstance(weight_config, TensorConfig) and weight_config.shape:
vocab_size = weight_config.shape[0]
self.numpy_tensor = numpy.random.randint(0, vocab_size, size=self.shape).astype(
self.dtype
)
if vocab_size == 0:
self.numpy_tensor = numpy.zeros(self.shape, dtype=self.dtype)
else:
self.numpy_tensor = numpy.random.randint(
0, vocab_size, size=self.shape
).astype(self.dtype)
elif self.check_arg(api_config, 1, "weight"):
if self.dtype.startswith("complex"):
real_dtype = "float32" if self.dtype == "complex64" else "float64"
Expand Down Expand Up @@ -1778,9 +1781,12 @@ def get_padding_offset(bsz, max_seq_len, seq_lens_this_time):
soft_labels = soft_labels / soft_labels.sum(axis=1, keepdims=True)
self.numpy_tensor = soft_labels.astype(self.dtype)
else:
self.numpy_tensor = numpy.random.randint(
0, num_classes, size=self.shape
).astype(self.dtype)
if num_classes == 0:
self.numpy_tensor = numpy.zeros(self.shape, dtype=self.dtype)
else:
self.numpy_tensor = numpy.random.randint(
0, num_classes, size=self.shape
).astype(self.dtype)
elif self.check_arg(api_config, 3, "weight"):
self.numpy_tensor = numpy.random.random(size=self.shape)
self.numpy_tensor = self.numpy_tensor / self.numpy_tensor.sum()
Expand Down Expand Up @@ -2635,9 +2641,12 @@ def get_padding_offset(bsz, max_seq_len, seq_lens_this_time):
if axis is None:
axis = 0
inputs = self.get_arg(api_config, 0, "x")
self.numpy_tensor = numpy.random.randint(
0, inputs.shape[axis], size=self.shape
).astype(self.dtype)
if inputs.shape[axis] == 0:
self.numpy_tensor = numpy.zeros(self.shape, dtype=self.dtype)
else:
self.numpy_tensor = numpy.random.randint(
0, inputs.shape[axis], size=self.shape
).astype(self.dtype)

elif api_config.api_name in {"paddle.Tensor.index_put", "paddle.index_put"}:
if self.check_arg(api_config, 1, "indices") and not self.get_arg(
Expand Down Expand Up @@ -2981,38 +2990,50 @@ def get_exponent_max(value, dtype_max, default_max=5):
seqlen, topk = self.shape[0], self.shape[1]
# Generate valid routemap vectorized for large seqlen
routemap = numpy.full(self.shape, -1, dtype="int32")
# Each row randomly assigns 1~topk experts to random positions
n_assigned = numpy.random.randint(1, topk + 1, size=seqlen)
# For each possible n_assigned value, batch process all rows with that count
for n in range(1, topk + 1):
mask = n_assigned == n
count = int(mask.sum())
if count == 0:
continue
# Generate random expert indices for these rows
expert_indices = numpy.array(
[
numpy.random.choice(num_experts, size=n, replace=False)
for _ in range(count)
],
dtype="int32",
)
# Generate random positions for these rows
position_indices = numpy.array(
[
numpy.random.choice(topk, size=n, replace=False)
for _ in range(count)
],
dtype="int32",
)
row_indices = numpy.where(mask)[0]
for j in range(n):
routemap[row_indices, position_indices[:, j]] = expert_indices[:, j]
self.numpy_tensor = routemap
# Update tokens_per_expert to match the generated routemap
tokens_count = [int(numpy.sum(routemap == e)) for e in range(num_experts)]
tokens_per_expert = self.get_arg(api_config, 5, "tokens_per_expert")
tokens_per_expert[:] = tokens_count
if topk == 0:
# 0-size topk dimension: routemap stays all -1
self.numpy_tensor = routemap
tokens_per_expert = self.get_arg(api_config, 5, "tokens_per_expert")
if tokens_per_expert is not None:
tokens_per_expert[:] = [0] * num_experts
else:
# Each row randomly assigns 1~min(topk, num_experts) experts
max_assign = min(topk, num_experts)
n_assigned = numpy.random.randint(1, max_assign + 1, size=seqlen)
# For each possible n_assigned value, batch process all rows with that count
for n in range(1, max_assign + 1):
mask = n_assigned == n
count = int(mask.sum())
if count == 0:
continue
# Generate random expert indices for these rows
expert_indices = numpy.array(
[
numpy.random.choice(num_experts, size=n, replace=False)
for _ in range(count)
],
dtype="int32",
)
# Generate random positions for these rows
position_indices = numpy.array(
[
numpy.random.choice(topk, size=n, replace=False)
for _ in range(count)
],
dtype="int32",
)
row_indices = numpy.where(mask)[0]
for j in range(n):
routemap[row_indices, position_indices[:, j]] = expert_indices[
:, j
]
self.numpy_tensor = routemap
# Update tokens_per_expert to match the generated routemap
tokens_count = [
int(numpy.sum(routemap == e)) for e in range(num_experts)
]
tokens_per_expert = self.get_arg(api_config, 5, "tokens_per_expert")
tokens_per_expert[:] = tokens_count
# expert_prob_topk (arg3): float32, shape [seqlen, topk], value in [0, 1]
elif self.check_arg(api_config, 3, "expert_prob_topk"):
routemap_config = self.get_arg(api_config, 2, "expert_routemap_topk")
Expand Down Expand Up @@ -3047,30 +3068,37 @@ def get_exponent_max(value, dtype_max, default_max=5):
seqlen, topk = self.shape[0], self.shape[1]
# Generate valid routemap vectorized for large seqlen
routemap = numpy.full(self.shape, -1, dtype="int32")
n_assigned = numpy.random.randint(1, topk + 1, size=seqlen)
for n in range(1, topk + 1):
mask = n_assigned == n
count = int(mask.sum())
if count == 0:
continue
expert_indices = numpy.array(
[
numpy.random.choice(num_experts, size=n, replace=False)
for _ in range(count)
],
dtype="int32",
)
position_indices = numpy.array(
[
numpy.random.choice(topk, size=n, replace=False)
for _ in range(count)
],
dtype="int32",
)
row_indices = numpy.where(mask)[0]
for j in range(n):
routemap[row_indices, position_indices[:, j]] = expert_indices[:, j]
self.numpy_tensor = routemap
if topk == 0:
# 0-size topk dimension: routemap stays all -1
self.numpy_tensor = routemap
else:
max_assign = min(topk, num_experts)
n_assigned = numpy.random.randint(1, max_assign + 1, size=seqlen)
for n in range(1, max_assign + 1):
mask = n_assigned == n
count = int(mask.sum())
if count == 0:
continue
expert_indices = numpy.array(
[
numpy.random.choice(num_experts, size=n, replace=False)
for _ in range(count)
],
dtype="int32",
)
position_indices = numpy.array(
[
numpy.random.choice(topk, size=n, replace=False)
for _ in range(count)
],
dtype="int32",
)
row_indices = numpy.where(mask)[0]
for j in range(n):
routemap[row_indices, position_indices[:, j]] = expert_indices[
:, j
]
self.numpy_tensor = routemap
# zipped_expertwise_rowmap (arg1): int32, shape [seqlen, num_experts]
# Needs to be valid rowmap based on routemap
elif self.check_arg(api_config, 1, "zipped_expertwise_rowmap"):
Expand Down
8 changes: 8 additions & 0 deletions tester/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ def __getattr__(self, name):

def classify_runtime_error(error_msg):
error_msg_lower = error_msg.lower()
if error_msg.startswith("[torch_assert_OOM]"):
return "oom", False
oom_markers = tuple(marker.lower() for marker in CUDA_OOM) + (
"cannot allocate memory",
"std::bad_alloc",
Expand Down Expand Up @@ -256,6 +258,12 @@ def report_runtime_error(self, err, default_log_type, phase="", allow_ignore_pad
write_to_log(log_type, self.api_config.config)
return log_type, fatal

def report_compare_error(self, err, phase="", default_log_type="paddle_accuracy"):
log_type, fatal = self.report_runtime_error(err, default_log_type, phase)
if fatal:
raise err
return log_type, fatal

def need_skip(self, paddle_only=False):
# not support
if "sparse" in self.api_config.api_name:
Expand Down
13 changes: 3 additions & 10 deletions tester/paddle_cinn_vs_dygraph.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,11 +298,7 @@ def compare(self, dygraph_output, static_output, is_backward=False):
try:
self.paddle_assert_accuracy(dygraph_output, static_output)
except Exception as err:
print(
f"[accuracy error] {backward_str}{self.api_config.config}\n{err!s}",
flush=True,
)
write_to_log("paddle_accuracy", self.api_config.config)
self.report_compare_error(err, backward_str.strip())
return False
elif isinstance(dygraph_output, (list, tuple)):
if not isinstance(static_output, (list, tuple)):
Expand Down Expand Up @@ -341,11 +337,8 @@ def compare(self, dygraph_output, static_output, is_backward=False):
try:
self.paddle_assert_accuracy(dygraph_item, static_item)
except Exception as err:
print(
f"[accuracy error] {backward_str}{self.api_config.config}\n{err!s}",
flush=True,
)
write_to_log("paddle_accuracy", self.api_config.config)
phase = f"{backward_str.strip()} idx={i}" if backward_str else f"idx={i}"
self.report_compare_error(err, phase)
return False
elif dygraph_output is None and static_output is None:
pass
Expand Down
Loading
Loading