diff --git a/tests/trainer/ppo/test_metric_utils_on_cpu.py b/tests/trainer/ppo/test_metric_utils_on_cpu.py index d9ca0717cd7..90fb11f84f9 100644 --- a/tests/trainer/ppo/test_metric_utils_on_cpu.py +++ b/tests/trainer/ppo/test_metric_utils_on_cpu.py @@ -568,6 +568,47 @@ def test_process_validation_metrics_with_pred(self): # For bootstrap with n=2, the majority vote could be either A or B # depending on the random sampling, so we don't check the exact value + def test_process_validation_metrics_none_filled_sparse_key(self): + """Sparse reward_extra_info keys are null-filled for samples that omit + them; aggregation must drop the None values instead of crashing on + np.mean([..., None, ...]). See issue #6830.""" + data_sources = ["source1", "source1", "source1"] + sample_inputs = ["prompt1", "prompt1", "prompt1"] # one uid group of 3 + infos_dict = { + "score": [0.8, 0.9, 0.7], + # emitted for only the first sample, null-filled for the rest + "sparse": [1.0, None, None], + # never emitted for this group + "all_none": [None, None, None], + } + + result = process_validation_metrics(data_sources, sample_inputs, infos_dict, seed=42) + + # Dense key aggregates over all 3. + self.assertAlmostEqual(result["source1"]["score"]["mean@3"], 0.8) + # Sparse key aggregates over the single present value. + self.assertAlmostEqual(result["source1"]["sparse"]["mean@1"], 1.0) + # All-None key is skipped entirely (no metrics emitted). + self.assertNotIn("all_none", result["source1"]) + + def test_process_validation_metrics_none_keeps_pred_aligned(self): + """A sparse numeric key with >1 present value and a None (not at the + end) must keep its predictions aligned so majority voting still zips + 1:1 instead of raising. See issue #6830.""" + data_sources = ["source1", "source1", "source1"] + sample_inputs = ["prompt1", "prompt1", "prompt1"] # one uid group of 3 + infos_dict = { + "score": [1.0, None, 2.0], # None in the middle + "pred": ["A", "B", "A"], + } + + result = process_validation_metrics(data_sources, sample_inputs, infos_dict, seed=42) + + # Aggregates over the two present values (no crash, no misalignment). + self.assertAlmostEqual(result["source1"]["score"]["mean@2"], 1.5) + # Majority-vote metrics are still produced for the filtered pair set. + self.assertIn("maj@2/mean", result["source1"]["score"]) + if __name__ == "__main__": unittest.main() diff --git a/verl/trainer/ppo/metric_utils.py b/verl/trainer/ppo/metric_utils.py index 8a812522906..874f9673f6d 100644 --- a/verl/trainer/ppo/metric_utils.py +++ b/verl/trainer/ppo/metric_utils.py @@ -638,6 +638,21 @@ def gen_ns(n_resps: int) -> list[int]: var_dict = uid_dict.setdefault(uid, {}) for var_name, var_vals in var2vals.items(): + # Drop None entries before aggregating. Sparse reward_extra_info + # keys (emitted for only some samples) are null-filled to a + # uniform schema in `_validate`, so a uid group can contain None; + # np.mean([..., None, ...]) would raise TypeError. Keep the + # aligned predictions in sync so majority voting still zips + # 1:1. See https://github.com/volcengine/verl/issues/6830. + vote_pred_vals = pred_vals + if any(v is None for v in var_vals): + if pred_vals is not None and len(pred_vals) == len(var_vals): + kept = [(v, p) for v, p in zip(var_vals, pred_vals, strict=True) if v is not None] + var_vals = [v for v, _ in kept] + vote_pred_vals = [p for _, p in kept] + else: + var_vals = [v for v in var_vals if v is not None] + # skip empty or string values if not var_vals or isinstance(var_vals[0], str): continue @@ -673,7 +688,8 @@ def gen_ns(n_resps: int) -> list[int]: if has_pred: # create vote_data vote_data = [ - {"val": val, "pred": pred} for val, pred in zip(var_vals, pred_vals, strict=True) + {"val": val, "pred": pred} + for val, pred in zip(var_vals, vote_pred_vals, strict=True) ] # compute maj metrics [(maj_n_mean, maj_n_std)] = bootstrap_metric(