From 3ae72a05d4f27a8b756c512d56896185f030fd1e Mon Sep 17 00:00:00 2001
From: abinggo <107740309+abinggo@users.noreply.github.com>
Date: Thu, 25 Jun 2026 15:31:50 +0800
Subject: [PATCH] [trainer] Drop None values in process_validation_metrics for
 sparse reward keys
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A reward_extra_info key emitted for only some validation samples is null-filled
to a uniform schema in `_validate`, so a per-uid group can contain None. The
aggregation skip-filter only caught empty lists and strings, so
`np.mean([..., None, ...])` raised `TypeError: unsupported operand type(s) for
/: 'NoneType' and 'int'` and crashed validation — any reward function that
conditionally adds an extra-info key triggers it.

Drop the None entries before aggregating. When predictions are present, filter
the (value, prediction) pairs together so majority voting still zips 1:1; a uid
whose only value was None is skipped entirely. No behavior change when a group
has no None.

Fixes #6830

Signed-off-by: abinggo <107740309+abinggo@users.noreply.github.com>
---
 tests/trainer/ppo/test_metric_utils_on_cpu.py | 41 +++++++++++++++++++
 verl/trainer/ppo/metric_utils.py              | 18 +++++++-
 2 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/tests/trainer/ppo/test_metric_utils_on_cpu.py b/tests/trainer/ppo/test_metric_utils_on_cpu.py
index d9ca0717cd7..90fb11f84f9 100644
--- a/tests/trainer/ppo/test_metric_utils_on_cpu.py
+++ b/tests/trainer/ppo/test_metric_utils_on_cpu.py
@@ -568,6 +568,47 @@ def test_process_validation_metrics_with_pred(self):
         # For bootstrap with n=2, the majority vote could be either A or B
         # depending on the random sampling, so we don't check the exact value
 
+    def test_process_validation_metrics_none_filled_sparse_key(self):
+        """Sparse reward_extra_info keys are null-filled for samples that omit
+        them; aggregation must drop the None values instead of crashing on
+        np.mean([..., None, ...]). See issue #6830."""
+        data_sources = ["source1", "source1", "source1"]
+        sample_inputs = ["prompt1", "prompt1", "prompt1"]  # one uid group of 3
+        infos_dict = {
+            "score": [0.8, 0.9, 0.7],
+            # emitted for only the first sample, null-filled for the rest
+            "sparse": [1.0, None, None],
+            # never emitted for this group
+            "all_none": [None, None, None],
+        }
+
+        result = process_validation_metrics(data_sources, sample_inputs, infos_dict, seed=42)
+
+        # Dense key aggregates over all 3.
+        self.assertAlmostEqual(result["source1"]["score"]["mean@3"], 0.8)
+        # Sparse key aggregates over the single present value.
+        self.assertAlmostEqual(result["source1"]["sparse"]["mean@1"], 1.0)
+        # All-None key is skipped entirely (no metrics emitted).
+        self.assertNotIn("all_none", result["source1"])
+
+    def test_process_validation_metrics_none_keeps_pred_aligned(self):
+        """A sparse numeric key with >1 present value and a None (not at the
+        end) must keep its predictions aligned so majority voting still zips
+        1:1 instead of raising. See issue #6830."""
+        data_sources = ["source1", "source1", "source1"]
+        sample_inputs = ["prompt1", "prompt1", "prompt1"]  # one uid group of 3
+        infos_dict = {
+            "score": [1.0, None, 2.0],  # None in the middle
+            "pred": ["A", "B", "A"],
+        }
+
+        result = process_validation_metrics(data_sources, sample_inputs, infos_dict, seed=42)
+
+        # Aggregates over the two present values (no crash, no misalignment).
+        self.assertAlmostEqual(result["source1"]["score"]["mean@2"], 1.5)
+        # Majority-vote metrics are still produced for the filtered pair set.
+        self.assertIn("maj@2/mean", result["source1"]["score"])
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/verl/trainer/ppo/metric_utils.py b/verl/trainer/ppo/metric_utils.py
index 8a812522906..874f9673f6d 100644
--- a/verl/trainer/ppo/metric_utils.py
+++ b/verl/trainer/ppo/metric_utils.py
@@ -638,6 +638,21 @@ def gen_ns(n_resps: int) -> list[int]:
             var_dict = uid_dict.setdefault(uid, {})
 
             for var_name, var_vals in var2vals.items():
+                # Drop None entries before aggregating. Sparse reward_extra_info
+                # keys (emitted for only some samples) are null-filled to a
+                # uniform schema in `_validate`, so a uid group can contain None;
+                # np.mean([..., None, ...]) would raise TypeError. Keep the
+                # aligned predictions in sync so majority voting still zips
+                # 1:1. See https://github.com/volcengine/verl/issues/6830.
+                vote_pred_vals = pred_vals
+                if any(v is None for v in var_vals):
+                    if pred_vals is not None and len(pred_vals) == len(var_vals):
+                        kept = [(v, p) for v, p in zip(var_vals, pred_vals, strict=True) if v is not None]
+                        var_vals = [v for v, _ in kept]
+                        vote_pred_vals = [p for _, p in kept]
+                    else:
+                        var_vals = [v for v in var_vals if v is not None]
+
                 # skip empty or string values
                 if not var_vals or isinstance(var_vals[0], str):
                     continue
@@ -673,7 +688,8 @@ def gen_ns(n_resps: int) -> list[int]:
                         if has_pred:
                             # create vote_data
                             vote_data = [
-                                {"val": val, "pred": pred} for val, pred in zip(var_vals, pred_vals, strict=True)
+                                {"val": val, "pred": pred}
+                                for val, pred in zip(var_vals, vote_pred_vals, strict=True)
                             ]
                             # compute maj metrics
                             [(maj_n_mean, maj_n_std)] = bootstrap_metric(