[trainer] Fix process_validation_metrics crash on None-filled sparse reward keys

tests/trainer/ppo/test_metric_utils_on_cpu.py

@@ -568,6 +568,47 @@ def test_process_validation_metrics_with_pred(self):
         # For bootstrap with n=2, the majority vote could be either A or B
         # depending on the random sampling, so we don't check the exact value
 
+    def test_process_validation_metrics_none_filled_sparse_key(self):
+        """Sparse reward_extra_info keys are null-filled for samples that omit
+        them; aggregation must drop the None values instead of crashing on
+        np.mean([..., None, ...]). See issue #6830."""
+        data_sources = ["source1", "source1", "source1"]
+        sample_inputs = ["prompt1", "prompt1", "prompt1"]  # one uid group of 3
+        infos_dict = {
+            "score": [0.8, 0.9, 0.7],
+            # emitted for only the first sample, null-filled for the rest
+            "sparse": [1.0, None, None],
+            # never emitted for this group
+            "all_none": [None, None, None],
+        }
+
+        result = process_validation_metrics(data_sources, sample_inputs, infos_dict, seed=42)
+
+        # Dense key aggregates over all 3.
+        self.assertAlmostEqual(result["source1"]["score"]["mean@3"], 0.8)
+        # Sparse key aggregates over the single present value.
+        self.assertAlmostEqual(result["source1"]["sparse"]["mean@1"], 1.0)
+        # All-None key is skipped entirely (no metrics emitted).
+        self.assertNotIn("all_none", result["source1"])
+
+    def test_process_validation_metrics_none_keeps_pred_aligned(self):
+        """A sparse numeric key with >1 present value and a None (not at the
+        end) must keep its predictions aligned so majority voting still zips
+        1:1 instead of raising. See issue #6830."""
+        data_sources = ["source1", "source1", "source1"]
+        sample_inputs = ["prompt1", "prompt1", "prompt1"]  # one uid group of 3
+        infos_dict = {
+            "score": [1.0, None, 2.0],  # None in the middle
+            "pred": ["A", "B", "A"],
+        }
+
+        result = process_validation_metrics(data_sources, sample_inputs, infos_dict, seed=42)
+
+        # Aggregates over the two present values (no crash, no misalignment).
+        self.assertAlmostEqual(result["source1"]["score"]["mean@2"], 1.5)
+        # Majority-vote metrics are still produced for the filtered pair set.
+        self.assertIn("maj@2/mean", result["source1"]["score"])
+
 
 if __name__ == "__main__":
     unittest.main()

verl/trainer/ppo/metric_utils.py

@@ -638,6 +638,21 @@ def gen_ns(n_resps: int) -> list[int]:
             var_dict = uid_dict.setdefault(uid, {})
 
             for var_name, var_vals in var2vals.items():
+                # Drop None entries before aggregating. Sparse reward_extra_info
+                # keys (emitted for only some samples) are null-filled to a
+                # uniform schema in `_validate`, so a uid group can contain None;
+                # np.mean([..., None, ...]) would raise TypeError. Keep the
+                # aligned predictions in sync so majority voting still zips
+                # 1:1. See https://github.com/volcengine/verl/issues/6830.
+                vote_pred_vals = pred_vals
+                if any(v is None for v in var_vals):
+                    if pred_vals is not None and len(pred_vals) == len(var_vals):
+                        kept = [(v, p) for v, p in zip(var_vals, pred_vals, strict=True) if v is not None]
+                        var_vals = [v for v, _ in kept]
+                        vote_pred_vals = [p for _, p in kept]
+                    else:
+                        var_vals = [v for v in var_vals if v is not None]
+
                 # skip empty or string values
                 if not var_vals or isinstance(var_vals[0], str):
                     continue
@@ -673,7 +688,8 @@ def gen_ns(n_resps: int) -> list[int]:
                         if has_pred:
                             # create vote_data
                             vote_data = [
-                                {"val": val, "pred": pred} for val, pred in zip(var_vals, pred_vals, strict=True)
+                                {"val": val, "pred": pred}
+                                for val, pred in zip(var_vals, vote_pred_vals, strict=True)
                             ]
                             # compute maj metrics
                             [(maj_n_mean, maj_n_std)] = bootstrap_metric(