Patch newton model reqs

source/isaaclab/isaaclab/scene/interactive_scene.py

@@ -28,7 +28,7 @@
     RigidObjectCollection,
     RigidObjectCollectionCfg,
 )
-from isaaclab.physics.scene_data_requirements import resolve_scene_data_requirements
+from isaaclab.physics.scene_data_requirements import aggregate_requirements, resolve_scene_data_requirements
 from isaaclab.sensors import ContactSensorCfg, FrameTransformerCfg, SensorBase, SensorBaseCfg
 from isaaclab.sim import SimulationContext
 from isaaclab.sim.utils.stage import get_current_stage, get_current_stage_id
@@ -226,6 +226,8 @@ def clone_environments(self, copy_from_source: bool = False):
             If True, clones are independent copies of the source prim and won't reflect its changes (start-up time
             may increase). Defaults to False.
         """
+        self._refresh_visualizer_clone_fn_from_requirements()
+
         # PhysX-only: set env id bit count for replicated physics. Newton handles env separation in its own API.
         # Intentionally matches both physx and ovphysx (both are PhysX-based)
         if self.cfg.replicate_physics and "physx" in self.physics_backend:
@@ -252,6 +254,25 @@ def clone_environments(self, copy_from_source: bool = False):
             if self.cloner_cfg.clone_usd:
                 cloner.usd_replicate(self.stage, *replicate_args)
 
+    def _refresh_visualizer_clone_fn_from_requirements(self) -> None:
+        """Refresh clone-time visualizer prebuild hook from current scene-data requirements."""
+        sensor_req = resolve_scene_data_requirements(
+            visualizer_types=(),
+            renderer_types=self._sensor_renderer_types(),
+        )
+        requirements = aggregate_requirements((self.sim.get_scene_data_requirements(), sensor_req))
+        if requirements != self.sim.get_scene_data_requirements():
+            self.sim.update_scene_data_requirements(requirements)
+
+        visualizer_clone_fn = cloner.resolve_visualizer_clone_fn(
+            physics_backend=self.physics_backend,
+            requirements=requirements,
+            stage=self.stage,
+            set_visualizer_artifact=self.sim.set_scene_data_visualizer_prebuilt_artifact,
+        )
+        if visualizer_clone_fn is not None:
+            self.cloner_cfg.visualizer_clone_fn = visualizer_clone_fn
+
     def _sensor_renderer_types(self) -> list[str]:
         """Return renderer type names used by scene sensors."""
         renderer_types: list[str] = []

source/isaaclab/isaaclab/sensors/camera/camera.py

@@ -114,6 +114,7 @@ def __init__(self, cfg: CameraCfg):
         self._check_supported_data_types(cfg)
         # initialize base class
         super().__init__(cfg)
+        self._register_renderer_scene_data_requirements()
 
         # toggle rendering of rtx sensors as True
         # this flag is read by SimulationContext to determine if rtx sensors should be rendered
@@ -154,6 +155,29 @@ def __init__(self, cfg: CameraCfg):
                     for prim in self.stage.Traverse():
                         prim.SetInstanceable(False)
 
+    def _register_renderer_scene_data_requirements(self) -> None:
+        """Register renderer requirements early enough for clone-time prebuilds."""
+        renderer_type = getattr(getattr(self.cfg, "renderer_cfg", None), "renderer_type", None)
+        if renderer_type is None:
+            return
+
+        from isaaclab.physics.scene_data_requirements import aggregate_requirements, requirement_for_renderer_type
+        from isaaclab.sim import SimulationContext
+
+        sim = SimulationContext.instance()
+        if sim is None:
+            return
+
+        try:
+            renderer_req = requirement_for_renderer_type(renderer_type)
+        except ValueError:
+            return
+
+        current_req = sim.get_scene_data_requirements()
+        merged_req = aggregate_requirements((current_req, renderer_req))
+        if merged_req != current_req:
+            sim.update_scene_data_requirements(merged_req)
+
     def __del__(self):
         """Unsubscribes from callbacks and cleans up renderer resources."""
         # unsubscribe callbacks

source/isaaclab/test/sim/test_physx_scene_data_provider_visualizer_contract.py

@@ -67,68 +67,11 @@ def test_load_prebuilt_artifact_populates_provider_state():
     assert provider._xform_mask_buf is None
 
 
-def test_load_prebuilt_artifact_missing_falls_back_to_usd_build():
-    """When no artifact is registered, the USD-traversal fallback is invoked."""
+def test_load_prebuilt_artifact_missing_sets_error_state():
+    """When no artifact is registered, model/state stay unset."""
     provider = _make_provider()
-    fallback_artifact = VisualizerPrebuiltArtifacts(
-        model="usd-built-model",
-        state="usd-built-state",
-        rigid_body_paths=["/World/envs/env_0/A"],
-        articulation_paths=[],
-        num_envs=2,
-    )
-    stored: list[VisualizerPrebuiltArtifacts] = []
-    provider._simulation_context = SimpleNamespace(
-        get_scene_data_visualizer_prebuilt_artifact=lambda: None,
-        set_scene_data_visualizer_prebuilt_artifact=stored.append,
-    )
-    provider._stage = None
-    provider._xform_views = {}
-    provider._view_body_index_map = {}
-    provider._view_order_tensors = {}
-    provider._pose_buf_num_bodies = 0
-    provider._positions_buf = None
-    provider._orientations_buf = None
-    provider._covered_buf = None
-    provider._xform_mask_buf = None
-
-    with (
-        patch.object(
-            PhysxSceneDataProvider,
-            "_build_newton_artifact_from_usd_fallback",
-            autospec=True,
-            return_value=fallback_artifact,
-        ),
-        patch(
-            "isaaclab_physx.scene_data_providers.physx_scene_data_provider.replace_newton_shape_colors",
-            lambda m, s: None,
-        ),
-    ):
-        provider._load_newton_model_from_prebuilt_artifact()
-
-    assert provider._last_newton_model_build_source == "usd_fallback"
-    assert provider._newton_model == "usd-built-model"
-    assert provider._newton_state == "usd-built-state"
-    assert provider._rigid_body_paths == ["/World/envs/env_0/A"]
-    assert provider._num_envs_at_last_newton_build == 2
-    # The fallback artifact is cached on the simulation context so subsequent providers see it.
-    assert stored == [fallback_artifact]
-
-
-def test_load_prebuilt_artifact_missing_and_fallback_failed_sets_missing_state():
-    """When both the prebuilt artifact and the USD-traversal fallback fail, model/state stay unset."""
-    provider = _make_provider()
-    provider._simulation_context = SimpleNamespace(
-        get_scene_data_visualizer_prebuilt_artifact=lambda: None,
-        set_scene_data_visualizer_prebuilt_artifact=lambda artifact: None,
-    )
-    with patch.object(
-        PhysxSceneDataProvider,
-        "_build_newton_artifact_from_usd_fallback",
-        autospec=True,
-        return_value=None,
-    ):
-        provider._load_newton_model_from_prebuilt_artifact()
+    provider._simulation_context = SimpleNamespace(get_scene_data_visualizer_prebuilt_artifact=lambda: None)
+    provider._load_newton_model_from_prebuilt_artifact()
     assert provider._last_newton_model_build_source == "missing"
     assert provider._newton_model is None
     assert provider._newton_state is None

source/isaaclab_physx/isaaclab_physx/scene_data_providers/physx_scene_data_provider.py

@@ -18,7 +18,6 @@
 from pxr import UsdGeom, UsdPhysics
 
 from isaaclab.physics.base_scene_data_provider import BaseSceneDataProvider
-from isaaclab.physics.scene_data_requirements import VisualizerPrebuiltArtifacts
 from isaaclab.sim.utils.newton_model_utils import replace_newton_shape_colors
 
 logger = logging.getLogger(__name__)
@@ -106,8 +105,6 @@ def __init__(self, stage, simulation_context) -> None:
                 "[PhysxSceneDataProvider] USD stage is None and not available from simulation_context. "
                 "Ensure the simulation context has a valid stage when using OV/Newton/Rerun/Viser visualizers."
             )
-        # Cached so the USD-traversal fallback can hand it to ``newton.ModelBuilder``.
-        self._up_axis = UsdGeom.GetStageUpAxis(self._stage)
         self._num_envs_at_last_newton_build: int | None = None  # for _refresh_newton_model_if_needed
 
         self._device = getattr(self._simulation_context, "device", "cuda:0")
@@ -125,7 +122,7 @@ def __init__(self, stage, simulation_context) -> None:
         self._xform_mask_buf = None
         # View index order as device tensors for vectorized scatter in _apply_view_poses.
         self._view_order_tensors: dict[str, Any] = {}
-        # Last load outcome (tests / debug): "prebuilt" | "usd_fallback" | "missing" | "error".
+        # Last load outcome (tests / debug): "prebuilt" | "missing" | "error".
         self._last_newton_model_build_source: str | None = None
         self._last_newton_model_build_elapsed_ms: float | None = None
 
@@ -168,40 +165,35 @@ def _model_body_paths(self, model) -> list[str]:
         return list(getattr(model, "body_label", None) or getattr(model, "body_key", []))
 
     def _load_newton_model_from_prebuilt_artifact(self) -> None:
-        """Load Newton model and state, preferring the prebuilt artifact and falling back to USD traversal.
-
-        The fast path consumes the artifact stashed on
-        :class:`~isaaclab.sim.SimulationContext` by the cloner's visualizer prebuild
-        hook. When the artifact is missing — for example when a Direct env adds a
-        camera in :meth:`_setup_scene` after the scene's clone-time requirement
-        resolution has already run — fall back to building the model directly from
-        the USD stage and stash the result on the simulation context so subsequent
-        callers hit the fast path.
-        """
+        """Load Newton model and state from the simulation context prebuilt artifact."""
         start_t = time.perf_counter()
         try:
             artifact = self._simulation_context.get_scene_data_visualizer_prebuilt_artifact()
-            if not artifact or artifact.model is None or artifact.state is None:
-                artifact = self._build_newton_artifact_from_usd_fallback()
-                if artifact is None:
-                    self._last_newton_model_build_source = "missing"
-                    logger.error(
-                        "[PhysxSceneDataProvider] No visualizer prebuilt artifact on the simulation context "
-                        "and the USD-traversal fallback failed; cannot sync PhysX to Newton."
-                    )
-                    self._clear_newton_model_state()
-                    return
-                self._simulation_context.set_scene_data_visualizer_prebuilt_artifact(artifact)
-                self._last_newton_model_build_source = "usd_fallback"
-            else:
-                self._last_newton_model_build_source = "prebuilt"
+            if not artifact:
+                self._last_newton_model_build_source = "missing"
+                logger.error(
+                    "[PhysxSceneDataProvider] No visualizer prebuilt artifact on the simulation context "
+                    "(expected VisualizerPrebuiltArtifacts from scene setup)."
+                )
+                self._clear_newton_model_state()
+                return
+
+            model = artifact.model
+            state = artifact.state
+            if model is None or state is None:
+                self._last_newton_model_build_source = "missing"
+                logger.error(
+                    "[PhysxSceneDataProvider] Prebuilt artifact is missing model or state; cannot sync PhysX to Newton."
+                )
+                self._clear_newton_model_state()
+                return
 
-            self._newton_model = artifact.model
-            self._newton_state = artifact.state
+            self._newton_model = model
+            self._newton_state = state
 
             replace_newton_shape_colors(self._newton_model, self._stage)
 
-            body_paths = list(artifact.rigid_body_paths) or self._model_body_paths(artifact.model)
+            body_paths = list(artifact.rigid_body_paths) or self._model_body_paths(model)
             self._rigid_body_paths = body_paths
             view_paths = list(body_paths)
             if artifact.articulation_paths:
@@ -220,9 +212,10 @@ def _load_newton_model_from_prebuilt_artifact(self) -> None:
             self._covered_buf = None
             self._xform_mask_buf = None
             self._num_envs_at_last_newton_build = int(artifact.num_envs)
+            self._last_newton_model_build_source = "prebuilt"
         except Exception as exc:
             self._last_newton_model_build_source = "error"
-            logger.error("[PhysxSceneDataProvider] Failed to load Newton model: %s", exc)
+            logger.error("[PhysxSceneDataProvider] Failed to load Newton model from prebuilt artifact: %s", exc)
             self._clear_newton_model_state()
         finally:
             elapsed_ms = (time.perf_counter() - start_t) * 1000.0
@@ -246,58 +239,6 @@ def _clear_newton_model_state(self) -> None:
         self._rigid_body_view_paths = []
         self._num_envs_at_last_newton_build = None
 
-    def _build_newton_artifact_from_usd_fallback(self) -> VisualizerPrebuiltArtifacts | None:
-        """Build a Newton model from USD when no prebuilt artifact is available.
-
-        Used by Direct envs that add their camera in :meth:`_setup_scene` after
-        :class:`~isaaclab.scene.InteractiveScene` has already resolved scene-data
-        requirements (with no sensors registered). Slower than the cloner-time
-        prebuild path because Newton has to traverse the full USD scene per
-        environment, but functionally equivalent and required for those envs.
-
-        Returns:
-            A :class:`~isaaclab.physics.scene_data_requirements.VisualizerPrebuiltArtifacts`
-            wrapping the freshly built Newton model, or ``None`` when the build
-            could not be performed.
-        """
-        try:
-            from newton import ModelBuilder
-        except ModuleNotFoundError as exc:
-            logger.error(
-                "[PhysxSceneDataProvider] Newton module not available; cannot build USD-fallback model. "
-                "Install the Newton backend to use newton/rerun/viser visualizers or the newton_warp renderer."
-            )
-            logger.debug("[PhysxSceneDataProvider] Newton import error: %s", exc)
-            return None
-
-        num_envs = self.get_num_envs()
-        if num_envs <= 0:
-            return None
-
-        try:
-            builder = ModelBuilder(up_axis=self._up_axis)
-            builder.add_usd(self._stage, ignore_paths=[r"/World/envs/.*"])
-            for env_id in range(num_envs):
-                builder.begin_world()
-                builder.add_usd(self._stage, root_path=f"/World/envs/env_{env_id}")
-                builder.end_world()
-
-            model = builder.finalize(device=self._device)
-            state = model.state()
-        except Exception as exc:
-            logger.error("[PhysxSceneDataProvider] USD-traversal Newton build failed: %s", exc)
-            return None
-
-        body_paths = self._model_body_paths(model)
-        articulation_paths = list(getattr(model, "articulation_label", None) or getattr(model, "articulation_key", []))
-        return VisualizerPrebuiltArtifacts(
-            model=model,
-            state=state,
-            rigid_body_paths=body_paths,
-            articulation_paths=articulation_paths,
-            num_envs=num_envs,
-        )
-
     def _setup_rigid_body_view(self) -> None:
         """Create PhysX RigidBodyView from Newton's body paths.
 

source/isaaclab_tasks/test/test_shadow_hand_vision_presets.py

@@ -326,18 +326,8 @@ def test_warp_with_invalid_camera_preset(shadow_hand_vision_presets, camera_pres
         id="physx-isaacsim_rtx-simple_shading_full_mdl",
     ),
     # ── PhysX physics + Warp: only rgb and depth are supported ──
-    # xfail: standard Shadow Hand USD contains PhysX tendons that Newton's ModelBuilder cannot parse,
-    # so the Newton model build fails and the Warp renderer cannot initialise.
-    pytest.param(
-        ("newton_renderer", "rgb", "physx"),
-        id="physx-warp-rgb",
-        marks=pytest.mark.xfail(raises=RuntimeError, reason="PhysX tendon schemas unsupported by Newton ModelBuilder"),
-    ),
-    pytest.param(
-        ("newton_renderer", "depth", "physx"),
-        id="physx-warp-depth",
-        marks=pytest.mark.xfail(raises=RuntimeError, reason="PhysX tendon schemas unsupported by Newton ModelBuilder"),
-    ),
+    pytest.param(("newton_renderer", "rgb", "physx"), id="physx-warp-rgb"),
+    pytest.param(("newton_renderer", "depth", "physx"), id="physx-warp-depth"),
     # ── Newton physics + Warp: Warp renderer is physics-backend agnostic ──
     pytest.param(("newton_renderer", "rgb", "newton"), id="newton-warp-rgb"),
     pytest.param(("newton_renderer", "depth", "newton"), id="newton-warp-depth"),