Decouple Camera from RTX/Replicator output-buffer logic

docs/source/how-to/save_camera_output.rst

@@ -14,7 +14,7 @@ directory.
 
    .. literalinclude:: ../../../scripts/tutorials/04_sensors/run_usd_camera.py
       :language: python
-      :emphasize-lines: 171-179, 229-247, 251-264
+      :emphasize-lines: 174-182, 232-250, 254-267
       :linenos:
 
 
@@ -25,6 +25,12 @@ Saving using Replicator Basic Writer
    The BasicWriter is part of the Omniverse Replicator ecosystem and is specific to the default
    Isaac RTX renderer backend. Other renderer backends may require different save workflows.
 
+.. note::
+   The ``colorize_*`` arguments below are set on
+   :attr:`~isaaclab.sensors.camera.CameraCfg.renderer_cfg` (an
+   :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg`); the same-named
+   fields on :class:`~isaaclab.sensors.camera.CameraCfg` are deprecated.
+
 To save camera outputs, we use the basic write class from Omniverse Replicator. This class allows us to save the
 images in a numpy format. For more information on the basic writer, please check the
 `documentation <https://docs.omniverse.nvidia.com/extensions/latest/ext_replicator/writer_examples.html>`_.

scripts/tutorials/04_sensors/run_usd_camera.py

@@ -65,6 +65,7 @@
 
 import numpy as np
 import torch
+from isaaclab_physx.renderers import IsaacRtxRendererCfg
 
 import omni.replicator.core as rep
 
@@ -97,9 +98,11 @@ def define_sensor() -> Camera:
             "instance_segmentation_fast",
             "instance_id_segmentation_fast",
         ],
-        colorize_semantic_segmentation=True,
-        colorize_instance_id_segmentation=True,
-        colorize_instance_segmentation=True,
+        renderer_cfg=IsaacRtxRendererCfg(
+            colorize_semantic_segmentation=True,
+            colorize_instance_id_segmentation=True,
+            colorize_instance_segmentation=True,
+        ),
         spawn=sim_utils.PinholeCameraCfg(
             focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(0.1, 1.0e5)
         ),
@@ -173,9 +176,9 @@ def run_simulator(sim: sim_utils.SimulationContext, scene_entities: dict):
     rep_writer = rep.BasicWriter(
         output_dir=output_dir,
         frame_padding=0,
-        colorize_instance_id_segmentation=camera.cfg.colorize_instance_id_segmentation,
-        colorize_instance_segmentation=camera.cfg.colorize_instance_segmentation,
-        colorize_semantic_segmentation=camera.cfg.colorize_semantic_segmentation,
+        colorize_instance_id_segmentation=camera.cfg.renderer_cfg.colorize_instance_id_segmentation,
+        colorize_instance_segmentation=camera.cfg.renderer_cfg.colorize_instance_segmentation,
+        colorize_semantic_segmentation=camera.cfg.renderer_cfg.colorize_semantic_segmentation,
     )
 
     # Camera positions, targets, orientations

source/isaaclab/config/extension.toml

@@ -1,7 +1,7 @@
 [package]
 
 # Note: Semantic Versioning is used: https://semver.org/
-version = "4.6.12"
+version = "4.6.13"
 
 # Description
 title = "Isaac Lab framework for Robot Learning"

source/isaaclab/docs/CHANGELOG.rst

@@ -1,6 +1,33 @@
 Changelog
 ---------
 
+4.6.13 (2026-04-23)
+~~~~~~~~~~~~~~~~~~~
+
+Changed
+^^^^^^^
+
+* Renderer backends now own output-buffer allocation via a new
+  ``BaseRenderer.create_output_buffers`` hook. Backends are the source of truth
+  for shape, dtype, and any aliasing (e.g. ``rgb`` as a view into ``rgba``).
+  Requested data types the backend cannot produce are dropped with a single
+  warning, instead of being silently discarded later.
+* :class:`~isaaclab_ov.renderers.OVRTXRenderer` now writes rendered tiles
+  directly into the torch storage backing ``camera.data.output``, eliminating
+  the per-frame ``wp.copy`` bridge.
+* Moved Kit/RTX-only logic out of :class:`~isaaclab.sensors.camera.Camera`
+  into :class:`~isaaclab_physx.renderers.IsaacRtxRenderer`.
+
+Deprecated
+^^^^^^^^^^
+
+* Deprecated RTX-flavored fields on :class:`~isaaclab.sensors.camera.CameraCfg`
+  (``semantic_filter``, ``colorize_semantic_segmentation``,
+  ``colorize_instance_segmentation``, ``colorize_instance_id_segmentation``,
+  ``semantic_segmentation_mapping``, ``depth_clipping_behavior``); set them on
+  :attr:`~isaaclab.sensors.camera.CameraCfg.renderer_cfg` instead.
+
+
 4.6.12 (2026-04-23)
 ~~~~~~~~~~~~~~~~~~~
 

source/isaaclab/isaaclab/renderers/base_renderer.py

@@ -20,6 +20,33 @@
 class BaseRenderer(ABC):
     """Abstract base class for renderer implementations."""
 
+    @abstractmethod
+    def create_output_buffers(
+        self,
+        data_types: list[str],
+        height: int,
+        width: int,
+        num_views: int,
+        device: torch.device | str,
+    ) -> dict[str, torch.Tensor]:
+        """Allocate output tensors for the supported subset of ``data_types``.
+
+        Implementations MUST omit any data-type names they cannot produce and
+        allocate on ``device``. They MAY include aliased entries that share
+        storage with another entry (e.g. ``rgb`` as a view into ``rgba``).
+
+        Args:
+            data_types: Names of the requested data types.
+            height: Image height in pixels.
+            width: Image width in pixels.
+            num_views: Number of camera views (batch dimension).
+            device: Torch device on which to allocate the buffers.
+
+        Returns:
+            Mapping from data-type name to a pre-allocated tensor.
+        """
+        pass
+
     @abstractmethod
     def prepare_stage(self, stage: Any, num_envs: int) -> None:
         """Prepare the stage for rendering before create_render_data is called.

source/isaaclab/isaaclab/sensors/camera/camera.py

@@ -12,21 +12,19 @@
 import numpy as np
 import torch
 import warp as wp
-from packaging import version
 
-from pxr import Sdf, UsdGeom
+from pxr import UsdGeom
 
 import isaaclab.utils.sensors as sensor_utils
 from isaaclab.app.settings_manager import get_settings_manager
 from isaaclab.renderers import BaseRenderer, Renderer
 from isaaclab.sim.views import FrameView
-from isaaclab.utils import has_kit, to_camel_case
+from isaaclab.utils import to_camel_case
 from isaaclab.utils.math import (
     convert_camera_frame_orientation_convention,
     create_rotation_matrix_from_view,
     quat_from_matrix,
 )
-from isaaclab.utils.version import get_isaac_sim_version
 
 from ..sensor_base import SensorBase
 from .camera_data import CameraData
@@ -94,12 +92,6 @@ class Camera(SensorBase):
     }
     """The set of sensor types that are not supported by the camera class."""
 
-    SIMPLE_SHADING_TYPES: set[str] = {
-        "simple_shading_constant_diffuse",
-        "simple_shading_diffuse_mdl",
-        "simple_shading_full_mdl",
-    }
-
     def __init__(self, cfg: CameraCfg):
         """Initializes the camera sensor.
 
@@ -115,10 +107,15 @@ def __init__(self, cfg: CameraCfg):
         # initialize base class
         super().__init__(cfg)
 
-        # toggle rendering of rtx sensors as True
-        # this flag is read by SimulationContext to determine if rtx sensors should be rendered
-        settings = get_settings_manager()
-        settings.set_bool("/isaaclab/render/rtx_sensors", True)
+        # TODO: Camera should not branch on a specific renderer_type string. Replace with a
+        # generic opt-in flag on RendererCfg (e.g. ``requires_kit_rtx_sensors_flag``) that
+        # RTX-family cfgs set to True, so this branch carries no renderer-specific knowledge.
+        # The flag must flip at scene-construction time (before sim.reset()) because
+        # SimulationContext.is_rendering and several env classes branch on it pre-reset;
+        # flipping inside the renderer's __init__ (which only runs at sim.reset()) would
+        # silently break that timing.
+        if self.cfg.renderer_cfg.renderer_type == "isaac_rtx":
+            get_settings_manager().set_bool("/isaaclab/render/rtx_sensors", True)
 
         # Compute camera orientation (convention conversion) and spawn
         rot = torch.tensor(self.cfg.offset.rot, dtype=torch.float32, device="cpu").unsqueeze(0)
@@ -138,22 +135,6 @@ def __init__(self, cfg: CameraCfg):
         self._renderer: BaseRenderer | None = None
         self._render_data = None
 
-        if not has_kit():
-            return
-        # HACK: We need to disable instancing for semantic_segmentation and instance_segmentation_fast to work
-        # checks for Isaac Sim v4.5 as this issue exists there
-        if get_isaac_sim_version() == version.parse("4.5"):
-            if "semantic_segmentation" in self.cfg.data_types or "instance_segmentation_fast" in self.cfg.data_types:
-                logger.warning(
-                    "Isaac Sim 4.5 introduced a bug in Camera and TiledCamera when outputting instance and semantic"
-                    " segmentation outputs for instanceable assets. As a workaround, the instanceable flag on assets"
-                    " will be disabled in the current workflow and may lead to longer load times and increased memory"
-                    " usage."
-                )
-                with Sdf.ChangeBlock():
-                    for prim in self.stage.Traverse():
-                        prim.SetInstanceable(False)
-
     def __del__(self):
         """Unsubscribes from callbacks and cleans up renderer resources."""
         # unsubscribe callbacks
@@ -168,10 +149,6 @@ def __str__(self) -> str:
         return (
             f"Camera @ '{self.cfg.prim_path}': \n"
             f"\tdata types   : {list(self.data.output.keys())} \n"
-            f"\tsemantic filter : {self.cfg.semantic_filter}\n"
-            f"\tcolorize semantic segm.   : {self.cfg.colorize_semantic_segmentation}\n"
-            f"\tcolorize instance segm.   : {self.cfg.colorize_instance_segmentation}\n"
-            f"\tcolorize instance id segm.: {self.cfg.colorize_instance_id_segmentation}\n"
             f"\tupdate period (s): {self.cfg.update_period}\n"
             f"\tshape        : {self.image_shape}\n"
             f"\tnumber of sensors : {self._view.count}"
@@ -385,16 +362,10 @@ def _initialize_impl(self):
 
         Raises:
             RuntimeError: If the number of camera prims in the view does not match the number of environments.
-            RuntimeError: If cameras are not enabled (missing ``--enable_cameras`` flag).
+            RuntimeError: Propagated from the renderer constructor when the active backend's
+                runtime requirements are not satisfied (e.g. the RTX backend requires the
+                simulation app to be launched with ``--enable_cameras``).
         """
-        renderer_type = getattr(self.cfg.renderer_cfg, "renderer_type", "default")
-        needs_kit_cameras = renderer_type in ("default", "isaac_rtx")
-        if needs_kit_cameras and not get_settings_manager().get("/isaaclab/cameras_enabled"):
-            raise RuntimeError(
-                "A camera was spawned without the --enable_cameras flag. Please use --enable_cameras to enable"
-                " rendering."
-            )
-
         # Initialize parent class
         super()._initialize_impl()
 
@@ -485,73 +456,24 @@ def _create_buffers(self):
         self._data.quat_w_world = torch.zeros((self._view.count, 4), device=self._device)
         self._update_poses(self._ALL_INDICES)
         self._data.image_shape = self.image_shape
-        # -- output data (eagerly pre-allocated so renderer.set_outputs() can hold tensor references)
-        data_dict = dict()
-        if "rgba" in self.cfg.data_types or "rgb" in self.cfg.data_types:
-            data_dict["rgba"] = torch.zeros(
-                (self._view.count, self.cfg.height, self.cfg.width, 4), device=self.device, dtype=torch.uint8
-            ).contiguous()
-        if "rgb" in self.cfg.data_types:
-            data_dict["rgb"] = data_dict["rgba"][..., :3]
-        if "albedo" in self.cfg.data_types:
-            data_dict["albedo"] = torch.zeros(
-                (self._view.count, self.cfg.height, self.cfg.width, 4), device=self.device, dtype=torch.uint8
-            ).contiguous()
-        for data_type in self.SIMPLE_SHADING_TYPES:
-            if data_type in self.cfg.data_types:
-                data_dict[data_type] = torch.zeros(
-                    (self._view.count, self.cfg.height, self.cfg.width, 3), device=self.device, dtype=torch.uint8
-                ).contiguous()
-        if "distance_to_image_plane" in self.cfg.data_types:
-            data_dict["distance_to_image_plane"] = torch.zeros(
-                (self._view.count, self.cfg.height, self.cfg.width, 1), device=self.device, dtype=torch.float32
-            ).contiguous()
-        if "depth" in self.cfg.data_types:
-            data_dict["depth"] = torch.zeros(
-                (self._view.count, self.cfg.height, self.cfg.width, 1), device=self.device, dtype=torch.float32
-            ).contiguous()
-        if "distance_to_camera" in self.cfg.data_types:
-            data_dict["distance_to_camera"] = torch.zeros(
-                (self._view.count, self.cfg.height, self.cfg.width, 1), device=self.device, dtype=torch.float32
-            ).contiguous()
-        if "normals" in self.cfg.data_types:
-            data_dict["normals"] = torch.zeros(
-                (self._view.count, self.cfg.height, self.cfg.width, 3), device=self.device, dtype=torch.float32
-            ).contiguous()
-        if "motion_vectors" in self.cfg.data_types:
-            data_dict["motion_vectors"] = torch.zeros(
-                (self._view.count, self.cfg.height, self.cfg.width, 2), device=self.device, dtype=torch.float32
-            ).contiguous()
-        if "semantic_segmentation" in self.cfg.data_types:
-            if self.cfg.colorize_semantic_segmentation:
-                data_dict["semantic_segmentation"] = torch.zeros(
-                    (self._view.count, self.cfg.height, self.cfg.width, 4), device=self.device, dtype=torch.uint8
-                ).contiguous()
-            else:
-                data_dict["semantic_segmentation"] = torch.zeros(
-                    (self._view.count, self.cfg.height, self.cfg.width, 1), device=self.device, dtype=torch.int32
-                ).contiguous()
-        if "instance_segmentation_fast" in self.cfg.data_types:
-            if self.cfg.colorize_instance_segmentation:
-                data_dict["instance_segmentation_fast"] = torch.zeros(
-                    (self._view.count, self.cfg.height, self.cfg.width, 4), device=self.device, dtype=torch.uint8
-                ).contiguous()
-            else:
-                data_dict["instance_segmentation_fast"] = torch.zeros(
-                    (self._view.count, self.cfg.height, self.cfg.width, 1), device=self.device, dtype=torch.int32
-                ).contiguous()
-        if "instance_id_segmentation_fast" in self.cfg.data_types:
-            if self.cfg.colorize_instance_id_segmentation:
-                data_dict["instance_id_segmentation_fast"] = torch.zeros(
-                    (self._view.count, self.cfg.height, self.cfg.width, 4), device=self.device, dtype=torch.uint8
-                ).contiguous()
-            else:
-                data_dict["instance_id_segmentation_fast"] = torch.zeros(
-                    (self._view.count, self.cfg.height, self.cfg.width, 1), device=self.device, dtype=torch.int32
-                ).contiguous()
-
-        self._data.output = data_dict
-        self._data.info = {name: None for name in self.cfg.data_types}
+        # -- output data: ask the renderer to allocate buffers for the requested data types.
+        buffers = self._renderer.create_output_buffers(
+            self.cfg.data_types,
+            self.cfg.height,
+            self.cfg.width,
+            self._view.count,
+            self.device,
+        )
+        # Surface any requested data types the active renderer cannot produce.
+        unsupported = [name for name in self.cfg.data_types if name not in buffers]
+        if unsupported:
+            logger.warning(
+                "Renderer %s does not support the following requested data types and will not produce them: %s",
+                type(self._renderer).__name__,
+                unsupported,
+            )
+        self._data.output = buffers
+        self._data.info = {name: None for name in buffers}
         self._renderer.set_outputs(self._render_data, self._data.output)
 
     def _update_intrinsic_matrices(self, env_ids: Sequence[int]):