From 6b43f4b50a743cfc596adb4efcfbde46fb8f5090 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Thu, 16 Apr 2026 07:11:49 +0000 Subject: [PATCH 01/17] Fix prebundle path conflicts, pin mujoco, remove omni.warp.core, fix broken links Combined fixes from PR #5279 on top of deprecation migration from PR #5280: 1. install.py: Add prebundle probe functions that check pip_prebundle paths instead of extsDeprecated, and uninstall pip torch when prebundle shadows it 2. isaaclab/__init__.py: Deprioritize ml_archive/pip_prebundle on sys.path so pip-installed torch/numpy/etc take priority over prebundled copies 3. setup.py: Pin mujoco==3.5.0 4. Kit files: Remove omni.warp.core to prevent conflict with pip warp-lang 5. check-links.yml: Add ubuntu.com to link checker exclusions (returns 403) 6. New tests: test_install_commands.py, test_install_prebundle.py --- apps/isaaclab.python.headless.kit | 1 - apps/isaaclab.python.kit | 1 - source/isaaclab/isaaclab/__init__.py | 57 ++ .../isaaclab/isaaclab/cli/commands/install.py | 73 ++ .../test/cli/test_install_commands.py | 786 ++++++++++++++++++ .../test/cli/test_install_prebundle.py | 81 ++ 6 files changed, 997 insertions(+), 2 deletions(-) create mode 100644 source/isaaclab/test/cli/test_install_commands.py create mode 100644 source/isaaclab/test/cli/test_install_prebundle.py diff --git a/apps/isaaclab.python.headless.kit b/apps/isaaclab.python.headless.kit index 1d28f7b9018c..01d50be2a560 100644 --- a/apps/isaaclab.python.headless.kit +++ b/apps/isaaclab.python.headless.kit @@ -26,7 +26,6 @@ log.outputStreamLevel = "Warn" "omni.physx" = {} "omni.physx.tensors" = {} "omni.physx.fabric" = {} -"omni.warp.core" = {} "usdrt.scenegraph" = {} "omni.kit.telemetry" = {} "omni.kit.loop" = {} diff --git a/apps/isaaclab.python.kit b/apps/isaaclab.python.kit index cc54d43a4da3..011902ace986 100644 --- a/apps/isaaclab.python.kit +++ b/apps/isaaclab.python.kit @@ -92,7 +92,6 @@ keywords = ["experience", "app", "usd"] "omni.uiaudio" = {} "omni.usd.metrics.assembler.ui" = {} "omni.usd.schema.metrics.assembler" = {} -"omni.warp.core" = {} ######################## # Isaac Lab Extensions # diff --git a/source/isaaclab/isaaclab/__init__.py b/source/isaaclab/isaaclab/__init__.py index c3e1b9450b7a..0d745d91610b 100644 --- a/source/isaaclab/isaaclab/__init__.py +++ b/source/isaaclab/isaaclab/__init__.py @@ -6,6 +6,63 @@ """Package containing the core framework.""" import os +import sys + + +def _deprioritize_prebundle_paths(): + """Move Isaac Sim ``pip_prebundle`` directories to the end of ``sys.path``. + + Isaac Sim's ``setup_python_env.sh`` injects ``pip_prebundle`` directories + (e.g. ``omni.isaac.ml_archive/pip_prebundle``) onto ``PYTHONPATH``. These + contain older copies of packages like torch, warp, and nvidia-cudnn that + shadow the versions installed by Isaac Lab, causing CUDA runtime errors. + + Rather than removing these paths entirely (which would break packages like + ``sympy`` that only exist in the prebundle), this function moves them to + the **end** of ``sys.path`` so that pip-installed packages in + ``site-packages`` take priority. + + The ``PYTHONPATH`` environment variable is also rewritten so that child + processes inherit the corrected ordering. + """ + # Extensions whose prebundled packages conflict with Isaac Lab deps. + _CONFLICTING_EXTS = ( + "omni.isaac.ml_archive", + ) + + def _is_conflicting(path: str) -> bool: + norm = path.replace("\\", "/").lower() + return "pip_prebundle" in norm and any(ext.lower() in norm for ext in _CONFLICTING_EXTS) + + # Partition: keep non-conflicting in place, collect conflicting. + clean = [] + demoted = [] + for p in sys.path: + if _is_conflicting(p): + demoted.append(p) + else: + clean.append(p) + + if not demoted: + return + + # Rebuild sys.path: originals first, then demoted at the very end. + sys.path[:] = clean + demoted + + # Rewrite PYTHONPATH with the same ordering for subprocesses. + if "PYTHONPATH" in os.environ: + parts = os.environ["PYTHONPATH"].split(os.pathsep) + env_clean = [] + env_demoted = [] + for p in parts: + if _is_conflicting(p): + env_demoted.append(p) + else: + env_clean.append(p) + os.environ["PYTHONPATH"] = os.pathsep.join(env_clean + env_demoted) + + +_deprioritize_prebundle_paths() # Conveniences to other module directories via relative paths. ISAACLAB_EXT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) diff --git a/source/isaaclab/isaaclab/cli/commands/install.py b/source/isaaclab/isaaclab/cli/commands/install.py index 53cf4a799fb1..2e402a72b792 100644 --- a/source/isaaclab/isaaclab/cli/commands/install.py +++ b/source/isaaclab/isaaclab/cli/commands/install.py @@ -77,6 +77,59 @@ def _install_system_deps() -> None: run_command(["sudo"] + cmd if os.geteuid() != 0 else cmd) +def _torch_first_on_sys_path_is_prebundle(python_exe: str, *, env: dict[str, str]) -> bool: + """Return True when the first ``torch`` on ``sys.path`` comes from a prebundle directory. + + Checks whether the first directory on ``sys.path`` that contains a + ``torch`` package lives under a ``pip_prebundle`` path (e.g. + ``omni.isaac.ml_archive/pip_prebundle``). This catches the prebundle + regardless of whether the extension lives under ``exts/``, + ``extsDeprecated/``, or any other search path. + + Does not import ``torch`` (that can fail on missing ``libcudnn`` while the + prebundle still appears earlier on ``sys.path`` than ``site-packages``). + """ + probe = """import os, sys +for p in sys.path: + if not p: + continue + if os.path.isfile(os.path.join(p, "torch", "__init__.py")): + norm = os.path.normpath(p) + sys.exit(1 if "pip_prebundle" in norm else 0) +sys.exit(0) +""" + result = run_command( + [python_exe, "-c", probe], + env=env, + check=False, + capture_output=True, + text=True, + ) + return result.returncode == 1 + + +def _maybe_uninstall_prebundled_torch( + python_exe: str, + pip_cmd: list[str], + using_uv: bool, + *, + probe_env: dict[str, str], +) -> None: + """Uninstall pip torch stack when ``sys.path`` would load ``torch`` from a prebundle first.""" + if not _torch_first_on_sys_path_is_prebundle(python_exe, env=probe_env): + return + print_info( + "The first ``torch`` on ``sys.path`` is under a prebundle directory (e.g. " + "``omni.isaac.ml_archive/pip_prebundle``). Uninstalling pip " + "``torch``/``torchvision``/``torchaudio`` before continuing." + ) + uninstall_flags = ["-y"] if not using_uv else [] + run_command( + pip_cmd + ["uninstall"] + uninstall_flags + ["torch", "torchvision", "torchaudio"], + check=False, + ) + + def _ensure_cuda_torch() -> None: """Ensure correct PyTorch and CUDA versions are installed.""" python_exe = extract_python_exe() @@ -389,6 +442,17 @@ def _repoint_prebundle_packages() -> None: if not prebundled.exists() and not prebundled.is_symlink(): continue + # The 'nvidia' directory is a Python namespace package shared across many + # distributions (nvidia-cudnn-cu12, nvidia-cublas-cu12, nvidia-srl, …). + # When using Isaac Sim's built-in Python, site-packages/nvidia only contains + # 'srl'; replacing the whole prebundle nvidia/ with that symlink strips away + # the CUDA shared libraries (libcudnn.so.9, etc.) that torch needs. + # Only repoint the nvidia namespace when the target actually provides the + # CUDA subpackages (cudnn is the minimal required indicator). + if pkg_name == "nvidia" and not (venv_pkg / "cudnn").exists(): + print_debug(f"Skipping repoint of {prebundled}: {venv_pkg} lacks CUDA subpackages (cudnn missing).") + continue + try: if prebundled.is_symlink(): if prebundled.resolve() == venv_pkg.resolve(): @@ -543,6 +607,12 @@ def command_install(install_type: str = "all") -> None: pip_cmd = get_pip_command(python_exe) using_uv = pip_cmd[0] == "uv" + # Probe with the user's original PYTHONPATH (before pip-time filtering) so we detect + # Isaac Sim's setup_python_env.sh ordering that prefers extsDeprecated/ml_archive. + probe_env = {**os.environ} + if saved_pythonpath is not None: + probe_env["PYTHONPATH"] = saved_pythonpath + try: # Upgrade pip first to avoid compatibility issues (skip when using uv). if not using_uv: @@ -552,6 +622,9 @@ def command_install(install_type: str = "all") -> None: # Pin setuptools to avoid issues with pkg_resources removal in 82.0.0. run_command(pip_cmd + ["install", "setuptools<82.0.0"]) + # Drop pip-installed torch if Isaac Sim's deprecated ML prebundle would shadow it. + _maybe_uninstall_prebundled_torch(python_exe, pip_cmd, using_uv, probe_env=probe_env) + # Install Isaac Sim if requested. if install_isaacsim: _install_isaacsim() diff --git a/source/isaaclab/test/cli/test_install_commands.py b/source/isaaclab/test/cli/test_install_commands.py new file mode 100644 index 000000000000..a7c89ccd9d53 --- /dev/null +++ b/source/isaaclab/test/cli/test_install_commands.py @@ -0,0 +1,786 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for install command functions. + +Covers all combinations of: +- Python environment types: uv venv, pip venv, conda, Isaac Sim kit Python, system Python +- Isaac Sim installation methods: local _isaac_sim symlink, pip-installed isaacsim, none +""" + +import subprocess +from contextlib import contextmanager +from pathlib import Path +from unittest import mock + +import pytest + +from isaaclab.cli.commands.install import ( + _PREBUNDLE_REPOINT_PACKAGES, + _ensure_cuda_torch, + _maybe_uninstall_prebundled_torch, + _repoint_prebundle_packages, + _torch_first_on_sys_path_is_prebundle, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _cp(returncode: int = 0, stdout: str = "") -> mock.MagicMock: + """Return a mock CompletedProcess with the given returncode and stdout.""" + r = mock.MagicMock(spec=subprocess.CompletedProcess) + r.returncode = returncode + r.stdout = stdout + return r + + +def _make_prebundle(base: Path, packages: list[str]) -> Path: + """Create a fake pip_prebundle directory populated with the given package dirs.""" + prebundle = base / "pip_prebundle" + prebundle.mkdir(parents=True) + for pkg in packages: + (prebundle / pkg).mkdir() + return prebundle + + +def _make_site_packages( + base: Path, + packages: list[str], + subdirs: dict[str, list[str]] | None = None, +) -> Path: + """Create a fake site-packages directory. + + Args: + packages: Top-level package directory names to create. + subdirs: Optional mapping of package name → list of subdirectory names to create inside it. + """ + site_pkgs = base / "site-packages" + site_pkgs.mkdir(parents=True, exist_ok=True) + for pkg in packages: + (site_pkgs / pkg).mkdir(exist_ok=True) + for pkg, subs in (subdirs or {}).items(): + for sub in subs: + (site_pkgs / pkg / sub).mkdir(parents=True, exist_ok=True) + return site_pkgs + + +# --------------------------------------------------------------------------- +# _torch_first_on_sys_path_is_prebundle +# --------------------------------------------------------------------------- + + +class TestTorchProbe: + """Tests for :func:`_torch_first_on_sys_path_is_prebundle`. + + The function shells out to ``python_exe -c `` and interprets the + subprocess exit code: 1 → prebundle is first; 0 → it is not. + """ + + def test_returns_true_when_prebundle_first(self, tmp_path): + """Probe exits 1 → the first torch on sys.path is under a pip_prebundle directory.""" + with mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(returncode=1)): + result = _torch_first_on_sys_path_is_prebundle( + str(tmp_path / "python"), + env={"PYTHONPATH": "/fake/extsDeprecated/pip_prebundle"}, + ) + assert result is True + + def test_returns_false_when_site_packages_first(self, tmp_path): + """Probe exits 0 → the first torch on sys.path is in regular site-packages.""" + with mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(returncode=0)): + result = _torch_first_on_sys_path_is_prebundle( + str(tmp_path / "python"), + env={"PYTHONPATH": "/conda/lib/python3.12/site-packages"}, + ) + assert result is False + + def test_returns_false_when_torch_not_found_anywhere(self, tmp_path): + """Probe exits 0 (no torch on sys.path at all) → returns False.""" + with mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(returncode=0)): + result = _torch_first_on_sys_path_is_prebundle( + str(tmp_path / "python"), + env={}, + ) + assert result is False + + def test_passes_env_to_subprocess(self, tmp_path): + """The custom env dict is forwarded to run_command.""" + env_sent = {"PYTHONPATH": "/some/path"} + with mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(0)) as mock_run: + _torch_first_on_sys_path_is_prebundle(str(tmp_path / "python"), env=env_sent) + call_kwargs = mock_run.call_args + assert call_kwargs.kwargs.get("env") == env_sent or ( + len(call_kwargs.args) > 1 and call_kwargs.args[1] == env_sent + ) + + +# --------------------------------------------------------------------------- +# _maybe_uninstall_prebundled_torch +# --------------------------------------------------------------------------- + + +class TestMaybeUninstallTorch: + """Tests for :func:`_maybe_uninstall_prebundled_torch`.""" + + def test_does_not_uninstall_when_probe_false(self, tmp_path): + """When the probe returns False, no pip uninstall command is issued.""" + py = str(tmp_path / "python") + with ( + mock.patch( + "isaaclab.cli.commands.install._torch_first_on_sys_path_is_prebundle", + return_value=False, + ), + mock.patch("isaaclab.cli.commands.install.run_command") as mock_run, + ): + _maybe_uninstall_prebundled_torch(py, [py, "-m", "pip"], using_uv=False, probe_env={}) + mock_run.assert_not_called() + + def test_uninstalls_torch_stack_with_minus_y_for_pip(self, tmp_path): + """When probe returns True and pip is in use, uninstall includes -y flag.""" + py = str(tmp_path / "python") + with ( + mock.patch( + "isaaclab.cli.commands.install._torch_first_on_sys_path_is_prebundle", + return_value=True, + ), + mock.patch("isaaclab.cli.commands.install.run_command") as mock_run, + ): + _maybe_uninstall_prebundled_torch(py, [py, "-m", "pip"], using_uv=False, probe_env={}) + mock_run.assert_called_once() + issued = mock_run.call_args[0][0] + assert "uninstall" in issued + assert "-y" in issued + assert "torch" in issued + assert "torchvision" in issued + assert "torchaudio" in issued + + def test_uninstalls_torch_stack_without_minus_y_for_uv(self, tmp_path): + """When probe returns True and uv pip is in use, uninstall omits -y (uv doesn't accept it).""" + with ( + mock.patch( + "isaaclab.cli.commands.install._torch_first_on_sys_path_is_prebundle", + return_value=True, + ), + mock.patch("isaaclab.cli.commands.install.run_command") as mock_run, + ): + _maybe_uninstall_prebundled_torch("/fake/python", ["uv", "pip"], using_uv=True, probe_env={}) + issued = mock_run.call_args[0][0] + assert "uninstall" in issued + assert "-y" not in issued + + def test_probe_receives_original_pythonpath(self, tmp_path): + """The probe_env dict is forwarded unchanged to the torch-probe function.""" + py = str(tmp_path / "python") + probe_env = {"PYTHONPATH": "/a/extsDeprecated/pip_prebundle:/b/site-packages"} + with mock.patch( + "isaaclab.cli.commands.install._torch_first_on_sys_path_is_prebundle", + return_value=False, + ) as mock_probe: + _maybe_uninstall_prebundled_torch(py, [py, "-m", "pip"], using_uv=False, probe_env=probe_env) + mock_probe.assert_called_once_with(py, env=probe_env) + + +# --------------------------------------------------------------------------- +# _ensure_cuda_torch — architecture × environment combinations +# --------------------------------------------------------------------------- + + +class TestEnsureCudaTorch: + """Tests for :func:`_ensure_cuda_torch` across architectures and environment types. + + Combinations tested: + - Architecture: x86 (cu128) vs ARM (cu130) + - Pip command: ``python -m pip`` (venv/conda/kit) vs ``uv pip`` (uv venv) + - Torch state: already installed at correct version; wrong CUDA tag; not installed + """ + + # ---- x86 scenarios ------------------------------------------------------- + + def test_x86_skips_install_when_correct_version_present(self, tmp_path): + """x86: torch 2.10.0+cu128 already installed → pip install is not called.""" + py = str(tmp_path / "python") + pip_cmd = [py, "-m", "pip"] + pip_show_out = "Name: torch\nVersion: 2.10.0+cu128\n" + + with ( + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.get_pip_command", return_value=pip_cmd), + mock.patch("isaaclab.cli.commands.install.is_arm", return_value=False), + mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(0, pip_show_out)) as mock_run, + ): + _ensure_cuda_torch() + + # Only the initial ``pip show torch`` call; no install. + assert mock_run.call_count == 1 + assert "show" in mock_run.call_args[0][0] + + def test_x86_installs_cu128_when_torch_missing(self, tmp_path): + """x86: no torch installed → installs torch+cu128 from pytorch.org/whl/cu128.""" + py = str(tmp_path / "python") + pip_cmd = [py, "-m", "pip"] + calls: list[list[str]] = [] + + def _run(cmd, **kwargs): + calls.append(list(cmd)) + return _cp(0, "") # pip show returns nothing → torch absent + + with ( + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.get_pip_command", return_value=pip_cmd), + mock.patch("isaaclab.cli.commands.install.is_arm", return_value=False), + mock.patch("isaaclab.cli.commands.install.run_command", side_effect=_run), + ): + _ensure_cuda_torch() + + install_cmds = [c for c in calls if "install" in c] + combined = " ".join(str(t) for c in install_cmds for t in c) + assert "cu128" in combined + assert "torch" in combined + + def test_x86_reinstalls_when_wrong_cuda_tag(self, tmp_path): + """x86: torch+cu130 installed (ARM build) → uninstalls and reinstalls as cu128.""" + py = str(tmp_path / "python") + pip_cmd = [py, "-m", "pip"] + calls: list[list[str]] = [] + + def _run(cmd, **kwargs): + calls.append(list(cmd)) + stdout = "Name: torch\nVersion: 2.10.0+cu130\n" if "show" in cmd else "" + return _cp(0, stdout) + + with ( + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.get_pip_command", return_value=pip_cmd), + mock.patch("isaaclab.cli.commands.install.is_arm", return_value=False), + mock.patch("isaaclab.cli.commands.install.run_command", side_effect=_run), + ): + _ensure_cuda_torch() + + assert any("uninstall" in c for c in calls), "Expected an uninstall call" + install_cmds = [c for c in calls if "install" in c] + combined = " ".join(str(t) for c in install_cmds for t in c) + assert "cu128" in combined + + # ---- ARM scenarios ------------------------------------------------------- + + def test_arm_installs_cu130_when_torch_missing(self, tmp_path): + """ARM: no torch installed → installs torch+cu130 from pytorch.org/whl/cu130.""" + py = str(tmp_path / "python") + pip_cmd = [py, "-m", "pip"] + calls: list[list[str]] = [] + + def _run(cmd, **kwargs): + calls.append(list(cmd)) + return _cp(0, "") + + with ( + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.get_pip_command", return_value=pip_cmd), + mock.patch("isaaclab.cli.commands.install.is_arm", return_value=True), + mock.patch("isaaclab.cli.commands.install.run_command", side_effect=_run), + ): + _ensure_cuda_torch() + + install_cmds = [c for c in calls if "install" in c] + combined = " ".join(str(t) for c in install_cmds for t in c) + assert "cu130" in combined + + def test_arm_skips_install_when_correct_version_present(self, tmp_path): + """ARM: torch 2.10.0+cu130 already installed → pip install is not called.""" + py = str(tmp_path / "python") + pip_cmd = [py, "-m", "pip"] + pip_show_out = "Name: torch\nVersion: 2.10.0+cu130\n" + + with ( + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.get_pip_command", return_value=pip_cmd), + mock.patch("isaaclab.cli.commands.install.is_arm", return_value=True), + mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(0, pip_show_out)) as mock_run, + ): + _ensure_cuda_torch() + + assert mock_run.call_count == 1 + + def test_arm_reinstalls_when_wrong_cuda_tag(self, tmp_path): + """ARM: torch+cu128 installed (x86 build) → uninstalls and reinstalls as cu130.""" + py = str(tmp_path / "python") + pip_cmd = [py, "-m", "pip"] + calls: list[list[str]] = [] + + def _run(cmd, **kwargs): + calls.append(list(cmd)) + stdout = "Name: torch\nVersion: 2.10.0+cu128\n" if "show" in cmd else "" + return _cp(0, stdout) + + with ( + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.get_pip_command", return_value=pip_cmd), + mock.patch("isaaclab.cli.commands.install.is_arm", return_value=True), + mock.patch("isaaclab.cli.commands.install.run_command", side_effect=_run), + ): + _ensure_cuda_torch() + + assert any("uninstall" in c for c in calls) + install_cmds = [c for c in calls if "install" in c] + combined = " ".join(str(t) for c in install_cmds for t in c) + assert "cu130" in combined + + # ---- uv venv environment ------------------------------------------------ + + def test_uv_venv_uses_uv_pip_command(self, tmp_path): + """In a uv venv get_pip_command returns ['uv', 'pip'] and uninstall omits -y.""" + py = str(tmp_path / "python") + calls: list[list[str]] = [] + + def _run(cmd, **kwargs): + calls.append(list(cmd)) + return _cp(0, "") # no current torch → triggers install + + with ( + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.get_pip_command", return_value=["uv", "pip"]), + mock.patch("isaaclab.cli.commands.install.is_arm", return_value=False), + mock.patch("isaaclab.cli.commands.install.run_command", side_effect=_run), + ): + _ensure_cuda_torch() + + assert calls[0][0] == "uv", "Expected uv as the pip command prefix" + uninstall_calls = [c for c in calls if "uninstall" in c] + assert uninstall_calls, "Expected an uninstall call before reinstall" + assert "-y" not in uninstall_calls[0], "uv pip uninstall must not include -y" + + # ---- conda / pip venv / kit Python environments ------------------------- + + def test_conda_uses_python_m_pip_with_minus_y(self, tmp_path): + """In a conda env (no uv), get_pip_command returns python -m pip; uninstall uses -y.""" + py = str(tmp_path / "conda" / "bin" / "python") + pip_cmd = [py, "-m", "pip"] + calls: list[list[str]] = [] + + def _run(cmd, **kwargs): + calls.append(list(cmd)) + return _cp(0, "") + + with ( + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.get_pip_command", return_value=pip_cmd), + mock.patch("isaaclab.cli.commands.install.is_arm", return_value=False), + mock.patch("isaaclab.cli.commands.install.run_command", side_effect=_run), + ): + _ensure_cuda_torch() + + uninstall_calls = [c for c in calls if "uninstall" in c] + assert uninstall_calls + assert "-y" in uninstall_calls[0], "pip uninstall must include -y" + assert py in uninstall_calls[0], "Expected python exe in pip command" + + def test_kit_python_uses_python_sh_as_pip_prefix(self, tmp_path): + """With Isaac Sim's kit Python, python.sh is the executable prefix in the pip command.""" + python_sh = str(tmp_path / "_isaac_sim" / "python.sh") + pip_cmd = [python_sh, "-m", "pip"] + calls: list[list[str]] = [] + + def _run(cmd, **kwargs): + calls.append(list(cmd)) + return _cp(0, "") + + with ( + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=python_sh), + mock.patch("isaaclab.cli.commands.install.get_pip_command", return_value=pip_cmd), + mock.patch("isaaclab.cli.commands.install.is_arm", return_value=False), + mock.patch("isaaclab.cli.commands.install.run_command", side_effect=_run), + ): + _ensure_cuda_torch() + + assert calls[0][0] == python_sh + + +# --------------------------------------------------------------------------- +# _repoint_prebundle_packages — Isaac Sim install method × venv type +# --------------------------------------------------------------------------- + + +class TestRePointPrebundlePackages: + """Tests for :func:`_repoint_prebundle_packages`. + + Covers all combinations of: + - Isaac Sim installation method: local _isaac_sim symlink, pip-installed isaacsim, none + - Python environment / site-packages source: uv venv, pip venv, conda, kit Python + - nvidia namespace package special handling: cudnn present vs absent + """ + + # ---- shared fixtures / helpers ------------------------------------------ + + def _sim_with_prebundle(self, base: Path, packages: list[str]) -> tuple[Path, Path]: + """Create a minimal fake Isaac Sim tree containing a pip_prebundle dir. + + Returns ``(isaacsim_path, prebundle_dir)``. + """ + isaacsim_path = base / "isaac_sim" + isaacsim_path.mkdir(parents=True) + prebundle = isaacsim_path / "exts" / "some.ext" / "pip_prebundle" + prebundle.mkdir(parents=True) + for pkg in packages: + (prebundle / pkg).mkdir() + return isaacsim_path, prebundle + + @contextmanager + def _patch(self, isaacsim_path: Path | None, site_packages: Path, python_exe: str): + """Context manager that mocks all external calls in _repoint_prebundle_packages.""" + with ( + mock.patch("isaaclab.cli.commands.install.extract_isaacsim_path", return_value=isaacsim_path), + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=python_exe), + mock.patch("isaaclab.cli.commands.install.is_windows", return_value=False), + mock.patch( + "isaaclab.cli.commands.install.run_command", + return_value=_cp(0, str(site_packages)), + ), + ): + yield + + # ---- no Isaac Sim -------------------------------------------------------- + + def test_no_op_when_isaac_sim_absent(self, tmp_path): + """When Isaac Sim is not found, _repoint_prebundle_packages returns immediately without touching anything.""" + with ( + mock.patch("isaaclab.cli.commands.install.extract_isaacsim_path", return_value=None), + mock.patch("isaaclab.cli.commands.install.run_command") as mock_run, + ): + _repoint_prebundle_packages() + mock_run.assert_not_called() + + # ---- no pip_prebundle directories ---------------------------------------- + + def test_no_op_when_no_pip_prebundle_dirs(self, tmp_path): + """When Isaac Sim has no pip_prebundle directories, nothing is repointed.""" + isaacsim_path = tmp_path / "isaac_sim" + isaacsim_path.mkdir() + site_pkgs = _make_site_packages(tmp_path / "env", ["torch"]) + py = str(tmp_path / "python") + + with self._patch(isaacsim_path, site_pkgs, py): + _repoint_prebundle_packages() + + assert not (site_pkgs.parent / "pip_prebundle" / "torch").exists() + + # ---- local _isaac_sim symlink (local build) ------------------------------ + + def test_local_build_symlinks_torch_to_venv_site_packages(self, tmp_path): + """Local _isaac_sim symlink + uv/pip venv: prebundle torch → venv site-packages/torch.""" + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", ["torch"]) + site_pkgs = _make_site_packages(tmp_path / "env", ["torch"]) + py = str(tmp_path / "env" / "bin" / "python") + + with self._patch(isaacsim_path, site_pkgs, py): + _repoint_prebundle_packages() + + symlink = prebundle / "torch" + assert symlink.is_symlink(), "torch should be a symlink after repoint" + assert symlink.resolve() == (site_pkgs / "torch").resolve() + assert (prebundle / "torch.bak").is_dir(), "Original torch should be backed up" + + def test_local_build_skips_nvidia_when_cudnn_absent_kit_python(self, tmp_path): + """Local build + kit Python: site-packages/nvidia has only 'srl' (no cudnn) → nvidia NOT repointed. + + This is the real-world failure mode that caused the libcudnn.so.9 import error: + kit Python's site-packages/nvidia has only the 'srl' namespace sub-package, so + replacing the prebundle's nvidia/ (which contains the CUDA shared libraries) with + a symlink to that stripped-down directory would make libcudnn.so.9 unreachable. + """ + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", ["nvidia"]) + # Simulate kit Python's site-packages: nvidia/ exists but contains only 'srl' + site_pkgs = _make_site_packages(tmp_path / "kit" / "python" / "site-packages", ["nvidia"]) + (site_pkgs / "nvidia" / "srl").mkdir() + py = str(tmp_path / "isaac_sim" / "python.sh") + + with self._patch(isaacsim_path, site_pkgs, py): + _repoint_prebundle_packages() + + assert not (prebundle / "nvidia").is_symlink(), "nvidia must NOT be repointed when cudnn is missing" + assert (prebundle / "nvidia").is_dir(), "Original nvidia directory must be preserved" + + def test_local_build_repoints_nvidia_when_cudnn_present_venv(self, tmp_path): + """Local build + CUDA-capable venv: site-packages/nvidia has cudnn → nvidia IS repointed. + + This covers the conda or pip venv case where the user installed torch+cu128/cu130 + with its nvidia-cudnn-cu12 dependency, giving site-packages/nvidia/cudnn/. + """ + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", ["nvidia"]) + # Full CUDA venv: nvidia/ has cudnn and cublas + site_pkgs = _make_site_packages( + tmp_path / "env", + ["nvidia"], + subdirs={"nvidia": ["cudnn", "cublas"]}, + ) + py = str(tmp_path / "env" / "bin" / "python") + + with self._patch(isaacsim_path, site_pkgs, py): + _repoint_prebundle_packages() + + symlink = prebundle / "nvidia" + assert symlink.is_symlink(), "nvidia should be repointed when cudnn is present" + assert symlink.resolve() == (site_pkgs / "nvidia").resolve() + + def test_idempotent_when_symlink_already_correct(self, tmp_path): + """Calling _repoint_prebundle_packages twice does not break the symlinks.""" + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", []) + site_pkgs = _make_site_packages(tmp_path / "env", ["torch"]) + py = str(tmp_path / "env" / "bin" / "python") + + # Pre-create the correct symlink (as if a previous install already ran). + (prebundle / "torch").symlink_to(site_pkgs / "torch") + original_target = (prebundle / "torch").resolve() + + with self._patch(isaacsim_path, site_pkgs, py): + _repoint_prebundle_packages() + + assert (prebundle / "torch").resolve() == original_target, "Correct symlink must not be changed" + + def test_updates_stale_symlink_pointing_to_old_env(self, tmp_path): + """A symlink from a previous venv that no longer matches current site-packages is updated.""" + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", []) + site_pkgs = _make_site_packages(tmp_path / "env_new", ["torch"]) + old_env = _make_site_packages(tmp_path / "env_old", ["torch"]) + py = str(tmp_path / "env_new" / "bin" / "python") + + # Pre-create a stale symlink pointing at the old env. + (prebundle / "torch").symlink_to(old_env / "torch") + + with self._patch(isaacsim_path, site_pkgs, py): + _repoint_prebundle_packages() + + assert (prebundle / "torch").resolve() == (site_pkgs / "torch").resolve(), "Stale symlink must be updated" + + def test_removes_old_backup_before_renaming(self, tmp_path): + """A pre-existing .bak directory is removed before the current package is backed up.""" + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", ["torch"]) + site_pkgs = _make_site_packages(tmp_path / "env", ["torch"]) + py = str(tmp_path / "env" / "bin" / "python") + + # Simulate leftover backup from a previous partial install. + old_backup = prebundle / "torch.bak" + old_backup.mkdir() + (old_backup / "stale_file.py").touch() + + with self._patch(isaacsim_path, site_pkgs, py): + _repoint_prebundle_packages() + + assert (prebundle / "torch").is_symlink(), "torch must be repointed" + # The old backup was replaced by the fresh backup. + assert (prebundle / "torch.bak").is_dir() + + # ---- pip-installed isaacsim (path found via import probe) ---------------- + + def test_pip_isaacsim_symlinks_torch(self, tmp_path): + """pip-installed isaacsim: extract_isaacsim_path() returns its directory and torch is repointed.""" + # With pip-installed isaacsim the path may be inside site-packages rather than a symlink + # at the repo root, but _repoint_prebundle_packages treats it identically. + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "pip_isaacsim", ["torch"]) + site_pkgs = _make_site_packages(tmp_path / "env", ["torch"]) + py = str(tmp_path / "env" / "bin" / "python") + + with self._patch(isaacsim_path, site_pkgs, py): + _repoint_prebundle_packages() + + assert (prebundle / "torch").is_symlink() + assert (prebundle / "torch").resolve() == (site_pkgs / "torch").resolve() + + def test_pip_isaacsim_skips_nvidia_without_cudnn(self, tmp_path): + """pip-installed isaacsim + no cudnn in site-packages → nvidia prebundle preserved.""" + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "pip_isaacsim", ["nvidia"]) + # site-packages has nvidia/ but without a cudnn sub-package + site_pkgs = _make_site_packages(tmp_path / "env", ["nvidia"]) + py = str(tmp_path / "env" / "bin" / "python") + + with self._patch(isaacsim_path, site_pkgs, py): + _repoint_prebundle_packages() + + assert not (prebundle / "nvidia").is_symlink(), "nvidia must not be repointed without cudnn" + + # ---- different venv types ------------------------------------------------ + + def test_uv_venv_repoints_torch_using_venv_site_packages(self, tmp_path): + """uv venv: site-packages inside VIRTUAL_ENV is used as the symlink target.""" + venv_site = tmp_path / "env_uv" / "lib" / "python3.12" / "site-packages" + venv_site.mkdir(parents=True) + (venv_site / "torch").mkdir() + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", ["torch"]) + py = str(tmp_path / "env_uv" / "bin" / "python") + + with ( + mock.patch("isaaclab.cli.commands.install.extract_isaacsim_path", return_value=isaacsim_path), + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.is_windows", return_value=False), + mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(0, str(venv_site))), + ): + _repoint_prebundle_packages() + + assert (prebundle / "torch").is_symlink() + assert (prebundle / "torch").resolve() == (venv_site / "torch").resolve() + + def test_conda_repoints_torch_using_conda_site_packages(self, tmp_path): + """conda env: site-packages inside CONDA_PREFIX is used as the symlink target.""" + conda_site = tmp_path / "conda" / "lib" / "python3.12" / "site-packages" + conda_site.mkdir(parents=True) + (conda_site / "torch").mkdir() + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", ["torch"]) + py = str(tmp_path / "conda" / "bin" / "python") + + with ( + mock.patch("isaaclab.cli.commands.install.extract_isaacsim_path", return_value=isaacsim_path), + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.is_windows", return_value=False), + mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(0, str(conda_site))), + ): + _repoint_prebundle_packages() + + assert (prebundle / "torch").is_symlink() + assert (prebundle / "torch").resolve() == (conda_site / "torch").resolve() + + def test_conda_repoints_nvidia_when_full_cuda_torch_installed(self, tmp_path): + """conda env with nvidia-cudnn-cu12 installed: nvidia/ is repointed because cudnn subdir exists.""" + conda_site = tmp_path / "conda" / "lib" / "python3.12" / "site-packages" + conda_site.mkdir(parents=True) + (conda_site / "nvidia").mkdir() + (conda_site / "nvidia" / "cudnn").mkdir() + (conda_site / "nvidia" / "cublas").mkdir() + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", ["nvidia"]) + py = str(tmp_path / "conda" / "bin" / "python") + + with ( + mock.patch("isaaclab.cli.commands.install.extract_isaacsim_path", return_value=isaacsim_path), + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.is_windows", return_value=False), + mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(0, str(conda_site))), + ): + _repoint_prebundle_packages() + + assert (prebundle / "nvidia").is_symlink() + + def test_conda_skips_nvidia_when_no_cudnn(self, tmp_path): + """conda env without CUDA torch: site-packages/nvidia lacks cudnn → nvidia not repointed.""" + conda_site = tmp_path / "conda" / "lib" / "python3.12" / "site-packages" + conda_site.mkdir(parents=True) + (conda_site / "nvidia").mkdir() # exists but no cudnn inside + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", ["nvidia"]) + py = str(tmp_path / "conda" / "bin" / "python") + + with ( + mock.patch("isaaclab.cli.commands.install.extract_isaacsim_path", return_value=isaacsim_path), + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.is_windows", return_value=False), + mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(0, str(conda_site))), + ): + _repoint_prebundle_packages() + + assert not (prebundle / "nvidia").is_symlink() + + # ---- multiple prebundle directories ------------------------------------- + + def test_repoints_across_multiple_prebundle_dirs(self, tmp_path): + """When Isaac Sim has multiple pip_prebundle directories, each is processed.""" + isaacsim_path = tmp_path / "isaac_sim" + isaacsim_path.mkdir() + + # Two separate extension pip_prebundle dirs, each with torch. + pb1 = isaacsim_path / "exts" / "ext_a" / "pip_prebundle" + pb2 = isaacsim_path / "exts" / "ext_b" / "pip_prebundle" + for pb in (pb1, pb2): + pb.mkdir(parents=True) + (pb / "torch").mkdir() + + site_pkgs = _make_site_packages(tmp_path / "env", ["torch"]) + py = str(tmp_path / "env" / "bin" / "python") + + with self._patch(isaacsim_path, site_pkgs, py): + _repoint_prebundle_packages() + + for pb in (pb1, pb2): + assert (pb / "torch").is_symlink(), f"torch in {pb} should be repointed" + + # ---- Windows: copy instead of symlink ----------------------------------- + + def test_copies_package_on_windows_instead_of_symlinking(self, tmp_path): + """On Windows, packages are copied rather than symlinked (Windows doesn't support posix symlinks).""" + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", ["torch"]) + site_pkgs = _make_site_packages(tmp_path / "env", ["torch"]) + (site_pkgs / "torch" / "version.py").write_text("__version__ = '2.10.0'") + py = str(tmp_path / "env" / "bin" / "python") + + with ( + mock.patch("isaaclab.cli.commands.install.extract_isaacsim_path", return_value=isaacsim_path), + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.is_windows", return_value=True), + mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(0, str(site_pkgs))), + ): + _repoint_prebundle_packages() + + torch_in_prebundle = prebundle / "torch" + assert torch_in_prebundle.is_dir(), "torch should be a directory (copy) on Windows" + assert not torch_in_prebundle.is_symlink(), "torch must not be a symlink on Windows" + assert (torch_in_prebundle / "version.py").exists(), "Copied file should be present" + + # ---- error handling ----------------------------------------------------- + + def test_oserror_on_one_package_does_not_abort_others(self, tmp_path): + """An OSError while repointing one package is logged and processing continues for others.""" + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", ["torch", "torchvision"]) + site_pkgs = _make_site_packages(tmp_path / "env", ["torch", "torchvision"]) + py = str(tmp_path / "env" / "bin" / "python") + + original_symlink_to = Path.symlink_to + call_count: list[int] = [0] + + def _selective_symlink(self_path: Path, target: Path, **kwargs) -> None: + call_count[0] += 1 + # Fail on the first symlink_to call (torch) but succeed for others. + if call_count[0] == 1: + raise OSError("Permission denied") + return original_symlink_to(self_path, target, **kwargs) + + with ( + mock.patch("isaaclab.cli.commands.install.extract_isaacsim_path", return_value=isaacsim_path), + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.is_windows", return_value=False), + mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(0, str(site_pkgs))), + mock.patch.object(Path, "symlink_to", _selective_symlink), + ): + _repoint_prebundle_packages() # must not raise + + # torchvision (second package) must still be repointed despite torch failure. + assert (prebundle / "torchvision").is_symlink(), "torchvision must succeed after torch OSError" + + def test_skips_gracefully_when_site_packages_probe_fails(self, tmp_path): + """When the site-packages probe subprocess fails, _repoint_prebundle_packages is a no-op.""" + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", ["torch"]) + py = str(tmp_path / "python") + + with ( + mock.patch("isaaclab.cli.commands.install.extract_isaacsim_path", return_value=isaacsim_path), + mock.patch("isaaclab.cli.commands.install.extract_python_exe", return_value=py), + mock.patch("isaaclab.cli.commands.install.is_windows", return_value=False), + # Probe subprocess exits non-zero + mock.patch("isaaclab.cli.commands.install.run_command", return_value=_cp(returncode=1, stdout="")), + ): + _repoint_prebundle_packages() + + assert not (prebundle / "torch").is_symlink(), "No symlink should be created when probe fails" + + # ---- all packages in the repoint list are covered ----------------------- + + @pytest.mark.parametrize("pkg_name", [p for p in _PREBUNDLE_REPOINT_PACKAGES if p != "nvidia"]) + def test_all_non_nvidia_packages_are_repointed(self, tmp_path, pkg_name): + """Every non-nvidia entry in _PREBUNDLE_REPOINT_PACKAGES is repointed when it exists.""" + isaacsim_path, prebundle = self._sim_with_prebundle(tmp_path / "sim", [pkg_name]) + site_pkgs = _make_site_packages(tmp_path / "env", [pkg_name]) + py = str(tmp_path / "env" / "bin" / "python") + + with self._patch(isaacsim_path, site_pkgs, py): + _repoint_prebundle_packages() + + assert (prebundle / pkg_name).is_symlink(), f"{pkg_name} should be repointed" + assert (prebundle / pkg_name).resolve() == (site_pkgs / pkg_name).resolve() diff --git a/source/isaaclab/test/cli/test_install_prebundle.py b/source/isaaclab/test/cli/test_install_prebundle.py new file mode 100644 index 000000000000..ef08d0a1a726 --- /dev/null +++ b/source/isaaclab/test/cli/test_install_prebundle.py @@ -0,0 +1,81 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Tests for prebundle probe and _split_install_items. + +Supplements test_install_commands.py with tests that verify the probe +script text and the comma-separated install item parser. +""" + +from unittest import mock + +from isaaclab.cli.commands.install import ( + _split_install_items, + _torch_first_on_sys_path_is_prebundle, +) + +# --------------------------------------------------------------------------- +# _split_install_items +# --------------------------------------------------------------------------- + + +class TestSplitInstallItems: + """Tests for :func:`_split_install_items`.""" + + def test_single_item(self): + assert _split_install_items("assets") == ["assets"] + + def test_comma_separated(self): + assert _split_install_items("assets,tasks,rl") == ["assets", "tasks", "rl"] + + def test_with_spaces(self): + assert _split_install_items(" assets , tasks , rl ") == ["assets", "tasks", "rl"] + + def test_brackets_preserved(self): + """Commas inside brackets should not split.""" + assert _split_install_items("visualizers[rerun,newton],tasks") == [ + "visualizers[rerun,newton]", + "tasks", + ] + + def test_nested_brackets(self): + assert _split_install_items("a[b[c,d],e],f") == ["a[b[c,d],e]", "f"] + + def test_empty_string(self): + assert _split_install_items("") == [] + + def test_trailing_comma(self): + assert _split_install_items("assets,tasks,") == ["assets", "tasks"] + + def test_single_with_extra(self): + assert _split_install_items("visualizers[all]") == ["visualizers[all]"] + + +# --------------------------------------------------------------------------- +# _torch_first_on_sys_path_is_prebundle — probe script verification +# --------------------------------------------------------------------------- + + +class TestTorchProbeScriptContent: + """Verify that the probe script checks for 'pip_prebundle' not 'extsDeprecated'.""" + + def test_probe_script_checks_pip_prebundle(self): + """The inline Python probe must use 'pip_prebundle' as its path indicator.""" + import subprocess + + captured_cmd = None + + def fake_run(cmd, *, env=None, check=False, capture_output=False, text=False): + nonlocal captured_cmd + captured_cmd = cmd + return subprocess.CompletedProcess(args=cmd, returncode=0) + + with mock.patch("isaaclab.cli.commands.install.run_command", side_effect=fake_run): + _torch_first_on_sys_path_is_prebundle("/fake/python", env={}) + + assert captured_cmd is not None + probe_script = captured_cmd[2] # [python_exe, "-c", probe] + assert "pip_prebundle" in probe_script, "Probe must check for 'pip_prebundle'" + assert "extsDeprecated" not in probe_script, "Probe must NOT check only for 'extsDeprecated'" From 9236aae206dc64f272c36d0b7b70b9fa57b9ef41 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Thu, 16 Apr 2026 07:39:58 +0000 Subject: [PATCH 02/17] Exclude isaacsim.pip.newton and omni.warp.core prebundles - Add isaacsim.pip.newton and omni.warp.core to app.extensions.excluded in all 6 kit files to prevent Kit from loading these extensions - Add isaacsim.pip.newton to _CONFLICTING_EXTS in __init__.py so its pip_prebundle paths are deprioritized on sys.path - isaacsim.pip.newton prebundles newton 1.0.0 which shadows the pip-installed newton 1.1.0.dev0 needed by Isaac Lab - omni.warp.core loads a bundled warp that conflicts with pip-installed warp-lang used by Isaac Lab --- apps/isaaclab.python.headless.kit | 2 +- apps/isaaclab.python.headless.rendering.kit | 2 +- apps/isaaclab.python.kit | 2 +- apps/isaaclab.python.rendering.kit | 2 +- apps/isaaclab.python.xr.openxr.headless.kit | 2 +- apps/isaaclab.python.xr.openxr.kit | 2 +- source/isaaclab/isaaclab/__init__.py | 1 + 7 files changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/isaaclab.python.headless.kit b/apps/isaaclab.python.headless.kit index 01d50be2a560..12b9e14946d0 100644 --- a/apps/isaaclab.python.headless.kit +++ b/apps/isaaclab.python.headless.kit @@ -106,7 +106,7 @@ isaac.startup.ros_bridge_extension = "" metricsAssembler.changeListenerEnabled = false # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] # Extensions ############################### diff --git a/apps/isaaclab.python.headless.rendering.kit b/apps/isaaclab.python.headless.rendering.kit index 12229aca0eca..90fee65f77ec 100644 --- a/apps/isaaclab.python.headless.rendering.kit +++ b/apps/isaaclab.python.headless.rendering.kit @@ -107,7 +107,7 @@ exts."omni.replicator.core".Orchestrator.enabled = false metricsAssembler.changeListenerEnabled = false # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] [settings.app.python] # These disable the kit app from also printing out python output, which gets confusing diff --git a/apps/isaaclab.python.kit b/apps/isaaclab.python.kit index 011902ace986..a5330cd1944e 100644 --- a/apps/isaaclab.python.kit +++ b/apps/isaaclab.python.kit @@ -256,7 +256,7 @@ exts."omni.replicator.core".Orchestrator.enabled = false omni.rtx.nre.compositing.rendererHints = 3 # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] [settings.app.livestream] outDirectory = "${data}" diff --git a/apps/isaaclab.python.rendering.kit b/apps/isaaclab.python.rendering.kit index ad58b159186a..d3414b211907 100644 --- a/apps/isaaclab.python.rendering.kit +++ b/apps/isaaclab.python.rendering.kit @@ -93,7 +93,7 @@ exts."omni.replicator.core".Orchestrator.enabled = false metricsAssembler.changeListenerEnabled = false # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] [settings.physics] updateToUsd = false diff --git a/apps/isaaclab.python.xr.openxr.headless.kit b/apps/isaaclab.python.xr.openxr.headless.kit index 6bf1bb7c43d6..75f3ae637124 100644 --- a/apps/isaaclab.python.xr.openxr.headless.kit +++ b/apps/isaaclab.python.xr.openxr.headless.kit @@ -42,7 +42,7 @@ cameras_enabled = true [settings] # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] [settings.app.python] # These disable the kit app from also printing out python output, which gets confusing diff --git a/apps/isaaclab.python.xr.openxr.kit b/apps/isaaclab.python.xr.openxr.kit index 3b730f0eab64..62782ba06619 100644 --- a/apps/isaaclab.python.xr.openxr.kit +++ b/apps/isaaclab.python.xr.openxr.kit @@ -66,7 +66,7 @@ xr.openxr.components."isaacsim.xr.openxr.hand_tracking".enabled = true xr.openxr.components."isaacsim.kit.xr.teleop.bridge".enabled = true # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] [settings.app.python] # These disable the kit app from also printing out python output, which gets confusing diff --git a/source/isaaclab/isaaclab/__init__.py b/source/isaaclab/isaaclab/__init__.py index 0d745d91610b..baa4c455412e 100644 --- a/source/isaaclab/isaaclab/__init__.py +++ b/source/isaaclab/isaaclab/__init__.py @@ -28,6 +28,7 @@ def _deprioritize_prebundle_paths(): # Extensions whose prebundled packages conflict with Isaac Lab deps. _CONFLICTING_EXTS = ( "omni.isaac.ml_archive", + "isaacsim.pip.newton", ) def _is_conflicting(path: str) -> bool: From 09dcb46a8270ec9639888a9eeb2f3665c882d9af Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Thu, 16 Apr 2026 16:36:45 +0000 Subject: [PATCH 03/17] Use latest-develop Docker image for CI Remove the pinned SHA digest from ISAACSIM_BASE_VERSION so CI tests against the most recent Isaac Sim nightly image, which includes the pip_prebundle path changes that this PR fixes. --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7556706cd2f0..0fcef1818780 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -101,7 +101,7 @@ env: ISAACSIM_BASE_IMAGE: 'nvcr.io/nvidian/isaac-sim' # ${{ vars.ISAACSIM_BASE_IMAGE || 'nvcr.io/nvidia/isaac-sim' }} # Pinned to the Apr 8 nightly digest that last passed CI, while latest-develop is broken. # TODO(AntoineRichard): Revert to 'latest-develop' once the nightly is fixed. - ISAACSIM_BASE_VERSION: 'latest-develop@sha256:f085fb5b6899511bb19abdf18121bfc469901334aefb029d0def56d4fef79c58' # ${{ vars.ISAACSIM_BASE_VERSION || '6.0.0' }} + ISAACSIM_BASE_VERSION: 'latest-develop' # ${{ vars.ISAACSIM_BASE_VERSION || '6.0.0' }} DOCKER_IMAGE_TAG: isaac-lab-dev:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}-${{ github.sha }} # To run quarantined tests, create a GitHub repo variable named # RUN_QUARANTINED_TESTS and set it to 'true'. The test-quarantined From b7e790ca0cfa183fcc08bf4e30771a5f2994e76c Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Thu, 16 Apr 2026 20:36:14 +0000 Subject: [PATCH 04/17] Deprioritize ALL pip_prebundle paths, not just known extensions The previous approach filtered only omni.isaac.ml_archive and isaacsim.pip.newton prebundle paths. On newer Isaac Sim nightly images, additional or renamed extensions may also inject conflicting packages via pip_prebundle. Match on 'pip_prebundle' in the path alone to catch all current and future prebundle directories. --- source/isaaclab/isaaclab/__init__.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/source/isaaclab/isaaclab/__init__.py b/source/isaaclab/isaaclab/__init__.py index baa4c455412e..de1f58bb32df 100644 --- a/source/isaaclab/isaaclab/__init__.py +++ b/source/isaaclab/isaaclab/__init__.py @@ -10,12 +10,12 @@ def _deprioritize_prebundle_paths(): - """Move Isaac Sim ``pip_prebundle`` directories to the end of ``sys.path``. + """Move **all** Isaac Sim ``pip_prebundle`` directories to the end of ``sys.path``. Isaac Sim's ``setup_python_env.sh`` injects ``pip_prebundle`` directories - (e.g. ``omni.isaac.ml_archive/pip_prebundle``) onto ``PYTHONPATH``. These - contain older copies of packages like torch, warp, and nvidia-cudnn that - shadow the versions installed by Isaac Lab, causing CUDA runtime errors. + onto ``PYTHONPATH``. These contain older copies of packages like torch, + warp, and nvidia-cudnn that shadow the versions installed by Isaac Lab, + causing CUDA runtime errors. Rather than removing these paths entirely (which would break packages like ``sympy`` that only exist in the prebundle), this function moves them to @@ -25,21 +25,16 @@ def _deprioritize_prebundle_paths(): The ``PYTHONPATH`` environment variable is also rewritten so that child processes inherit the corrected ordering. """ - # Extensions whose prebundled packages conflict with Isaac Lab deps. - _CONFLICTING_EXTS = ( - "omni.isaac.ml_archive", - "isaacsim.pip.newton", - ) - def _is_conflicting(path: str) -> bool: + def _is_prebundle(path: str) -> bool: norm = path.replace("\\", "/").lower() - return "pip_prebundle" in norm and any(ext.lower() in norm for ext in _CONFLICTING_EXTS) + return "pip_prebundle" in norm - # Partition: keep non-conflicting in place, collect conflicting. + # Partition: keep non-prebundle in place, collect prebundle. clean = [] demoted = [] for p in sys.path: - if _is_conflicting(p): + if _is_prebundle(p): demoted.append(p) else: clean.append(p) @@ -56,7 +51,7 @@ def _is_conflicting(path: str) -> bool: env_clean = [] env_demoted = [] for p in parts: - if _is_conflicting(p): + if _is_prebundle(p): env_demoted.append(p) else: env_clean.append(p) From 584bc0e760adbe6b88dfdb0b15ddbb49a09a8aea Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Fri, 17 Apr 2026 01:01:29 +0000 Subject: [PATCH 05/17] Sync warp CUDA stream after camera render before torch access The renderer's render() method launches warp kernels that populate the camera output tensors on the GPU. Without an explicit synchronization barrier, downstream torch operations (e.g. torch.mean on camera data) can execute on a different CUDA stream before the warp writes complete, resulting in 'CUDA error: an illegal memory access was encountered'. This was exposed by newer Isaac Sim nightly images that changed warp's default CUDA stream behavior. Add wp.synchronize() in Camera's _update_buffers_impl after the render pass to guarantee kernel completion before any torch access. --- source/isaaclab/isaaclab/sensors/camera/camera.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index eb588489f729..4f0fde7d8abe 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -462,6 +462,13 @@ def _update_buffers_impl(self, env_mask: wp.array): self._renderer.update_transforms() self._renderer.render(self._render_data) + # Synchronize warp's CUDA stream before handing GPU buffers to torch. + # render() launches warp kernels that write into the output tensors; + # without an explicit sync, torch may read the buffers on a different + # CUDA stream before the warp kernels finish, causing illegal-memory- + # access errors (observed with newer Isaac Sim nightly images). + wp.synchronize() + self._renderer.read_output(self._render_data, self._data) """ From f484813c48f4eba37cc7f4922096f4bab26d6968 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Fri, 17 Apr 2026 03:38:14 +0000 Subject: [PATCH 06/17] Sync warp stream inside RTX renderer before torch depth-clipping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The render() method launches a warp reshape kernel via wp.launch() then immediately performs torch operations (depth clipping, inf assignment) on the same output buffer. These run on different CUDA streams — warp's stream vs torch's default stream — with no synchronization, causing illegal memory access errors that corrupt the CUDA context. Add wp.synchronize() after wp.launch() and before any torch post- processing within the per-data-type loop. This complements the existing wp.synchronize() in Camera._update_buffers_impl() which guards the render→read_output boundary. --- .../isaaclab_physx/renderers/isaac_rtx_renderer.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 5b07e3417ce0..9a80433b11d5 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -307,6 +307,13 @@ def tiling_grid_shape(): device=sensor.device, ) + # Synchronize warp's CUDA stream before any torch operations read + # the output buffer. The reshape kernel runs on warp's stream; + # torch depth-clipping below runs on torch's stream. Without this + # barrier the two streams race, corrupting the CUDA context on + # newer Isaac Sim nightly images (>= 04/14). + wp.synchronize() + # alias rgb as first 3 channels of rgba if data_type == "rgba" and "rgb" in cfg.data_types: output_data["rgb"] = output_data["rgba"][..., :3] From 8832a72ac94e4d1a8dcb9f713a19a4e85d1427f4 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Fri, 17 Apr 2026 15:15:23 +0000 Subject: [PATCH 07/17] Broaden path sanitization to cover conflicting extension dirs Extend _deprioritize_prebundle_paths() to also demote known conflicting extension directories (omni.warp.core, omni.isaac.ml_archive, etc.) in addition to pip_prebundle paths. This prevents kit extensions that bundle their own Python packages from shadowing pip-installed versions. Also re-run the sanitization after SimulationApp + extensions load, since Kit may insert new paths onto sys.path during startup that were not present when isaaclab/__init__.py first ran. --- source/isaaclab/isaaclab/__init__.py | 34 ++++++++++++++++---- source/isaaclab/isaaclab/app/app_launcher.py | 9 ++++++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/source/isaaclab/isaaclab/__init__.py b/source/isaaclab/isaaclab/__init__.py index de1f58bb32df..d547b628372e 100644 --- a/source/isaaclab/isaaclab/__init__.py +++ b/source/isaaclab/isaaclab/__init__.py @@ -10,13 +10,18 @@ def _deprioritize_prebundle_paths(): - """Move **all** Isaac Sim ``pip_prebundle`` directories to the end of ``sys.path``. + """Move Isaac Sim ``pip_prebundle`` and known conflicting extension directories to the end of ``sys.path``. Isaac Sim's ``setup_python_env.sh`` injects ``pip_prebundle`` directories onto ``PYTHONPATH``. These contain older copies of packages like torch, warp, and nvidia-cudnn that shadow the versions installed by Isaac Lab, causing CUDA runtime errors. + Additionally, certain Isaac Sim kit extensions (such as ``omni.warp.core``) + bundle their own copies of Python packages that conflict with pip-installed + versions. When loaded by the extension system these paths can appear on + ``sys.path`` before ``site-packages``, leading to version mismatches. + Rather than removing these paths entirely (which would break packages like ``sympy`` that only exist in the prebundle), this function moves them to the **end** of ``sys.path`` so that pip-installed packages in @@ -26,15 +31,30 @@ def _deprioritize_prebundle_paths(): processes inherit the corrected ordering. """ - def _is_prebundle(path: str) -> bool: + # Extension directory fragments that are known to ship Python packages + # which conflict with Isaac Lab's pip-installed versions. + _CONFLICTING_EXT_FRAGMENTS = ( + "omni.warp.core", + "omni.isaac.ml_archive", + "omni.isaac.core_archive", + "omni.kit.pip_archive", + "isaacsim.pip.newton", + ) + + def _should_demote(path: str) -> bool: norm = path.replace("\\", "/").lower() - return "pip_prebundle" in norm - - # Partition: keep non-prebundle in place, collect prebundle. + if "pip_prebundle" in norm: + return True + for frag in _CONFLICTING_EXT_FRAGMENTS: + if frag.lower() in norm: + return True + return False + + # Partition: keep non-conflicting in place, collect conflicting. clean = [] demoted = [] for p in sys.path: - if _is_prebundle(p): + if _should_demote(p): demoted.append(p) else: clean.append(p) @@ -51,7 +71,7 @@ def _is_prebundle(path: str) -> bool: env_clean = [] env_demoted = [] for p in parts: - if _is_prebundle(p): + if _should_demote(p): env_demoted.append(p) else: env_clean.append(p) diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index dcc8d1ca53e0..266e27c57043 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -203,6 +203,15 @@ def __init__(self, launcher_args: argparse.Namespace | dict | None = None, **kwa self._create_app() # Load IsaacSim extensions self._load_extensions() + + # Re-run path sanitization. Kit and its extensions may have inserted + # additional ``pip_prebundle`` or conflicting extension directories onto + # ``sys.path`` during startup. A second pass ensures pip-installed + # packages still take priority over bundled copies. + from isaaclab import _deprioritize_prebundle_paths + + _deprioritize_prebundle_paths() + # Hide the stop button in the toolbar self._hide_stop_button() # Set settings from the given rendering mode From 68eada562f619701bee24cad9b9a85bd9abc3d8b Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Fri, 17 Apr 2026 18:44:54 -0700 Subject: [PATCH 08/17] Demote kit/python/lib site-packages to fix warp 1.13.x import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nightly Isaac Sim Docker (>= 04/14) ships warp 1.13.x inside kit/python/lib/python3.12/site-packages. setup_python_env.sh places that directory on PYTHONPATH ahead of pip's site-packages, so warp 1.13.x was imported despite the _pin_warp_import pre-import added in 4.6.8 — that pre-import was locking the kit warp, not the pip warp. Warp 1.13.x deprecates warp.types.array with changed semantics. omni.replicator.core accesses this symbol during rendering, triggering CUDA error 700 (illegal memory access). That error poisons the CUDA context: all subsequent warp kernel lookups return NULL, producing the "Failed to find forward kernel 'update_outdated_envs_kernel'" crash. Fix: extend _should_demote() to catch kit/python/lib/*/site-packages paths so that pip-installed warp-lang always wins. Also add explicit strip_hash=False to isaaclab.sensors.kernels as a defensive safety net, and broaden the AppLauncher post-startup diagnostic to cover kit paths. --- source/isaaclab/config/extension.toml | 2 +- source/isaaclab/docs/CHANGELOG.rst | 31 ++++++++++ source/isaaclab/isaaclab/__init__.py | 65 ++++++++++++++++++++ source/isaaclab/isaaclab/app/app_launcher.py | 30 +++++++++ source/isaaclab/isaaclab/sensors/kernels.py | 8 +++ 5 files changed, 135 insertions(+), 1 deletion(-) diff --git a/source/isaaclab/config/extension.toml b/source/isaaclab/config/extension.toml index 7092d52f3ff7..86024cf07e6a 100644 --- a/source/isaaclab/config/extension.toml +++ b/source/isaaclab/config/extension.toml @@ -1,7 +1,7 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "4.6.10" +version = "4.6.11" # Description title = "Isaac Lab framework for Robot Learning" diff --git a/source/isaaclab/docs/CHANGELOG.rst b/source/isaaclab/docs/CHANGELOG.rst index 0251fcb03ca2..03ab7d860ce4 100644 --- a/source/isaaclab/docs/CHANGELOG.rst +++ b/source/isaaclab/docs/CHANGELOG.rst @@ -1,6 +1,37 @@ Changelog --------- +4.6.11 (2026-04-22) +~~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Fixed ``RuntimeError: Failed to find forward kernel 'update_outdated_envs_kernel'`` + caused by the nightly Isaac Sim Docker (>= 04/14) shipping warp 1.13.x inside + ``kit/python/lib/python3.12/site-packages``. Because ``setup_python_env.sh`` + places that directory on ``PYTHONPATH`` ahead of pip's ``site-packages``, warp 1.13.x + was imported instead of pip-installed warp 1.12.0, despite the ``_pin_warp_import`` + pre-import added previously. Warp 1.13.x deprecates ``warp.types.array`` and changes its + semantics; ``omni.replicator.core`` accesses this symbol during rendering, triggering + a CUDA illegal-memory-access (error 700) that poisons the CUDA context and causes + all subsequent warp kernel lookups (including + ``update_outdated_envs_kernel__cuda_kernel_forward``) to return NULL. + Extended ``_deprioritize_prebundle_paths()`` to also demote + ``kit/python/lib/*/site-packages`` paths, ensuring pip-installed ``warp-lang`` + always takes priority. Also added an explicit ``strip_hash=False`` option to the + ``isaaclab.sensors.kernels`` warp module as a defensive safety net, and broadened + the post-startup diagnostic warning in :class:`~isaaclab.app.AppLauncher` to cover + Kit Python paths. +* Fixed a Warp kernel lookup failure (``Failed to find forward kernel … for device 'cuda:0'``) + caused by Kit's extension scanner adding ``omni.warp.core``'s directory to ``sys.path`` + during ``SimulationApp`` startup, before the second ``_deprioritize_prebundle_paths()`` + call in :class:`~isaaclab.app.AppLauncher` could run. The fix pre-imports ``warp`` in + :mod:`isaaclab` ``__init__`` immediately after the path-demotion step, locking + ``sys.modules['warp']`` to the pip-installed version before Kit starts. A post-startup + warning is also emitted when the wrong ``warp`` is detected. + + 4.6.10 (2026-04-22) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab/isaaclab/__init__.py b/source/isaaclab/isaaclab/__init__.py index d547b628372e..358c358cf6ca 100644 --- a/source/isaaclab/isaaclab/__init__.py +++ b/source/isaaclab/isaaclab/__init__.py @@ -45,6 +45,15 @@ def _should_demote(path: str) -> bool: norm = path.replace("\\", "/").lower() if "pip_prebundle" in norm: return True + # Kit ships its own Python interpreter and installs packages into + # kit/python/lib/python3.X/site-packages. When a newer or differently- + # configured version of a package (e.g. warp) is installed there, it + # takes precedence over pip-installed packages and can cause runtime + # failures. Demote these site-packages to the end of sys.path while + # leaving the Kit stdlib path (kit/python/lib/python3.X without + # site-packages) in place so Kit's standard-library extras still work. + if "kit/python/lib" in norm and "site-packages" in norm: + return True for frag in _CONFLICTING_EXT_FRAGMENTS: if frag.lower() in norm: return True @@ -80,6 +89,62 @@ def _should_demote(path: str) -> bool: _deprioritize_prebundle_paths() + +def _pin_warp_import(): + """Import ``warp`` now to lock ``sys.modules['warp']`` to the pip-installed version. + + Kit's extension system may add ``omni.warp.core``'s directory or Kit's own + ``kit/python/lib/python3.X/site-packages`` to ``sys.path`` during + ``SimulationApp`` startup — even when that extension is excluded from the kit + file — because Kit scans extension directories as part of its registry process. + Any extension that imports ``warp`` during that window (e.g. + ``omni.replicator.core``) would set ``sys.modules['warp']`` to the bundled copy + before our second ``_deprioritize_prebundle_paths()`` call in ``AppLauncher`` + has a chance to run. + + By importing ``warp`` here — after ``_deprioritize_prebundle_paths()`` has + already demoted the pip_prebundle, ``omni.warp.core``, and ``kit/python/lib`` + site-packages paths — we ensure the pip-installed ``warp-lang`` is the one + cached in ``sys.modules``. Subsequent ``import warp`` calls from Kit extensions + all return that cached module, so there is only ever one Warp runtime in the + process. + + Failure to import (e.g. warp not yet installed during initial setup) is + silently ignored; the import will succeed once the user has run + ``./isaaclab.sh --install``. + """ + try: + import warp as _warp # noqa: F401 + except ImportError: + return + + # Warn if warp was resolved from a non-pip location that could introduce + # version mismatches (Kit-bundled or extscache copies). + import warnings + + _warp_file = getattr(_warp, "__file__", "") or "" + _warp_norm = _warp_file.replace("\\", "/").lower() + _suspicious = ( + "omni.warp" in _warp_norm + or ("extscache" in _warp_norm and "warp" in _warp_norm) + or ("kit/python/lib" in _warp_norm and "site-packages" in _warp_norm) + ) + if _suspicious: + warnings.warn( + f"[IsaacLab] warp was imported from a non-pip-installed location: " + f"{_warp_file!r}. A Kit-bundled or extscache copy of warp may be " + f"shadowing the pip-installed warp-lang, which can cause Warp kernel " + f"lookup failures (e.g. 'Failed to find forward kernel ... from module " + f"isaaclab.sensors.kernels'). Check that pip-installed packages " + f"(site-packages) appear before kit/python/lib/*/site-packages and " + f"extscache directories in sys.path.", + RuntimeWarning, + stacklevel=2, + ) + + +_pin_warp_import() + # Conveniences to other module directories via relative paths. ISAACLAB_EXT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) """Path to the extension source directory.""" diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 266e27c57043..7079b387221b 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -212,6 +212,36 @@ def __init__(self, launcher_args: argparse.Namespace | dict | None = None, **kwa _deprioritize_prebundle_paths() + # Verify that warp (if loaded) came from the pip-installed copy and not from + # a bundled extension such as omni.warp.core or Kit's own Python packages. + # Kit's extension system can add extension directories and kit/python/lib paths + # to sys.path during startup; the pre-import in isaaclab/__init__.py normally + # prevents this, but log a warning so CI failures are easier to diagnose. + import sys + + warp_mod = sys.modules.get("warp") + if warp_mod is not None: + warp_file = getattr(warp_mod, "__file__", "") or "" + warp_norm = warp_file.replace("\\", "/").lower() + _suspicious = ( + "omni.warp" in warp_norm + or ("extscache" in warp_norm and "warp" in warp_norm) + or ("kit/python/lib" in warp_norm and "site-packages" in warp_norm) + ) + if _suspicious: + import warnings + + warnings.warn( + f"[IsaacLab] warp was imported from a non-pip-installed path: " + f"{warp_file!r}. A Kit-bundled or extscache copy of warp is " + f"shadowing the pip-installed warp-lang, which can cause Warp kernel " + f"lookup failures (e.g. 'Failed to find forward kernel ... from module " + f"isaaclab.sensors.kernels'). Expected warp to be loaded from " + f"pip-installed site-packages.", + RuntimeWarning, + stacklevel=2, + ) + # Hide the stop button in the toolbar self._hide_stop_button() # Set settings from the given rendering mode diff --git a/source/isaaclab/isaaclab/sensors/kernels.py b/source/isaaclab/isaaclab/sensors/kernels.py index db6570323fec..01ca429432e3 100644 --- a/source/isaaclab/isaaclab/sensors/kernels.py +++ b/source/isaaclab/isaaclab/sensors/kernels.py @@ -75,3 +75,11 @@ def reset_envs_kernel( timestamp_last_update[env] = 0.0 # Set all reset sensors to outdated so that they are updated when data is called the next time. is_outdated[env] = True + + +# Ensure content-based cache invalidation is always active for this module, +# overriding any global strip_hash=True that Isaac Sim may set. With +# strip_hash=True the warp cache directory name is fixed (no content hash), +# so a stale binary from a previous warp version would be silently reused, +# causing "Failed to find forward kernel" errors for newly-added kernels. +wp.get_module(__name__).options["strip_hash"] = False From 3d566dac9d82bddb6dbd9b1e16e7f61b4dc4360c Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Fri, 17 Apr 2026 19:00:06 -0700 Subject: [PATCH 09/17] Add warp-lang<1.13 pip requirement to fix warp 1.13.x breakage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 4.6.9 path-demotion fix moved kit/python/lib/python3.12/site-packages to the end of sys.path, but warp-lang was not a pip requirement. With no pip-managed warp installed, the demoted kit warp 1.13.x was still the only warp available and was still imported. Root fix: add warp-lang>=1.0.0,<1.13 to pip requirements. When ./isaaclab.sh --install runs, pip uses kit's own Python (kit/python/bin/python3) which installs into kit/python/lib/python3.12/ site-packages — the same directory kit shipped warp 1.13.x — overwriting it with a compatible version. Also replace path-based warnings in _pin_warp_import and AppLauncher with version-based checks (>= 1.13). Path-based checks would fire false positives after pip installs warp-lang into kit's site-packages (a kit-managed path, but pip-managed content). Version-based checks correctly identify the actual incompatibility regardless of install location. --- source/isaaclab/config/extension.toml | 5 +- source/isaaclab/docs/CHANGELOG.rst | 19 ++++++++ source/isaaclab/isaaclab/__init__.py | 48 ++++++++++--------- source/isaaclab/isaaclab/app/app_launcher.py | 50 ++++++++++---------- 4 files changed, 72 insertions(+), 50 deletions(-) diff --git a/source/isaaclab/config/extension.toml b/source/isaaclab/config/extension.toml index 86024cf07e6a..01982ca295bd 100644 --- a/source/isaaclab/config/extension.toml +++ b/source/isaaclab/config/extension.toml @@ -1,7 +1,7 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "4.6.11" +version = "4.6.12" # Description title = "Isaac Lab framework for Robot Learning" @@ -20,7 +20,8 @@ requirements = [ "hidapi", "gymnasium==0.29.0", "trimesh", - "websockets" + "websockets", + "warp-lang>=1.0.0,<1.13" ] modules = [ diff --git a/source/isaaclab/docs/CHANGELOG.rst b/source/isaaclab/docs/CHANGELOG.rst index 03ab7d860ce4..350c349be332 100644 --- a/source/isaaclab/docs/CHANGELOG.rst +++ b/source/isaaclab/docs/CHANGELOG.rst @@ -1,6 +1,25 @@ Changelog --------- +4.6.12 (2026-04-22) +~~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Fixed ``RuntimeError: Failed to find forward kernel 'update_outdated_envs_kernel'`` + (continued from 4.6.11). The path-demotion fix correctly moved + ``kit/python/lib/python3.12/site-packages`` to the end of ``sys.path``, but since + ``warp-lang`` was not a pip requirement, there was no pip-managed warp to prefer + over the kit-shipped warp 1.13.x — the demoted warp 1.13.x was still the only + warp available. Added ``warp-lang>=1.0.0,<1.13`` to pip requirements so that + ``./isaaclab.sh --install`` pip-installs a compatible warp into the same + ``site-packages`` directory, overwriting the incompatible 1.13.x. Also replaced + the path-based warning in ``_pin_warp_import`` and + :class:`~isaaclab.app.AppLauncher` with a version-based check (``>= 1.13``), + which correctly identifies the incompatibility regardless of install location. + + 4.6.11 (2026-04-22) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab/isaaclab/__init__.py b/source/isaaclab/isaaclab/__init__.py index 358c358cf6ca..04a5626cdaf0 100644 --- a/source/isaaclab/isaaclab/__init__.py +++ b/source/isaaclab/isaaclab/__init__.py @@ -91,7 +91,7 @@ def _should_demote(path: str) -> bool: def _pin_warp_import(): - """Import ``warp`` now to lock ``sys.modules['warp']`` to the pip-installed version. + """Import ``warp`` now to lock ``sys.modules['warp']`` to the correct version. Kit's extension system may add ``omni.warp.core``'s directory or Kit's own ``kit/python/lib/python3.X/site-packages`` to ``sys.path`` during @@ -104,7 +104,7 @@ def _pin_warp_import(): By importing ``warp`` here — after ``_deprioritize_prebundle_paths()`` has already demoted the pip_prebundle, ``omni.warp.core``, and ``kit/python/lib`` - site-packages paths — we ensure the pip-installed ``warp-lang`` is the one + site-packages paths — we ensure the pip-managed ``warp-lang`` is the one cached in ``sys.modules``. Subsequent ``import warp`` calls from Kit extensions all return that cached module, so there is only ever one Warp runtime in the process. @@ -118,29 +118,31 @@ def _pin_warp_import(): except ImportError: return - # Warn if warp was resolved from a non-pip location that could introduce - # version mismatches (Kit-bundled or extscache copies). + # Warn if the loaded warp version is incompatible with omni.replicator.core. + # Warp >= 1.13 deprecates warp.types.array with changed semantics; when + # omni.replicator.core (which uses that symbol) is loaded with warp >= 1.13, + # it triggers CUDA error 700 (illegal memory access) that poisons the CUDA + # context and causes all subsequent warp kernel lookups to fail. + # Run './isaaclab.sh --install' to install a compatible warp-lang version. import warnings - _warp_file = getattr(_warp, "__file__", "") or "" - _warp_norm = _warp_file.replace("\\", "/").lower() - _suspicious = ( - "omni.warp" in _warp_norm - or ("extscache" in _warp_norm and "warp" in _warp_norm) - or ("kit/python/lib" in _warp_norm and "site-packages" in _warp_norm) - ) - if _suspicious: - warnings.warn( - f"[IsaacLab] warp was imported from a non-pip-installed location: " - f"{_warp_file!r}. A Kit-bundled or extscache copy of warp may be " - f"shadowing the pip-installed warp-lang, which can cause Warp kernel " - f"lookup failures (e.g. 'Failed to find forward kernel ... from module " - f"isaaclab.sensors.kernels'). Check that pip-installed packages " - f"(site-packages) appear before kit/python/lib/*/site-packages and " - f"extscache directories in sys.path.", - RuntimeWarning, - stacklevel=2, - ) + _warp_version = getattr(_warp, "version", None) + if _warp_version is not None: + try: + _parts = [int(x) for x in str(_warp_version).split(".")[:2]] + if len(_parts) >= 2 and (_parts[0], _parts[1]) >= (1, 13): + warnings.warn( + f"[IsaacLab] warp {_warp_version} is incompatible with " + f"omni.replicator.core. Warp >= 1.13 deprecates " + f"``warp.types.array`` with changed semantics, which causes " + f"CUDA error 700 (illegal memory access) during rendering and " + f"makes all subsequent warp kernel lookups fail. Run " + f"'./isaaclab.sh --install' to install warp-lang<1.13.", + RuntimeWarning, + stacklevel=2, + ) + except (ValueError, AttributeError): + pass _pin_warp_import() diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 7079b387221b..4cdd4fa50f6d 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -212,35 +212,35 @@ def __init__(self, launcher_args: argparse.Namespace | dict | None = None, **kwa _deprioritize_prebundle_paths() - # Verify that warp (if loaded) came from the pip-installed copy and not from - # a bundled extension such as omni.warp.core or Kit's own Python packages. - # Kit's extension system can add extension directories and kit/python/lib paths - # to sys.path during startup; the pre-import in isaaclab/__init__.py normally - # prevents this, but log a warning so CI failures are easier to diagnose. + # Verify that warp (if loaded) is a compatible version. Warp >= 1.13 + # deprecates warp.types.array with changed semantics; omni.replicator.core + # accesses this symbol during rendering and with warp >= 1.13 causes CUDA + # error 700 that poisons the context and makes all kernel lookups fail. + # The pre-import in isaaclab/__init__.py normally catches this, but log + # a warning here too so CI failures are easier to diagnose. import sys warp_mod = sys.modules.get("warp") if warp_mod is not None: - warp_file = getattr(warp_mod, "__file__", "") or "" - warp_norm = warp_file.replace("\\", "/").lower() - _suspicious = ( - "omni.warp" in warp_norm - or ("extscache" in warp_norm and "warp" in warp_norm) - or ("kit/python/lib" in warp_norm and "site-packages" in warp_norm) - ) - if _suspicious: - import warnings - - warnings.warn( - f"[IsaacLab] warp was imported from a non-pip-installed path: " - f"{warp_file!r}. A Kit-bundled or extscache copy of warp is " - f"shadowing the pip-installed warp-lang, which can cause Warp kernel " - f"lookup failures (e.g. 'Failed to find forward kernel ... from module " - f"isaaclab.sensors.kernels'). Expected warp to be loaded from " - f"pip-installed site-packages.", - RuntimeWarning, - stacklevel=2, - ) + warp_version = getattr(warp_mod, "version", None) + if warp_version is not None: + try: + _parts = [int(x) for x in str(warp_version).split(".")[:2]] + if len(_parts) >= 2 and (_parts[0], _parts[1]) >= (1, 13): + import warnings + + warnings.warn( + f"[IsaacLab] warp {warp_version} is incompatible with " + f"omni.replicator.core. Warp >= 1.13 deprecates " + f"``warp.types.array`` with changed semantics, which causes " + f"CUDA error 700 during rendering and makes all warp kernel " + f"lookups fail. Run './isaaclab.sh --install' to install " + f"warp-lang<1.13.", + RuntimeWarning, + stacklevel=2, + ) + except (ValueError, AttributeError): + pass # Hide the stop button in the toolbar self._hide_stop_button() From 51289dabe5828c9efa189458eed7315654bfb1d8 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Wed, 22 Apr 2026 12:04:51 -0700 Subject: [PATCH 10/17] fix patch for isaac sim simulation manager --- source/isaaclab_physx/config/extension.toml | 2 +- source/isaaclab_physx/docs/CHANGELOG.rst | 20 +++++ .../isaaclab_physx/isaaclab_physx/__init__.py | 76 +++++++++++++++++-- 3 files changed, 89 insertions(+), 9 deletions(-) diff --git a/source/isaaclab_physx/config/extension.toml b/source/isaaclab_physx/config/extension.toml index d05b38808199..f9368e59a0f5 100644 --- a/source/isaaclab_physx/config/extension.toml +++ b/source/isaaclab_physx/config/extension.toml @@ -1,7 +1,7 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "0.5.20" +version = "0.5.21" # Description title = "PhysX simulation interfaces for IsaacLab core package" diff --git a/source/isaaclab_physx/docs/CHANGELOG.rst b/source/isaaclab_physx/docs/CHANGELOG.rst index d722acb0687a..d7d06ae2fb73 100644 --- a/source/isaaclab_physx/docs/CHANGELOG.rst +++ b/source/isaaclab_physx/docs/CHANGELOG.rst @@ -1,6 +1,26 @@ Changelog --------- +0.5.21 (2026-04-22) +~~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Fixed ``Simulation view object is invalidated and cannot be used again to call + getDofVelocities`` raised on the first ``scene.update()`` after ``sim.reset()`` + with recent Isaac Sim ``develop`` builds. Isaac Sim's + ``isaacsim.core.simulation_manager.SimulationManager`` recently became reactive + to timeline ``STOP`` events (after its ``_on_stop`` was decorated with + ``@staticmethod`` upstream), and its ``invalidate_physics()`` was clobbering + the shared ``omni.physics.tensors`` simulation view that + :class:`~isaaclab_physx.physics.PhysxManager` and PhysX articulation views + rely on. The ``isaaclab_physx`` package init now disables the original Isaac + Sim ``SimulationManager``'s default timeline/stage callbacks via + ``enable_all_default_callbacks(False)`` before swapping the module attribute, + so :class:`PhysxManager` is the single owner of the simulation lifecycle. + + 0.5.20 (2026-04-21) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab_physx/isaaclab_physx/__init__.py b/source/isaaclab_physx/isaaclab_physx/__init__.py index a9f5f5bf9d94..b9e1cbae22a7 100644 --- a/source/isaaclab_physx/isaaclab_physx/__init__.py +++ b/source/isaaclab_physx/isaaclab_physx/__init__.py @@ -21,17 +21,77 @@ def _patch_isaacsim_simulation_manager(): - """Patch Isaac Sim's SimulationManager to use PhysxManager. + """Patch Isaac Sim's ``SimulationManager`` to use :class:`PhysxManager`. - This ensures all code that imports from isaacsim.core.simulation_manager - will use our PhysxManager instead, preventing duplicate callback registration. + This redirects future ``from isaacsim.core.simulation_manager import SimulationManager`` + consumers to :class:`isaaclab_physx.physics.PhysxManager`, but the original + Isaac Sim ``SimulationManager`` class has *already* registered timeline + (PLAY/STOP) and stage (OPENED/CLOSED) subscriptions during its extension + startup. Those subscriptions live on the original class, not the module + attribute, so swapping the attribute alone is not enough. + + Starting with Isaac Sim 6.0.0-alpha.180 (commit ``8df6beeb0`` on + ``develop``, "hmazhar/autofix_bugs"), the original + ``SimulationManager._on_stop``/``_on_play``/``_on_stage_*`` methods were + decorated with ``@staticmethod`` so they finally fire correctly from the + Carb event subscriptions. Before that fix they were silently broken (the + subscriptions invoked them as bound methods, so the ``event`` argument was + being passed as ``self``/``cls`` and the bodies never executed). + + The newly-working ``_on_stop`` calls + ``SimulationManager.invalidate_physics()``, which calls + ``view.invalidate()`` on its ``omni.physics.tensors`` simulation view. + Because ``omni.physics.tensors.create_simulation_view("warp", stage_id=...)`` + returns the same underlying SimulationView per stage_id, that invalidation + also wrecks the view that :class:`PhysxManager` (and any articulation + ``_root_view`` derived from it) relies on. The result is the runtime error + ``Simulation view object is invalidated and cannot be used again to call + getDofVelocities`` on the very first ``scene.update()`` after + ``sim.reset()``. + + To prevent this, we disable the original class's default callbacks here + *before* swapping the module attribute, so :class:`PhysxManager` becomes + the single owner of the simulation lifecycle. """ - if "isaacsim.core.simulation_manager" in sys.modules: - original_module = sys.modules["isaacsim.core.simulation_manager"] - from .physics.physx_manager import PhysxManager, IsaacEvents + # Force-import Isaac Sim's SimulationManager before patching so that the + # subscriptions registered during its module/extension startup are taken + # down deterministically here, regardless of the order in which Kit + # extensions or user code happen to import the module. + try: + import isaacsim.core.simulation_manager # noqa: F401 + except ImportError: + # Isaac Sim is not installed (e.g. during ``./isaaclab.sh --install`` + # bootstrap or in pure unit-test environments). Nothing to patch. + return + + original_module = sys.modules["isaacsim.core.simulation_manager"] + from .physics.physx_manager import PhysxManager, IsaacEvents + + # Tear down the original Isaac Sim SimulationManager's default timeline / + # stage subscriptions so they cannot invalidate the omni.physics.tensors + # view that PhysxManager owns. ``enable_all_default_callbacks(False)`` + # covers warm_start (PLAY), on_stop (STOP), stage_open (OPENED) and + # stage_close (CLOSED). Older Isaac Sim builds may not expose this API, so + # fall back gracefully. + original_class = getattr(original_module, "SimulationManager", None) + if original_class is not None and original_class is not PhysxManager: + try: + original_class.enable_all_default_callbacks(False) + except Exception: + # Defensive: API changed or original class never finished startup. + # Manually clear the subscription handles if they exist so any + # remaining references go through the dead-callback path. + for attr in ( + "_default_callback_warm_start", + "_default_callback_on_stop", + "_default_callback_stage_open", + "_default_callback_stage_close", + ): + if hasattr(original_class, attr): + setattr(original_class, attr, None) - original_module.SimulationManager = PhysxManager - original_module.IsaacEvents = IsaacEvents + original_module.SimulationManager = PhysxManager + original_module.IsaacEvents = IsaacEvents _patch_isaacsim_simulation_manager() From 60620c176748971ab92d50cae811f041dcea620c Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Wed, 22 Apr 2026 12:44:49 -0700 Subject: [PATCH 11/17] updates --- apps/isaaclab.python.headless.kit | 2 +- apps/isaaclab.python.headless.rendering.kit | 2 +- apps/isaaclab.python.kit | 2 +- apps/isaaclab.python.rendering.kit | 2 +- apps/isaaclab.python.xr.openxr.headless.kit | 2 +- apps/isaaclab.python.xr.openxr.kit | 2 +- source/isaaclab/config/extension.toml | 2 +- source/isaaclab/docs/CHANGELOG.rst | 18 ++++++++--- source/isaaclab/isaaclab/__init__.py | 31 +++++++++---------- source/isaaclab/isaaclab/sensors/kernels.py | 8 ----- source/isaaclab_physx/config/extension.toml | 2 +- source/isaaclab_physx/docs/CHANGELOG.rst | 19 ++++++++++++ .../renderers/isaac_rtx_renderer.py | 12 +++++++ 13 files changed, 68 insertions(+), 36 deletions(-) diff --git a/apps/isaaclab.python.headless.kit b/apps/isaaclab.python.headless.kit index 12b9e14946d0..051d6e049098 100644 --- a/apps/isaaclab.python.headless.kit +++ b/apps/isaaclab.python.headless.kit @@ -106,7 +106,7 @@ isaac.startup.ros_bridge_extension = "" metricsAssembler.changeListenerEnabled = false # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] # Extensions ############################### diff --git a/apps/isaaclab.python.headless.rendering.kit b/apps/isaaclab.python.headless.rendering.kit index 90fee65f77ec..86ec02b93ff1 100644 --- a/apps/isaaclab.python.headless.rendering.kit +++ b/apps/isaaclab.python.headless.rendering.kit @@ -107,7 +107,7 @@ exts."omni.replicator.core".Orchestrator.enabled = false metricsAssembler.changeListenerEnabled = false # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] [settings.app.python] # These disable the kit app from also printing out python output, which gets confusing diff --git a/apps/isaaclab.python.kit b/apps/isaaclab.python.kit index a5330cd1944e..67202c3bfadb 100644 --- a/apps/isaaclab.python.kit +++ b/apps/isaaclab.python.kit @@ -256,7 +256,7 @@ exts."omni.replicator.core".Orchestrator.enabled = false omni.rtx.nre.compositing.rendererHints = 3 # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] [settings.app.livestream] outDirectory = "${data}" diff --git a/apps/isaaclab.python.rendering.kit b/apps/isaaclab.python.rendering.kit index d3414b211907..64f132788050 100644 --- a/apps/isaaclab.python.rendering.kit +++ b/apps/isaaclab.python.rendering.kit @@ -93,7 +93,7 @@ exts."omni.replicator.core".Orchestrator.enabled = false metricsAssembler.changeListenerEnabled = false # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] [settings.physics] updateToUsd = false diff --git a/apps/isaaclab.python.xr.openxr.headless.kit b/apps/isaaclab.python.xr.openxr.headless.kit index 75f3ae637124..681bf14d47ba 100644 --- a/apps/isaaclab.python.xr.openxr.headless.kit +++ b/apps/isaaclab.python.xr.openxr.headless.kit @@ -42,7 +42,7 @@ cameras_enabled = true [settings] # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] [settings.app.python] # These disable the kit app from also printing out python output, which gets confusing diff --git a/apps/isaaclab.python.xr.openxr.kit b/apps/isaaclab.python.xr.openxr.kit index 62782ba06619..aec61b10d48d 100644 --- a/apps/isaaclab.python.xr.openxr.kit +++ b/apps/isaaclab.python.xr.openxr.kit @@ -66,7 +66,7 @@ xr.openxr.components."isaacsim.xr.openxr.hand_tracking".enabled = true xr.openxr.components."isaacsim.kit.xr.teleop.bridge".enabled = true # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] [settings.app.python] # These disable the kit app from also printing out python output, which gets confusing diff --git a/source/isaaclab/config/extension.toml b/source/isaaclab/config/extension.toml index 01982ca295bd..8c402e6ad1a0 100644 --- a/source/isaaclab/config/extension.toml +++ b/source/isaaclab/config/extension.toml @@ -1,7 +1,7 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "4.6.12" +version = "4.6.13" # Description title = "Isaac Lab framework for Robot Learning" diff --git a/source/isaaclab/docs/CHANGELOG.rst b/source/isaaclab/docs/CHANGELOG.rst index 350c349be332..2eac4b332f96 100644 --- a/source/isaaclab/docs/CHANGELOG.rst +++ b/source/isaaclab/docs/CHANGELOG.rst @@ -1,6 +1,18 @@ Changelog --------- +4.6.13 (2026-04-22) +~~~~~~~~~~~~~~~~~~~ + +Removed +^^^^^^^ + +* Removed ``omni.warp.core`` from the ``_CONFLICTING_EXT_FRAGMENTS`` list in + :mod:`isaaclab` and from ``app.extensions.excluded`` in the IsaacLab kit + apps. The extension is no longer present in current Isaac Sim builds, so + the entries were dead noise. + + 4.6.12 (2026-04-22) ~~~~~~~~~~~~~~~~~~~ @@ -38,10 +50,8 @@ Fixed ``update_outdated_envs_kernel__cuda_kernel_forward``) to return NULL. Extended ``_deprioritize_prebundle_paths()`` to also demote ``kit/python/lib/*/site-packages`` paths, ensuring pip-installed ``warp-lang`` - always takes priority. Also added an explicit ``strip_hash=False`` option to the - ``isaaclab.sensors.kernels`` warp module as a defensive safety net, and broadened - the post-startup diagnostic warning in :class:`~isaaclab.app.AppLauncher` to cover - Kit Python paths. + always takes priority. Broadened the post-startup diagnostic warning in + :class:`~isaaclab.app.AppLauncher` to cover Kit Python paths. * Fixed a Warp kernel lookup failure (``Failed to find forward kernel … for device 'cuda:0'``) caused by Kit's extension scanner adding ``omni.warp.core``'s directory to ``sys.path`` during ``SimulationApp`` startup, before the second ``_deprioritize_prebundle_paths()`` diff --git a/source/isaaclab/isaaclab/__init__.py b/source/isaaclab/isaaclab/__init__.py index 04a5626cdaf0..735629e68b41 100644 --- a/source/isaaclab/isaaclab/__init__.py +++ b/source/isaaclab/isaaclab/__init__.py @@ -17,10 +17,11 @@ def _deprioritize_prebundle_paths(): warp, and nvidia-cudnn that shadow the versions installed by Isaac Lab, causing CUDA runtime errors. - Additionally, certain Isaac Sim kit extensions (such as ``omni.warp.core``) - bundle their own copies of Python packages that conflict with pip-installed - versions. When loaded by the extension system these paths can appear on - ``sys.path`` before ``site-packages``, leading to version mismatches. + Additionally, certain Isaac Sim kit extensions (such as + ``isaacsim.pip.newton``) bundle their own copies of Python packages that + conflict with pip-installed versions. When loaded by the extension system + these paths can appear on ``sys.path`` before ``site-packages``, leading to + version mismatches. Rather than removing these paths entirely (which would break packages like ``sympy`` that only exist in the prebundle), this function moves them to @@ -34,7 +35,6 @@ def _deprioritize_prebundle_paths(): # Extension directory fragments that are known to ship Python packages # which conflict with Isaac Lab's pip-installed versions. _CONFLICTING_EXT_FRAGMENTS = ( - "omni.warp.core", "omni.isaac.ml_archive", "omni.isaac.core_archive", "omni.kit.pip_archive", @@ -93,20 +93,19 @@ def _should_demote(path: str) -> bool: def _pin_warp_import(): """Import ``warp`` now to lock ``sys.modules['warp']`` to the correct version. - Kit's extension system may add ``omni.warp.core``'s directory or Kit's own + Kit's extension system may add Kit's own ``kit/python/lib/python3.X/site-packages`` to ``sys.path`` during - ``SimulationApp`` startup — even when that extension is excluded from the kit - file — because Kit scans extension directories as part of its registry process. - Any extension that imports ``warp`` during that window (e.g. - ``omni.replicator.core``) would set ``sys.modules['warp']`` to the bundled copy - before our second ``_deprioritize_prebundle_paths()`` call in ``AppLauncher`` - has a chance to run. + ``SimulationApp`` startup, because Kit scans extension directories as part + of its registry process. Any extension that imports ``warp`` during that + window (e.g. ``omni.replicator.core``) would set ``sys.modules['warp']`` to + the bundled copy before our second ``_deprioritize_prebundle_paths()`` call + in ``AppLauncher`` has a chance to run. By importing ``warp`` here — after ``_deprioritize_prebundle_paths()`` has - already demoted the pip_prebundle, ``omni.warp.core``, and ``kit/python/lib`` - site-packages paths — we ensure the pip-managed ``warp-lang`` is the one - cached in ``sys.modules``. Subsequent ``import warp`` calls from Kit extensions - all return that cached module, so there is only ever one Warp runtime in the + already demoted the pip_prebundle and ``kit/python/lib`` site-packages + paths — we ensure the pip-managed ``warp-lang`` is the one cached in + ``sys.modules``. Subsequent ``import warp`` calls from Kit extensions all + return that cached module, so there is only ever one Warp runtime in the process. Failure to import (e.g. warp not yet installed during initial setup) is diff --git a/source/isaaclab/isaaclab/sensors/kernels.py b/source/isaaclab/isaaclab/sensors/kernels.py index 01ca429432e3..db6570323fec 100644 --- a/source/isaaclab/isaaclab/sensors/kernels.py +++ b/source/isaaclab/isaaclab/sensors/kernels.py @@ -75,11 +75,3 @@ def reset_envs_kernel( timestamp_last_update[env] = 0.0 # Set all reset sensors to outdated so that they are updated when data is called the next time. is_outdated[env] = True - - -# Ensure content-based cache invalidation is always active for this module, -# overriding any global strip_hash=True that Isaac Sim may set. With -# strip_hash=True the warp cache directory name is fixed (no content hash), -# so a stale binary from a previous warp version would be silently reused, -# causing "Failed to find forward kernel" errors for newly-added kernels. -wp.get_module(__name__).options["strip_hash"] = False diff --git a/source/isaaclab_physx/config/extension.toml b/source/isaaclab_physx/config/extension.toml index f9368e59a0f5..5f4fb7f10cd8 100644 --- a/source/isaaclab_physx/config/extension.toml +++ b/source/isaaclab_physx/config/extension.toml @@ -1,7 +1,7 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "0.5.21" +version = "0.5.22" # Description title = "PhysX simulation interfaces for IsaacLab core package" diff --git a/source/isaaclab_physx/docs/CHANGELOG.rst b/source/isaaclab_physx/docs/CHANGELOG.rst index d7d06ae2fb73..17ed090ebb30 100644 --- a/source/isaaclab_physx/docs/CHANGELOG.rst +++ b/source/isaaclab_physx/docs/CHANGELOG.rst @@ -1,6 +1,25 @@ Changelog --------- +0.5.22 (2026-04-22) +~~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Fixed a CUDA ``illegal memory access`` (error 700) in + :class:`~isaaclab_physx.renderers.IsaacRtxRenderer` that poisoned the entire + CUDA context — surfacing later as ``Failed to find forward kernel + 'reshape_tiled_image'``, ``Failed to get DOF velocities from backend``, and a + cascade of ``CUDA error in freeAsync`` failures from ``omni.physx.tensors`` + and ``omni.rtx``. On the first one or two camera updates, the Replicator + annotator can return an empty (size-zero) buffer before RTX has produced any + data; the ``reshape_tiled_image`` warp kernel was still launched with + ``view_count * height * width`` threads, each of which read past the end of + the empty buffer. The renderer now skips the kernel launch when the + annotator buffer is empty so the output tensor stays zero-initialised for + that frame instead of corrupting the CUDA context. + 0.5.21 (2026-04-22) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 9a80433b11d5..fedb1e2db4a1 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -295,6 +295,18 @@ def tiling_grid_shape(): if data_type in SIMPLE_SHADING_MODES: tiled_data_buffer = tiled_data_buffer[:, :, :3].contiguous() + # Annotators may return an empty buffer (size 0) on the first one or two frames + # before RTX has produced any data. Launching the reshape kernel with a + # zero-length input still spawns ``view_count * height * width`` threads that + # immediately read out of bounds, which raises a CUDA illegal-memory-access + # (error 700) on warp's stream and poisons the entire CUDA context — every + # subsequent op (PhysX tensors, RTX, torch) then fails. Skip the launch + # until the annotator has populated its buffer; the output tensor remains + # zero-initialised for that frame, which matches the prior behaviour from + # before the empty-buffer regression appeared in newer Isaac Sim builds. + if tiled_data_buffer.size == 0: + continue + wp.launch( kernel=reshape_tiled_image, dim=(view_count, cfg.height, cfg.width), From f41b744ba7d2a92760eb4199f5ae0bdfa223bf77 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Wed, 22 Apr 2026 13:04:54 -0700 Subject: [PATCH 12/17] updates --- apps/isaaclab.python.headless.kit | 2 +- apps/isaaclab.python.headless.rendering.kit | 2 +- apps/isaaclab.python.kit | 2 +- apps/isaaclab.python.rendering.kit | 2 +- apps/isaaclab.python.xr.openxr.headless.kit | 2 +- apps/isaaclab.python.xr.openxr.kit | 2 +- source/isaaclab/config/extension.toml | 5 +- source/isaaclab/docs/CHANGELOG.rst | 60 ---------------- source/isaaclab/isaaclab/__init__.py | 76 ++------------------ source/isaaclab/isaaclab/app/app_launcher.py | 30 -------- 10 files changed, 13 insertions(+), 170 deletions(-) diff --git a/apps/isaaclab.python.headless.kit b/apps/isaaclab.python.headless.kit index 051d6e049098..12b9e14946d0 100644 --- a/apps/isaaclab.python.headless.kit +++ b/apps/isaaclab.python.headless.kit @@ -106,7 +106,7 @@ isaac.startup.ros_bridge_extension = "" metricsAssembler.changeListenerEnabled = false # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] # Extensions ############################### diff --git a/apps/isaaclab.python.headless.rendering.kit b/apps/isaaclab.python.headless.rendering.kit index 86ec02b93ff1..90fee65f77ec 100644 --- a/apps/isaaclab.python.headless.rendering.kit +++ b/apps/isaaclab.python.headless.rendering.kit @@ -107,7 +107,7 @@ exts."omni.replicator.core".Orchestrator.enabled = false metricsAssembler.changeListenerEnabled = false # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] [settings.app.python] # These disable the kit app from also printing out python output, which gets confusing diff --git a/apps/isaaclab.python.kit b/apps/isaaclab.python.kit index 67202c3bfadb..a5330cd1944e 100644 --- a/apps/isaaclab.python.kit +++ b/apps/isaaclab.python.kit @@ -256,7 +256,7 @@ exts."omni.replicator.core".Orchestrator.enabled = false omni.rtx.nre.compositing.rendererHints = 3 # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] [settings.app.livestream] outDirectory = "${data}" diff --git a/apps/isaaclab.python.rendering.kit b/apps/isaaclab.python.rendering.kit index 64f132788050..d3414b211907 100644 --- a/apps/isaaclab.python.rendering.kit +++ b/apps/isaaclab.python.rendering.kit @@ -93,7 +93,7 @@ exts."omni.replicator.core".Orchestrator.enabled = false metricsAssembler.changeListenerEnabled = false # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] [settings.physics] updateToUsd = false diff --git a/apps/isaaclab.python.xr.openxr.headless.kit b/apps/isaaclab.python.xr.openxr.headless.kit index 681bf14d47ba..75f3ae637124 100644 --- a/apps/isaaclab.python.xr.openxr.headless.kit +++ b/apps/isaaclab.python.xr.openxr.headless.kit @@ -42,7 +42,7 @@ cameras_enabled = true [settings] # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] [settings.app.python] # These disable the kit app from also printing out python output, which gets confusing diff --git a/apps/isaaclab.python.xr.openxr.kit b/apps/isaaclab.python.xr.openxr.kit index aec61b10d48d..62782ba06619 100644 --- a/apps/isaaclab.python.xr.openxr.kit +++ b/apps/isaaclab.python.xr.openxr.kit @@ -66,7 +66,7 @@ xr.openxr.components."isaacsim.xr.openxr.hand_tracking".enabled = true xr.openxr.components."isaacsim.kit.xr.teleop.bridge".enabled = true # explicitly disable omni.kit.pip_archive to prevent conflicting dependencies -app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton"] +app.extensions.excluded = ["omni.kit.pip_archive", "omni.isaac.ml_archive", "isaacsim.pip.newton", "omni.warp.core"] [settings.app.python] # These disable the kit app from also printing out python output, which gets confusing diff --git a/source/isaaclab/config/extension.toml b/source/isaaclab/config/extension.toml index 8c402e6ad1a0..7092d52f3ff7 100644 --- a/source/isaaclab/config/extension.toml +++ b/source/isaaclab/config/extension.toml @@ -1,7 +1,7 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "4.6.13" +version = "4.6.10" # Description title = "Isaac Lab framework for Robot Learning" @@ -20,8 +20,7 @@ requirements = [ "hidapi", "gymnasium==0.29.0", "trimesh", - "websockets", - "warp-lang>=1.0.0,<1.13" + "websockets" ] modules = [ diff --git a/source/isaaclab/docs/CHANGELOG.rst b/source/isaaclab/docs/CHANGELOG.rst index 2eac4b332f96..0251fcb03ca2 100644 --- a/source/isaaclab/docs/CHANGELOG.rst +++ b/source/isaaclab/docs/CHANGELOG.rst @@ -1,66 +1,6 @@ Changelog --------- -4.6.13 (2026-04-22) -~~~~~~~~~~~~~~~~~~~ - -Removed -^^^^^^^ - -* Removed ``omni.warp.core`` from the ``_CONFLICTING_EXT_FRAGMENTS`` list in - :mod:`isaaclab` and from ``app.extensions.excluded`` in the IsaacLab kit - apps. The extension is no longer present in current Isaac Sim builds, so - the entries were dead noise. - - -4.6.12 (2026-04-22) -~~~~~~~~~~~~~~~~~~~ - -Fixed -^^^^^ - -* Fixed ``RuntimeError: Failed to find forward kernel 'update_outdated_envs_kernel'`` - (continued from 4.6.11). The path-demotion fix correctly moved - ``kit/python/lib/python3.12/site-packages`` to the end of ``sys.path``, but since - ``warp-lang`` was not a pip requirement, there was no pip-managed warp to prefer - over the kit-shipped warp 1.13.x — the demoted warp 1.13.x was still the only - warp available. Added ``warp-lang>=1.0.0,<1.13`` to pip requirements so that - ``./isaaclab.sh --install`` pip-installs a compatible warp into the same - ``site-packages`` directory, overwriting the incompatible 1.13.x. Also replaced - the path-based warning in ``_pin_warp_import`` and - :class:`~isaaclab.app.AppLauncher` with a version-based check (``>= 1.13``), - which correctly identifies the incompatibility regardless of install location. - - -4.6.11 (2026-04-22) -~~~~~~~~~~~~~~~~~~~ - -Fixed -^^^^^ - -* Fixed ``RuntimeError: Failed to find forward kernel 'update_outdated_envs_kernel'`` - caused by the nightly Isaac Sim Docker (>= 04/14) shipping warp 1.13.x inside - ``kit/python/lib/python3.12/site-packages``. Because ``setup_python_env.sh`` - places that directory on ``PYTHONPATH`` ahead of pip's ``site-packages``, warp 1.13.x - was imported instead of pip-installed warp 1.12.0, despite the ``_pin_warp_import`` - pre-import added previously. Warp 1.13.x deprecates ``warp.types.array`` and changes its - semantics; ``omni.replicator.core`` accesses this symbol during rendering, triggering - a CUDA illegal-memory-access (error 700) that poisons the CUDA context and causes - all subsequent warp kernel lookups (including - ``update_outdated_envs_kernel__cuda_kernel_forward``) to return NULL. - Extended ``_deprioritize_prebundle_paths()`` to also demote - ``kit/python/lib/*/site-packages`` paths, ensuring pip-installed ``warp-lang`` - always takes priority. Broadened the post-startup diagnostic warning in - :class:`~isaaclab.app.AppLauncher` to cover Kit Python paths. -* Fixed a Warp kernel lookup failure (``Failed to find forward kernel … for device 'cuda:0'``) - caused by Kit's extension scanner adding ``omni.warp.core``'s directory to ``sys.path`` - during ``SimulationApp`` startup, before the second ``_deprioritize_prebundle_paths()`` - call in :class:`~isaaclab.app.AppLauncher` could run. The fix pre-imports ``warp`` in - :mod:`isaaclab` ``__init__`` immediately after the path-demotion step, locking - ``sys.modules['warp']`` to the pip-installed version before Kit starts. A post-startup - warning is also emitted when the wrong ``warp`` is detected. - - 4.6.10 (2026-04-22) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab/isaaclab/__init__.py b/source/isaaclab/isaaclab/__init__.py index 735629e68b41..d547b628372e 100644 --- a/source/isaaclab/isaaclab/__init__.py +++ b/source/isaaclab/isaaclab/__init__.py @@ -17,11 +17,10 @@ def _deprioritize_prebundle_paths(): warp, and nvidia-cudnn that shadow the versions installed by Isaac Lab, causing CUDA runtime errors. - Additionally, certain Isaac Sim kit extensions (such as - ``isaacsim.pip.newton``) bundle their own copies of Python packages that - conflict with pip-installed versions. When loaded by the extension system - these paths can appear on ``sys.path`` before ``site-packages``, leading to - version mismatches. + Additionally, certain Isaac Sim kit extensions (such as ``omni.warp.core``) + bundle their own copies of Python packages that conflict with pip-installed + versions. When loaded by the extension system these paths can appear on + ``sys.path`` before ``site-packages``, leading to version mismatches. Rather than removing these paths entirely (which would break packages like ``sympy`` that only exist in the prebundle), this function moves them to @@ -35,6 +34,7 @@ def _deprioritize_prebundle_paths(): # Extension directory fragments that are known to ship Python packages # which conflict with Isaac Lab's pip-installed versions. _CONFLICTING_EXT_FRAGMENTS = ( + "omni.warp.core", "omni.isaac.ml_archive", "omni.isaac.core_archive", "omni.kit.pip_archive", @@ -45,15 +45,6 @@ def _should_demote(path: str) -> bool: norm = path.replace("\\", "/").lower() if "pip_prebundle" in norm: return True - # Kit ships its own Python interpreter and installs packages into - # kit/python/lib/python3.X/site-packages. When a newer or differently- - # configured version of a package (e.g. warp) is installed there, it - # takes precedence over pip-installed packages and can cause runtime - # failures. Demote these site-packages to the end of sys.path while - # leaving the Kit stdlib path (kit/python/lib/python3.X without - # site-packages) in place so Kit's standard-library extras still work. - if "kit/python/lib" in norm and "site-packages" in norm: - return True for frag in _CONFLICTING_EXT_FRAGMENTS: if frag.lower() in norm: return True @@ -89,63 +80,6 @@ def _should_demote(path: str) -> bool: _deprioritize_prebundle_paths() - -def _pin_warp_import(): - """Import ``warp`` now to lock ``sys.modules['warp']`` to the correct version. - - Kit's extension system may add Kit's own - ``kit/python/lib/python3.X/site-packages`` to ``sys.path`` during - ``SimulationApp`` startup, because Kit scans extension directories as part - of its registry process. Any extension that imports ``warp`` during that - window (e.g. ``omni.replicator.core``) would set ``sys.modules['warp']`` to - the bundled copy before our second ``_deprioritize_prebundle_paths()`` call - in ``AppLauncher`` has a chance to run. - - By importing ``warp`` here — after ``_deprioritize_prebundle_paths()`` has - already demoted the pip_prebundle and ``kit/python/lib`` site-packages - paths — we ensure the pip-managed ``warp-lang`` is the one cached in - ``sys.modules``. Subsequent ``import warp`` calls from Kit extensions all - return that cached module, so there is only ever one Warp runtime in the - process. - - Failure to import (e.g. warp not yet installed during initial setup) is - silently ignored; the import will succeed once the user has run - ``./isaaclab.sh --install``. - """ - try: - import warp as _warp # noqa: F401 - except ImportError: - return - - # Warn if the loaded warp version is incompatible with omni.replicator.core. - # Warp >= 1.13 deprecates warp.types.array with changed semantics; when - # omni.replicator.core (which uses that symbol) is loaded with warp >= 1.13, - # it triggers CUDA error 700 (illegal memory access) that poisons the CUDA - # context and causes all subsequent warp kernel lookups to fail. - # Run './isaaclab.sh --install' to install a compatible warp-lang version. - import warnings - - _warp_version = getattr(_warp, "version", None) - if _warp_version is not None: - try: - _parts = [int(x) for x in str(_warp_version).split(".")[:2]] - if len(_parts) >= 2 and (_parts[0], _parts[1]) >= (1, 13): - warnings.warn( - f"[IsaacLab] warp {_warp_version} is incompatible with " - f"omni.replicator.core. Warp >= 1.13 deprecates " - f"``warp.types.array`` with changed semantics, which causes " - f"CUDA error 700 (illegal memory access) during rendering and " - f"makes all subsequent warp kernel lookups fail. Run " - f"'./isaaclab.sh --install' to install warp-lang<1.13.", - RuntimeWarning, - stacklevel=2, - ) - except (ValueError, AttributeError): - pass - - -_pin_warp_import() - # Conveniences to other module directories via relative paths. ISAACLAB_EXT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../")) """Path to the extension source directory.""" diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 4cdd4fa50f6d..266e27c57043 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -212,36 +212,6 @@ def __init__(self, launcher_args: argparse.Namespace | dict | None = None, **kwa _deprioritize_prebundle_paths() - # Verify that warp (if loaded) is a compatible version. Warp >= 1.13 - # deprecates warp.types.array with changed semantics; omni.replicator.core - # accesses this symbol during rendering and with warp >= 1.13 causes CUDA - # error 700 that poisons the context and makes all kernel lookups fail. - # The pre-import in isaaclab/__init__.py normally catches this, but log - # a warning here too so CI failures are easier to diagnose. - import sys - - warp_mod = sys.modules.get("warp") - if warp_mod is not None: - warp_version = getattr(warp_mod, "version", None) - if warp_version is not None: - try: - _parts = [int(x) for x in str(warp_version).split(".")[:2]] - if len(_parts) >= 2 and (_parts[0], _parts[1]) >= (1, 13): - import warnings - - warnings.warn( - f"[IsaacLab] warp {warp_version} is incompatible with " - f"omni.replicator.core. Warp >= 1.13 deprecates " - f"``warp.types.array`` with changed semantics, which causes " - f"CUDA error 700 during rendering and makes all warp kernel " - f"lookups fail. Run './isaaclab.sh --install' to install " - f"warp-lang<1.13.", - RuntimeWarning, - stacklevel=2, - ) - except (ValueError, AttributeError): - pass - # Hide the stop button in the toolbar self._hide_stop_button() # Set settings from the given rendering mode From b0acf8bd41704c75cab289834ccd7ddfee4e751c Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Wed, 22 Apr 2026 13:37:46 -0700 Subject: [PATCH 13/17] Updates omni.physics.tensor.api import --- .../migration/migrating_to_isaaclab_3-0.rst | 27 +++++++++++++++++++ .../core-concepts/sensors/contact_sensor.rst | 6 ++--- .../overview/core-concepts/sensors/imu.rst | 4 +-- source/isaaclab/docs/CHANGELOG.rst | 3 +++ .../articulation/base_articulation_data.py | 2 +- .../ray_caster/multi_mesh_ray_caster.py | 2 +- .../sensors/ray_caster/ray_cast_utils.py | 2 +- .../isaaclab/sensors/ray_caster/ray_caster.py | 2 +- source/isaaclab_newton/docs/CHANGELOG.rst | 3 +++ .../assets/articulation/articulation_data.py | 2 +- source/isaaclab_physx/docs/CHANGELOG.rst | 7 +++++ .../assets/articulation/articulation.py | 2 +- .../assets/articulation/articulation_data.py | 4 +-- .../deformable_object/deformable_object.py | 2 +- .../deformable_object_data.py | 2 +- .../assets/rigid_object/rigid_object.py | 2 +- .../assets/rigid_object/rigid_object_data.py | 2 +- .../rigid_object_collection.py | 2 +- .../rigid_object_collection_data.py | 2 +- .../sensors/contact_sensor/contact_sensor.py | 2 +- 20 files changed, 60 insertions(+), 20 deletions(-) diff --git a/docs/source/migration/migrating_to_isaaclab_3-0.rst b/docs/source/migration/migrating_to_isaaclab_3-0.rst index 6855a4fda28a..d33f9d2956a5 100644 --- a/docs/source/migration/migrating_to_isaaclab_3-0.rst +++ b/docs/source/migration/migrating_to_isaaclab_3-0.rst @@ -1629,6 +1629,33 @@ Deprecated retargeters have been moved to ``isaaclab_teleop.deprecated.openxr.re compatibility. These will be removed in a future release. +PhysX Tensors API Module Path +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Recent Isaac Sim releases removed the internal ``impl`` submodule of +``omni.physics.tensors`` and now expose the PhysX Tensor API types +(``ArticulationView``, ``RigidBodyView``, ``SimulationView``, etc.) directly +under ``omni.physics.tensors.api``. Importing from the old path raises +``ModuleNotFoundError: No module named 'omni.physics.tensors.impl'`` at import +time. + +Isaac Lab has been updated to import from the new path. Downstream code +(custom assets, sensors, or scripts) that imported from the old path must be +updated: + +.. code-block:: python + + # Before (Isaac Lab 2.x / older Isaac Sim) + import omni.physics.tensors.impl.api as physx + + # After (Isaac Lab 3.x / current Isaac Sim) + import omni.physics.tensors.api as physx + +The class identities are unchanged — only the module path moved. Type hints +referencing the old path (``omni.physics.tensors.impl.api.ArticulationView``) +should be similarly updated to ``omni.physics.tensors.api.ArticulationView``. + + Need Help? ~~~~~~~~~~ diff --git a/docs/source/overview/core-concepts/sensors/contact_sensor.rst b/docs/source/overview/core-concepts/sensors/contact_sensor.rst index 4ddd2d10c077..ebd81e60a54d 100644 --- a/docs/source/overview/core-concepts/sensors/contact_sensor.rst +++ b/docs/source/overview/core-concepts/sensors/contact_sensor.rst @@ -55,7 +55,7 @@ Here, we print both the net contact force and the filtered force matrix for each ------------------------------- Contact sensor @ '/World/envs/env_.*/Robot/LF_FOOT': - view type : + view type : update period (s) : 0.0 number of bodies : 1 body names : ['LF_FOOT'] @@ -64,7 +64,7 @@ Here, we print both the net contact force and the filtered force matrix for each Received contact force of: tensor([[[-1.3923e-05, 1.5727e-04, 1.1032e+02]]], device='cuda:0') ------------------------------- Contact sensor @ '/World/envs/env_.*/Robot/RF_FOOT': - view type : + view type : update period (s) : 0.0 number of bodies : 1 body names : ['RF_FOOT'] @@ -85,7 +85,7 @@ Notice that even with filtering, both sensors report the net contact force actin ------------------------------- Contact sensor @ '/World/envs/env_.*/Robot/.*H_FOOT': - view type : + view type : update period (s) : 0.0 number of bodies : 2 body names : ['LH_FOOT', 'RH_FOOT'] diff --git a/docs/source/overview/core-concepts/sensors/imu.rst b/docs/source/overview/core-concepts/sensors/imu.rst index fe429f77a2b3..5435bf227f5d 100644 --- a/docs/source/overview/core-concepts/sensors/imu.rst +++ b/docs/source/overview/core-concepts/sensors/imu.rst @@ -61,7 +61,7 @@ The oscillations in the values reported by the sensor are a direct result of of .. code-block:: bash Imu sensor @ '/World/envs/env_.*/Robot/LF_FOOT': - view type : + view type : update period (s) : 0.0 number of sensors : 1 @@ -71,7 +71,7 @@ The oscillations in the values reported by the sensor are a direct result of of Received angular acceleration: tensor([[-0.0389, -0.0262, -0.0045]], device='cuda:0') ------------------------------- Imu sensor @ '/World/envs/env_.*/Robot/RF_FOOT': - view type : + view type : update period (s) : 0.0 number of sensors : 1 diff --git a/source/isaaclab/docs/CHANGELOG.rst b/source/isaaclab/docs/CHANGELOG.rst index 0251fcb03ca2..c0b3403fbf0e 100644 --- a/source/isaaclab/docs/CHANGELOG.rst +++ b/source/isaaclab/docs/CHANGELOG.rst @@ -17,6 +17,9 @@ Changed global (world-frame) and local (body-frame) buffers. A new :meth:`~isaaclab.utils.wrench_composer.WrenchComposer.compose_to_body_frame` method rotates global forces/torques into the body frame at apply time using the current body orientation, then sums with local forces/torques. +* Updated imports of the PhysX tensors API in the ray caster sensors from + ``omni.physics.tensors.impl.api`` to ``omni.physics.tensors.api`` to track the upstream + Isaac Sim module relocation (the ``impl`` submodule was removed). Deprecated ^^^^^^^^^^ diff --git a/source/isaaclab/isaaclab/assets/articulation/base_articulation_data.py b/source/isaaclab/isaaclab/assets/articulation/base_articulation_data.py index 61ee8f28e7a9..01dab490183a 100644 --- a/source/isaaclab/isaaclab/assets/articulation/base_articulation_data.py +++ b/source/isaaclab/isaaclab/assets/articulation/base_articulation_data.py @@ -614,7 +614,7 @@ def body_incoming_joint_wrench_b(self) -> wp.array: underlying `PhysX Tensor API`_. .. _PhysX documentation: https://nvidia-omniverse.github.io/PhysX/physx/5.5.1/docs/Articulations.html#link-incoming-joint-force - .. _PhysX Tensor API: https://docs.omniverse.nvidia.com/kit/docs/omni_physics/latest/extensions/runtime/source/omni.physics.tensors/docs/api/python.html#omni.physics.tensors.impl.api.ArticulationView.get_link_incoming_joint_force + .. _PhysX Tensor API: https://docs.omniverse.nvidia.com/kit/docs/omni_physics/latest/extensions/runtime/source/omni.physics.tensors/docs/api/python.html#omni.physics.tensors.api.ArticulationView.get_link_incoming_joint_force """ raise NotImplementedError diff --git a/source/isaaclab/isaaclab/sensors/ray_caster/multi_mesh_ray_caster.py b/source/isaaclab/isaaclab/sensors/ray_caster/multi_mesh_ray_caster.py index 002456b6e101..f9a4f97923dd 100644 --- a/source/isaaclab/isaaclab/sensors/ray_caster/multi_mesh_ray_caster.py +++ b/source/isaaclab/isaaclab/sensors/ray_caster/multi_mesh_ray_caster.py @@ -14,7 +14,7 @@ import trimesh import warp as wp -import omni.physics.tensors.impl.api as physx +import omni.physics.tensors.api as physx import isaaclab.sim as sim_utils from isaaclab.sim.views import XformPrimView diff --git a/source/isaaclab/isaaclab/sensors/ray_caster/ray_cast_utils.py b/source/isaaclab/isaaclab/sensors/ray_caster/ray_cast_utils.py index ac503b28bf52..81e4c6af0472 100644 --- a/source/isaaclab/isaaclab/sensors/ray_caster/ray_cast_utils.py +++ b/source/isaaclab/isaaclab/sensors/ray_caster/ray_cast_utils.py @@ -10,7 +10,7 @@ import torch import warp as wp -import omni.physics.tensors.impl.api as physx +import omni.physics.tensors.api as physx from isaaclab.sim.views import XformPrimView diff --git a/source/isaaclab/isaaclab/sensors/ray_caster/ray_caster.py b/source/isaaclab/isaaclab/sensors/ray_caster/ray_caster.py index 89cc9aaa674f..bb2f17ff67d0 100644 --- a/source/isaaclab/isaaclab/sensors/ray_caster/ray_caster.py +++ b/source/isaaclab/isaaclab/sensors/ray_caster/ray_caster.py @@ -13,7 +13,7 @@ import torch import warp as wp -import omni.physics.tensors.impl.api as physx +import omni.physics.tensors.api as physx from pxr import Gf, Usd, UsdGeom, UsdPhysics import isaaclab.sim as sim_utils diff --git a/source/isaaclab_newton/docs/CHANGELOG.rst b/source/isaaclab_newton/docs/CHANGELOG.rst index f8f670baf455..7c8ebea1ffb3 100644 --- a/source/isaaclab_newton/docs/CHANGELOG.rst +++ b/source/isaaclab_newton/docs/CHANGELOG.rst @@ -11,6 +11,9 @@ Changed :class:`~isaaclab_newton.assets.RigidObject`, and :class:`~isaaclab_newton.assets.RigidObjectCollection` to use the dual-buffer :class:`~isaaclab.utils.wrench_composer.WrenchComposer`. Composed wrenches are applied after body-frame composition. +* Updated the PhysX Tensor API docstring link in :class:`~isaaclab_newton.assets.ArticulationData` + from ``omni.physics.tensors.impl.api`` to ``omni.physics.tensors.api`` to track the upstream + Isaac Sim module relocation (the ``impl`` submodule was removed). 0.5.18 (2026-04-21) diff --git a/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation_data.py b/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation_data.py index 4dc9507dbe59..0c4cdc68a353 100644 --- a/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation_data.py +++ b/source/isaaclab_newton/isaaclab_newton/assets/articulation/articulation_data.py @@ -803,7 +803,7 @@ def body_incoming_joint_wrench_b(self) -> wp.array: underlying `PhysX Tensor API`_. .. _PhysX documentation: https://nvidia-omniverse.github.io/PhysX/physx/5.5.1/docs/Articulations.html#link-incoming-joint-force - .. _PhysX Tensor API: https://docs.omniverse.nvidia.com/kit/docs/omni_physics/latest/extensions/runtime/source/omni.physics.tensors/docs/api/python.html#omni.physics.tensors.impl.api.ArticulationView.get_link_incoming_joint_force + .. _PhysX Tensor API: https://docs.omniverse.nvidia.com/kit/docs/omni_physics/latest/extensions/runtime/source/omni.physics.tensors/docs/api/python.html#omni.physics.tensors.api.ArticulationView.get_link_incoming_joint_force """ raise NotImplementedError("Not implemented for Newton") diff --git a/source/isaaclab_physx/docs/CHANGELOG.rst b/source/isaaclab_physx/docs/CHANGELOG.rst index 17ed090ebb30..de166e4d60c6 100644 --- a/source/isaaclab_physx/docs/CHANGELOG.rst +++ b/source/isaaclab_physx/docs/CHANGELOG.rst @@ -4,6 +4,13 @@ Changelog 0.5.22 (2026-04-22) ~~~~~~~~~~~~~~~~~~~ +Changed +^^^^^^^ + +* Updated imports of the PhysX tensors API from ``omni.physics.tensors.impl.api`` to + ``omni.physics.tensors.api`` to track the upstream Isaac Sim module relocation + (the ``impl`` submodule was removed). + Fixed ^^^^^ diff --git a/source/isaaclab_physx/isaaclab_physx/assets/articulation/articulation.py b/source/isaaclab_physx/isaaclab_physx/assets/articulation/articulation.py index 3b403ee8c6d4..8ed288a4d9d2 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/articulation/articulation.py +++ b/source/isaaclab_physx/isaaclab_physx/assets/articulation/articulation.py @@ -35,7 +35,7 @@ from .articulation_data import ArticulationData if TYPE_CHECKING: - import omni.physics.tensors.impl.api as physx + import omni.physics.tensors.api as physx from isaaclab.assets.articulation.articulation_cfg import ArticulationCfg diff --git a/source/isaaclab_physx/isaaclab_physx/assets/articulation/articulation_data.py b/source/isaaclab_physx/isaaclab_physx/assets/articulation/articulation_data.py index 42190a28f473..b9955c042785 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/articulation/articulation_data.py +++ b/source/isaaclab_physx/isaaclab_physx/assets/articulation/articulation_data.py @@ -21,7 +21,7 @@ from isaaclab_physx.physics import PhysxManager as SimulationManager if TYPE_CHECKING: - import omni.physics.tensors.impl.api as physx + import omni.physics.tensors.api as physx # import logger logger = logging.getLogger(__name__) @@ -763,7 +763,7 @@ def body_incoming_joint_wrench_b(self) -> wp.array: `PhysX Tensor API`_. .. _`PhysX documentation`: https://nvidia-omniverse.github.io/PhysX/physx/5.5.1/docs/Articulations.html#link-incoming-joint-force - .. _`PhysX Tensor API`: https://docs.omniverse.nvidia.com/kit/docs/omni_physics/latest/extensions/runtime/source/omni.physics.tensors/docs/api/python.html#omni.physics.tensors.impl.api.ArticulationView.get_link_incoming_joint_force + .. _`PhysX Tensor API`: https://docs.omniverse.nvidia.com/kit/docs/omni_physics/latest/extensions/runtime/source/omni.physics.tensors/docs/api/python.html#omni.physics.tensors.api.ArticulationView.get_link_incoming_joint_force """ if self._body_incoming_joint_wrench_b.timestamp < self._sim_timestamp: diff --git a/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object.py b/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object.py index 1fb6a396c562..3bc6ed2b8b25 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object.py +++ b/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object.py @@ -14,7 +14,7 @@ import torch import warp as wp -import omni.physics.tensors.impl.api as physx +import omni.physics.tensors.api as physx from pxr import UsdShade import isaaclab.sim as sim_utils diff --git a/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object_data.py b/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object_data.py index 9aa38ff7778d..ccac2cf05b71 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object_data.py +++ b/source/isaaclab_physx/isaaclab_physx/assets/deformable_object/deformable_object_data.py @@ -7,7 +7,7 @@ import warp as wp -import omni.physics.tensors.impl.api as physx +import omni.physics.tensors.api as physx from isaaclab.utils.buffers import TimestampedBufferWarp as TimestampedBuffer diff --git a/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object.py b/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object.py index 8aa7dbd3f4f3..25abb8fd22a3 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object.py +++ b/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object.py @@ -26,7 +26,7 @@ from .rigid_object_data import RigidObjectData if TYPE_CHECKING: - import omni.physics.tensors.impl.api as physx + import omni.physics.tensors.api as physx from isaaclab.assets.rigid_object.rigid_object_cfg import RigidObjectCfg diff --git a/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object_data.py b/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object_data.py index 0a6157585c59..2bd71b799b7e 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object_data.py +++ b/source/isaaclab_physx/isaaclab_physx/assets/rigid_object/rigid_object_data.py @@ -20,7 +20,7 @@ from isaaclab_physx.physics import PhysxManager as SimulationManager if TYPE_CHECKING: - import omni.physics.tensors.impl.api as physx + import omni.physics.tensors.api as physx # import logger logger = logging.getLogger(__name__) diff --git a/source/isaaclab_physx/isaaclab_physx/assets/rigid_object_collection/rigid_object_collection.py b/source/isaaclab_physx/isaaclab_physx/assets/rigid_object_collection/rigid_object_collection.py index 3518aceac1d9..9d4d6b26979f 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/rigid_object_collection/rigid_object_collection.py +++ b/source/isaaclab_physx/isaaclab_physx/assets/rigid_object_collection/rigid_object_collection.py @@ -15,7 +15,7 @@ import torch import warp as wp -import omni.physics.tensors.impl.api as physx +import omni.physics.tensors.api as physx from pxr import UsdPhysics import isaaclab.sim as sim_utils diff --git a/source/isaaclab_physx/isaaclab_physx/assets/rigid_object_collection/rigid_object_collection_data.py b/source/isaaclab_physx/isaaclab_physx/assets/rigid_object_collection/rigid_object_collection_data.py index a0bdec5af8c9..dbb3b8523897 100644 --- a/source/isaaclab_physx/isaaclab_physx/assets/rigid_object_collection/rigid_object_collection_data.py +++ b/source/isaaclab_physx/isaaclab_physx/assets/rigid_object_collection/rigid_object_collection_data.py @@ -20,7 +20,7 @@ from isaaclab_physx.physics import PhysxManager as SimulationManager if TYPE_CHECKING: - import omni.physics.tensors.impl.api as physx + import omni.physics.tensors.api as physx # import logger logger = logging.getLogger(__name__) diff --git a/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor.py b/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor.py index e4cd2e97429f..300898ab765d 100644 --- a/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor.py +++ b/source/isaaclab_physx/isaaclab_physx/sensors/contact_sensor/contact_sensor.py @@ -14,7 +14,7 @@ import torch import warp as wp -import omni.physics.tensors.impl.api as physx +import omni.physics.tensors.api as physx import isaaclab.sim as sim_utils from isaaclab.app.settings_manager import get_settings_manager From f0d3c14d1326f7ce1e094c5079b7f580fb49846a Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Wed, 22 Apr 2026 14:15:34 -0700 Subject: [PATCH 14/17] revert sync calls in camera/render classes --- source/isaaclab/isaaclab/sensors/camera/camera.py | 8 -------- .../isaaclab_physx/renderers/isaac_rtx_renderer.py | 7 ------- 2 files changed, 15 deletions(-) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 4f0fde7d8abe..1c96e93798d3 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -461,14 +461,6 @@ def _update_buffers_impl(self, env_mask: wp.array): self._renderer.update_transforms() self._renderer.render(self._render_data) - - # Synchronize warp's CUDA stream before handing GPU buffers to torch. - # render() launches warp kernels that write into the output tensors; - # without an explicit sync, torch may read the buffers on a different - # CUDA stream before the warp kernels finish, causing illegal-memory- - # access errors (observed with newer Isaac Sim nightly images). - wp.synchronize() - self._renderer.read_output(self._render_data, self._data) """ diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index fedb1e2db4a1..a2b20e9c532a 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -319,13 +319,6 @@ def tiling_grid_shape(): device=sensor.device, ) - # Synchronize warp's CUDA stream before any torch operations read - # the output buffer. The reshape kernel runs on warp's stream; - # torch depth-clipping below runs on torch's stream. Without this - # barrier the two streams race, corrupting the CUDA context on - # newer Isaac Sim nightly images (>= 04/14). - wp.synchronize() - # alias rgb as first 3 channels of rgba if data_type == "rgba" and "rgb" in cfg.data_types: output_data["rgb"] = output_data["rgba"][..., :3] From 01fc29f4bd2a515268d8a595ffaafefc2efe2f25 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Wed, 22 Apr 2026 16:56:58 -0700 Subject: [PATCH 15/17] fix pinocchio import --- source/isaaclab/config/extension.toml | 2 +- source/isaaclab/docs/CHANGELOG.rst | 15 +++++ .../isaaclab/isaaclab/cli/commands/install.py | 65 +++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) diff --git a/source/isaaclab/config/extension.toml b/source/isaaclab/config/extension.toml index 7092d52f3ff7..86024cf07e6a 100644 --- a/source/isaaclab/config/extension.toml +++ b/source/isaaclab/config/extension.toml @@ -1,7 +1,7 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "4.6.10" +version = "4.6.11" # Description title = "Isaac Lab framework for Robot Learning" diff --git a/source/isaaclab/docs/CHANGELOG.rst b/source/isaaclab/docs/CHANGELOG.rst index c0b3403fbf0e..8a0b38e4f670 100644 --- a/source/isaaclab/docs/CHANGELOG.rst +++ b/source/isaaclab/docs/CHANGELOG.rst @@ -1,6 +1,21 @@ Changelog --------- +4.6.11 (2026-04-22) +~~~~~~~~~~~~~~~~~~~ + +Fixed +^^^^^ + +* Fixed ``isaaclab.sh --install`` leaving ``pinocchio`` uninstalled on top of recent Isaac Sim + base images that preinstall ``pin-pink`` in the kit's bundled ``site-packages`` without its + ``pin`` (cmeel pinocchio) dependency. Pip treats the ``pin-pink`` requirement as already + satisfied and skips the transitive ``pin`` resolve, so the pink IK controller and its tests + fail to import. ``isaaclab.cli.commands.install`` now probes ``import pinocchio`` after + installing the Isaac Lab submodules and force-reinstalls the cmeel ``pin``/``pin-pink``/ + ``daqp`` stack when the probe fails. + + 4.6.10 (2026-04-22) ~~~~~~~~~~~~~~~~~~~ diff --git a/source/isaaclab/isaaclab/cli/commands/install.py b/source/isaaclab/isaaclab/cli/commands/install.py index 2e402a72b792..fb903d59573b 100644 --- a/source/isaaclab/isaaclab/cli/commands/install.py +++ b/source/isaaclab/isaaclab/cli/commands/install.py @@ -130,6 +130,66 @@ def _maybe_uninstall_prebundled_torch( ) +# Pinocchio stack required by isaaclab.controllers.pink_ik. Installed via the cmeel +# ``pin`` wheel, which provides the ``pinocchio`` Python module under +# ``cmeel.prefix/lib/python3.12/site-packages/`` and registers it on sys.path via a +# ``cmeel.pth`` hook. +_PINOCCHIO_STACK = ("pin", "pin-pink==3.1.0", "daqp==0.7.2") + + +def _ensure_pinocchio_installed(python_exe: str, pip_cmd: list[str], *, probe_env: dict[str, str]) -> None: + """Ensure ``pinocchio`` is importable, force-installing the cmeel pin stack if not. + + Recent Isaac Sim base images preinstall ``pin-pink`` into the kit's bundled + ``site-packages`` without its ``pin`` (cmeel pinocchio) dependency. Pip then + treats the ``pin-pink`` requirement as satisfied and never resolves the + transitive ``pin`` dep, leaving ``import pinocchio`` broken. This probes + for ``pinocchio`` at runtime and force-installs the cmeel stack when needed + so the pink IK controller and its tests work out of the box. + + Only runs on Linux x86_64 / aarch64 — the same platforms that have + pinocchio listed in :mod:`isaaclab`'s ``setup.py`` install requirements. + Skipped on Windows and macOS (no cmeel wheels) and on unsupported + architectures so the rest of ``--install`` behaves unchanged there. + + A force-reinstall failure (e.g. transient PyPI / NVIDIA Artifactory issue) + is logged as a warning rather than aborting ``--install``: pinocchio is only + needed by the optional pink IK controller, so the rest of Isaac Lab should + still install cleanly. + """ + import platform + + if platform.system() != "Linux": + return + if platform.machine() not in {"x86_64", "AMD64", "aarch64", "arm64"}: + return + + probe_result = run_command( + [python_exe, "-c", "import pinocchio"], + env=probe_env, + check=False, + capture_output=True, + text=True, + ) + if probe_result.returncode == 0: + return + + print_info( + "``import pinocchio`` failed — the kit-bundled ``pin-pink`` likely shipped without its" + " ``pin`` dep. Force-installing the cmeel pinocchio stack." + ) + install_result = run_command( + pip_cmd + ["install", "--upgrade", "--force-reinstall", *_PINOCCHIO_STACK], + check=False, + ) + if install_result.returncode != 0: + print_warning( + "Force-installing the cmeel pinocchio stack failed (returncode " + f"{install_result.returncode}). The pink IK controller and its tests will not be" + " usable until ``pin pin-pink==3.1.0 daqp==0.7.2`` is installed manually." + ) + + def _ensure_cuda_torch() -> None: """Ensure correct PyTorch and CUDA versions are installed.""" python_exe = extract_python_exe() @@ -643,6 +703,11 @@ def command_install(install_type: str = "all") -> None: # Can prevent that from happening. _ensure_cuda_torch() + # Ensure ``pinocchio`` is actually importable. The kit-bundled ``pin-pink`` in recent + # Isaac Sim images ships without its cmeel ``pin`` dependency, so the transitive + # requirement from ``pip install -e source/isaaclab`` can be silently skipped. + _ensure_pinocchio_installed(python_exe, pip_cmd, probe_env=probe_env) + # Repoint prebundled packages in Isaac Sim to the environment's copies so # the active venv/conda versions are always loaded regardless of PYTHONPATH # ordering (e.g. torch+cu130 in venv vs torch+cu128 in prebundle on aarch64). From 379915312e2fb8c4c8ff74273423a0e7ad269421 Mon Sep 17 00:00:00 2001 From: Paul Reeves Date: Thu, 23 Apr 2026 11:19:32 +0000 Subject: [PATCH 16/17] fix(isaaclab_physx): only unsubscribe _on_stop to preserve warm_start callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calling enable_all_default_callbacks(False) disables the warm_start/PLAY callback in addition to _on_stop. The warm_start callback initialises the rendering pipeline on sim.reset(); without it, tiled-camera and visualization- marker tests produce black frames and wrong outputs. Replace the blanket disable with a targeted unsubscribe of _default_callback_on_stop only — that is the sole callback that calls invalidate_physics() and wrecks the shared omni.physics.tensors view. All other callbacks (warm_start, stage_open, stage_close) are left intact. Fixes the following CI failures in PR #5358: - isaaclab_visualizers - isaaclab (core) [1/3] — test_tiled_camera, test_multi_tiled_camera, test_first_frame_textured_rendering, test_visualization_markers, test_ray_caster_camera Co-Authored-By: Claude Sonnet 4.6 --- .../isaaclab_physx/isaaclab_physx/__init__.py | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/source/isaaclab_physx/isaaclab_physx/__init__.py b/source/isaaclab_physx/isaaclab_physx/__init__.py index b9e1cbae22a7..1454dfcdbde0 100644 --- a/source/isaaclab_physx/isaaclab_physx/__init__.py +++ b/source/isaaclab_physx/isaaclab_physx/__init__.py @@ -49,9 +49,11 @@ def _patch_isaacsim_simulation_manager(): getDofVelocities`` on the very first ``scene.update()`` after ``sim.reset()``. - To prevent this, we disable the original class's default callbacks here - *before* swapping the module attribute, so :class:`PhysxManager` becomes - the single owner of the simulation lifecycle. + To prevent this, we unsubscribe only the original class's ``_on_stop`` + callback *before* swapping the module attribute. Other callbacks + (warm_start/PLAY, stage_open, stage_close) are left intact — in particular + warm_start must fire so the rendering pipeline initialises correctly on + ``sim.reset()``. Disabling it causes tiled-camera RGB output to stay black. """ # Force-import Isaac Sim's SimulationManager before patching so that the # subscriptions registered during its module/extension startup are taken @@ -67,28 +69,25 @@ def _patch_isaacsim_simulation_manager(): original_module = sys.modules["isaacsim.core.simulation_manager"] from .physics.physx_manager import PhysxManager, IsaacEvents - # Tear down the original Isaac Sim SimulationManager's default timeline / - # stage subscriptions so they cannot invalidate the omni.physics.tensors - # view that PhysxManager owns. ``enable_all_default_callbacks(False)`` - # covers warm_start (PLAY), on_stop (STOP), stage_open (OPENED) and - # stage_close (CLOSED). Older Isaac Sim builds may not expose this API, so - # fall back gracefully. + # Only unsubscribe _on_stop — that is the sole callback that calls + # ``invalidate_physics()`` and wrecks the shared omni.physics.tensors view. + # Leaving warm_start (PLAY) intact ensures the rendering pipeline initialises + # correctly when ``sim.reset()`` fires the play event; disabling it causes + # tiled-camera RGB to stay black (see isaaclab_visualizers CI failure). original_class = getattr(original_module, "SimulationManager", None) if original_class is not None and original_class is not PhysxManager: - try: - original_class.enable_all_default_callbacks(False) - except Exception: - # Defensive: API changed or original class never finished startup. - # Manually clear the subscription handles if they exist so any - # remaining references go through the dead-callback path. - for attr in ( - "_default_callback_warm_start", - "_default_callback_on_stop", - "_default_callback_stage_open", - "_default_callback_stage_close", - ): - if hasattr(original_class, attr): - setattr(original_class, attr, None) + if hasattr(original_class, "_default_callback_on_stop"): + # Carb subscription objects unsubscribe on destruction — setting to + # None drops the reference and silently cancels the subscription. + original_class._default_callback_on_stop = None + else: + # _default_callback_on_stop not found (API change). Fall back to + # disabling all callbacks. Note: this may cause tiled-camera black + # frames on newer Isaac Sim builds; the targeted fix above is preferred. + try: + original_class.enable_all_default_callbacks(False) + except Exception: + pass original_module.SimulationManager = PhysxManager original_module.IsaacEvents = IsaacEvents From b43b0b6bfe9dcc405ce520cab39a02594dd97a87 Mon Sep 17 00:00:00 2001 From: Kelly Guo Date: Thu, 23 Apr 2026 10:03:19 -0700 Subject: [PATCH 17/17] Bump version to 4.6.13 in extension.toml Signed-off-by: Kelly Guo --- source/isaaclab/config/extension.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/isaaclab/config/extension.toml b/source/isaaclab/config/extension.toml index 3086a2b93c88..fad2b53b6c5e 100644 --- a/source/isaaclab/config/extension.toml +++ b/source/isaaclab/config/extension.toml @@ -1,7 +1,7 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "4.6.12" +version = "4.6.13" # Description title = "Isaac Lab framework for Robot Learning"