diff --git a/scripts/benchmarks/nsys_trace.json b/scripts/benchmarks/nsys_trace.json new file mode 100644 index 00000000000..b4020521e91 --- /dev/null +++ b/scripts/benchmarks/nsys_trace.json @@ -0,0 +1,167 @@ +[ + { + "_comment": "=== PYTHON IMPORTS (tracks module loading) ===", + "domain": "Python-Imports", + "color": "0x9E9E9E", + "module": "importlib", + "functions": ["import_module"] + }, + { + "domain": "Python-Imports", + "color": "0x9E9E9E", + "module": "importlib._bootstrap", + "functions": ["_find_and_load", "_load_unlocked"] + }, + { + "_comment": "=== WARP (verified working) ===", + "domain": "Warp", + "color": "0xF44336", + "module": "warp", + "functions": ["launch", "synchronize", "copy", "zeros", "empty"] + }, + { + "_comment": "=== PYTORCH (verified working) ===", + "domain": "PyTorch", + "color": "0x3F51B5", + "module": "torch.autograd", + "functions": ["backward"] + }, + { + "_comment": "=== USD (verified working) ===", + "domain": "USD", + "color": "0x795548", + "module": "pxr.UsdGeom", + "functions": ["Xform", "Mesh", "Sphere", "Cube"] + }, + { + "_comment": "=== ISAACLAB ENVIRONMENTS ===", + "domain": "IsaacLab-Env", + "color": "0x9C27B0", + "module": "isaaclab.envs.manager_based_rl_env", + "functions": [ + {"function": "ManagerBasedRLEnv.__init__", "color": "0x9C27B0"}, + {"function": "ManagerBasedRLEnv.step", "color": "0xAB47BC"}, + {"function": "ManagerBasedRLEnv.reset", "color": "0xBA68C8"} + ] + }, + { + "domain": "IsaacLab-Env", + "color": "0x9C27B0", + "module": "isaaclab.envs.manager_based_env", + "functions": [ + {"function": "ManagerBasedEnv.__init__", "color": "0x9C27B0"}, + {"function": "ManagerBasedEnv.step", "color": "0xAB47BC"}, + {"function": "ManagerBasedEnv.reset", "color": "0xBA68C8"}, + {"function": "ManagerBasedEnv._reset_idx", "color": "0xCE93D8"} + ] + }, + { + "_comment": "=== ISAACLAB SIMULATION ===", + "domain": "IsaacLab-Sim", + "color": "0x4CAF50", + "module": "isaaclab.sim.simulation_context", + "functions": [ + {"function": "SimulationContext.__init__", "color": "0x4CAF50"}, + {"function": "SimulationContext.reset", "color": "0x66BB6A"}, + {"function": "SimulationContext.step", "color": "0x81C784"} + ] + }, + { + "_comment": "=== ISAACLAB SCENE ===", + "domain": "IsaacLab-Scene", + "color": "0x2196F3", + "module": "isaaclab.scene.interactive_scene", + "functions": [ + {"function": "InteractiveScene.__init__", "color": "0x2196F3"}, + {"function": "InteractiveScene.reset", "color": "0x42A5F5"}, + {"function": "InteractiveScene.write_data_to_sim", "color": "0x64B5F6"}, + {"function": "InteractiveScene.update", "color": "0x90CAF9"} + ] + }, + { + "_comment": "=== ISAACLAB MANAGERS ===", + "domain": "IsaacLab-Managers", + "color": "0xFF9800", + "module": "isaaclab.managers.observation_manager", + "functions": [ + {"function": "ObservationManager.__init__", "color": "0xFF9800"}, + {"function": "ObservationManager.reset", "color": "0xFFA726"}, + {"function": "ObservationManager.compute", "color": "0xFFB74D"} + ] + }, + { + "domain": "IsaacLab-Managers", + "color": "0xFF9800", + "module": "isaaclab.managers.action_manager", + "functions": [ + {"function": "ActionManager.__init__", "color": "0xFF9800"}, + {"function": "ActionManager.reset", "color": "0xFFA726"}, + {"function": "ActionManager.process_action", "color": "0xFFB74D"}, + {"function": "ActionManager.apply_action", "color": "0xFFCC80"} + ] + }, + { + "domain": "IsaacLab-Managers", + "color": "0xFF9800", + "module": "isaaclab.managers.reward_manager", + "functions": [ + {"function": "RewardManager.__init__", "color": "0xFF9800"}, + {"function": "RewardManager.reset", "color": "0xFFA726"}, + {"function": "RewardManager.compute", "color": "0xFFB74D"} + ] + }, + { + "_comment": "=== ISAACLAB ASSETS ===", + "domain": "IsaacLab-Assets", + "color": "0x607D8B", + "module": "isaaclab.assets.articulation.articulation", + "functions": [ + {"function": "Articulation.__init__", "color": "0x607D8B"}, + {"function": "Articulation.reset", "color": "0x78909C"}, + {"function": "Articulation.write_data_to_sim", "color": "0x90A4AE"}, + {"function": "Articulation.update", "color": "0xB0BEC5"} + ] + }, + { + "_comment": "=== ISAACLAB SENSORS ===", + "domain": "IsaacLab-Sensors", + "color": "0x00BCD4", + "module": "isaaclab.sensors.camera.camera", + "functions": [ + {"function": "Camera.__init__", "color": "0x00BCD4"}, + {"function": "Camera.reset", "color": "0x26C6DA"}, + {"function": "Camera.update", "color": "0x4DD0E1"} + ] + }, + { + "_comment": "=== RSL-RL ===", + "domain": "RSL-RL", + "color": "0x673AB7", + "module": "rsl_rl.runners.on_policy_runner", + "functions": [ + {"function": "OnPolicyRunner.__init__", "color": "0x673AB7"}, + {"function": "OnPolicyRunner.learn", "color": "0x7E57C2"} + ] + }, + { + "domain": "RSL-RL", + "color": "0x673AB7", + "module": "rsl_rl.algorithms.ppo", + "functions": [ + {"function": "PPO.__init__", "color": "0x673AB7"}, + {"function": "PPO.act", "color": "0x7E57C2"}, + {"function": "PPO.update", "color": "0x9575CD"} + ] + }, + { + "_comment": "=== NEWTON WARP RENDERER ===", + "domain": "NewtonWarpRenderer", + "color": "0xE91E63", + "module": "isaaclab_newton.renderers.newton_warp_renderer", + "functions": [ + {"function": "NewtonWarpRenderer.update_transforms", "color": "0x2196F3"}, + {"function": "NewtonWarpRenderer.render", "color": "0x4CAF50"}, + {"function": "NewtonWarpRenderer.read_output", "color": "0xFF9800"} + ] + } +] diff --git a/scripts/benchmarks/test/test_nsys_trace.py b/scripts/benchmarks/test/test_nsys_trace.py new file mode 100644 index 00000000000..787896fb253 --- /dev/null +++ b/scripts/benchmarks/test/test_nsys_trace.py @@ -0,0 +1,147 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sync tests for ``scripts/benchmarks/nsys_trace.json``. + +The JSON lists Python functions that nsys annotates during profiling via the +``--python-functions-trace`` flag. Entries reference IsaacLab (and adjacent) +source by dotted paths. These tests catch two kinds of drift: + +* Referenced functions that no longer resolve (hard failure) +* Covered classes that have public methods not listed in the JSON (warning — + signals that new coverage may need to be added) +""" + +from isaaclab.app import AppLauncher + +simulation_app = AppLauncher(headless=True).app + +"""Rest everything follows.""" + +import importlib +import inspect +import json +import warnings +from collections import defaultdict +from pathlib import Path + +import pytest + +TRACE_JSON_PATH = Path(__file__).resolve().parents[1] / "nsys_trace.json" + + +def _load_trace_entries() -> list[dict]: + """Return the parsed JSON entries from the trace file.""" + with TRACE_JSON_PATH.open() as f: + return json.load(f) + + +def _function_name_and_module(entry_module: str, func_spec) -> tuple[str, str]: + """Normalize a function spec to ``(module, dotted_name)``. + + ``func_spec`` may be a bare string or a dict that optionally overrides + ``module`` (per the nsys --python-functions-trace schema). + """ + if isinstance(func_spec, str): + return entry_module, func_spec + return func_spec.get("module", entry_module), func_spec["function"] + + +def _iter_function_pairs(entries: list[dict]) -> list[tuple[str, str]]: + """Yield ``(module, dotted_function_path)`` for every function in the JSON.""" + pairs: list[tuple[str, str]] = [] + for entry in entries: + entry_module = entry["module"] + for func_spec in entry["functions"]: + pairs.append(_function_name_and_module(entry_module, func_spec)) + return pairs + + +def _resolve(module_name: str, dotted_path: str): + """Import ``module_name`` and walk ``dotted_path`` via getattr.""" + obj = importlib.import_module(module_name) + for attr in dotted_path.split("."): + obj = getattr(obj, attr) + return obj + + +def _group_methods_by_class(pairs: list[tuple[str, str]]) -> dict[tuple[str, str], set[str]]: + """Group referenced method names by ``(module, class_name)``. + + Top-level functions (paths without a dot) are skipped — no class context. + """ + grouped: dict[tuple[str, str], set[str]] = defaultdict(set) + for module_name, dotted_path in pairs: + parts = dotted_path.split(".") + if len(parts) >= 2: + class_name, method_name = parts[0], parts[-1] + grouped[(module_name, class_name)].add(method_name) + return grouped + + +def _is_own_public_method(cls: type, name: str, member: object) -> bool: + """True if ``member`` is a public method defined directly on ``cls``.""" + if not inspect.isfunction(member): + return False + if name not in cls.__dict__: + return False + if name.startswith("_"): + return False + return True + + +_FUNCTION_PAIRS = _iter_function_pairs(_load_trace_entries()) + + +@pytest.mark.parametrize( + "module_name, dotted_path", + _FUNCTION_PAIRS, + ids=[f"{m}:{p}" for m, p in _FUNCTION_PAIRS], +) +def test_function_resolves(module_name: str, dotted_path: str): + """Every function referenced in the trace JSON must resolve to a callable. + + A missing reference silently loses profiling coverage, so this fails loudly. + Modules that aren't importable in the current environment (e.g. optional + RL frameworks) are skipped rather than failed. + """ + try: + importlib.import_module(module_name) + except ImportError as exc: + pytest.skip(f"Module '{module_name}' not importable here: {exc}") + + try: + resolved = _resolve(module_name, dotted_path) + except AttributeError as exc: + pytest.fail(f"'{module_name}.{dotted_path}' not found: {exc}") + + assert callable(resolved), f"'{module_name}.{dotted_path}' resolved but is not callable" + + +def test_warn_unreferenced_methods_on_covered_classes(): + """Emit a warning for each public method that isn't listed in the JSON. + + Scope: classes that already have at least one method referenced. If the + class gained a new public method since the JSON was last updated, it shows + up here as a nudge to add (or intentionally omit) it. Inherited, dunder, + and private methods are excluded to keep the signal actionable. + """ + grouped = _group_methods_by_class(_FUNCTION_PAIRS) + + for (module_name, class_name), referenced in sorted(grouped.items()): + try: + cls = _resolve(module_name, class_name) + except (ImportError, AttributeError): + continue + if not inspect.isclass(cls): + continue + + own_public = {name for name, member in inspect.getmembers(cls) if _is_own_public_method(cls, name, member)} + unreferenced = own_public - referenced + if unreferenced: + warnings.warn( + f"{module_name}.{class_name} has public methods not listed in nsys_trace.json: {sorted(unreferenced)}", + stacklevel=2, + )