diff --git a/.dockerignore b/.dockerignore
index b6687c95fa7..51d4ec61a45 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -28,3 +28,4 @@ _isaac_sim?
docker/.isaac-lab-docker-history
# ignore uv environment
env_isaaclab
+tools/wheel_builder/build/
diff --git a/.gitattributes b/.gitattributes
index e3c0ead689d..68372999154 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -12,3 +12,4 @@
*.hdf5 filter=lfs diff=lfs merge=lfs -text
*.bat text eol=crlf
+*.sh text eol=lf
diff --git a/.github/ISSUE_TEMPLATE/proposal.md b/.github/ISSUE_TEMPLATE/proposal.md
index c07d7f56dc8..a6dd4081906 100644
--- a/.github/ISSUE_TEMPLATE/proposal.md
+++ b/.github/ISSUE_TEMPLATE/proposal.md
@@ -26,8 +26,8 @@ A clear and concise description of any alternative solutions or features you've
Describe the versions where you are observing the missing feature in:
-- Isaac Lab Version: [e.g. 2.3.2]
-- Isaac Sim Version: [e.g. 5.1, this can be obtained by `cat ${ISAACSIM_PATH}/VERSION`]
+- Isaac Lab Version: [e.g. 3.0.0]
+- Isaac Sim Version: [e.g. 6.0, this can be obtained by `cat ${ISAACSIM_PATH}/VERSION`]
### Additional context
diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md
index 6e0582f3737..ff0ac5b8f12 100644
--- a/.github/ISSUE_TEMPLATE/question.md
+++ b/.github/ISSUE_TEMPLATE/question.md
@@ -17,5 +17,5 @@ For questions that are related to running and understanding Isaac Sim, please po
Describe the versions that you are currently using:
-- Isaac Lab Version: [e.g. 2.3.2]
-- Isaac Sim Version: [e.g. 5.1, this can be obtained by `cat ${ISAACSIM_PATH}/VERSION`]
+- Isaac Lab Version: [e.g. 3.0.0]
+- Isaac Sim Version: [e.g. 6.0, this can be obtained by `cat ${ISAACSIM_PATH}/VERSION`]
diff --git a/.github/actions/combine-results/action.yml b/.github/actions/combine-results/action.yml
deleted file mode 100644
index 8ed66e3b460..00000000000
--- a/.github/actions/combine-results/action.yml
+++ /dev/null
@@ -1,103 +0,0 @@
-# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
-# All rights reserved.
-#
-# SPDX-License-Identifier: BSD-3-Clause
-
-name: 'Combine XML Test Results'
-description: 'Combines multiple XML test result files into a single file'
-
-inputs:
- tests-dir:
- description: 'Directory containing test result files'
- default: 'tests'
- required: false
- output-file:
- description: 'Output combined XML file path'
- required: true
- reports-dir:
- description: 'Directory to store the combined results'
- default: 'reports'
- required: false
-
-runs:
- using: composite
- steps:
- - name: Combine XML Test Results
- shell: sh
- run: |
- # Function to combine multiple XML test results
- combine_xml_results() {
- local tests_dir="$1"
- local output_file="$2"
- local reports_dir="$3"
-
- echo "Combining test results from: $tests_dir"
- echo "Output file: $output_file"
- echo "Reports directory: $reports_dir"
-
- # Check if reports directory exists
- if [ ! -d "$reports_dir" ]; then
- echo "⚠️ Reports directory does not exist: $reports_dir"
- mkdir -p "$reports_dir"
- fi
-
- # Check if tests directory exists
- if [ ! -d "$tests_dir" ]; then
- echo "⚠️ Tests directory does not exist: $tests_dir"
- echo "Creating fallback XML..."
- echo 'Tests directory was not found' > "$output_file"
- return
- fi
-
- # Find all XML files in the tests directory
- echo "Searching for XML files in: $tests_dir"
- xml_files=$(find "$tests_dir" -name "*.xml" -type f 2>/dev/null | sort)
-
- if [ -z "$xml_files" ]; then
- echo "⚠️ No XML files found in: $tests_dir"
- echo "Creating fallback XML..."
- echo 'No XML test result files were found' > "$output_file"
- return
- fi
-
- # Count XML files found
- file_count=$(echo "$xml_files" | wc -l)
- echo "✅ Found $file_count XML file(s):"
- echo "$xml_files" | while read -r file; do
- echo " - $file ($(wc -c < "$file") bytes)"
- done
-
- # Create combined XML
- echo "🔄 Combining $file_count XML files..."
- echo '' > "$output_file"
- echo '' >> "$output_file"
-
- # Process each XML file
- combined_count=0
- echo "$xml_files" | while read -r file; do
- if [ -f "$file" ]; then
- echo " Processing: $file"
- # Remove XML declaration and outer testsuites wrapper from each file
- # Remove first line (XML declaration) and strip outer / tags
- sed '1d; s/^//; s/<\/testsuites>$//' "$file" >> "$output_file" 2>/dev/null || {
- echo " ⚠️ Warning: Could not process $file, skipping..."
- }
- combined_count=$((combined_count + 1))
- fi
- done
-
- echo '' >> "$output_file"
- echo "✅ Successfully combined $combined_count files into: $output_file"
-
- # Verify output file was created
- if [ -f "$output_file" ]; then
- echo "✅ Final output file created: $output_file"
- echo "📊 Output file size: $(wc -c < "$output_file") bytes"
- else
- echo "❌ Failed to create output file: $output_file"
- exit 1
- fi
- }
-
- # Call the function with provided parameters
- combine_xml_results "${{ inputs.tests-dir }}" "${{ inputs.output-file }}" "${{ inputs.reports-dir }}"
diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml
index 2db402d4204..7f88241cfb8 100644
--- a/.github/actions/docker-build/action.yml
+++ b/.github/actions/docker-build/action.yml
@@ -18,7 +18,7 @@ inputs:
required: true
dockerfile-path:
description: 'Path to Dockerfile'
- default: 'docker/Dockerfile.curobo'
+ default: 'docker/Dockerfile.base'
required: false
context-path:
description: 'Build context path'
@@ -52,13 +52,18 @@ runs:
local dockerfile_path="$4"
local context_path="$5"
+ # Skip build if image already exists locally (e.g. built by a prior job on the same runner)
+ if docker image inspect "$image_tag" > /dev/null 2>&1; then
+ echo "Image $image_tag already exists locally, skipping build."
+ return 0
+ fi
+
echo "Building Docker image: $image_tag"
echo "Using Dockerfile: $dockerfile_path"
echo "Build context: $context_path"
# Build Docker image
docker buildx build --progress=plain --platform linux/amd64 \
- -t isaac-lab-dev \
-t $image_tag \
--build-arg ISAACSIM_BASE_IMAGE_ARG="$isaacsim_base_image" \
--build-arg ISAACSIM_VERSION_ARG="$isaacsim_version" \
@@ -71,7 +76,8 @@ runs:
--load $context_path
echo "✅ Docker image built successfully: $image_tag"
- docker images | grep isaac-lab-dev
+ echo "Current local Docker images:"
+ docker images
}
# Call the function with provided parameters
diff --git a/.github/actions/ecr-build-push-pull/README.md b/.github/actions/ecr-build-push-pull/README.md
new file mode 100644
index 00000000000..2e306c978b8
--- /dev/null
+++ b/.github/actions/ecr-build-push-pull/README.md
@@ -0,0 +1,24 @@
+# ecr-build-push-pull
+
+Builds a Docker image and pushes it to ECR, or pulls it if the tag already exists.
+ECR is also used as the BuildKit layer cache.
+
+## Usage
+
+```yaml
+- uses: ./.github/actions/ecr-build-push-pull
+ with:
+ image-tag: ${{ env.DOCKER_IMAGE_TAG }}
+ isaacsim-base-image: nvcr.io/nvidia/isaac-sim
+ isaacsim-version: 6.0.0
+ dockerfile-path: docker/Dockerfile.base
+ cache-tag: cache-base
+ ecr-url: (optional, complete url for ECR storage)
+```
+
+## ECR URL resolution order
+
+1. `ecr-url` input
+2. `ECR_CACHE_URL` environment variable on the runner
+3. SSM parameter `/github-runner//ecr-cache-url`
+4. If none resolve, ECR is skipped and the image is built locally
diff --git a/.github/actions/ecr-build-push-pull/action.yml b/.github/actions/ecr-build-push-pull/action.yml
new file mode 100644
index 00000000000..b661d4b9fd6
--- /dev/null
+++ b/.github/actions/ecr-build-push-pull/action.yml
@@ -0,0 +1,321 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+name: 'ECR Build-Push-Pull'
+description: >
+ Builds a Docker image and pushes it to ECR, using ECR as the layer cache.
+ If the image already exists in ECR (same tag), pulls it instead of building.
+ Drop-in replacement for docker-build/action.yml with ECR-backed caching.
+
+inputs:
+ image-tag:
+ description: 'Tag for the Docker image (e.g. my-image:latest).'
+ required: true
+ isaacsim-base-image:
+ description: 'IsaacSim base image (passed as ISAACSIM_BASE_IMAGE_ARG build-arg).'
+ required: true
+ isaacsim-version:
+ description: 'IsaacSim version (passed as ISAACSIM_VERSION_ARG build-arg).'
+ required: true
+ dockerfile-path:
+ description: 'Path to Dockerfile, relative to the repository root'
+ default: 'docker/Dockerfile.base'
+ required: false
+ ecr-url:
+ description: >
+ ECR repository URL (e.g. "123456789.dkr.ecr.us-west-2.amazonaws.com/my-repo").
+ Resolved in the following order:
+ 1. ecr-url input, if provided.
+ 2. ECR_CACHE_URL environment variable on the runner.
+ 3. SSM parameter /github-runner//ecr-cache-url.
+ 4. If still empty, ECR cache is skipped and the image is built locally.
+ required: false
+ default: ''
+ cache-tag:
+ description: Tag used for the ECR layer cache image (e.g. "cache-base", "cache-curobo").
+ required: false
+ default: 'cache'
+runs:
+ using: composite
+ steps:
+
+ ##### 1: Setup docker config + Login to nvcr.io #####
+
+ # Create a temp docker config with credsStore disabled before any login.
+ # The runner's credential store backend is broken ("not implemented") and
+ # causes all docker login calls to fail unless we bypass it upfront.
+ # The temp config is exported as DOCKER_CONFIG so all subsequent steps
+ # (including ECR login in step 3) inherit it automatically.
+
+ - name: Setup docker config and login to nvcr.io
+ shell: bash
+ run: |
+ DOCKER_CONFIG_DIR=$(mktemp -d)
+ if [ -f "${HOME}/.docker/config.json" ]; then
+ python3 -c "import json; cfg=json.load(open('${HOME}/.docker/config.json')); cfg['credsStore']=''; cfg.pop('credHelpers',None); json.dump(cfg,open('${DOCKER_CONFIG_DIR}/config.json','w'))"
+ else
+ echo '{"credsStore":""}' > "${DOCKER_CONFIG_DIR}/config.json"
+ fi
+ echo "DOCKER_CONFIG=${DOCKER_CONFIG_DIR}" >> "$GITHUB_ENV"
+ export DOCKER_CONFIG="${DOCKER_CONFIG_DIR}"
+
+ if [ -n "${{ env.NGC_API_KEY }}" ]; then
+ echo "🔵 Logging into nvcr.io..."
+ docker login -u \$oauthtoken -p ${{ env.NGC_API_KEY }} nvcr.io
+ else
+ echo "🟠 NGC_API_KEY not set - skipping nvcr.io login (normal for fork PRs)"
+ fi
+
+ ##### 2: Resolve ECR URL #####
+
+ # Tries: explicit input >> ECR_CACHE_URL env var >> SSM parameter on EC2.
+ # Exports ECR_URL to GITHUB_ENV and sets output `available`.
+
+ - name: Resolve ECR URL
+ id: resolve-ecr
+ shell: bash
+ env:
+ INPUT_ECR_URL: ${{ inputs.ecr-url }}
+ run: |
+ ECR_URL="${INPUT_ECR_URL:-}"
+
+ if [ -z "${ECR_URL}" ]; then
+ echo "🔵 ecr-url input not set, trying ECR_CACHE_URL env var..."
+ ECR_URL="${ECR_CACHE_URL:-}"
+ [ -n "${ECR_URL}" ] && echo "🟢 Using ECR_CACHE_URL env var: ${ECR_URL}"
+ fi
+
+ if [ -z "${ECR_URL}" ]; then
+ echo "🔵 ECR_CACHE_URL env var not set, trying SSM..."
+ IMDS_TOKEN=$(curl -sf -X PUT "http://169.254.169.254/latest/api/token" \
+ -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") || true
+ INSTANCE_ID=$(curl -sf -H "X-aws-ec2-metadata-token: ${IMDS_TOKEN}" \
+ "http://169.254.169.254/latest/meta-data/instance-id") || true
+ INSTANCE_REGION=$(curl -sf -H "X-aws-ec2-metadata-token: ${IMDS_TOKEN}" \
+ "http://169.254.169.254/latest/meta-data/placement/region") || true
+
+ if [ -n "${INSTANCE_ID}" ]; then
+ ECR_URL=$(aws ssm get-parameter \
+ --name "/github-runner/${INSTANCE_ID}/ecr-cache-url" \
+ --region "${INSTANCE_REGION}" \
+ --query 'Parameter.Value' --output text 2>/dev/null) || ECR_URL=""
+ if [ -n "${ECR_URL}" ]; then
+ echo "🟢 Resolved ECR URL from SSM (/github-runner/${INSTANCE_ID}/ecr-cache-url): ${ECR_URL}"
+ else
+ echo "🔵 SSM parameter not found for instance ${INSTANCE_ID}"
+ fi
+ else
+ echo "🔵 Not running on EC2 or IMDS unavailable, skipping SSM lookup"
+ fi
+ fi
+
+ if [ -n "${ECR_URL}" ]; then
+ echo "ECR_URL=${ECR_URL}" >> "$GITHUB_ENV"
+ echo "available=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "🟠 ECR URL cannot be resolved. Building locally without ECR cache."
+ fi
+
+ ##### 3: Setup ECR authentication #####
+
+ # Validates the ECR URL, derives ECR image tags, and logs into ECR.
+ # DOCKER_CONFIG (with credsStore disabled) is already set by step 1.
+
+ - name: Setup ECR authentication
+ if: steps.resolve-ecr.outputs.available == 'true'
+ shell: bash
+ run: |
+ REGISTRY=$(echo "${ECR_URL}" | cut -d'/' -f1)
+ AWS_REGION=$(echo "${REGISTRY}" | sed 's/.*\.dkr\.ecr\.\(.*\)\.amazonaws\.com/\1/')
+
+ if [ "${AWS_REGION}" = "${REGISTRY}" ]; then
+ echo "🔴 Invalid ECR URL - cannot extract AWS region: ${ECR_URL}"
+ echo "🔴 Expected format: .dkr.ecr..amazonaws.com/"
+ exit 1
+ fi
+
+ ECR_TAG=$(echo "${{ inputs.image-tag }}" | tr ':/' '--')
+ ECR_IMAGE="${ECR_URL}:${ECR_TAG}"
+ CACHE_IMAGE="${ECR_URL}:${{ inputs.cache-tag }}"
+
+ echo "ECR_IMAGE=${ECR_IMAGE}" >> "$GITHUB_ENV"
+ echo "CACHE_IMAGE=${CACHE_IMAGE}" >> "$GITHUB_ENV"
+
+ echo "🔵 Logging into ECR registry..."
+ aws ecr get-login-password --region "${AWS_REGION}" | \
+ docker login --username AWS --password-stdin "${REGISTRY}"
+
+ ##### 4: Check if exact image exists in ECR #####
+
+ # Lightweight manifest check - fetches only the image manifest (~KB),
+ # not the actual layers. If the exact per-commit image already exists
+ # in ECR, sets output `hit: true` to skip all subsequent build/push steps.
+
+ - name: Check exact image in ECR
+ id: pull-exact
+ if: steps.resolve-ecr.outputs.available == 'true'
+ shell: bash
+ run: |
+ echo "🔵 Checking if commit-tagged image exists in ECR >> ${ECR_IMAGE}"
+ if docker manifest inspect "${ECR_IMAGE}" >/dev/null 2>&1; then
+ echo "🟢 Commit-tagged image found in ECR, skipping build!"
+ echo "hit=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "🟠 Image ${ECR_IMAGE} not found in ECR, will try deps-cache strategy..."
+ fi
+
+ # Pull the image when the manifest check succeeded but the image is not
+ # available locally (test jobs need it for `docker run`). Build jobs
+ # that only push to ECR will already have the image or don't need it.
+ - name: Pull exact image from ECR
+ if: steps.pull-exact.outputs.hit == 'true'
+ shell: bash
+ run: |
+ if docker image inspect "${{ inputs.image-tag }}" >/dev/null 2>&1; then
+ echo "🟢 Image already available locally, skipping pull"
+ else
+ echo "🔵 Pulling ${ECR_IMAGE} from ECR..."
+ docker pull "${ECR_IMAGE}"
+ docker tag "${ECR_IMAGE}" "${{ inputs.image-tag }}"
+ echo "🟢 Image pulled and tagged as ${{ inputs.image-tag }}"
+ fi
+
+ ##### 5: Check deps cache #####
+
+ # Hashes installation-relevant files + the base image digest to produce a stable
+ # deps- ECR tag. If the image exists in ECR, the build job succeeds
+ # immediately and test jobs pull the deps image with a source volume mount.
+
+ # Edit DEPS_FILES or DEPS_MANIFEST_PATTERN when install
+ # inputs change (new packages, new manifests, etc.).
+
+ - name: Check deps cache
+ id: deps-cache
+ if: steps.resolve-ecr.outputs.available == 'true' && steps.pull-exact.outputs.hit != 'true'
+ shell: bash
+ run: |
+ ##### Deps-hash configuration #####
+ # Exact files/dirs whose full content is hashed. The Dockerfile is first.
+ DEPS_FILES=(
+ "${{ inputs.dockerfile-path }}"
+ isaaclab.sh
+ environment.yml
+ source/isaaclab/isaaclab/cli
+ )
+ # Manifest files matched repo-wide via git ls-files.
+ DEPS_MANIFEST_PATTERN='(setup\.py|pyproject\.toml|setup\.cfg|extension\.toml|requirements[^/]*\.txt|uv\.lock)$'
+
+ # Resolve the actual base image digest so a new push of a mutable tag
+ # (e.g. latest-develop) invalidates the deps cache automatically.
+ BASE_IMAGE_DIGEST=$(docker buildx imagetools inspect \
+ "${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}" \
+ --format '{{json .Manifest.Digest}}' 2>/dev/null | tr -d '"' || true)
+ if [ -n "${BASE_IMAGE_DIGEST}" ]; then
+ BASE_IMAGE_UNIQ_ID="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}:${BASE_IMAGE_DIGEST}"
+ else
+ echo "🟠 Could not resolve base image digest, falling back to tag string"
+ BASE_IMAGE_UNIQ_ID="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}"
+ fi
+
+ echo "🔵 Base image ID: ${BASE_IMAGE_UNIQ_ID}"
+
+ MANIFEST_FILES=$(git ls-files | grep -E "${DEPS_MANIFEST_PATTERN}" || true)
+ FILE_HASH=$(git ls-files -s "${DEPS_FILES[@]}" ${MANIFEST_FILES} 2>/dev/null \
+ | sha256sum | cut -c1-16)
+ DEPS_HASH=$(printf '%s %s' "${FILE_HASH}" "${BASE_IMAGE_UNIQ_ID}" | sha256sum | cut -c1-16)
+ DEPS_ECR_IMAGE="${ECR_URL}:deps-${DEPS_HASH}"
+ echo "🔵 Deps hash: ${DEPS_HASH}"
+ echo "🔵 Checking if deps image ${DEPS_ECR_IMAGE} exists in ECR..."
+
+ # Lightweight manifest check - fetches only the image manifest (~KB),
+ # not the actual layers, so this completes in seconds.
+ if docker manifest inspect "${DEPS_ECR_IMAGE}" >/dev/null 2>&1; then
+ echo "🟢 Deps cache HIT!!! Image exists in ECR: ${DEPS_ECR_IMAGE}"
+ # Create a commit-tagged alias pointing to the same manifest (registry-side,
+ # no layer download). Test jobs will pull this tag normally.
+ echo "🔵 Tagging as commit image ${ECR_IMAGE}..."
+ docker buildx imagetools create -t "${ECR_IMAGE}" "${DEPS_ECR_IMAGE}"
+ echo "🟢 Tagged ${ECR_IMAGE} >> ${DEPS_ECR_IMAGE}"
+ echo "deps-cache-hit=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "🟠 Deps cache MISS 😿😿😿 (${DEPS_HASH}). Will build now. 🐢🐢🐢"
+ echo "DEPS_ECR_IMAGE=${DEPS_ECR_IMAGE}" >> "$GITHUB_ENV"
+ echo "PUSH_DEPS_IMAGE=true" >> "$GITHUB_ENV"
+ fi
+
+ ##### 6: Full build #####
+
+ # Runs when neither the exact image nor the deps cache was available.
+ # Uses ECR layer cache (--cache-from/--cache-to) when ECR is available.
+
+ - name: Full build
+ if: steps.pull-exact.outputs.hit != 'true' && steps.deps-cache.outputs.deps-cache-hit != 'true'
+ shell: bash
+ run: |
+ BUILD_ARGS=(
+ --progress=plain
+ --platform linux/amd64
+ -f "${{ inputs.dockerfile-path }}"
+ --build-arg "ISAACSIM_BASE_IMAGE_ARG=${{ inputs.isaacsim-base-image }}"
+ --build-arg "ISAACSIM_VERSION_ARG=${{ inputs.isaacsim-version }}"
+ --build-arg "ISAACSIM_ROOT_PATH_ARG=/isaac-sim"
+ --build-arg "ISAACLAB_PATH_ARG=/workspace/isaaclab"
+ --build-arg "DOCKER_USER_HOME_ARG=/root"
+ -t "${{ inputs.image-tag }}"
+ )
+ if [ -n "${ECR_URL:-}" ]; then
+ BUILD_ARGS+=(
+ --cache-from "type=registry,ref=${CACHE_IMAGE}"
+ --cache-to "type=registry,ref=${CACHE_IMAGE},mode=max"
+ -t "${ECR_IMAGE}"
+ )
+ fi
+
+ BUILDER_NAME="ci-builder-${{ github.run_id }}-${{ github.job }}"
+ docker buildx create --use --driver docker-container --name "${BUILDER_NAME}" \
+ || docker buildx use "${BUILDER_NAME}"
+ trap 'docker buildx rm "${BUILDER_NAME}" || true' EXIT
+
+ echo "🔵 Building ${{ inputs.image-tag }}..."
+ docker buildx build --load "${BUILD_ARGS[@]}" .
+
+ ##### 7: Push to ECR #####
+
+ # Pushes the per-commit ECR image after a successful full build.
+ # Skipped if the image was pulled in (4).
+
+ - name: Push to ECR
+ if: >
+ steps.resolve-ecr.outputs.available == 'true' &&
+ steps.pull-exact.outputs.hit != 'true' &&
+ steps.deps-cache.outputs.deps-cache-hit != 'true'
+ shell: bash
+ run: |
+ echo "🔵 Pushing ${ECR_IMAGE} to ECR..."
+ docker push "${ECR_IMAGE}"
+ echo "🟢 Pushed ${ECR_IMAGE}"
+
+ ##### 8: Push deps tag #####
+
+ # Tags the freshly built image as deps- so future runs with identical
+ # install inputs hit the fast path (step 5) instead of doing a full build.
+
+ - name: Push deps tag
+ if: env.PUSH_DEPS_IMAGE == 'true'
+ shell: bash
+ run: |
+ echo "🔵 Pushing deps image for future cache hits: ${DEPS_ECR_IMAGE}"
+ docker tag "${{ inputs.image-tag }}" "${DEPS_ECR_IMAGE}"
+ docker push "${DEPS_ECR_IMAGE}"
+
+ ##### 9: Cleanup docker config #####
+
+ - name: Cleanup docker config
+ if: always()
+ shell: bash
+ run: |
+ if [ -n "${DOCKER_CONFIG}" ] && [ -d "${DOCKER_CONFIG}" ]; then
+ rm -rf "${DOCKER_CONFIG}"
+ fi
diff --git a/.github/actions/run-package-tests/action.yml b/.github/actions/run-package-tests/action.yml
new file mode 100644
index 00000000000..50c3d225d63
--- /dev/null
+++ b/.github/actions/run-package-tests/action.yml
@@ -0,0 +1,180 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+name: 'Run Package Tests'
+description: >
+ Pulls the Docker image from ECR, runs pytest inside the container, uploads
+ results as an artifact, and fails fork PRs on test failures.
+ NOTE: The calling job must check out the code before using this action.
+
+inputs:
+ image-tag:
+ description: 'Docker image tag'
+ required: true
+ isaacsim-base-image:
+ description: 'IsaacSim base image'
+ required: true
+ isaacsim-version:
+ description: 'IsaacSim version'
+ required: true
+ dockerfile-path:
+ description: 'Path to Dockerfile'
+ default: 'docker/Dockerfile.base'
+ required: false
+ cache-tag:
+ description: 'ECR cache tag'
+ default: 'cache-base'
+ required: false
+ filter-pattern:
+ description: 'Pattern to filter test files (e.g., isaaclab_tasks)'
+ default: ''
+ required: false
+ shard-index:
+ description: 'Zero-based shard index'
+ default: ''
+ required: false
+ shard-count:
+ description: 'Total number of shards'
+ default: ''
+ required: false
+ curobo-only:
+ description: 'Run only cuRobo and SkillGen tests'
+ default: 'false'
+ required: false
+ quarantined-only:
+ description: 'Run only quarantined tests'
+ default: 'false'
+ required: false
+ include-files:
+ description: 'Comma-separated list of specific test files to include'
+ default: ''
+ required: false
+ pytest-options:
+ description: 'Additional pytest options'
+ default: ''
+ required: false
+ container-name:
+ description: 'Docker container name prefix (run-id is appended automatically)'
+ required: true
+runs:
+ using: composite
+ steps:
+ # Display some details on AWS instance we're running on
+ - name: AWS Instance Info
+ shell: bash
+ run: |
+ # get instance ID for debugging purposes (if running on EC2)
+ IMDS_TOKEN=$(curl -sf -X PUT "http://169.254.169.254/latest/api/token" \
+ -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") || true
+ if [ -n "$IMDS_TOKEN" ]; then
+ INSTANCE_ID=$(curl -sf -H "X-aws-ec2-metadata-token: ${IMDS_TOKEN}" \
+ "http://169.254.169.254/latest/meta-data/instance-id") || true
+ INSTANCE_REGION=$(curl -sf -H "X-aws-ec2-metadata-token: ${IMDS_TOKEN}" \
+ "http://169.254.169.254/latest/meta-data/placement/region") || true
+ INSTANCE_TYPE=$(curl -sf -H "X-aws-ec2-metadata-token: ${IMDS_TOKEN}" \
+ "http://169.254.169.254/latest/meta-data/instance-type") || true
+ echo "⚪ Instance ID: ${INSTANCE_ID:-Not running on EC2}"
+ echo "⚪ Instance Region: ${INSTANCE_REGION:-Not running on EC2}"
+ echo "⚪ Instance Type: ${INSTANCE_TYPE:-Not running on EC2}"
+ echo "⚪ Connect: https://${INSTANCE_REGION}.console.aws.amazon.com/ec2-instance-connect/ssh/home?region=${INSTANCE_REGION}&connType=standard&instanceId=${INSTANCE_ID}&osUser=ubuntu&sshPort=22&addressFamily=ipv4"
+ else
+ echo "🟠 Could not obtain IMDS token, probably not running on EC2"
+ fi
+
+ - name: Record pull start time
+ id: pull-start
+ shell: bash
+ run: echo "time=$(date +%s)" >> "$GITHUB_OUTPUT"
+
+ - name: Pull image from ECR
+ uses: ./.github/actions/ecr-build-push-pull
+ with:
+ image-tag: ${{ inputs.image-tag }}
+ isaacsim-base-image: ${{ inputs.isaacsim-base-image }}
+ isaacsim-version: ${{ inputs.isaacsim-version }}
+ dockerfile-path: ${{ inputs.dockerfile-path }}
+ cache-tag: ${{ inputs.cache-tag }}
+
+ - name: Report pull duration
+ if: always()
+ shell: bash
+ run: |
+ start_time="${{ steps.pull-start.outputs.time }}"
+ if [ -z "$start_time" ]; then
+ echo "🟠 Could not calculate pull duration (start time not recorded)" >> "$GITHUB_STEP_SUMMARY"
+ exit 0
+ fi
+ elapsed=$(( $(date +%s) - start_time ))
+ printf "🔵 Image pull took %dm %ds\n" $((elapsed/60)) $((elapsed%60))
+ echo "🔵 Docker Image Pulled in ${elapsed}s" >> "$GITHUB_STEP_SUMMARY"
+
+ - name: Run Tests
+ uses: ./.github/actions/run-tests
+ with:
+ test-path: "tools"
+ result-file: "${{ github.job }}-report.xml"
+ container-name: "${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}"
+ image-tag: ${{ inputs.image-tag }}
+ pytest-options: ${{ inputs.pytest-options }}
+ filter-pattern: ${{ inputs.filter-pattern }}
+ shard-index: ${{ inputs.shard-index }}
+ shard-count: ${{ inputs.shard-count }}
+ curobo-only: ${{ inputs.curobo-only }}
+ quarantined-only: ${{ inputs.quarantined-only }}
+ include-files: ${{ inputs.include-files }}
+ volume-mount-source: ${{ github.workspace }}
+
+ - name: Check Test Results
+ if: always()
+ shell: bash
+ run: |
+ if [ -f "reports/${{ github.job }}-report.xml" ]; then
+ if grep -qE 'failures="[1-9][0-9]*"' reports/${{ github.job }}-report.xml || grep -qE 'errors="[1-9][0-9]*"' reports/${{ github.job }}-report.xml; then
+ echo "Tests failed for PR from fork. The test report is in the logs. Failing the job."
+ exit 1
+ fi
+ else
+ echo "No test results file found. This might indicate test execution failed."
+ exit 1
+ fi
+
+ - name: Kill container on cancellation
+ if: cancelled()
+ shell: bash
+ run: |
+ CONTAINER="${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}"
+ echo "🟠 Workflow cancelled, force-killing container ${CONTAINER}..."
+ docker kill "${CONTAINER}" 2>/dev/null || true
+ docker rm -f "${CONTAINER}" 2>/dev/null || true
+
+ - name: Cleanup old Docker images
+ if: always()
+ shell: bash
+ run: |
+ # Don't let cleanup errors fail a passing test job
+ set +e
+
+ echo "🔵 Cleaning up old Docker images..."
+
+ echo "⚪ Disk usage before cleanup:"
+ df -h /var/lib/docker
+
+ ISAACSIM_IMAGE="${{ inputs.isaacsim-base-image }}:${{ inputs.isaacsim-version }}"
+
+ # Pin the base IsaacSim image so prune -a doesn't remove it
+ ISAACSIM_IMAGE_PIN_CONTAINER="isaacsim-pin-${{ inputs.container-name }}-${{ github.run_id }}-${{ github.run_attempt }}"
+ echo "🔵 Pinning IsaacSim base image: ${ISAACSIM_IMAGE} to temporary container ${ISAACSIM_IMAGE_PIN_CONTAINER}"
+ docker create --name "${ISAACSIM_IMAGE_PIN_CONTAINER}" "${ISAACSIM_IMAGE}" true 2>/dev/null || true
+
+ echo "🔵 Removing Docker images older than 3 days..."
+ docker image prune -a -f --filter "until=72h"
+
+ echo "🔵 Removing temporary container ${ISAACSIM_IMAGE_PIN_CONTAINER}..."
+ docker rm "${ISAACSIM_IMAGE_PIN_CONTAINER}" 2>/dev/null || true
+
+ echo "🟢 Docker image cleanup complete."
+
+ echo "⚪ Disk usage after cleanup:"
+ df -h /var/lib/docker
diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml
index 46712286014..feafa86d765 100644
--- a/.github/actions/run-tests/action.yml
+++ b/.github/actions/run-tests/action.yml
@@ -31,6 +31,30 @@ inputs:
description: 'Pattern to filter test files (e.g., isaaclab_tasks)'
default: ''
required: false
+ curobo-only:
+ description: 'Run only cuRobo and SkillGen tests (requires the cuRobo Docker image)'
+ default: 'false'
+ required: false
+ quarantined-only:
+ description: 'Run only tests listed in QUARANTINED_TESTS (skipped in normal jobs)'
+ default: 'false'
+ required: false
+ include-files:
+ description: 'Comma-separated list of specific test file paths to include (e.g., source/pkg/test/test_a.py,source/pkg/test/test_b.py)'
+ default: ''
+ required: false
+ shard-index:
+ description: 'Zero-based index of this shard (used with shard-count to split tests across parallel jobs)'
+ default: ''
+ required: false
+ shard-count:
+ description: 'Total number of shards (used with shard-index to split tests across parallel jobs)'
+ default: ''
+ required: false
+ volume-mount-source:
+ description: 'Host path to bind-mount at /workspace/isaaclab (for deps-cache-hit mode)'
+ default: ''
+ required: false
runs:
using: composite
@@ -47,6 +71,12 @@ runs:
local reports_dir="$5"
local pytest_options="$6"
local filter_pattern="$7"
+ local curobo_only="$8"
+ local include_files="$9"
+ local quarantined_only="${10}"
+ local shard_index="${11}"
+ local shard_count="${12}"
+ local volume_mount_source="${13}"
echo "Running tests in: $test_path"
if [ -n "$pytest_options" ]; then
@@ -55,6 +85,15 @@ runs:
if [ -n "$filter_pattern" ]; then
echo "With filter pattern: $filter_pattern"
fi
+ if [ "$curobo_only" = "true" ]; then
+ echo "cuRobo-only mode enabled: running only cuRobo and SkillGen tests"
+ fi
+ if [ -n "$include_files" ]; then
+ echo "Include files: $include_files"
+ fi
+ if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then
+ echo "Shard: $shard_index of $shard_count"
+ fi
# Create reports directory
mkdir -p "$reports_dir"
@@ -73,9 +112,33 @@ runs:
-e PYTHONIOENCODING=utf-8 \
-e TEST_RESULT_FILE=$result_file"
+ if [ "$curobo_only" = "true" ]; then
+ docker_env_vars="$docker_env_vars -e TEST_CUROBO_ONLY=true"
+ echo "Setting TEST_CUROBO_ONLY=true"
+ fi
+
+ if [ "$quarantined_only" = "true" ]; then
+ docker_env_vars="$docker_env_vars -e TEST_QUARANTINED_ONLY=true"
+ echo "Setting TEST_QUARANTINED_ONLY=true"
+ fi
+
+ if [ -n "$include_files" ]; then
+ # Strip spaces so the value is safe to embed in an unquoted docker_env_vars string.
+ # conftest.py splits on commas and strips whitespace, so compact form works fine.
+ include_files_compact="${include_files// /}"
+ docker_env_vars="$docker_env_vars -e TEST_INCLUDE_FILES=$include_files_compact"
+ echo "Setting TEST_INCLUDE_FILES=$include_files_compact"
+ fi
+
+ if [ -n "$shard_index" ] && [ -n "$shard_count" ]; then
+ docker_env_vars="$docker_env_vars -e TEST_SHARD_INDEX=$shard_index -e TEST_SHARD_COUNT=$shard_count"
+ echo "Setting TEST_SHARD_INDEX=$shard_index TEST_SHARD_COUNT=$shard_count"
+ fi
+
if [ -n "$filter_pattern" ]; then
- if [[ "$filter_pattern" == not* ]]; then
- # Handle "not pattern" case
+ if [[ "$filter_pattern" == "not "* ]]; then
+ # Handle "not " case - note the trailing space to avoid
+ # matching words that happen to start with "not".
exclude_pattern="${filter_pattern#not }"
docker_env_vars="$docker_env_vars -e TEST_EXCLUDE_PATTERN=$exclude_pattern"
echo "Setting exclude pattern: $exclude_pattern"
@@ -90,9 +153,22 @@ runs:
echo "Docker environment variables: '$docker_env_vars'"
- # Run tests in container with error handling
- echo "🚀 Starting Docker container for tests..."
- if docker run --name $container_name \
+ # Volume mount for deps-cache-hit mode: bind-mount the checked-out
+ # source code over /workspace/isaaclab instead of baking it into the image.
+ docker_volume_args=""
+ if [ -n "$volume_mount_source" ]; then
+ docker_volume_args="-v ${volume_mount_source}:/workspace/isaaclab"
+ echo "🔵 Volume-mounting ${volume_mount_source} >> /workspace/isaaclab"
+ fi
+
+ # Run tests in a detached container and follow logs. Running detached
+ # means the container lifecycle is independent of the shell - if the
+ # runner kills this step on cancellation, the `if: always()` cleanup
+ # step can still `docker kill` the container reliably.
+ # `docker logs -f` is the foreground process and is trivially killable.
+ echo "🔵 Starting Docker container for tests..."
+ docker run -d --name $container_name \
+ --init --stop-timeout 5 \
--entrypoint bash --gpus all --network=host \
--security-opt=no-new-privileges:true \
--memory=$(echo "$(free -m | awk '/^Mem:/{print $2}') * 0.9 / 1" | bc)m \
@@ -100,58 +176,128 @@ runs:
--oom-kill-disable=false \
--ulimit nofile=65536:65536 \
--ulimit nproc=4096:4096 \
+ $docker_volume_args \
$docker_env_vars \
$image_tag \
-c "
set -e
cd /workspace/isaaclab
mkdir -p tests
+ rm _isaac_sim || true
+ ln -s /isaac-sim _isaac_sim
echo 'Starting pytest with path: $test_path'
- /isaac-sim/python.sh -m pytest --ignore=tools/conftest.py $test_path $pytest_options -v --junitxml=tests/$result_file || echo 'Pytest completed with exit code: $?'
- "; then
- echo "✅ Docker container completed successfully"
+ ./isaaclab.sh -p -m pytest --ignore=tools/conftest.py --ignore=source/isaaclab/test/install_ci $test_path $pytest_options -v --junitxml=tests/$result_file
+ "
+
+ # Stream container logs to CI output (this is the killable foreground process).
+ docker logs -f $container_name &
+ local logs_pid=$!
+
+ # Wait for the container to exit and capture its exit code.
+ DOCKER_EXIT=$(docker wait $container_name) || DOCKER_EXIT=1
+
+ # Stop following logs.
+ kill $logs_pid 2>/dev/null || true
+ wait $logs_pid 2>/dev/null || true
+
+ if [ $DOCKER_EXIT -eq 0 ]; then
+ echo "🟢 Docker container completed successfully"
else
- echo "⚠️ Docker container failed, but continuing to copy results..."
+ echo "🟠 Docker container failed (exit $DOCKER_EXIT), but continuing to copy results..."
fi
- # Copy test results with error handling
- echo "📋 Attempting to copy test results..."
- if docker cp $container_name:/workspace/isaaclab/tests/$result_file "$reports_dir/$result_file" 2>/dev/null; then
- echo "✅ Test results copied successfully"
+ # Copy test results with error handling.
+ # When volume-mounted, test output lands on the host filesystem directly
+ # (docker cp cannot see bind-mounted paths after the container stops).
+ echo "🔵 Attempting to copy test results..."
+ if [ -n "$volume_mount_source" ] && [ -f "${volume_mount_source}/tests/$result_file" ]; then
+ cp "${volume_mount_source}/tests/$result_file" "$reports_dir/$result_file"
+ echo "🟢 Test results copied from volume mount to $reports_dir/$result_file"
+ elif cp_err=$(docker cp $container_name:/workspace/isaaclab/tests/$result_file "$reports_dir/$result_file" 2>&1); then
+ echo "🟢 Test results copied successfully to $reports_dir/$result_file"
else
- echo "❌ Failed to copy specific result file, trying to copy all test results..."
- if docker cp $container_name:/workspace/isaaclab/tests/ "$reports_dir/" 2>/dev/null; then
- echo "✅ All test results copied successfully"
+ echo "🔴 Failed to copy specific result file: $cp_err"
+ echo "🔴 Trying to copy all test results..."
+ if cp_err=$(docker cp $container_name:/workspace/isaaclab/tests/ "$reports_dir/" 2>&1); then
+ echo "🟢 All test results copied successfully to $reports_dir/"
# Look for any XML files and use the first one found
if [ -f "$reports_dir/full_report.xml" ]; then
mv "$reports_dir/full_report.xml" "$reports_dir/$result_file"
- echo "✅ Found and renamed full_report.xml to $result_file"
- elif [ -f "$reports_dir/test-reports-"*".xml" ]; then
- # Combine individual test reports if no full report exists
- echo "📊 Combining individual test reports..."
- echo '' > "$reports_dir/$result_file"
+ echo "🟢 Found and renamed full_report.xml to $result_file"
+ elif ls "$reports_dir"/test-reports-*.xml 1>/dev/null 2>&1; then
+ # Combine individual test reports if no full report exists.
+ # Extract each block intact to produce valid JUnit XML.
+ echo "🔵 Combining individual test reports..."
+ echo '' > "$reports_dir/$result_file"
+ echo '' >> "$reports_dir/$result_file"
for xml_file in "$reports_dir"/test-reports-*.xml; do
if [ -f "$xml_file" ]; then
echo " Processing: $xml_file"
- sed '1d; /^> "$reports_dir/$result_file" 2>/dev/null || true
+ # Copy everything between and (inclusive)
+ sed -n '//p' "$xml_file" >> "$reports_dir/$result_file" || echo "🟠 Warning: failed to extract testsuite from $xml_file"
fi
done
echo '' >> "$reports_dir/$result_file"
- echo "✅ Combined individual test reports into $result_file"
+ echo "🟢 Combined individual test reports into $result_file"
else
- echo "❌ No test result files found, creating fallback"
+ echo "🔴 No test result files found, creating fallback"
echo "Container may have failed to generate any results" > "$reports_dir/$result_file"
fi
else
- echo "❌ Failed to copy any test results, creating fallback"
+ echo "🔴 Failed to copy any test results, creating fallback"
echo "Container may have failed to generate results" > "$reports_dir/$result_file"
fi
fi
+ # Copy comparison images (saved by test_rendering_correctness.py).
+ local img_dir="$reports_dir/comparison-images"
+ if [ -n "$volume_mount_source" ] && [ -d "${volume_mount_source}/tests/comparison-images" ]; then
+ cp -r "${volume_mount_source}/tests/comparison-images" "$img_dir"
+ echo "🟢 Comparison images copied to $img_dir"
+ elif docker cp "$container_name:/workspace/isaaclab/tests/comparison-images" "$img_dir" 2>/dev/null; then
+ echo "🟢 Comparison images copied from container to $img_dir"
+ fi
+
# Clean up container
- echo "🧹 Cleaning up Docker container..."
- docker rm $container_name 2>/dev/null || echo "⚠️ Container cleanup failed, but continuing..."
+ echo "🔵 Cleaning up Docker container..."
+ docker rm $container_name 2>/dev/null || echo "🟠 Container cleanup failed, but continuing..."
+
+ return $DOCKER_EXIT
}
# Call the function with provided parameters
- run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}"
+ run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "${{ inputs.pytest-options }}" "${{ inputs.filter-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}"
+
+ - name: Write job summary
+ if: always()
+ shell: bash
+ run: |
+ report="${{ inputs.reports-dir }}/${{ inputs.result-file }}"
+ if [ ! -f "$report" ]; then
+ echo "🟠 No test report found" >> "$GITHUB_STEP_SUMMARY"
+ exit 0
+ fi
+
+ # Parse JUnit XML and produce a markdown summary.
+ # Exit code 2 means the report was corrupt/unreadable - surface the
+ # error in the summary but don't mask the real test exit code.
+ if ! python3 .github/actions/run-tests/junit_summary.py "$report" >> "$GITHUB_STEP_SUMMARY"; then
+ echo "::warning::junit_summary.py failed to parse $report - test results may be missing from the summary"
+ fi
+
+ - name: Upload comparison images
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: comparison-images-${{ inputs.container-name }}
+ path: ${{ inputs.reports-dir }}/comparison-images/
+ if-no-files-found: ignore
+ retention-days: 7
+
+ - name: Clean up Docker container
+ if: always()
+ shell: bash
+ run: |
+ echo "🔵 Cleaning up Docker container..."
+ docker kill "${{ inputs.container-name }}" 2>/dev/null || true
+ docker rm -f "${{ inputs.container-name }}" 2>/dev/null || true
diff --git a/.github/actions/run-tests/junit_summary.py b/.github/actions/run-tests/junit_summary.py
new file mode 100755
index 00000000000..70480acccf7
--- /dev/null
+++ b/.github/actions/run-tests/junit_summary.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Parse a JUnit XML report and print a markdown summary (for $GITHUB_STEP_SUMMARY)."""
+
+import sys
+import xml.etree.ElementTree as ET
+
+if len(sys.argv) < 2:
+ print("Usage: junit_summary.py ", file=sys.stderr)
+ sys.exit(1)
+
+try:
+ tree = ET.parse(sys.argv[1])
+ root = tree.getroot()
+except ET.ParseError as exc:
+ print(f"🔴 Failed to parse test report: {exc}")
+ sys.exit(0) # non-fatal so the step summary still renders
+except OSError as exc:
+ print(f"🔴 Failed to read test report: {exc}")
+ sys.exit(0)
+
+passed, failed, errored, skipped = [], [], [], []
+# Keyed by (test_name, label) -> {"diff_pct": str, "ssim": str, "passed": bool}
+comparison_scores = {}
+total_time = 0.0
+
+
+def safe_float(val, default=0.0):
+ try:
+ return float(val)
+ except (ValueError, TypeError):
+ return default
+
+
+for tc in root.iter("testcase"):
+ classname = tc.get("classname", "")
+ tc_name = tc.get("name", "unknown")
+ name = f"{classname}.{tc_name}" if classname else tc_name
+ t = safe_float(tc.get("time", 0))
+ total_time += t
+ tc_failed = tc.find("failure") is not None
+ tc_errored = tc.find("error") is not None
+ if tc_failed:
+ failed.append((name, t, tc.find("failure").get("message", "")))
+ elif tc_errored:
+ errored.append((name, t, tc.find("error").get("message", "")))
+ elif (skip_el := tc.find("skipped")) is not None:
+ skipped.append((name, t, skip_el.get("message", "")))
+ else:
+ passed.append((name, t))
+
+ # Collect diff_pct:*, ssim:*, and img_*:* properties emitted by test_rendering_correctness.
+ props = tc.find("properties")
+ if props is not None:
+ for prop in props.findall("property"):
+ prop_name = prop.get("name", "")
+ for prefix in ("diff_pct:", "ssim:", "threshold:", "img_result:", "img_golden:"):
+ if prop_name.startswith(prefix):
+ label = prop_name[len(prefix) :]
+ key = (name, label)
+ if key not in comparison_scores:
+ comparison_scores[key] = {
+ "diff_pct": "",
+ "ssim": "",
+ "threshold": "",
+ "passed": not (tc_failed or tc_errored),
+ "img_result": "",
+ "img_golden": "",
+ }
+ field = prefix.rstrip(":")
+ comparison_scores[key][field] = prop.get("value", "")
+
+mins, secs = divmod(total_time, 60)
+time_str = f"{int(mins)}m:{secs:.0f}s"
+
+
+def sanitize_msg(msg, max_len=300):
+ """Collapse newlines, escape pipe characters, and truncate for markdown tables."""
+ return msg.replace("\n", " ").replace("\r", "").replace("|", "\\|")[:max_len]
+
+
+def fmt_name(name):
+ """Format a test name for markdown: strip ``source.`` prefix, allow word-breaking."""
+ if name.startswith("source."):
+ name = name[len("source.") :]
+ # Insert zero-width spaces after dots and brackets so tables can wrap.
+ return name.replace(".", ".\u200b").replace("[", "[\u200b").replace("]", "]\u200b")
+
+
+if failed or errored:
+ print(f"🔴 {len(failed) + len(errored)} FAILED, {len(passed)} PASSED ({time_str})")
+elif not passed and not skipped:
+ print("🟠 No test cases found in report")
+
+if failed or errored:
+ print("")
+ print("| Status | Test | Time | Message |")
+ print("|--------|------|------|---------|")
+ for name, t, msg in sorted(failed, key=lambda x: x[1], reverse=True):
+ print(f"| ASSERTION | {fmt_name(name)} | {t:.1f}s | {sanitize_msg(msg)} |")
+ for name, t, msg in sorted(errored, key=lambda x: x[1], reverse=True):
+ print(f"| ERROR | {fmt_name(name)} | {t:.1f}s | {sanitize_msg(msg)} |")
+
+if passed:
+ print(f"\n🟢 {len(passed)} PASSED ({time_str})
")
+ print("")
+ print("
")
+ print("")
+ print("| Test | Time |")
+ print("|------|------|")
+ for name, t in sorted(passed, key=lambda x: x[1], reverse=True):
+ print(f"| {fmt_name(name)} | {t:.1f}s |")
+ print("")
+ print("🟠 {len(skipped)} SKIPPED
")
+ print("")
+ print("
")
+ print("")
+ print("| Test | Reason |")
+ print("|------|--------|")
+ for name, t, msg in skipped:
+ print(f"| {fmt_name(name)} | {sanitize_msg(msg)} |")
+ print("")
+ print("- **
+ Apple Vision Pro connects to Isaac Lab via the native `Isaac XR Teleop Sample Client`_ app.
- #. Now you can say "Play", "Stop", and "Reset" to control teleoperation while the app is
- connected.
+ .. important::
-#. Teleoperate the simulated robot by moving your hands.
+ Apple Vision Pro requires the ``auto-native`` device profile. Pass the ``avp``
+ shorthand when launching the teleop script:
- .. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/cloudxr_bimanual_teleop.gif
- :align: center
- :alt: Isaac Lab teleoperation of a bimanual dexterous robot with CloudXR
-
- .. note::
-
- The red dots represent the tracked position of the hand joints. Latency or offset between the
- motion of the dots and the robot may be caused by the limits of the robot joints and/or robot
- controller.
-
- .. note::
- When the inverse kinematics solver fails to find a valid solution, an error message will appear
- in the XR device display. To recover from this state, click the **Reset** button to return
- the robot to its original pose and continue teleoperation.
+ .. code-block:: bash
- .. figure:: ../_static/setup/cloudxr_avp_ik_error.jpg
- :align: center
- :figwidth: 80%
- :alt: IK Error Message Display in XR Device
+ ./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py \
+ --task Isaac-PickPlace-GR1T2-WaistEnabled-Abs-v0 \
+ --visualizer kit --xr \
+ --cloudxr_env avp
+ See :ref:`isaac-teleop-cloudxr-profiles` for details on the shipped profiles.
+ .. _build-apple-vision-pro:
-#. When you are finished with the example, click **Disconnect** to disconnect from Isaac Lab.
+ .. rubric:: Build and Install the Client App
-.. admonition:: Learn More about Teleoperation and Imitation Learning in Isaac Lab
+ Requirements:
- See :ref:`teleoperation-imitation-learning` to learn how to record teleoperated demonstrations
- and build teleoperation and imitation learning workflows with Isaac Lab.
+ * Apple Vision Pro with visionOS 26, Apple M3 Pro chip (11-core CPU), 16 GB unified memory
+ * Apple Silicon Mac with macOS Sequoia 15.6+ and Xcode 26.0
+ On your Mac:
-.. _manus-vive-handtracking:
-
-Manus + Vive Hand Tracking
-~~~~~~~~~~~~~~~~~~~~~~~~~~
+ #. Clone the `Isaac XR Teleop Sample Client`_ repository:
-Manus gloves and HTC Vive trackers can provide hand tracking when optical hand tracking from a headset is occluded.
-This setup expects Manus gloves with a Manus SDK license and Vive trackers attached to the gloves.
-Requires Isaac Sim 5.1 or later.
+ .. code-block:: bash
-Run the teleoperation example with Manus + Vive tracking:
+ git clone git@github.com:isaac-sim/isaac-xr-teleop-sample-client-apple.git
-.. dropdown:: Installation instructions
- :open:
+ #. Check out the version that matches your Isaac Lab version:
- Vive tracker integration is provided through the libsurvive library.
+ +-------------------+---------------------+
+ | Isaac Lab Version | Client App Version |
+ +-------------------+---------------------+
+ | 3.0 | v3.0.0 |
+ +-------------------+---------------------+
+ | 2.3 | v2.3.0 |
+ +-------------------+---------------------+
- To install, clone the repository, build the python package, and install the required udev rules.
- In your Isaac Lab virtual environment, run the following commands:
+ .. code-block:: bash
- .. code-block:: bash
+ git checkout
- git clone https://github.com/collabora/libsurvive.git
- cd libsurvive
- pip install scikit-build
- python setup.py install
+ #. Follow the README in the repository to build and install the app on your Apple Vision
+ Pro.
- sudo cp ./useful_files/81-vive.rules /etc/udev/rules.d/
- sudo udevadm control --reload-rules && sudo udevadm trigger
+ .. _teleoperate-apple-vision-pro:
+ .. rubric:: Teleoperate with Apple Vision Pro
- The Manus integration is provided through the Isaac Sim teleoperation input plugin framework.
- Install the plugin by following the build and installation steps in `isaac-teleop-device-plugins `_.
+ .. tip::
-In the same terminal from which you will launch Isaac Lab, set:
+ **Before wearing the headset**, verify connectivity from your Mac:
-.. code-block:: bash
+ .. code:: bash
- export ISAACSIM_HANDTRACKER_LIB=/build-manus-default/lib/libIsaacSimManusHandTracking.so
+ nc -vz 48010
-Once the plugin is installed, run the teleoperation example:
+ Expected output: ``Connection to port 48010 [tcp/*] succeeded!``
-.. code-block:: bash
+ On your Isaac Lab workstation, ensure Isaac Lab and CloudXR are running as described in
+ :ref:`run-isaac-lab-with-the-cloudxr-runtime`.
- ./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py \
- --task Isaac-PickPlace-GR1T2-Abs-v0 \
- --teleop_device manusvive \
- --xr \
- --enable_pinocchio
+ On your Apple Vision Pro:
-The recommended workflow, is to start Isaac Lab, click **Start AR**, and then put on the Manus gloves, vive trackers, and
-headset. Once you are ready to begin the session, use voice commands to launch the Isaac XR teleop sample client and
-connect to Isaac Lab.
+ #. Open the Isaac XR Teleop Sample Client.
-Isaac Lab automatically calibrates the Vive trackers using wrist pose data from the Apple Vision Pro during the initial
-frames of the session. If calibration fails, for example, if the red dots do not accurately follow the teleoperator's
-hands, restart Isaac Lab and begin with your hands in a palm-up position to improve calibration reliability.
+ .. figure:: ../_static/setup/cloudxr_avp_connect_ui.jpg
+ :align: center
+ :figwidth: 50%
+ :alt: Apple Vision Pro connect UI
-For optimal performance, position the lighthouse above the hands, tilted slightly downward.
-Ensure the lighthouse remains stable; a stand is recommended to prevent wobbling.
+ #. Enter the IP address of your Isaac Lab workstation and click **Connect**.
-Ensure that while the task is being teleoperated, the hands remain stable and visible to the lighthouse at all times.
-See: `Installing the Base Stations `_
-and `Tips for Setting Up the Base Stations `_
+ .. note::
-.. note::
+ The Apple Vision Pro and workstation must be IP-reachable from one another. We
+ recommend a dedicated Wifi 6 router.
- On first launch of the Manus Vive device, the Vive lighthouses may take a few seconds to calibrate. Keep the Vive trackers
- stable and visible to the lighthouse during this time. If the light houses are moved or if tracking fails or is unstable,
- calibration can be forced by deleting the calibration file at: ``$XDG_RUNTIME_DIR/libsurvive/config.json``. If XDG_RUNTIME_DIR
- is not set, the default directory is ``~/.config/libsurvive``.
+ #. After a brief period you should see the simulation rendered in the headset along with
+ teleoperation controls.
- For more information consult the libsurvive documentation: `libsurvive `_.
+ .. figure:: ../_static/setup/cloudxr_avp_teleop_ui.jpg
+ :align: center
+ :figwidth: 50%
+ :alt: Apple Vision Pro teleop UI
-For optimal performance, position the lighthouse above the hands, tilted slightly downward.
-One lighthouse is sufficient if both hands are visible.
-Ensure the lighthouse remains stable; a stand is recommended to prevent wobbling.
+ #. Click **Play** to begin teleoperating. Use **Play**, **Stop**, and **Reset** to control
+ the session.
-.. note::
+ .. tip::
- To avoid resource contention and crashes, ensure Manus and Vive devices are connected to different USB controllers/buses.
- Use ``lsusb -t`` to identify different buses and connect devices accordingly.
+ For bimanual tasks, visionOS voice control enables hands-free UI:
- Vive trackers are automatically calculated to map to the left and right wrist joints obtained from a stable
- OpenXR hand tracking wrist pose.
- This auto-mapping calculation supports up to 2 Vive trackers;
- if more than 2 Vive trackers are detected, it uses the first two trackers detected for calibration, which may not be correct.
+ #. **Settings** > **Accessibility** > **Voice Control** > Turn on **Voice Control**
+ #. Enable **
- ** under **Commands** > **Basic Navigation**
+ #. Say "Play", "Stop", or "Reset" while the app is connected.
-.. _develop-xr-isaac-lab:
+ #. Teleoperate the robot by moving your hands.
-Develop for XR in Isaac Lab
----------------------------
+ .. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/cloudxr_bimanual_teleop.gif
+ :align: center
+ :alt: Bimanual dexterous teleoperation with CloudXR
-This section will walk you through how to develop XR environments in Isaac Lab for building
-teleoperation workflows.
+ .. note::
+ If the IK solver fails, an error message appears in the headset. Click **Reset** to
+ return the robot to its original pose and continue.
-.. _run-isaac-lab-with-xr:
+ .. figure:: ../_static/setup/cloudxr_avp_ik_error.jpg
+ :align: center
+ :figwidth: 80%
+ :alt: IK error message in XR device
-Run Isaac Lab with XR Extensions Enabled
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ #. Click **Disconnect** when finished.
-In order to enable extensions necessary for XR, and to see the AR Panel in the UI, Isaac Lab must be
-loaded with an XR experience file. This can be done automatically by passing the ``--xr`` flag to
-any Isaac Lab script that uses :class:`app.AppLauncher`.
-For example: you can enable and use XR in any of the :ref:`tutorials` by invoking them with the
-additional ``--xr`` flag.
+.. _manus-vive-handtracking:
+Manus Gloves
+------------
-.. _configure-scene-placement:
+Manus gloves provide high-fidelity finger tracking via the Manus SDK. This is useful when optical
+hand tracking from the headset is occluded or when higher-precision finger data is needed.
-Configure XR Scene Placement
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. important::
-Placement of the robot simulation within the XR device's local coordinate frame can be achieved
-using an XR anchor, and is configurable using the ``xr`` field (type :class:`openxr.XrCfg`) in the
-environment configuration.
+ Manus gloves and other external push-device peripherals require
+ ``NV_CXR_ENABLE_PUSH_DEVICES=1``. The shipped ``.env`` profiles set this to ``0``
+ (optimised for headset optical hand tracking). To use Manus gloves, create a custom
+ ``.env`` file with the value set to ``1`` and pass it via ``--cloudxr_env``:
-Specifically: the pose specified by the ``anchor_pos`` and ``anchor_rot`` fields of the
-:class:`openxr.XrCfg` will appear at the origin of the XR device's local coordinate frame, which
-should be on the floor.
+ .. code-block:: bash
-.. note::
+ # Copy a shipped profile and enable push devices
+ cp $(python -c "from isaaclab_teleop import CLOUDXR_JS_ENV; print(CLOUDXR_JS_ENV)") ~/manus.env
+ sed -i 's/NV_CXR_ENABLE_PUSH_DEVICES=0/NV_CXR_ENABLE_PUSH_DEVICES=1/' ~/manus.env
- On Apple Vision Pro, the local coordinate frame can be reset to a point on the floor beneath the
- user by holding the digital crown.
+ ./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py \
+ --task Isaac-PickPlace-GR1T2-WaistEnabled-Abs-v0 \
+ --visualizer kit --xr \
+ --cloudxr_env ~/manus.env
-For example: if a robot should appear at the position of the user, the ``anchor_pos`` and
-``anchor_rot`` properties should be set to a pose on the floor directly beneath the robot.
+ See :ref:`isaac-teleop-cloudxr-profiles` for full details on customising profiles.
.. note::
- The XR anchor configuration is applied in :class:`openxr.OpenXRDevice` by creating a prim at the
- position of the anchor, and modifying the ``xr/profile/ar/anchorMode`` and
- ``/xrstage/profile/ar/customAnchor`` settings.
-
- If you are running a script that does not use :class:`openxr.OpenXRDevice`, you will need to do
- this explicitly.
-
-
-.. _optimize-xr-performance:
-
-Optimize XR Performance
-~~~~~~~~~~~~~~~~~~~~~~~
+ Manus glove support has been migrated into Isaac Teleop as a native plugin. The previous
+ ``isaac-teleop-device-plugins`` repository and the ``libsurvive``-based Vive tracker integration
+ are no longer required.
-.. dropdown:: Configure the physics and render time step
- :open:
+Requirements:
- In order to provide a high-fidelity immersive experience, it is recommended to ensure that the
- simulation render time step roughly matches the XR device display time step.
+* Manus gloves with a Manus SDK license
- It is also important to ensure that this time step can be simulated and rendered in real time.
+The Manus plugin is included in the ``isaacteleop`` package and activated automatically when
+configured in the environment's retargeting pipeline. Manus tracking data flows through the same
+API as headset-based optical hand tracking in Isaac Teleop, so the same retargeters and pipelines
+work with both input sources.
- The Apple Vision Pro display runs at 90Hz, but many Isaac Lab simulations will not achieve 90Hz
- performance when rendering stereo views for XR; so for best experience on Apple Vision Pro, we
- suggest running with a simulation dt of 90Hz and a render interval of 2, meaning that the
- simulation is rendered once for every two simulation steps, or at 45Hz, if performance allows.
+For plugin configuration details, see the `Manus plugin documentation
+`_.
- You can still set the simulation dt lower or higher depending on your requirements, but this may
- result in the simulation appearing faster or slower when rendered in XR.
+The recommended workflow:
- Overriding the time step configuration for an environment can be done by modifying the
- :class:`sim.SimulationCfg` in the environment's ``__post_init__`` function. For instance:
+#. Start Isaac Lab and click **Start AR**.
+#. Put on the Manus gloves and headset.
+#. Use voice commands to launch the Isaac XR Teleop Sample Client and connect to Isaac Lab.
- .. code-block:: python
- @configclass
- class XrTeleopEnvCfg(ManagerBasedRLEnvCfg):
+Run with Docker
+---------------
- def __post_init__(self):
- self.sim.dt = 1.0 / 90
- self.sim.render_interval = 2
+Teleoperation runs in a **single container**. Build the image yourself and run a single container.
+Do **not** use Docker Compose, which is a multi-container setup as we had in Isaac Lab 2.x. All
+components run inside one container with Isaac Lab in this release.
- Also note that by default the CloudXR Runtime attempts to dynamically adjust its pacing based on
- how long Isaac Lab takes to render. If render times are highly variable, this can lead to the
- simulation appearing to speed up or slow down when rendered in XR. If this is an issue, the
- CloudXR Runtime can be configured to use a fixed time step by setting the environment variable
- ``NV_PACER_FIXED_TIME_STEP_MS`` to an integer quantity when starting the CloudXR Runtime Docker
- containere.
+The CloudXR runtime auto-launches when a teleop script is started, so no separate
+runtime command is needed.
+Run the teleop script (e.g. ``record_demos.py`` to record demonstrations):
-.. dropdown:: Try running physics on CPU
- :open:
-
- It is currently recommended to try running Isaac Lab teleoperation scripts with the ``--device
- cpu`` flag. This will cause Physics calculations to be done on the CPU, which may be reduce
- latency when only a single environment is present in the simulation.
-
-
-.. _control-robot-with-xr:
-
-Control the Robot with XR Device Inputs
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Isaac Lab provides a flexible architecture for using XR tracking data to control
-simulated robots. This section explains the components of this architecture and how they work together.
-
-.. _openxr-device-architecture:
-
-OpenXR Device
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The :class:`isaaclab.devices.OpenXRDevice` is the core component that enables XR-based teleoperation in Isaac Lab.
-This device interfaces with CloudXR to receive tracking data from the XR headset and transform it into robot control
-commands.
-
-At its heart, XR teleoperation requires mapping (or "retargeting") user inputs, such as hand movements and poses,
-into robot control signals. Isaac Lab makes this straightforward through its OpenXRDevice and Retargeter architecture.
-The OpenXRDevice captures hand tracking data via Isaac Sim's OpenXR API, then passes this data through one or more
-Retargeters that convert it into robot actions.
-
-The OpenXRDevice also integrates with the XR device's user interface when using CloudXR, allowing users to trigger
-simulation events directly from their XR environment.
-
-.. _control-robot-with-xr-retargeters:
-
-Retargeting Architecture
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Retargeters are specialized components that convert raw tracking data into meaningful control signals
-for robots. They implement the :class:`isaaclab.devices.RetargeterBase` interface and are passed to
-the OpenXRDevice during initialization.
-
-Isaac Lab provides three main retargeters for hand tracking:
-
-.. dropdown:: Se3RelRetargeter (:class:`isaaclab.devices.openxr.retargeters.Se3RelRetargeter`)
-
- * Generates incremental robot commands from relative hand movements
- * Best for precise manipulation tasks
-
-.. dropdown:: Se3AbsRetargeter (:class:`isaaclab.devices.openxr.retargeters.Se3AbsRetargeter`)
-
- * Maps hand position directly to robot end-effector position
- * Enables 1:1 spatial control
-
-.. dropdown:: GripperRetargeter (:class:`isaaclab.devices.openxr.retargeters.GripperRetargeter`)
-
- * Controls gripper state based on thumb-index finger distance
- * Used alongside position retargeters for full robot control
-
-.. dropdown:: GR1T2Retargeter (:class:`isaaclab.devices.openxr.retargeters.GR1T2Retargeter`)
-
- * Retargets OpenXR hand tracking data to GR1T2 hand end-effector commands
- * Handles both left and right hands, converting hand poses to joint angles for the GR1T2 robot's hands
- * Supports visualization of tracked hand joints
-
-.. dropdown:: UnitreeG1Retargeter (:class:`isaaclab.devices.openxr.retargeters.UnitreeG1Retargeter`)
-
- * Retargets OpenXR hand tracking data to Unitree G1 using Inspire 5-finger hand end-effector commands
- * Handles both left and right hands, converting hand poses to joint angles for the G1 robot's hands
- * Supports visualization of tracked hand joints
-
-Retargeters can be combined to control different robot functions simultaneously.
-
-Using Retargeters with Hand Tracking
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Here's an example of setting up hand tracking:
-
-.. code-block:: python
-
- from isaaclab.devices import OpenXRDevice, OpenXRDeviceCfg
- from isaaclab.devices.openxr.retargeters import Se3AbsRetargeter, GripperRetargeter
-
- # Create retargeters
- position_retargeter = Se3AbsRetargeter(
- bound_hand=DeviceBase.TrackingTarget.HAND_RIGHT,
- zero_out_xy_rotation=True,
- use_wrist_position=False # Use pinch position (thumb-index midpoint) instead of wrist
- )
- gripper_retargeter = GripperRetargeter(bound_hand=DeviceBase.TrackingTarget.HAND_RIGHT)
-
- # Create OpenXR device with hand tracking and both retargeters
- device = OpenXRDevice(
- OpenXRDeviceCfg(xr_cfg=env_cfg.xr),
- retargeters=[position_retargeter, gripper_retargeter],
- )
-
- # Main control loop
- while True:
- # Get the latest commands from the XR device
- commands = device.advance()
- if commands is None:
- continue
-
- # Apply the commands to the environment
- obs, reward, terminated, truncated, info = env.step(commands)
-
- if terminated or truncated:
- break
-
-Here's a diagram for the dataflow and algorithm used in humanoid teleoperation. Using Apple Vision Pro, we collect 26 keypoints for each hand.
-The wrist keypoint is used to control the hand end-effector, while the remaining hand keypoints are used for hand retargeting.
-
-.. figure:: ../_static/teleop/teleop_diagram.jpg
- :align: center
- :figwidth: 80%
- :alt: teleop_diagram
-
-For dex-retargeting, we are currently using the Dexpilot optimizer, which relies on the five fingertips and the palm for retargeting. It is essential
-that the links used for retargeting are defined exactly at the fingertips—not in the middle of the fingers—to ensure accurate optimization.Please refer
-to the image below for hand asset selection, find a suitable hand asset, or add fingertip links in IsaacLab as needed.
-
-.. figure:: ../_static/teleop/hand_asset.jpg
- :align: center
- :figwidth: 60%
- :alt: hand_asset
-
-.. _control-robot-with-xr-callbacks:
-
-Adding Callbacks for XR UI Events
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The OpenXRDevice can handle events triggered by user interactions with XR UI elements like buttons and menus.
-When a user interacts with these elements, the device triggers registered callback functions:
-
-.. code-block:: python
-
- # Register callbacks for teleop control events
- device.add_callback("RESET", reset_callback)
- device.add_callback("START", start_callback)
- device.add_callback("STOP", stop_callback)
-
-When the user interacts with the XR UI, these callbacks will be triggered to control the simulation
-or recording process. You can also add custom messages from the client side using custom keys that will
-trigger these callbacks, allowing for programmatic control of the simulation alongside direct user interaction.
-The custom keys can be any string value that matches the callback registration.
-
-
-Teleop Environment Configuration
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-XR-based teleoperation can be integrated with Isaac Lab's environment configuration system using the
-``teleop_devices`` field in your environment configuration:
-
-.. code-block:: python
-
- from dataclasses import field
- from isaaclab.envs import ManagerBasedEnvCfg
- from isaaclab.devices import DevicesCfg, OpenXRDeviceCfg
- from isaaclab.devices.openxr import XrCfg
- from isaaclab.devices.openxr.retargeters import Se3AbsRetargeterCfg, GripperRetargeterCfg
-
- @configclass
- class MyEnvironmentCfg(ManagerBasedEnvCfg):
- """Configuration for a teleoperation-enabled environment."""
-
- # Add XR configuration with custom anchor position
- xr: XrCfg = XrCfg(
- anchor_pos=[0.0, 0.0, 0.0],
- anchor_rot=[1.0, 0.0, 0.0, 0.0]
- )
-
- # Define teleoperation devices
- teleop_devices: DevicesCfg = field(default_factory=lambda: DevicesCfg(
- # Configuration for hand tracking with absolute position control
- handtracking=OpenXRDeviceCfg(
- xr_cfg=None, # Will use environment's xr config
- retargeters=[
- Se3AbsRetargeterCfg(
- bound_hand=0, # HAND_LEFT enum value
- zero_out_xy_rotation=True,
- use_wrist_position=False,
- ),
- GripperRetargeterCfg(bound_hand=0),
- ]
- ),
- # Add other device configurations as needed
- ))
-
-
-Teleop Device Factory
-^^^^^^^^^^^^^^^^^^^^^
-
-To create a teleoperation device from your environment configuration, use the ``create_teleop_device`` factory function:
-
-.. code-block:: python
-
- from isaaclab.devices import create_teleop_device
- from isaaclab.envs import ManagerBasedEnv
-
- # Create environment from configuration
- env_cfg = MyEnvironmentCfg()
- env = ManagerBasedEnv(env_cfg)
-
- # Define callbacks for teleop events
- callbacks = {
- "RESET": lambda: print("Reset simulation"),
- "START": lambda: print("Start teleoperation"),
- "STOP": lambda: print("Stop teleoperation"),
- }
-
- # Create teleop device from configuration with callbacks
- device_name = "handtracking" # Must match a key in teleop_devices
- device = create_teleop_device(
- device_name,
- env_cfg.teleop_devices,
- callbacks=callbacks
- )
-
- # Use device in control loop
- while True:
- # Get the latest commands from the device
- commands = device.advance()
- if commands is None:
- continue
-
- # Apply commands to environment
- obs, reward, terminated, truncated, info = env.step(commands)
-
-
-Extending the Retargeting System
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The retargeting system is designed to be extensible. You can create custom retargeters by following these steps:
-
-1. Create a configuration dataclass for your retargeter:
-
-.. code-block:: python
-
- from dataclasses import dataclass
- from isaaclab.devices.retargeter_base import RetargeterCfg
-
- @dataclass
- class MyCustomRetargeterCfg(RetargeterCfg):
- """Configuration for my custom retargeter."""
- scaling_factor: float = 1.0
- filter_strength: float = 0.5
- # Add any other configuration parameters your retargeter needs
-
-2. Implement your retargeter class by extending the RetargeterBase:
-
-.. code-block:: python
-
- from isaaclab.devices.retargeter_base import RetargeterBase
- from isaaclab.devices import OpenXRDevice
- import torch
- from typing import Any
-
- class MyCustomRetargeter(RetargeterBase):
- """A custom retargeter that processes OpenXR tracking data."""
-
- def __init__(self, cfg: MyCustomRetargeterCfg):
- """Initialize retargeter with configuration.
-
- Args:
- cfg: Configuration object for retargeter settings.
- """
- super().__init__()
- self.scaling_factor = cfg.scaling_factor
- self.filter_strength = cfg.filter_strength
- # Initialize any other required attributes
-
- def retarget(self, data: dict) -> Any:
- """Transform raw tracking data into robot control commands.
-
- Args:
- data: Dictionary containing tracking data from OpenXRDevice.
- Keys are TrackingTarget enum values, values are joint pose dictionaries.
-
- Returns:
- Any: The transformed control commands for the robot.
- """
- # Access hand tracking data using TrackingTarget enum
- right_hand_data = data[DeviceBase.TrackingTarget.HAND_RIGHT]
-
- # Extract specific joint positions and orientations
- wrist_pose = right_hand_data.get("wrist")
- thumb_tip_pose = right_hand_data.get("thumb_tip")
- index_tip_pose = right_hand_data.get("index_tip")
-
- # Access head tracking data
- head_pose = data[DeviceBase.TrackingTarget.HEAD]
-
- # Process the tracking data and apply your custom logic
- # ...
-
- # Return control commands in appropriate format
- return torch.tensor([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0]) # Example output
-
-3. Register your retargeter by setting ``retargeter_type`` on the config class:
-
-.. code-block:: python
-
- # Import your retargeter at the top of your module
- from my_package.retargeters import MyCustomRetargeter, MyCustomRetargeterCfg
-
- # Link the config to the implementation for factory construction
- MyCustomRetargeterCfg.retargeter_type = MyCustomRetargeter
-
-4. Now you can use your custom retargeter in teleop device configurations:
-
-.. code-block:: python
-
- from isaaclab.devices import OpenXRDeviceCfg, DevicesCfg
- from isaaclab.devices.openxr import XrCfg
- from my_package.retargeters import MyCustomRetargeterCfg
-
- # Create XR configuration for proper scene placement
- xr_config = XrCfg(anchor_pos=[0.0, 0.0, 0.0], anchor_rot=[1.0, 0.0, 0.0, 0.0])
-
- # Define teleop devices with custom retargeter
- teleop_devices = DevicesCfg(
- handtracking=OpenXRDeviceCfg(
- xr_cfg=xr_config,
- retargeters=[
- MyCustomRetargeterCfg(
- scaling_factor=1.5,
- filter_strength=0.7,
- ),
- ]
- ),
- )
-
-As the OpenXR capabilities expand beyond hand tracking to include head tracking and other features,
-additional retargeters can be developed to map this data to various robot control paradigms.
-
-
-Creating Custom Teleop Devices
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-You can create and register your own custom teleoperation devices by following these steps:
-
-1. Create a configuration dataclass for your device:
-
-.. code-block:: python
-
- from dataclasses import dataclass
- from isaaclab.devices import DeviceCfg
-
- @dataclass
- class MyCustomDeviceCfg(DeviceCfg):
- """Configuration for my custom device."""
- sensitivity: float = 1.0
- invert_controls: bool = False
- # Add any other configuration parameters your device needs
-
-2. Implement your device class by inheriting from DeviceBase:
-
-.. code-block:: python
-
- from isaaclab.devices import DeviceBase
- import torch
-
- class MyCustomDevice(DeviceBase):
- """A custom teleoperation device."""
-
- def __init__(self, cfg: MyCustomDeviceCfg):
- """Initialize the device with configuration.
-
- Args:
- cfg: Configuration object for device settings.
- """
- super().__init__()
- self.sensitivity = cfg.sensitivity
- self.invert_controls = cfg.invert_controls
- # Initialize any other required attributes
- self._device_input = torch.zeros(7) # Example: 6D pose + gripper
-
- def reset(self):
- """Reset the device state."""
- self._device_input.zero_()
- # Reset any other state variables
-
- def add_callback(self, key: str, func):
- """Add callback function for a button/event.
-
- Args:
- key: Button or event name.
- func: Callback function to be called when event occurs.
- """
- # Implement callback registration
- pass
-
- def advance(self) -> torch.Tensor:
- """Get the latest commands from the device.
-
- Returns:
- torch.Tensor: Control commands (e.g., delta pose + gripper).
- """
- # Update internal state based on device input
- # Return command tensor
- return self._device_input
-
-3. Register your device with the teleoperation device factory by adding it to the ``DEVICE_MAP``:
-
-.. code-block:: python
-
- # Import your device at the top of your module
- from my_package.devices import MyCustomDevice, MyCustomDeviceCfg
-
- # Add your device to the factory
- from isaaclab.devices.teleop_device_factory import DEVICE_MAP
-
- # Register your device type with its constructor
- DEVICE_MAP[MyCustomDeviceCfg] = MyCustomDevice
-
-4. Now you can use your custom device in environment configurations:
-
-.. code-block:: python
-
- from dataclasses import field
- from isaaclab.envs import ManagerBasedEnvCfg
- from isaaclab.devices import DevicesCfg
- from my_package.devices import MyCustomDeviceCfg
-
- @configclass
- class MyEnvironmentCfg(ManagerBasedEnvCfg):
- """Environment configuration with custom teleop device."""
-
- teleop_devices: DevicesCfg = field(default_factory=lambda: DevicesCfg(
- my_custom_device=MyCustomDeviceCfg(
- sensitivity=1.5,
- invert_controls=True,
- ),
- ))
-
-
-.. _xr-known-issues:
-
-Known Issues
-------------
-
-* ``XR_ERROR_VALIDATION_FAILURE: xrWaitFrame(frameState->type == 0)`` when stopping AR Mode
-
- This error message can be safely ignored. It is caused by a race condition in the exit handler for
- AR Mode.
-
-* ``XR_ERROR_INSTANCE_LOST in xrPollEvent: Call to "xrt_session_poll_events" failed``
-
- This error may occur if the CloudXR runtime exits before Isaac Lab. Restart the CloudXR
- runtime to resume teleoperation.
-
-* ``[omni.usd] TF_PYTHON_EXCEPTION`` when starting/stopping AR Mode
-
- This error message can be safely ignored. It is caused by a race condition in the enter/exit
- handler for AR Mode.
-
-* ``Invalid version string in _ParseVersionString``
+.. code-block:: bash
- This error message can be caused by shader assets authored with older versions of USD, and can
- typically be ignored.
+ ./isaaclab.sh -p scripts/tools/record_demos.py \
+ --task Isaac-PickPlace-Locomanipulation-G1-Abs-v0 \
+ --num_demos 5 \
+ --dataset_file ./datasets/dataset.hdf5 \
+ --xr --visualizer kit
-* The XR device connects successfully, but no video is displayed, even though the Isaac Lab viewport responds to tracking.
+Then in the Isaac Sim UI, set the AR panel to **System OpenXR Runtime** and click **Start XR**.
- This error occurs when the GPU index differs between the host and the container, causing CUDA
- to load on the wrong GPU. To fix this, set ``NV_GPU_INDEX`` in the runtime container to ``0``, ``1``,
- or ``2`` to ensure the GPU selected by CUDA matches the host.
+For a fully headless experience, replace ``--visualizer kit`` with ``--headless`` and the XR
+teleop session will run automatically.
+.. admonition:: Next Steps
-Kubernetes Deployment
----------------------
+ * **Architecture, retargeting, and control schemes**: :ref:`isaac-teleop-feature`
+ * **Teleoperation for imitation learning**: :ref:`teleoperation-imitation-learning`
+ * **API reference**: :ref:`isaaclab_teleop-api`
-For information on deploying XR Teleop for Isaac Lab on a Kubernetes cluster, see :ref:`cloudxr-teleoperation-cluster`.
..
References
.. _`Apple Vision Pro`: https://www.apple.com/apple-vision-pro/
-.. _`Docker Compose`: https://docs.docker.com/compose/install/linux/#install-using-the-repository
-.. _`Docker`: https://docs.docker.com/desktop/install/linux-install/
.. _`NVIDIA CloudXR`: https://developer.nvidia.com/cloudxr-sdk
-.. _`NVIDIA Container Toolkit`: https://github.com/NVIDIA/nvidia-container-toolkit
.. _`Isaac XR Teleop Sample Client`: https://github.com/isaac-sim/isaac-xr-teleop-sample-client-apple
+.. _`CloudXR Network Setup`: https://docs.nvidia.com/cloudxr-sdk/latest/requirement/network_setup.html
+.. _`CloudXR.js`: https://docs.nvidia.com/cloudxr-sdk/latest/usr_guide/cloudxr_js/index.html
diff --git a/docs/source/how-to/configure_rendering.rst b/docs/source/how-to/configure_rendering.rst
index adfa8b5556c..ee2510fdc44 100644
--- a/docs/source/how-to/configure_rendering.rst
+++ b/docs/source/how-to/configure_rendering.rst
@@ -1,7 +1,16 @@
-Configuring Rendering Settings
-==============================
+Configuring RTX Rendering Settings
+====================================
-Isaac Lab offers 3 preset rendering modes: performance, balanced, and quality.
+.. note::
+
+ This guide covers the **RTX renderer** settings, which are used when running Isaac Lab with
+ Isaac Sim. The RTX renderer is based on NVIDIA's Omniverse RTX rendering pipeline and is
+ available for all camera sensors in the PhysX backend.
+
+ For the **Newton renderer** (used with the Newton backend or in kit-less mode), see
+ :ref:`overview_renderers` for the pluggable renderer architecture and available backends.
+
+Isaac Lab's RTX renderer offers 3 preset rendering modes: performance, balanced, and quality.
You can select a mode via a command line argument or from within a script, and customize settings as needed.
Adjust and fine-tune rendering to achieve the ideal balance for your workflow.
@@ -123,7 +132,8 @@ There are 2 ways to provide settings that overwrite presets.
Examples of RTX settings can be found from within the repo, in the render mode preset files located in ``apps/rendering_modes``.
- In addition, the RTX documentation can be found here - https://docs.omniverse.nvidia.com/materials-and-rendering/latest/rtx-renderer.html.
+ In addition, the full NVIDIA RTX renderer documentation can be found at
+ https://docs.omniverse.nvidia.com/materials-and-rendering/latest/rtx-renderer.html.
An example usage of ``carb_settings``.
@@ -149,3 +159,43 @@ Due to this, we recommend using per-tile or per-camera resolution of at least 10
For renders at lower resolutions, we advice setting the ``antialiasing_mode`` attribute in :class:`~sim.RenderCfg` to
``DLAA``, and also potentially enabling ``enable_dl_denoiser``. Both of these settings should help improve render
quality, but also comes at a cost of performance. Additional rendering parameters can also be specified in :class:`~sim.RenderCfg`.
+
+
+If you observe visual artifacts such as ghosting or disocclusion issues when using tiled rendering, you can try
+adjusting the ``disocclusionScale`` parameter. This setting controls how aggressively the renderer handles
+areas that become newly visible between frames:
+
+.. code-block:: python
+
+ render_cfg = sim_utils.RenderCfg(
+ carb_settings={
+ "/rtx/aovConverter/disocclusionScale": 10000,
+ }
+ )
+
+.. note::
+
+ This parameter is not commonly exposed as it may have side effects in certain scenarios.
+ Only use it as a last resort if other quality settings do not resolve the visual artifacts.
+ The value can be adjusted to a very high value to reduce disocclusion artifacts.
+
+
+Rendering UsdVol 3D Gaussian Scenes in Multiple Environments
+------------------------------------------------------------
+
+When using UsdVol volumes with 3D Gaussian particles (e.g. exported from
+`3DGRUT `_)
+in **multiple environments**, you must set the following so the renderer uses the correct compositing path:
+
+.. code-block:: python
+
+ render_cfg = sim_utils.RenderCfg(
+ carb_settings={
+ "omni.rtx.nre.compositing.rendererHints": 3,
+ }
+ )
+
+.. warning::
+
+ With multiple environments, each environment holds its own copy of the scene, increasing device memory use,
+ and environments are rendered one after another, which can substantially slow down rendering.
diff --git a/docs/source/how-to/haply_teleoperation.rst b/docs/source/how-to/haply_teleoperation.rst
index 1f8d1d6e252..46651077e33 100644
--- a/docs/source/how-to/haply_teleoperation.rst
+++ b/docs/source/how-to/haply_teleoperation.rst
@@ -70,7 +70,7 @@ Software Requirements
* Isaac Lab (follow the :ref:`installation guide `)
* Haply SDK (provided by Haply Robotics)
-* Python 3.10+
+* Python 3.12+
* ``websockets`` Python package (automatically installed with Isaac Lab)
diff --git a/docs/source/how-to/import_new_asset.rst b/docs/source/how-to/import_new_asset.rst
index 41eacc48673..e241be44e96 100644
--- a/docs/source/how-to/import_new_asset.rst
+++ b/docs/source/how-to/import_new_asset.rst
@@ -37,6 +37,12 @@ Using URDF Importer
For using the URDF importer in the GUI, please check the documentation at `URDF importer`_. For using the URDF importer from Python scripts, we include a utility tool called ``convert_urdf.py``. This script creates an instance of :class:`~sim.converters.UrdfConverterCfg` which
is then passed to the :class:`~sim.converters.UrdfConverter` class.
+.. note::
+ The URDF importer was upgraded to version 3.0 in Isaac Sim 6, replacing the previous C++
+ binding-based API with a Python pipeline (``urdf-usd-converter``). Assets are now made
+ instanceable by default — ``make_instanceable`` is no longer a configuration option.
+ See the :doc:`/source/migration/migrating_to_isaaclab_3-0` for a full list of breaking changes.
+
The URDF importer has various configuration parameters that can be set to control the behavior of the importer.
The default values for the importer's configuration parameters are specified are in the :class:`~sim.converters.UrdfConverterCfg` class, and they are listed below. We made a few commonly modified settings to be available as command-line arguments when calling the ``convert_urdf.py``, and they are marked with ``*`` in the list. For a comprehensive list of the configuration parameters, please check the the documentation at `URDF importer`_.
@@ -44,6 +50,7 @@ The default values for the importer's configuration parameters are specified are
This depends on whether you have a floating-base or fixed-base robot. The command-line flag is
``--fix-base`` where when set, the importer will fix the base of the robot, otherwise it will default to floating-base.
* :attr:`~sim.converters.UrdfConverterCfg.root_link_name` - The link on which the PhysX articulation root is placed.
+ **Deprecated in URDF importer 3.0** — this option is ignored.
* :attr:`~sim.converters.UrdfConverterCfg.merge_fixed_joints` * - Whether to merge the fixed joints.
Usually, this should be set to ``True`` to reduce the asset complexity. The command-line flag is
``--merge-joints`` where when set, the importer will merge the fixed joints, otherwise it will default to not merging the fixed joints.
@@ -59,7 +66,8 @@ The default values for the importer's configuration parameters are specified are
* :attr:`~sim.converters.UrdfConverterCfg.JointDriveCfg.PDGainsCfg` - To directly set the stiffness and damping.
* :attr:`~sim.converters.UrdfConverterCfg.JointDriveCfg.NaturalFrequencyGainsCfg` - To set the gains using the
- desired natural frequency response of the system.
+ desired natural frequency response of the system. **Deprecated in URDF importer 3.0** — use
+ ``PDGainsCfg`` instead.
For more detailed information on the configuration parameters, please check the documentation for :class:`~sim.converters.UrdfConverterCfg`.
@@ -75,8 +83,11 @@ pre-processed URDF and the original URDF are:
* We removed various collision bodies from the URDF to reduce the complexity of the asset.
* We changed all the joint's damping and friction parameters to ``0.0``. This ensures that we can perform
effort-control on the joints without PhysX adding additional damping.
-* We added the ```` tag to fixed joints. This ensures that the importer does
- not merge these fixed joints.
+* The ```` URDF tag is **no longer supported** in URDF importer 3.0. Fixed joint
+ merging is now a Python pre-processing step that merges all fixed joints when
+ ``merge_fixed_joints`` is enabled. If you need to preserve a specific fixed joint, disable
+ ``merge_fixed_joints`` entirely or restructure the URDF to use a non-fixed joint type
+ (e.g. revolute with zero-range limits).
The following shows the steps to clone the repository and run the converter:
@@ -97,7 +108,7 @@ The following shows the steps to clone the repository and run the converter:
# run the converter
./isaaclab.sh -p scripts/tools/convert_urdf.py \
../anymal_d_simple_description/urdf/anymal.urdf \
- source/isaaclab_assets/data/Robots/ANYbotics/anymal_d.usd \
+ source/isaaclab_assets/data/Robots/ANYbotics/ \
--merge-joints \
--joint-stiffness 0.0 \
--joint-damping 0.0 \
@@ -116,16 +127,17 @@ The following shows the steps to clone the repository and run the converter:
:: run the converter
isaaclab.bat -p scripts\tools\convert_urdf.py ^
..\anymal_d_simple_description\urdf\anymal.urdf ^
- source\isaaclab_assets\data\Robots\ANYbotics\anymal_d.usd ^
+ source\isaaclab_assets\data\Robots\ANYbotics\ ^
--merge-joints ^
--joint-stiffness 0.0 ^
--joint-damping 0.0 ^
--joint-target-type none
Executing the above script will create a USD file inside the
-``source/isaaclab_assets/data/Robots/ANYbotics/`` directory:
+``source/isaaclab_assets/data/Robots/ANYbotics/anymal/`` directory (the subdirectory name
+is derived automatically from the robot name in the URDF):
-* ``anymal_d.usd`` - This is the main asset file.
+* ``anymal.usda`` - This is the main asset file.
To run the script headless, you can add the ``--headless`` flag. This will not open the GUI and
@@ -155,15 +167,23 @@ We made a few commonly modified settings to be available as command-line argumen
``convert_mjcf.py``, and they are marked with ``*`` in the list. For a comprehensive list of the configuration
parameters, please check the the documentation at `MJCF importer`_.
-
-* :attr:`~sim.converters.MjcfConverterCfg.fix_base*` - Whether to fix the base of the robot.
- This depends on whether you have a floating-base or fixed-base robot. The command-line flag is
- ``--fix-base`` where when set, the importer will fix the base of the robot, otherwise it will default to floating-base.
-* :attr:`~sim.converters.MjcfConverterCfg.make_instanceable*` - Whether to create instanceable assets.
- Usually, this should be set to ``True``. The command-line flag is ``--make-instanceable`` where
- when set, the importer will create instanceable assets, otherwise it will default to non-instanceable.
-* :attr:`~sim.converters.MjcfConverterCfg.import_sites*` - Whether to parse the tag in the MJCF.
- Usually, this should be set to ``True``. The command-line flag is ``--import-sites`` where when set, the importer will parse the tag, otherwise it will default to not parsing the tag.
+.. note::
+ The MJCF importer was rewritten in Isaac Sim 5.0 to use the ``mujoco-usd-converter`` library.
+ Settings such as ``fix_base``, ``import_sites``, ``import_inertia_tensor``, and ``make_instanceable``
+ are no longer needed — the converter now handles these automatically based on the MJCF file content.
+
+* :attr:`~sim.converters.MjcfConverterCfg.merge_mesh` * - Whether to merge meshes where possible to
+ optimize the model. The command-line flag is ``--merge-mesh``.
+* :attr:`~sim.converters.MjcfConverterCfg.collision_from_visuals` * - Whether to generate collision
+ geometry from visual geometries. The command-line flag is ``--collision-from-visuals``.
+* :attr:`~sim.converters.MjcfConverterCfg.collision_type` * - Type of collision geometry to use
+ (e.g. ``"default"``, ``"Convex Hull"``, ``"Convex Decomposition"``). The command-line flag is
+ ``--collision-type``.
+* :attr:`~sim.converters.MjcfConverterCfg.self_collision` * - Whether to activate self-collisions
+ between links of the articulation. The command-line flag is ``--self-collision``.
+* :attr:`~sim.converters.MjcfConverterCfg.import_physics_scene` * - Import physics scene properties
+ (gravity, time step, etc.) from the MJCF file. Defaults to ``False``. The command-line flag is
+ ``--import-physics-scene``.
Example Usage
@@ -182,7 +202,7 @@ The following shows the steps to clone the repository and run the converter:
.. code-block:: bash
- # clone a repository with URDF files
+ # clone a repository with MJCF files
git clone git@github.com:google-deepmind/mujoco_menagerie.git
# go to top of the Isaac Lab repository
@@ -191,15 +211,14 @@ The following shows the steps to clone the repository and run the converter:
./isaaclab.sh -p scripts/tools/convert_mjcf.py \
../mujoco_menagerie/unitree_h1/h1.xml \
source/isaaclab_assets/data/Robots/Unitree/h1.usd \
- --import-sites \
- --make-instanceable
+ --merge-mesh
.. tab-item:: :icon:`fa-brands fa-windows` Windows
:sync: windows
.. code-block:: batch
- :: clone a repository with URDF files
+ :: clone a repository with MJCF files
git clone git@github.com:google-deepmind/mujoco_menagerie.git
:: go to top of the Isaac Lab repository
@@ -208,14 +227,12 @@ The following shows the steps to clone the repository and run the converter:
isaaclab.bat -p scripts\tools\convert_mjcf.py ^
..\mujoco_menagerie\unitree_h1\h1.xml ^
source\isaaclab_assets\data\Robots\Unitree\h1.usd ^
- --import-sites ^
- --make-instanceable
+ --merge-mesh
-Executing the above script will create USD files inside the
+Executing the above script will create the USD file inside the
``source/isaaclab_assets/data/Robots/Unitree/`` directory:
-* ``h1.usd`` - This is the main asset file. It contains all the non-mesh data.
-* ``Props/instanceable_assets.usd`` - This is the mesh data file.
+* ``h1.usd`` - This is the converted USD asset file.
.. figure:: ../_static/tutorials/tutorial_convert_mjcf.jpg
:align: center
diff --git a/docs/source/how-to/index.rst b/docs/source/how-to/index.rst
index 02c0ff99ae1..278e88c4b6c 100644
--- a/docs/source/how-to/index.rst
+++ b/docs/source/how-to/index.rst
@@ -24,6 +24,7 @@ how to import a new asset into Isaac Lab.
import_new_asset
write_articulation_cfg
+ robots
Creating a Fixed Asset
----------------------
@@ -47,6 +48,17 @@ useful when you want to create diverse environments with different objects.
multi_asset_spawning
+Cloning Environments
+--------------------
+
+This guide explains how Isaac Lab's template-based cloning system works, including
+cloning strategies, heterogeneous environments, and collision filtering.
+
+.. toctree::
+ :maxdepth: 1
+
+ cloning
+
Saving Camera Output
--------------------
@@ -137,11 +149,11 @@ additional resources that help you use Omniverse features in Isaac Lab.
master_omniverse
-Setting up CloudXR Teleoperation
---------------------------------
+Setting up Isaac Teleop with CloudXR
+------------------------------------
-This guide explains how to use CloudXR and Apple Vision Pro for immersive streaming and
-teleoperation in Isaac Lab.
+This guide explains how to install Isaac Teleop, start the CloudXR runtime, and connect XR
+devices for immersive teleoperation in Isaac Lab.
.. toctree::
:maxdepth: 1
diff --git a/docs/source/how-to/make_fixed_prim.rst b/docs/source/how-to/make_fixed_prim.rst
index 6d9d50232dc..5726eff2668 100644
--- a/docs/source/how-to/make_fixed_prim.rst
+++ b/docs/source/how-to/make_fixed_prim.rst
@@ -100,7 +100,7 @@ For instance, to spawn an ANYmal robot and make it static in the simulation worl
),
)
anymal_spawn_cfg.func(
- "/World/ANYmal", anymal_spawn_cfg, translation=(0.0, 0.0, 0.8), orientation=(1.0, 0.0, 0.0, 0.0)
+ "/World/ANYmal", anymal_spawn_cfg, translation=(0.0, 0.0, 0.8), orientation=(0.0, 0.0, 0.0, 1.0)
)
diff --git a/docs/source/how-to/multi_asset_spawning.rst b/docs/source/how-to/multi_asset_spawning.rst
index 0dbcf48ba78..650b641bd68 100644
--- a/docs/source/how-to/multi_asset_spawning.rst
+++ b/docs/source/how-to/multi_asset_spawning.rst
@@ -110,6 +110,8 @@ to understand the entire simulation scene. This helps speed up the simulation sc
However, in the case of spawning different assets in different environments, this assumption does not hold
anymore. Hence the flag :attr:`scene.InteractiveScene.replicate_physics` must be disabled.
+For a full guide on the template-based cloning system including strategies and collision filtering,
+see :doc:`cloning`.
.. literalinclude:: ../../../scripts/demos/multi_asset.py
:language: python
diff --git a/docs/source/how-to/optimize_stage_creation.rst b/docs/source/how-to/optimize_stage_creation.rst
index b262878d667..567b635405e 100644
--- a/docs/source/how-to/optimize_stage_creation.rst
+++ b/docs/source/how-to/optimize_stage_creation.rst
@@ -9,7 +9,7 @@ What These Features Do
**Fabric Cloning**
-- Clones environments using Fabric library (see `USD Fabric USDRT Documentation `_)
+- Clones environments using Fabric library (see `USD Fabric USDRT Documentation `_)
- Partially supported and enabled by default on some environments (see `Limitations`_ section for a list)
**Stage in Memory**
@@ -22,6 +22,7 @@ Usage Examples
--------------
Fabric cloning can be toggled by setting the :attr:`isaaclab.scene.InteractiveSceneCfg.clone_in_fabric` flag.
+For a full guide on the template-based cloning system, see :doc:`cloning`.
**Using Fabric Cloning with a RL environment**
@@ -47,26 +48,23 @@ Stage in memory can be toggled by setting the :attr:`isaaclab.sim.SimulationCfg.
# create env with stage in memory
env = ManagerBasedRLEnv(cfg=cfg)
-Note, if stage in memory is enabled without using an existing RL environment class, a few more steps are need.
-The stage creation steps should be wrapped in a :py:keyword:`with` statement to set the stage context.
-If the stage needs to be attached, the :meth:`~isaaclab.sim.utils.attach_stage_to_usd_context` function should
-be called after the stage is created.
+When using stage in memory without an existing RL environment class, wrap the stage creation steps
+in a :py:keyword:`with` statement to set the stage context. The stage is automatically attached
+to the USD context when ``SimulationContext`` is created with ``create_stage_in_memory=True``.
**Using Stage in Memory with a manual scene setup**
.. code-block:: python
# init simulation context with stage in memory
+ # Note: stage is automatically attached to USD context
sim = SimulationContext(cfg=SimulationCfg(create_stage_in_memory=True))
- # grab stage in memory and set stage context
- stage_in_memory = sim.get_initial_stage()
- with stage_utils.use_stage(stage_in_memory):
+ # grab stage and set stage context
+ with stage_utils.use_stage(sim.stage):
# create cartpole scene
scene_cfg = CartpoleSceneCfg(num_envs=1024)
scene = InteractiveScene(scene_cfg)
- # attach stage to memory after stage is created
- sim_utils.attach_stage_to_usd_context()
sim.play()
@@ -118,13 +116,11 @@ Limitations
- Cannot be currently enabled at the same time as **Fabric Cloning**.
-- Attaching stage in memory to the USD context can be slow, offsetting some or all of the performance benefits.
-
- - Note, attaching is only necessary when rendering is enabled. For example, in headless mode, attachment is not required.
+- The stage is automatically attached to the USD context at ``SimulationContext`` creation, ensuring proper
+ lifecycle events for viewport and physics systems.
- Certain low-level Kit APIs do not yet support stage in memory.
- - In most cases, when these APIs are hit, existing scripts will automatically early attach the stage and print a warning message.
- In one particular case, for some environments, the API call to color the ground plane is skipped, when stage in memory is enabled.
diff --git a/docs/source/how-to/record_animation.rst b/docs/source/how-to/record_animation.rst
index 2d675684fbf..61b4ca823d2 100644
--- a/docs/source/how-to/record_animation.rst
+++ b/docs/source/how-to/record_animation.rst
@@ -122,6 +122,6 @@ After the stop time is reached, a file will be saved to:
.. _Stage Recorder: https://docs.omniverse.nvidia.com/extensions/latest/ext_animation_stage-recorder.html
-.. _Fabric: https://docs.omniverse.nvidia.com/kit/docs/usdrt/latest/docs/usd_fabric_usdrt.html
+.. _Fabric: https://docs.omniverse.nvidia.com/kit/docs/usdrt.scenegraph/latest/usd_fabric_usdrt.html
.. _Omniverse Launcher: https://docs.omniverse.nvidia.com/launcher/latest/index.html
.. _tutorial on layering in Omniverse: https://www.youtube.com/watch?v=LTwmNkSDh-c&ab_channel=NVIDIAOmniverse
diff --git a/docs/source/how-to/record_video.rst b/docs/source/how-to/record_video.rst
index aba74363129..01ee6240bb0 100644
--- a/docs/source/how-to/record_video.rst
+++ b/docs/source/how-to/record_video.rst
@@ -3,6 +3,9 @@ Recording video clips during training
Isaac Lab supports recording video clips during training using the
`gymnasium.wrappers.RecordVideo `_ class.
+When the ``--video`` flag is enabled, Isaac Lab captures a perspective view of the scene. The backend
+is chosen automatically from the active physics and renderer stack: an Isaac Sim Kit camera or a
+Newton GL headless viewer.
This feature can be enabled by installing ``ffmpeg`` and using the following command line arguments with the training
script:
@@ -11,7 +14,6 @@ script:
* ``--video_length``: length of each recorded video (in steps)
* ``--video_interval``: interval between each video recording (in steps)
-Make sure to also add the ``--enable_cameras`` argument when running headless.
Note that enabling recording is equivalent to enabling rendering during training, which will slow down both startup and runtime performance.
Example usage:
@@ -23,3 +25,128 @@ Example usage:
The recorded videos will be saved in the same directory as the training checkpoints, under
``IsaacLab/logs////videos/train``.
+
+
+Overview
+--------
+
+The video recording feature is implemented using the ``VideoRecorder`` class. This class is responsible for resolving the video backend from the scene, capturing the video frames, and saving them to a file.
+
+* ``VideoRecorderCfg`` (``isaaclab.envs.utils.video_recorder_cfg``) holds resolution and world-space
+ perspective parameters ``camera_position`` and ``camera_target`` (defaults to a diagonal view of the
+ scene).
+* ``VideoRecorder`` (``isaaclab.envs.utils.video_recorder``) picks a video backend from the scene
+ (Kit vs Newton GL), builds the matching low-level capture object, and returns RGB frames via
+ ``render_rgb_array()``.
+* Direct RL, Direct MARL and manager-based RL environments copy the task's
+ :class:`~isaaclab.envs.common.ViewerCfg` ``eye`` and ``lookat`` into those fields before the
+ recorder is constructed, so training clips align with the task's intended viewport when
+ ``origin_type`` is ``"world"``.
+
+
+Configuration: ``VideoRecorderCfg``
+------------------------------------
+
+The dataclass lives in ``isaaclab.envs.utils.video_recorder_cfg``. Fields ``camera_position`` and
+``camera_target`` are the perspective ``eye`` and ``lookat`` points in meters.
+
+.. literalinclude:: ../../../source/isaaclab/isaaclab/envs/utils/video_recorder_cfg.py
+ :language: python
+ :lines: 20-48
+
+
+Task framing: ``ViewerCfg``
+----------------------------
+
+Tasks define the interactive viewer with :class:`~isaaclab.envs.common.ViewerCfg`. The ``eye`` and
+``lookat`` tuples are the same values the RL base classes copy into ``VideoRecorderCfg`` (see below).
+If your task uses ``origin_type="world"``, those tuples are world-space positions and match what the
+perspective recorder expects.
+
+.. literalinclude:: ../../../source/isaaclab/isaaclab/envs/common.py
+ :language: python
+ :lines: 20-28
+
+
+Backend selection: Kit vs Newton GL
+-------------------------------------
+
+``VideoRecorder`` resolves the implementation from the live :class:`~isaaclab.scene.InteractiveScene`.
+If the user provides the PhysX physics (``presets=physx,...``) or Isaac RTX (``presets=isaac_rtx_renderer,...``) in the sensor stack, the Kit path is selected (``omni.replicator`` on
+``/OmniverseKit_Persp``). The Newton GL path is selected when Newton physics is active (``presets=newton,...``) or the Newton
+Warp renderer (``presets=newton_renderer,...``) appears in the sensor stack - and neither PhysX nor Isaac RTX is present to claim the
+Kit path. OVRTX (``presets=ovrtx_renderer,...`` from ``isaaclab_ov``) can pair with IsaacSim or Newton physics; in that case the video backend is
+selected via the physics preset. If both Kit and Newton GL signals are present (e.g., ``presets=physx,isaac_rtx_renderer,...`` or ``presets=newton,newton_renderer,...``), the Kit path is chosen.
+
+.. literalinclude:: ../../../source/isaaclab/isaaclab/envs/utils/video_recorder.py
+ :language: python
+ :lines: 38-59
+
+
+Construction and dispatch
+--------------------------
+
+When ``env_render_mode`` is ``"rgb_array"`` (as when wrappers or scripts request RGB frames for
+video), the recorder instantiates the backend-specific helper and passes through ``camera_position``,
+``camera_target``, and window size.
+
+.. literalinclude:: ../../../source/isaaclab/isaaclab/envs/utils/video_recorder.py
+ :language: python
+ :lines: 70-114
+
+
+Customising the camera view
+----------------------------
+
+When ``--video`` is passed, the recording camera uses the same
+position and look-at target as the interactive viewer. The defaults come from
+:class:`~isaaclab.envs.common.ViewerCfg`:
+
+* ``eye = (7.5, 7.5, 7.5)`` — camera position in world space (metres)
+* ``lookat = (0.0, 0.0, 0.0)`` — camera look-at target in world space (metres)
+* Resolution ``1280x720``
+
+To change the recording angle, override the ``viewer`` field in your task's environment config.
+The RL base classes automatically copy ``eye`` and ``lookat`` into ``VideoRecorderCfg`` before
+recording starts (when ``origin_type`` is ``"world"``), so the video clip uses the same viewpoint
+as the interactive viewport:
+
+.. code-block:: python
+
+ from isaaclab.envs import ManagerBasedRLEnvCfg
+ from isaaclab.envs.common import ViewerCfg
+ from isaaclab.utils import configclass
+
+ @configclass
+ class MyTaskCfg(ManagerBasedRLEnvCfg):
+ viewer: ViewerCfg = ViewerCfg(
+ eye=(5.0, 5.0, 5.0),
+ lookat=(0.0, 0.0, 1.0),
+ )
+
+
+Summary
+-------
+
+.. list-table::
+ :widths: 40 22 38
+ :header-rows: 1
+
+ * - Stack example (``presets=...``)
+ - Video backend
+ - Capture mechanism
+ * - ``physx,...`` or ``isaac_rtx_renderer,...``
+ - Kit (``"kit"``)
+ - ``/OmniverseKit_Persp`` + Replicator RGB
+ * - ``newton,...`` or ``newton_renderer,...`` (no Kit signals)
+ - Newton GL (``"newton_gl"``)
+ - ``newton.viewer.ViewerGL`` on the SDP Newton model
+ * - ``newton,...,ovrtx_renderer,...`` (OVRTX + Newton physics)
+ - Newton GL (``"newton_gl"``)
+ - ``newton.viewer.ViewerGL`` on the SDP Newton model
+
+
+See also
+--------
+
+* :doc:`/source/features/visualization` - interactive visualizers
diff --git a/docs/source/how-to/robots.rst b/docs/source/how-to/robots.rst
new file mode 100644
index 00000000000..39b011156b5
--- /dev/null
+++ b/docs/source/how-to/robots.rst
@@ -0,0 +1,67 @@
+.. _isaac-lab-robots:
+
+Robot Configurations
+=======================
+
+Robots are entirely defined as instances of configurations within Isaac Lab. If you examine ``source/isaaclab_assets/isaaclab_assets/robots``, you will see a number of files, each of which
+contains configurations for the robot in question. The purpose of these individual files is to better define scope for all the different robots, but there is nothing preventing
+you from :ref:`adding your own ` to your project or even to the ``isaaclab`` repository! For example, consider the following configuration for
+the Dofbot
+
+.. code-block:: python
+
+ import isaaclab.sim as sim_utils
+ from isaaclab.actuators import ImplicitActuatorCfg
+ from isaaclab.assets.articulation import ArticulationCfg
+ from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR
+
+ DOFBOT_CONFIG = ArticulationCfg(
+ spawn=sim_utils.UsdFileCfg(
+ usd_path=f"{ISAAC_NUCLEUS_DIR}/Robots/Dofbot/dofbot.usd",
+ rigid_props=sim_utils.RigidBodyPropertiesCfg(
+ disable_gravity=False,
+ max_depenetration_velocity=5.0,
+ ),
+ articulation_props=sim_utils.ArticulationRootPropertiesCfg(
+ enabled_self_collisions=True, solver_position_iteration_count=8, solver_velocity_iteration_count=0
+ ),
+ ),
+ init_state=ArticulationCfg.InitialStateCfg(
+ joint_pos={
+ "joint1": 0.0,
+ "joint2": 0.0,
+ "joint3": 0.0,
+ "joint4": 0.0,
+ },
+ pos=(0.25, -0.25, 0.0),
+ ),
+ actuators={
+ "front_joints": ImplicitActuatorCfg(
+ joint_names_expr=["joint[1-2]"],
+ effort_limit_sim=100.0,
+ velocity_limit_sim=100.0,
+ stiffness=10000.0,
+ damping=100.0,
+ ),
+ "joint3_act": ImplicitActuatorCfg(
+ joint_names_expr=["joint3"],
+ effort_limit_sim=100.0,
+ velocity_limit_sim=100.0,
+ stiffness=10000.0,
+ damping=100.0,
+ ),
+ "joint4_act": ImplicitActuatorCfg(
+ joint_names_expr=["joint4"],
+ effort_limit_sim=100.0,
+ velocity_limit_sim=100.0,
+ stiffness=10000.0,
+ damping=100.0,
+ ),
+ },
+ )
+
+This completely defines the dofbot! You could copy this into a ``.py`` file and import it as a module and you would be able to use the dofbot in
+your own lab sims. One common feature you will see in any config defining things with state is the presence of an ``InitialStateCfg``. Remember, the configurations
+are what informs vectorization, and it's the ``InitialStateCfg`` that describes the state of the joints of our robot when it gets created in each environment. The
+``ImplicitActuatorCfg`` defines the joints of the robot using the default actuation model determined by the joint type. Not all joints need to be actuated, but you
+will get warnings if you don't. If you aren't planning on using those undefined joints, you can generally ignore these.
diff --git a/docs/source/how-to/save_camera_output.rst b/docs/source/how-to/save_camera_output.rst
index f92f9bcc93d..1c050afaa17 100644
--- a/docs/source/how-to/save_camera_output.rst
+++ b/docs/source/how-to/save_camera_output.rst
@@ -21,6 +21,10 @@ directory.
Saving using Replicator Basic Writer
------------------------------------
+.. note::
+ The BasicWriter is part of the Omniverse Replicator ecosystem and is specific to the default
+ Isaac RTX renderer backend. Other renderer backends may require different save workflows.
+
To save camera outputs, we use the basic write class from Omniverse Replicator. This class allows us to save the
images in a numpy format. For more information on the basic writer, please check the
`documentation `_.
@@ -32,18 +36,11 @@ images in a numpy format. For more information on the basic writer, please check
While stepping the simulator, the images can be saved to the defined folder. Since the BasicWriter only supports
saving data using NumPy format, we first need to convert the PyTorch sensors to NumPy arrays before packing
-them in a dictionary.
+them in a dictionary and writing with the BasicWriter.
.. literalinclude:: ../../../scripts/tutorials/04_sensors/run_usd_camera.py
:language: python
:start-at: # Save images from camera at camera_index
- :end-at: single_cam_info = camera.data.info[camera_index]
-
-After this step, we can save the images using the BasicWriter.
-
-.. literalinclude:: ../../../scripts/tutorials/04_sensors/run_usd_camera.py
- :language: python
- :start-at: # Pack data back into replicator format to save them using its writer
:end-at: rep_writer.write(rep_output)
diff --git a/docs/source/how-to/simulation_performance.rst b/docs/source/how-to/simulation_performance.rst
index 3dd113a1285..bab04b6e8c1 100644
--- a/docs/source/how-to/simulation_performance.rst
+++ b/docs/source/how-to/simulation_performance.rst
@@ -1,8 +1,16 @@
-Simulation Performance and Tuning
-====================================
+PhysX Simulation Performance and Tuning
+=========================================
-The performance of the simulation can be affected by various factors, including the number of objects in the scene,
-the complexity of the physics simulation, and the hardware being used. Here are some tips to improve performance:
+.. note::
+
+ This guide covers performance tuning for the **PhysX** backend, which is the default when
+ running Isaac Lab with Isaac Sim. For the **Newton** backend solver parameters (e.g.
+ ``njmax``, ``nconmax``, ``ls_iterations``), see the
+ :ref:`migrating-to-isaaclab-3-0` migration guide and the Newton physics documentation.
+
+The performance of the PhysX simulation can be affected by various factors, including the number
+of objects in the scene, the complexity of the physics simulation, and the hardware being used.
+Here are some tips to improve performance:
1. **Use Headless Mode**: Running the simulation in headless mode can significantly improve performance, especially
when rendering is not required. You can enable headless mode by using the ``--headless`` flag when running the
@@ -62,8 +70,8 @@ Additional Performance Guides
-----------------------------
There are many ways to "tune" the performance of the simulation, but the way you choose largely depends on what you are trying to simulate. In general, the first place
-you will want to look for performance gains is with the `physics engine `_. Next to rendering
-and running deep learning models, the physics engine is the most computationally costly. Tuning the physics sim to limit the scope to only the task of interest is a great place to
+you will want to look for performance gains is with the `PhysX engine `_. Next to rendering
+and running deep learning models, the PhysX engine is the most computationally costly. Tuning the PhysX sim to limit the scope to only the task of interest is a great place to
start hunting for performance gains.
We have recently released a new `gripper tuning guide `_ , specific to contact and grasp tuning. Please check it first if you intend to use robot grippers. For additional details, you should also checkout these guides!
diff --git a/docs/source/migration/migrating_deformables.rst b/docs/source/migration/migrating_deformables.rst
new file mode 100644
index 00000000000..efb7f7da689
--- /dev/null
+++ b/docs/source/migration/migrating_deformables.rst
@@ -0,0 +1,224 @@
+.. _migrating-deformables:
+
+Migration of Deformables
+========================
+
+.. currentmodule:: isaaclab
+
+In the newer versions of Omni Physics (107.0 and later), the old deformable body functionality has become deprecated.
+The following sections describe the changes to migrate to the new Omni Physics API, specifically moving away from
+Soft Bodies and towards Surface and Volume Deformables. We currently only support deformable bodies in the PhysX
+backend, hence these features are implemented in ``isaaclab_physx``.
+
+.. note::
+
+ The following changes are with respect to Isaac Lab v3.0.0 and Omni Physics v110.0. Please refer to the
+ `release notes`_ for any changes in the future releases.
+
+
+Surface and Volume Deformables
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+With the new Omni Physics API, deformable bodies are split into two distinct types, as described in the
+`Omni Physics documentation`_:
+
+- **Volume deformables**: 3D objects simulated with a tetrahedral FEM mesh (e.g., soft cubes, spheres, capsules).
+ These support kinematic targets on individual vertices. The simulation operates on a tetrahedral mesh internally,
+ while a separate triangle surface mesh handles rendering.
+- **Surface deformables**: 2D surfaces simulated directly on a triangle mesh (e.g., cloth, fabric, membranes).
+ These have additional material properties for controlling stretch, shear, and bend stiffness, but do not support
+ kinematic vertex targets.
+
+The type of deformable is determined by the **physics material** assigned to the object:
+
+- :class:`~isaaclab_physx.sim.DeformableBodyMaterialCfg` creates a **volume** deformable.
+- :class:`~isaaclab_physx.sim.SurfaceDeformableBodyMaterialCfg` creates a **surface** deformable.
+
+
+Migration from the Old API
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Import Changes
+^^^^^^^^^^^^^^
+
+All deformable-related classes have moved from ``isaaclab`` to ``isaaclab_physx``. The table below summarizes the
+import changes:
+
+.. list-table::
+ :header-rows: 1
+ :widths: 50 50
+
+ * - Old Import
+ - New Import
+ * - ``from isaaclab.sim import DeformableBodyPropertiesCfg``
+ - ``from isaaclab_physx.sim import DeformableBodyPropertiesCfg``
+ * - ``from isaaclab.sim import DeformableBodyMaterialCfg``
+ - ``from isaaclab_physx.sim import DeformableBodyMaterialCfg``
+
+
+Removed Properties
+^^^^^^^^^^^^^^^^^^
+
+The following properties have been **removed** from :class:`~isaaclab_physx.sim.DeformableBodyPropertiesCfg`:
+
+- ``collision_simplification`` and related parameters (``collision_simplification_remeshing``,
+ ``collision_simplification_target_triangle_count``, ``collision_simplification_force_conforming``,
+ ``collision_simplification_remove_open_edges``) — collision mesh generation is now handled automatically by
+ PhysX through ``deformableUtils.create_auto_volume_deformable_hierarchy()`` and
+ ``deformableUtils.create_auto_surface_deformable_hierarchy()``.
+- ``simulation_hexahedral_resolution`` — the simulation mesh resolution is no longer user-configurable;
+ PhysX determines it automatically.
+- ``vertex_velocity_damping`` — replaced by the more general ``linear_damping`` property from the
+ `PhysX deformable schema`_.
+- ``sleep_damping`` — replaced by ``settling_damping`` in the `PhysX deformable schema`_.
+
+Added Properties
+^^^^^^^^^^^^^^^^
+
+The following properties have been **added** to :class:`~isaaclab_physx.sim.DeformableBodyPropertiesCfg`:
+
+- ``linear_damping`` — linear damping coefficient [1/s].
+- ``max_linear_velocity`` — maximum allowable linear velocity [m/s]. A negative value lets the simulation choose
+ a per-vertex value dynamically (currently only supported for surface deformables).
+- ``settling_damping`` — additional damping applied when vertex velocity falls below ``settling_threshold`` [1/s].
+- ``enable_speculative_c_c_d`` — enables speculative continuous collision detection.
+- ``disable_gravity`` — per-deformable gravity control.
+- ``collision_pair_update_frequency`` — how often surface-to-surface collision pairs are updated per time step
+ (surface deformables only).
+- ``collision_iteration_multiplier`` — collision subiterations per solver iteration (surface deformables only).
+
+For a full description of all available properties, refer to the `PhysX deformable schema`_ and
+`OmniPhysics deformable schema`_ documentation.
+
+Material Changes
+^^^^^^^^^^^^^^^^
+
+The old :class:`DeformableBodyMaterialCfg` (from ``isaaclab.sim``) has been replaced by a new hierarchy in
+``isaaclab_physx``:
+
+- :class:`~isaaclab_physx.sim.DeformableBodyMaterialCfg` — for volume deformables. Contains ``density``,
+ ``static_friction``, ``dynamic_friction``, ``youngs_modulus``, ``poissons_ratio``, and ``elasticity_damping``.
+- :class:`~isaaclab_physx.sim.SurfaceDeformableBodyMaterialCfg` — extends the volume material config with
+ surface-specific properties: ``surface_thickness``, ``surface_stretch_stiffness``, ``surface_shear_stiffness``,
+ ``surface_bend_stiffness``, and ``bend_damping``.
+
+The old ``damping_scale`` property has been removed. Use ``elasticity_damping`` directly instead.
+
+DeformableObject View Change
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The internal PhysX view type has changed from ``physx.SoftBodyView`` to ``physx.DeformableBodyView``.
+The property ``root_physx_view`` has been deprecated in favor of ``root_view``.
+
+
+Code Examples
+~~~~~~~~~~~~~
+
+Volume Deformable (Before and After)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+**Before**:
+
+.. code-block:: python
+ :emphasize-lines: 1,2
+
+ import isaaclab.sim as sim_utils
+ from isaaclab.assets import DeformableObject, DeformableObjectCfg
+
+ cfg = DeformableObjectCfg(
+ prim_path="/World/Origin.*/Cube",
+ spawn=sim_utils.MeshCuboidCfg(
+ size=(0.2, 0.2, 0.2),
+ deformable_props=sim_utils.DeformableBodyPropertiesCfg(),
+ visual_material=sim_utils.PreviewSurfaceCfg(),
+ physics_material=sim_utils.DeformableBodyMaterialCfg(poissons_ratio=0.4, youngs_modulus=1e5),
+ ),
+ )
+ cube_object = DeformableObject(cfg=cfg)
+
+**After**:
+
+.. code-block:: python
+ :emphasize-lines: 1,2,3
+
+ import isaaclab.sim as sim_utils
+ from isaaclab_physx.assets import DeformableObject, DeformableObjectCfg
+ from isaaclab_physx.sim import DeformableBodyPropertiesCfg, DeformableBodyMaterialCfg
+
+ cfg = DeformableObjectCfg(
+ prim_path="/World/Origin.*/Cube",
+ spawn=sim_utils.MeshCuboidCfg(
+ size=(0.2, 0.2, 0.2),
+ deformable_props=DeformableBodyPropertiesCfg(),
+ visual_material=sim_utils.PreviewSurfaceCfg(),
+ physics_material=DeformableBodyMaterialCfg(poissons_ratio=0.4, youngs_modulus=1e5),
+ ),
+ )
+ cube_object = DeformableObject(cfg=cfg)
+
+Surface Deformable (New)
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Surface deformables use :class:`~isaaclab.sim.spawners.meshes.MeshSquareCfg` for 2D meshes, combined with
+:class:`~isaaclab_physx.sim.SurfaceDeformableBodyMaterialCfg`:
+
+.. code-block:: python
+
+ import isaaclab.sim as sim_utils
+ from isaaclab_physx.assets import DeformableObject, DeformableObjectCfg
+ from isaaclab_physx.sim import DeformableBodyPropertiesCfg, SurfaceDeformableBodyMaterialCfg
+
+ cfg = DeformableObjectCfg(
+ prim_path="/World/Origin.*/Cloth",
+ spawn=sim_utils.MeshSquareCfg(
+ size=1.5,
+ resolution=(21, 21),
+ deformable_props=DeformableBodyPropertiesCfg(),
+ visual_material=sim_utils.PreviewSurfaceCfg(),
+ physics_material=SurfaceDeformableBodyMaterialCfg(poissons_ratio=0.4, youngs_modulus=1e5),
+ ),
+ )
+ cloth_object = DeformableObject(cfg=cfg)
+
+USD File Deformable
+^^^^^^^^^^^^^^^^^^^
+
+Deformable properties can also be applied to imported USD assets using
+:class:`~isaaclab.sim.spawners.from_files.UsdFileCfg`:
+
+.. code-block:: python
+
+ import isaaclab.sim as sim_utils
+ from isaaclab_physx.assets import DeformableObject, DeformableObjectCfg
+ from isaaclab_physx.sim import DeformableBodyPropertiesCfg, DeformableBodyMaterialCfg
+
+ from isaaclab.utils.assets import ISAACLAB_NUCLEUS_DIR
+
+ cfg = DeformableObjectCfg(
+ prim_path="/World/Origin.*/Teddy",
+ spawn=sim_utils.UsdFileCfg(
+ usd_path=f"{ISAACLAB_NUCLEUS_DIR}/Objects/Teddy_Bear/teddy_bear.usd",
+ deformable_props=DeformableBodyPropertiesCfg(),
+ physics_material=DeformableBodyMaterialCfg(poissons_ratio=0.4, youngs_modulus=1e5),
+ scale=[0.05, 0.05, 0.05],
+ ),
+ )
+ teddy_object = DeformableObject(cfg=cfg)
+
+
+Limitations
+~~~~~~~~~~~
+
+- **Kinematic targets are volume-only.** Calling
+ :meth:`~isaaclab_physx.assets.DeformableObject.write_nodal_kinematic_target_to_sim_index` on a surface
+ deformable will raise a ``ValueError``.
+- **Surface-specific solver properties** (``collision_pair_update_frequency``,
+ ``collision_iteration_multiplier``) have no effect on volume deformables.
+- **Deformables are PhysX-only.** The ``isaaclab_physx`` extension is required; other physics backends
+ do not support deformable bodies through Isaac Lab yet.
+
+
+.. _Omni Physics documentation: https://docs.omniverse.nvidia.com/kit/docs/omni_physics/110.0/dev_guide/deformables/deformable_bodies.html
+.. _PhysX deformable schema: https://docs.omniverse.nvidia.com/kit/docs/omni_physics/110.0/dev_guide/deformables/physx_deformable_schema.html#physxbasedeformablebodyapi
+.. _OmniPhysics deformable schema: https://docs.omniverse.nvidia.com/kit/docs/omni_physics/110.0/dev_guide/deformables/omniphysics_deformable_schema.html
+.. _release notes: https://github.com/isaac-sim/IsaacLab/releases
diff --git a/docs/source/migration/migrating_to_isaaclab_3-0.rst b/docs/source/migration/migrating_to_isaaclab_3-0.rst
new file mode 100644
index 00000000000..6787a42766e
--- /dev/null
+++ b/docs/source/migration/migrating_to_isaaclab_3-0.rst
@@ -0,0 +1,1709 @@
+.. _migrating-to-isaaclab-3-0:
+
+Migrating to Isaac Lab 3.0
+==========================
+
+.. currentmodule:: isaaclab
+
+Isaac Lab 3.0 introduces a multi-backend architecture that separates simulation backend-specific code
+from the core Isaac Lab API. This allows for future support of different physics backends while
+maintaining a consistent user-facing API.
+
+This guide covers the main breaking changes and deprecations you need to address when migrating
+from Isaac Lab 2.x to Isaac Lab 3.0.
+
+
+Visualizer CLI and Headless Behavior
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In Isaac Lab 3.0, the ``--headless`` argument is deprecated. Instead, use ``--visualizer`` / ``--viz``
+to determine whether viewer apps are launched with an Isaac Lab command.
+
+Visualizers are lightweight viewer apps for monitoring, debugging, and recording workflows
+(see :doc:`/source/features/visualization`).
+
+The details below describe how CLI visualizer arguments resolve together with
+``SimulationCfg.visualizer_cfgs``.
+
+- ``--viz`` accepts **comma-separated** values (for example ``--viz kit,newton``).
+- If omitted, visualizers are resolved from ``SimulationCfg.visualizer_cfgs``.
+- ``--viz none`` explicitly disables all visualizers, including config-defined ones.
+- ``--headless`` is deprecated (still supported) and overrides ``--viz`` by forcing headless mode.
+
+For the full behavior of visualizer resolution, with the visualizer CLI arg, visualizer configs,
+and ``--headless``, see :ref:`visualization-common-modes`.
+
+
+Multi-Backend Architecture
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Isaac Lab 3.0 introduces a **factory-based multi-backend architecture** that allows asset classes
+to be backed by different physics engines — currently **PhysX** and **Newton**.
+
+When you instantiate an asset class from the ``isaaclab`` package (e.g., ``Articulation``,
+``RigidObject``), a factory automatically resolves and loads the correct backend implementation:
+
+.. code-block:: python
+
+ from isaaclab.assets import Articulation, ArticulationCfg
+
+ # The factory pattern creates the appropriate backend implementation.
+ # No import changes are needed — the same isaaclab imports work regardless of backend.
+ robot = Articulation(cfg=ArticulationCfg(...))
+
+The factory works by convention: for a class defined in ``isaaclab.assets.articulation``, it
+imports the matching class from ``isaaclab_{backend}.assets.articulation``. This means the
+``isaaclab_physx`` and ``isaaclab_newton`` packages mirror the ``isaaclab`` module structure.
+
+.. note::
+
+ The backend is currently set to ``"physx"`` by default. Newton backend support is being
+ actively developed. When backend selection is fully configurable, you will be able to
+ switch backends without changing any asset import paths.
+
+For a comprehensive overview of the factory pattern, backend selection, and how to add a new
+backend, see :doc:`/source/overview/core-concepts/multi_backend_architecture`.
+
+New ``isaaclab_physx`` and ``isaaclab_newton`` Extensions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Two new backend extensions have been introduced:
+
+- **``isaaclab_physx``** — PhysX-specific implementations of all asset and sensor classes.
+- **``isaaclab_newton``** — Newton-specific implementations of asset classes (Articulation and
+ RigidObject).
+
+The following classes have been moved to ``isaaclab_physx``:
+
+.. list-table::
+ :header-rows: 1
+ :widths: 50 50
+
+ * - Isaac Lab 2.x
+ - Isaac Lab 3.0
+ * - ``from isaaclab.assets import DeformableObject``
+ - ``from isaaclab_physx.assets import DeformableObject``
+ * - ``from isaaclab.assets import DeformableObjectCfg``
+ - ``from isaaclab_physx.assets import DeformableObjectCfg``
+ * - ``from isaaclab.assets import DeformableObjectData``
+ - ``from isaaclab_physx.assets import DeformableObjectData``
+ * - ``from isaaclab.assets import SurfaceGripper``
+ - ``from isaaclab_physx.assets import SurfaceGripper``
+ * - ``from isaaclab.assets import SurfaceGripperCfg``
+ - ``from isaaclab_physx.assets import SurfaceGripperCfg``
+
+.. note::
+
+ The ``isaaclab_physx`` extension is installed automatically with Isaac Lab. No additional
+ installation steps are required.
+
+
+Renaming of ``XformPrimView`` to ``FrameView``
+-----------------------------------------------
+
+Isaac Lab's ``XformPrimView`` and related classes have been renamed to ``FrameView`` to
+better reflect their purpose and avoid confusion with Isaac Sim's ``XFormPrim`` class
+hierarchy. The old ``XformPrimView`` name is kept as a deprecated alias.
+
+The rename applies across all backends:
+
+.. list-table::
+ :header-rows: 1
+ :widths: 50 50
+
+ * - Isaac Lab 2.x
+ - Isaac Lab 3.0
+ * - ``BaseXformPrimView``
+ - :class:`~isaaclab.sim.views.BaseFrameView`
+ * - ``UsdXformPrimView``
+ - :class:`~isaaclab.sim.views.UsdFrameView`
+ * - ``XformPrimView``
+ - :class:`~isaaclab.sim.views.FrameView`
+ * - ``FabricXformPrimView``
+ - :class:`~isaaclab_physx.sim.views.FabricFrameView`
+ * - ``NewtonSiteXformPrimView``
+ - :class:`~isaaclab_newton.sim.views.NewtonSiteFrameView`
+
+For most users, the only change needed is updating imports:
+
+.. code-block:: python
+
+ # Before
+ from isaaclab.sim.views import XformPrimView
+
+ # After
+ from isaaclab.sim.views import FrameView
+
+The :class:`~isaaclab.sim.views.FrameView` factory automatically dispatches to the correct
+backend (:class:`~isaaclab_physx.sim.views.FabricFrameView` for PhysX,
+:class:`~isaaclab_newton.sim.views.NewtonSiteFrameView` for Newton) based on the active
+physics backend. The deprecated ``XformPrimView`` alias continues to work but will be
+removed in a future release.
+
+
+Unchanged Imports
+-----------------
+
+The following asset classes remain in the ``isaaclab`` package and can still be imported as before:
+
+- :class:`~isaaclab.assets.Articulation`, :class:`~isaaclab.assets.ArticulationCfg`, :class:`~isaaclab.assets.ArticulationData`
+- :class:`~isaaclab.assets.RigidObject`, :class:`~isaaclab.assets.RigidObjectCfg`, :class:`~isaaclab.assets.RigidObjectData`
+- :class:`~isaaclab.assets.RigidObjectCollection`, :class:`~isaaclab.assets.RigidObjectCollectionCfg`, :class:`~isaaclab.assets.RigidObjectCollectionData`
+
+These classes now inherit from new abstract base classes but maintain full backward compatibility.
+
+The following sensor classes also remain in the ``isaaclab`` package with unchanged imports:
+
+- :class:`~isaaclab.sensors.ContactSensor`, :class:`~isaaclab.sensors.ContactSensorCfg`, :class:`~isaaclab.sensors.ContactSensorData`
+- :class:`~isaaclab.sensors.Imu`, :class:`~isaaclab.sensors.ImuCfg`, :class:`~isaaclab.sensors.ImuData`
+- :class:`~isaaclab.sensors.Pva`, :class:`~isaaclab.sensors.PvaCfg`, :class:`~isaaclab.sensors.PvaData`
+- :class:`~isaaclab.sensors.FrameTransformer`, :class:`~isaaclab.sensors.FrameTransformerCfg`, :class:`~isaaclab.sensors.FrameTransformerData`
+
+These sensor classes now use factory patterns that automatically instantiate the appropriate backend
+implementation (PhysX by default), maintaining full backward compatibility.
+
+.. note::
+
+ The ``Imu`` sensor in Isaac Lab 3.0 is **not** the same as the ``Imu`` sensor in 2.x.
+ The old ``Imu`` (full state sensor) has been renamed to :class:`~isaaclab.sensors.Pva`.
+ The new :class:`~isaaclab.sensors.Imu` is a lightweight sensor that only provides angular velocity
+ and linear acceleration. See :ref:`imu-to-pva-migration` below for details.
+
+If you need to import the PhysX sensor implementations directly (e.g., for type hints or subclassing),
+you can import from ``isaaclab_physx.sensors``:
+
+.. code-block:: python
+
+ # Direct PhysX implementation imports
+ from isaaclab_physx.sensors import ContactSensor, ContactSensorData
+ from isaaclab_physx.sensors import Imu, ImuData
+ from isaaclab_physx.sensors import Pva, PvaData
+ from isaaclab_physx.sensors import FrameTransformer, FrameTransformerData
+
+
+New ``isaaclab_newton`` Extension
+---------------------------------
+
+A new extension ``isaaclab_newton`` provides Newton physics backend implementations for:
+
+- :class:`~isaaclab_newton.assets.Articulation` and :class:`~isaaclab_newton.assets.ArticulationData`
+- :class:`~isaaclab_newton.assets.RigidObject` and :class:`~isaaclab_newton.assets.RigidObjectData`
+
+These classes implement the same base interfaces as their PhysX counterparts
+(:class:`~isaaclab.assets.BaseArticulation`, :class:`~isaaclab.assets.BaseRigidObject`),
+ensuring a consistent API across backends. They use the same warp-based data conventions
+(``wp.array`` with structured types, ``_index`` / ``_mask`` write methods).
+
+.. note::
+
+ The ``isaaclab_newton`` extension requires the ``newton`` package and its dependencies
+ (``mujoco``, ``mujoco-warp``). These are installed automatically when installing the
+ ``isaaclab_newton`` package.
+
+If you need to import Newton implementations directly (e.g., for type hints or subclassing):
+
+.. code-block:: python
+
+ from isaaclab_newton.assets import Articulation as NewtonArticulation
+ from isaaclab_newton.assets import RigidObject as NewtonRigidObject
+
+
+Deformable Object API Changes
+------------------------------
+
+Isaac Lab 3.0 updates the deformable body API to align with the current Omni Physics 110.0
+release. The old soft body API has been deprecated and replaced by two distinct deformable
+types: **volume deformables** (3D FEM tetrahedral meshes) and **surface deformables** (2D
+triangle cloth meshes). The deformable type is determined by the physics material assigned:
+
+- :class:`~isaaclab_physx.sim.DeformableBodyMaterialCfg` for volume deformables.
+- :class:`~isaaclab_physx.sim.SurfaceDeformableBodyMaterialCfg` for surface deformables.
+
+All deformable-related classes have moved from ``isaaclab`` to ``isaaclab_physx``, as shown
+in the import table above. Several properties on
+:class:`~isaaclab_physx.sim.DeformableBodyPropertiesCfg` have been removed or added to match
+the new Omni Physics schema.
+
+For a comprehensive guide covering the full deformable API migration — including removed and
+added properties, material changes, code examples for both volume and surface deformables, and
+current limitations — see :ref:`migrating-deformables`.
+
+
+.. _imu-to-pva-migration:
+
+IMU Sensor Renamed to PVA; New Lightweight IMU Sensor
+-----------------------------------------------------
+
+The old ``Imu`` sensor has been renamed to **PVA** (Pose Velocity Acceleration) because it provided
+full pose, velocity, and acceleration data — far more than a real inertial measurement unit measures.
+A new lightweight **IMU** sensor has been introduced that only provides the two physical quantities
+a real IMU measures: angular velocity (gyroscope) and linear acceleration (accelerometer).
+
+If you were using the old ``Imu`` sensor, you need to decide which new sensor to use:
+
+- Use :class:`~isaaclab.sensors.Pva` / :class:`~isaaclab.sensors.PvaCfg` if you need full state
+ data (pose, linear velocity, angular velocity, linear and angular acceleration, projected gravity).
+- Use :class:`~isaaclab.sensors.Imu` / :class:`~isaaclab.sensors.ImuCfg` if you only need angular
+ velocity and linear acceleration (as a real IMU provides).
+
+**Import changes:**
+
+.. code-block:: python
+
+ # Before (Isaac Lab 2.x) — the old IMU provided full state
+ from isaaclab.sensors import Imu, ImuCfg, ImuData
+
+ # After (Isaac Lab 3.x) — use PVA for the same full-state sensor
+ from isaaclab.sensors import Pva, PvaCfg, PvaData
+
+ # Or use the new lightweight IMU for angular velocity + linear acceleration only
+ from isaaclab.sensors import Imu, ImuCfg, ImuData
+
+**Configuration changes:**
+
+The ``gravity_bias`` configuration parameter has been removed from both sensors:
+
+- **PVA** reports raw kinematic acceleration (no gravity contribution), as the acceleration
+ is derived from finite differencing of velocities which do not include gravity.
+- **IMU** unconditionally includes gravity in its accelerometer readings, matching the behavior
+ of a real accelerometer. The gravity vector is automatically queried from the simulation.
+
+.. code-block:: python
+
+ # Before (Isaac Lab 2.x)
+ imu_cfg = ImuCfg(
+ prim_path="{ENV_REGEX_NS}/Robot/base",
+ gravity_bias=(0.0, 0.0, 9.81), # had to be configured manually
+ )
+
+ # After (Isaac Lab 3.x) — PVA (no gravity in acceleration)
+ pva_cfg = PvaCfg(prim_path="{ENV_REGEX_NS}/Robot/base")
+
+ # After (Isaac Lab 3.x) — IMU (gravity always included automatically)
+ imu_cfg = ImuCfg(prim_path="{ENV_REGEX_NS}/Robot/base")
+
+**Observation function changes:**
+
+.. code-block:: python
+
+ # Before (Isaac Lab 2.x)
+ from isaaclab.envs.mdp import imu_orientation, imu_projected_gravity
+
+ # After (Isaac Lab 3.x)
+ from isaaclab.envs.mdp import pva_orientation, pva_projected_gravity
+
+**Data property changes:**
+
+The new ``ImuData`` only provides ``ang_vel_b`` and ``lin_acc_b``. If you were accessing other
+properties (``pos_w``, ``quat_w``, ``lin_vel_b``, ``ang_acc_b``, ``projected_gravity_b``), switch
+to :class:`~isaaclab.sensors.PvaData` which provides all of them.
+
+
+Sensor Pose Properties Deprecation
+----------------------------------
+
+The ``pose_w``, ``pos_w``, and ``quat_w`` properties on :class:`~isaaclab.sensors.ContactSensorData`
+are deprecated and will be removed in a future release.
+
+If you need to track sensor poses in world frame, please use a dedicated sensor such as
+:class:`~isaaclab.sensors.FrameTransformer` instead.
+
+**Before (deprecated):**
+
+.. code-block:: python
+
+ # Using pose properties directly on sensor data
+ sensor_pos = contact_sensor.data.pos_w
+ sensor_quat = contact_sensor.data.quat_w
+
+**After (recommended):**
+
+.. code-block:: python
+
+ # Use FrameTransformer to track sensor pose
+ frame_transformer = FrameTransformer(FrameTransformerCfg(
+ prim_path="{ENV_REGEX_NS}/Robot/base",
+ target_frames=[
+ FrameTransformerCfg.FrameCfg(prim_path="{ENV_REGEX_NS}/Robot/sensor_link"),
+ ],
+ ))
+ sensor_pos = frame_transformer.data.target_pos_w
+ sensor_quat = frame_transformer.data.target_quat_w
+
+
+Multi-Backend Support: PresetCfg Pattern
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Isaac Lab 3.0 introduces a **PresetCfg pattern** for writing environment configurations
+that work with both the PhysX and Newton backends. Instead of hard-coding a single
+physics config, environments declare named configuration variants. The active variant
+is selected at launch via a Hydra CLI override.
+
+What is PresetCfg?
+------------------
+
+:class:`~isaaclab_tasks.utils.PresetCfg` is a base ``@configclass`` whose typed fields
+represent named variants of a configuration section. The field named ``default`` is used
+when no CLI override is given. Other fields are named presets selectable with
+``presets=`` on the command line:
+
+.. code-block:: python
+
+ from isaaclab_tasks.utils import PresetCfg
+ from isaaclab.utils import configclass
+
+ @configclass
+ class MyPhysicsCfg(PresetCfg):
+ default: PhysxCfg = PhysxCfg(...) # used when no override is given
+ physx: PhysxCfg = PhysxCfg(...) # selected by presets=physx
+ newton: NewtonCfg = NewtonCfg(...) # selected by presets=newton
+
+Selecting a preset at launch
+-----------------------------
+
+Pass ``presets=newton`` (or ``presets=physx``) on the CLI to swap the entire config section:
+
+.. code-block:: bash
+
+ # Run with Newton backend
+ python train.py task=Isaac-Franka-Cabinet-v0 presets=newton
+
+ # Run with default (PhysX) backend
+ python train.py task=Isaac-Franka-Cabinet-v0
+
+Adding Multi-Backend Support to an Environment
+-----------------------------------------------
+
+**Step 1 — Physics config**
+
+Replace a plain ``PhysxCfg(...)`` assignment in ``__post_init__`` with a ``PresetCfg``
+subclass that carries both a PhysX and a Newton variant.
+
+*Before:*
+
+.. code-block:: python
+
+ def __post_init__(self):
+ self.sim.dt = 1 / 60
+ self.sim.physics = PhysxCfg(bounce_threshold_velocity=0.2)
+
+*After:*
+
+.. code-block:: python
+
+ from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg
+ from isaaclab_physx.physics import PhysxCfg
+ from isaaclab_tasks.utils import PresetCfg
+
+ @configclass
+ class ReachPhysicsCfg(PresetCfg):
+ default: PhysxCfg = PhysxCfg(bounce_threshold_velocity=0.2)
+ physx: PhysxCfg = PhysxCfg(bounce_threshold_velocity=0.2)
+ newton: NewtonCfg = NewtonCfg(
+ solver_cfg=MJWarpSolverCfg(
+ njmax=20, nconmax=20, ls_iterations=20,
+ cone="pyramidal", ls_parallel=True,
+ integrator="implicitfast", impratio=1,
+ ),
+ num_substeps=1,
+ debug_mode=False,
+ )
+
+ # In the env cfg __post_init__:
+ def __post_init__(self):
+ self.sim.dt = 1 / 60
+ self.sim.physics = ReachPhysicsCfg()
+
+Key Newton solver parameters:
+
+.. list-table::
+ :header-rows: 1
+ :widths: 25 75
+
+ * - Parameter
+ - Effect
+ * - ``njmax``
+ - Max constraint rows; set ≥ expected contact count per env
+ * - ``nconmax``
+ - Max contacts per env
+ * - ``ls_iterations``
+ - Linear solver iterations (higher = more stable, slower)
+ * - ``cone``
+ - ``"pyramidal"`` (fast) or ``"elliptic"`` (more accurate)
+ * - ``integrator``
+ - ``"implicitfast"`` (recommended) or ``"euler"``
+ * - ``impratio``
+ - Impedance ratio; >1 improves soft contact stability
+ * - ``num_substeps``
+ - Physics substeps per environment step
+
+**Step 2 — Differentiating Newton and PhysX Configs**
+
+Not all configurations may be the same between Newton and PhysX simulations.
+We can provide a Newton-specific config such as:
+
+.. code-block:: python
+
+ @configclass
+ class EventCfg:
+ """Full event config (PhysX-compatible)."""
+ robot_physics_material = EventTerm(
+ func=mdp.randomize_rigid_body_material,
+ mode="startup",
+ params={...},
+ )
+ reset_all = EventTerm(func=mdp.reset_scene_to_default, mode="reset")
+ reset_robot_joints = EventTerm(
+ func=mdp.reset_joints_by_offset, mode="reset", params={...}
+ )
+
+
+ @configclass
+ class _EnvNewtonEventCfg:
+ """Newton-compatible events."""
+ reset_all = EventTerm(func=mdp.reset_scene_to_default, mode="reset")
+ reset_robot_joints = EventTerm(
+ func=mdp.reset_joints_by_offset, mode="reset", params={...}
+ )
+
+
+ @configclass
+ class EnvEventCfg(PresetCfg):
+ default: EventCfg = EventCfg()
+ physx: EventCfg = EventCfg()
+ newton: _EnvNewtonEventCfg = _EnvNewtonEventCfg()
+
+Then change the ``events`` field in your env cfg from ``EventCfg`` to ``EnvEventCfg``:
+
+.. code-block:: python
+
+ @configclass
+ class MyEnvCfg(ManagerBasedRLEnvCfg):
+ events: EnvEventCfg = EnvEventCfg() # was: EventCfg = EventCfg()
+
+
+RigidObjectCollection API Renaming
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :class:`~isaaclab_physx.assets.RigidObjectCollection` and
+:class:`~isaaclab_physx.assets.RigidObjectCollectionData` classes have undergone an API rename
+to provide consistency with other asset classes. The ``object_*`` naming convention has been
+deprecated in favor of ``body_*``.
+
+
+Method Renames
+--------------
+
+The following methods have been renamed. The old methods are deprecated and will be removed in a
+future release:
+
++------------------------------------------+------------------------------------------+
+| Deprecated (2.x) | New (3.0) |
++==========================================+==========================================+
+| ``write_object_state_to_sim()`` | ``write_body_state_to_sim()`` |
++------------------------------------------+------------------------------------------+
+| ``write_object_link_state_to_sim()`` | ``write_body_link_state_to_sim()`` |
++------------------------------------------+------------------------------------------+
+| ``write_object_pose_to_sim()`` | ``write_body_pose_to_sim()`` |
++------------------------------------------+------------------------------------------+
+| ``write_object_link_pose_to_sim()`` | ``write_body_link_pose_to_sim()`` |
++------------------------------------------+------------------------------------------+
+| ``write_object_com_pose_to_sim()`` | ``write_body_com_pose_to_sim()`` |
++------------------------------------------+------------------------------------------+
+| ``write_object_velocity_to_sim()`` | ``write_body_com_velocity_to_sim()`` |
++------------------------------------------+------------------------------------------+
+| ``write_object_com_velocity_to_sim()`` | ``write_body_com_velocity_to_sim()`` |
++------------------------------------------+------------------------------------------+
+| ``write_object_link_velocity_to_sim()`` | ``write_body_link_velocity_to_sim()`` |
++------------------------------------------+------------------------------------------+
+| ``find_objects()`` | ``find_bodies()`` |
++------------------------------------------+------------------------------------------+
+
+
+Property Renames (Data Class)
+-----------------------------
+
+The following properties on :class:`~isaaclab_physx.assets.RigidObjectCollectionData` have been
+renamed. The old properties are deprecated and will be removed in a future release:
+
++------------------------------------------+------------------------------------------+
+| Deprecated (2.x) | New (3.0) |
++==========================================+==========================================+
+| ``default_object_state`` | ``default_body_state`` |
++------------------------------------------+------------------------------------------+
+| ``object_names`` | ``body_names`` |
++------------------------------------------+------------------------------------------+
+| ``object_link_pose_w`` | ``body_link_pose_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_link_vel_w`` | ``body_link_vel_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_pose_w`` | ``body_com_pose_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_vel_w`` | ``body_com_vel_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_state_w`` | ``body_state_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_link_state_w`` | ``body_link_state_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_state_w`` | ``body_com_state_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_acc_w`` | ``body_com_acc_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_pose_b`` | ``body_com_pose_b`` |
++------------------------------------------+------------------------------------------+
+| ``object_link_pos_w`` | ``body_link_pos_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_link_quat_w`` | ``body_link_quat_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_link_lin_vel_w`` | ``body_link_lin_vel_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_link_ang_vel_w`` | ``body_link_ang_vel_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_pos_w`` | ``body_com_pos_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_quat_w`` | ``body_com_quat_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_lin_vel_w`` | ``body_com_lin_vel_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_ang_vel_w`` | ``body_com_ang_vel_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_lin_acc_w`` | ``body_com_lin_acc_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_ang_acc_w`` | ``body_com_ang_acc_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_pos_b`` | ``body_com_pos_b`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_quat_b`` | ``body_com_quat_b`` |
++------------------------------------------+------------------------------------------+
+| ``object_link_lin_vel_b`` | ``body_link_lin_vel_b`` |
++------------------------------------------+------------------------------------------+
+| ``object_link_ang_vel_b`` | ``body_link_ang_vel_b`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_lin_vel_b`` | ``body_com_lin_vel_b`` |
++------------------------------------------+------------------------------------------+
+| ``object_com_ang_vel_b`` | ``body_com_ang_vel_b`` |
++------------------------------------------+------------------------------------------+
+| ``object_pose_w`` | ``body_pose_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_pos_w`` | ``body_pos_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_quat_w`` | ``body_quat_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_vel_w`` | ``body_vel_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_lin_vel_w`` | ``body_lin_vel_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_ang_vel_w`` | ``body_ang_vel_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_lin_vel_b`` | ``body_lin_vel_b`` |
++------------------------------------------+------------------------------------------+
+| ``object_ang_vel_b`` | ``body_ang_vel_b`` |
++------------------------------------------+------------------------------------------+
+| ``object_acc_w`` | ``body_acc_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_lin_acc_w`` | ``body_lin_acc_w`` |
++------------------------------------------+------------------------------------------+
+| ``object_ang_acc_w`` | ``body_ang_acc_w`` |
++------------------------------------------+------------------------------------------+
+
+.. note::
+
+ All deprecated methods and properties will issue a deprecation warning when used. Your existing
+ code will continue to work, but you should migrate to the new API to avoid issues in future releases.
+
+
+Migration Example
+-----------------
+
+Here's a complete example showing how to update your code:
+
+**Before (Isaac Lab 2.x):**
+
+.. code-block:: python
+
+ from isaaclab.assets import DeformableObject, DeformableObjectCfg
+ from isaaclab.assets import SurfaceGripper, SurfaceGripperCfg
+ from isaaclab.assets import RigidObjectCollection
+
+ # Using deprecated root_physx_view
+ robot = scene["robot"]
+ masses = robot.root_physx_view.get_masses()
+
+ # Using deprecated object_* API
+ collection = scene["object_collection"]
+ poses = collection.data.object_pose_w
+ collection.write_object_state_to_sim(state, env_ids=env_ids, object_ids=object_ids)
+
+**After (Isaac Lab 3.0):**
+
+.. code-block:: python
+
+ from isaaclab_physx.assets import DeformableObject, DeformableObjectCfg
+ from isaaclab_physx.assets import SurfaceGripper, SurfaceGripperCfg
+ from isaaclab.assets import RigidObjectCollection # unchanged
+
+ # Using new root_view property
+ robot = scene["robot"]
+ masses = robot.root_view.get_masses()
+
+ # Using new body_* API
+ collection = scene["object_collection"]
+ poses = collection.data.body_pose_w
+ collection.write_body_state_to_sim(state, env_ids=env_ids, body_ids=object_ids)
+
+
+Quaternion Format
+~~~~~~~~~~~~~~~~~
+
+**The quaternion format changed from WXYZ to XYZW.**
+
++------------------+----------------------------------+----------------------------------+
+| Component | Old Format (WXYZ) | New Format (XYZW) |
++==================+==================================+==================================+
+| Order | ``(w, x, y, z)`` | ``(x, y, z, w)`` |
++------------------+----------------------------------+----------------------------------+
+| Identity | ``(1.0, 0.0, 0.0, 0.0)`` | ``(0.0, 0.0, 0.0, 1.0)`` |
++------------------+----------------------------------+----------------------------------+
+
+
+Why This Change?
+----------------
+
+The new XYZW format aligns with:
+
+- **Warp**: NVIDIA's spatial computing framework
+- **PhysX**: PhysX physics engine
+- **Newton**: Newton multi-solver framework
+
+This alignment removes the need for internal quaternion conversions, making the code simpler,
+faster, and less error-prone.
+
+
+What You Need to Update
+-----------------------
+
+Any hard-coded quaternion values in your code need to be converted from WXYZ to XYZW.
+This includes:
+
+1. **Configuration files** - ``rot`` parameters in asset configs
+2. **Task definitions** - Goal poses, initial states
+3. **Controller parameters** - Target orientations
+4. **Documentation** - Code examples with quaternions
+
+Also, if you were relying on the :func:`~isaaclab.utils.math.convert_quat` function to convert quaternions, this should
+no longer be needed. (This would happen if you were pulling values from the views directly.)
+
+Example: Updating Asset Configuration
+-------------------------------------
+
+**Before (WXYZ):**
+
+.. code-block:: python
+
+ from isaaclab.assets import AssetBaseCfg
+
+ cfg = AssetBaseCfg(
+ init_state=AssetBaseCfg.InitialStateCfg(
+ pos=(0.0, 0.0, 0.5),
+ rot=(1.0, 0.0, 0.0, 0.0), # OLD: w, x, y, z
+ ),
+ )
+
+**After (XYZW):**
+
+.. code-block:: python
+
+ from isaaclab.assets import AssetBaseCfg
+
+ cfg = AssetBaseCfg(
+ init_state=AssetBaseCfg.InitialStateCfg(
+ pos=(0.0, 0.0, 0.5),
+ rot=(0.0, 0.0, 0.0, 1.0), # NEW: x, y, z, w
+ ),
+ )
+
+
+Using the Quaternion Finder Tool
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We provide a tool to help you find and fix quaternions in your codebase automatically. This is not a bulletproof tool,
+but it should help you find most of the quaternions that need to be updated. You *should* review the results manually.
+
+.. warning::
+ Do not run the tool on the whole codebase! If you run the tool on our own packages (isaaclab, or isaaclab_tasks for
+ instance) it will find all the quaternions that we already converted. This tool is only meant to be used on your own
+ codebase with no overlap with our own packages.
+
+Finding Quaternions
+-------------------
+
+Run the tool to scan your code for potential quaternions:
+
+.. code-block:: bash
+
+ # Scan the 'source' directory (default)
+ python scripts/tools/find_quaternions.py
+
+ # Scan a specific path
+ python scripts/tools/find_quaternions.py --path my_project/
+
+ # Compare against a different branch
+ python scripts/tools/find_quaternions.py --base develop
+
+.. tip::
+ We recommend always running the tool with a custom base branch *and* a specific path.
+
+
+The tool will show you:
+
+- Quaternions that haven't been updated (marked as ``UNCHANGED``)
+- Whether each looks like a WXYZ identity quaternion (``WXYZ_IDENTITY``)
+- Whether the format is likely WXYZ (``LIKELY_WXYZ``)
+
+
+Understanding the Output
+------------------------
+
+.. code-block:: text
+
+ my_project/robot_cfg.py:42:8 ⚠ UNCHANGED [WXYZ_IDENTITY]
+ Values: [1.0, 0.0, 0.0, 0.0]
+ Source: rot=(1.0, 0.0, 0.0, 0.0),
+
+This tells you:
+
+- **File and line**: ``my_project/robot_cfg.py:42``
+- **Status**: ``UNCHANGED`` means this line hasn't been modified yet
+- **Flag**: ``WXYZ_IDENTITY`` means it's the identity quaternion in old WXYZ format
+- **Values**: The actual quaternion values found
+- **Source**: The line of code for context
+
+
+Filtering Results
+-----------------
+
+Focus on specific types of quaternions:
+
+.. code-block:: bash
+
+ # Only show identity quaternions [1, 0, 0, 0]
+ python scripts/tools/find_quaternions.py --check-identity
+
+ # Only show quaternions likely in WXYZ format
+ python scripts/tools/find_quaternions.py --likely-wxyz
+
+ # Show ALL potential quaternions (ignore format heuristics)
+ python scripts/tools/find_quaternions.py --all-quats
+
+
+Fixing Quaternions Automatically
+--------------------------------
+
+The tool can automatically convert quaternions from WXYZ to XYZW:
+
+.. code-block:: bash
+
+ # Interactive mode: prompts before each fix
+ python scripts/tools/find_quaternions.py --fix
+
+ # Only fix identity quaternions (safest option)
+ python scripts/tools/find_quaternions.py --fix-identity-only
+
+ # Preview changes without applying them
+ python scripts/tools/find_quaternions.py --fix --dry-run
+
+ # Apply all fixes without prompting
+ python scripts/tools/find_quaternions.py --fix --force
+
+
+Interactive Fix Example
+-----------------------
+
+When running with ``--fix``, you'll see something like:
+
+.. code-block:: text
+
+ ────────────────────────────────────────────────────────────────────────────────
+ 📍 my_project/robot_cfg.py:42 [WXYZ_IDENTITY]
+ ────────────────────────────────────────────────────────────────────────────────
+ 40 | init_state=AssetBaseCfg.InitialStateCfg(
+ 41 | pos=(0.0, 0.0, 0.5),
+ >>> 42 | rot=(1.0, 0.0, 0.0, 0.0),
+ 43 | ),
+ 44 | )
+ ────────────────────────────────────────────────────────────────────────────────
+ Change: [1.0, 0.0, 0.0, 0.0] → [0.0, 0.0, 0.0, 1.0]
+ Result: rot=(0.0, 0.0, 0.0, 1.0),
+ Apply this fix? [Y/n/a/q]:
+
+Options:
+
+- **Y** (yes): Apply this fix
+- **n** (no): Skip this one
+- **a** (all): Apply all remaining fixes without asking
+- **q** (quit): Stop fixing
+
+
+How the Tool Works
+------------------
+
+The tool uses several techniques to find quaternions:
+
+1. **Python files**: Parses the code using AST (Abstract Syntax Tree) to find
+ 4-element tuples and lists with numeric values.
+
+2. **JSON files**: Uses regex to find 4-element arrays.
+
+3. **RST documentation**: Searches for quaternion-like patterns in docs.
+
+To identify if something is a quaternion, the tool checks:
+
+- Is it exactly 4 numeric values?
+- Does the sum of squares ≈ 1? (unit quaternion property)
+- Does it match known patterns like identity quaternions?
+
+To determine if it's in WXYZ format:
+
+- Is the first value 1.0 and rest are 0? (WXYZ identity)
+- Is the first value a common cos(θ/2) value like 0.707, 0.866, etc.?
+- Is the pattern consistent with first-element being the scalar part?
+
+
+Best Practices for Migration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+1. **Start with a clean git state** - Commit your work before running fixes.
+
+2. **Run the tool first without ``--fix``** - Review what will be changed.
+
+3. **Fix identity quaternions first** - They're the most common and safest:
+
+ .. code-block:: bash
+
+ python scripts/tools/find_quaternions.py --fix-identity-only
+
+4. **Review non-identity quaternions manually** - Some 4-element lists might
+ not be quaternions (e.g., RGBA colors, bounding boxes).
+
+5. **Test your code** - Run your simulations to verify everything works correctly.
+
+6. **Check documentation** - Update any docs or comments that mention quaternion format.
+
+API Changes
+~~~~~~~~~~~
+
+The ``convert_quat`` function has been removed
+----------------------------------------------
+
+Previously, IsaacLab had a utility function to convert between quaternion formats:
+
+.. code-block:: python
+
+ # OLD - No longer needed
+ from isaaclab.utils.math import convert_quat
+ quat_xyzw = convert_quat(quat_wxyz, "xyzw")
+
+Since everything now uses XYZW natively, this function is no longer needed.
+If you were using it, simply remove the conversion calls.
+
+
+Math utility functions now expect XYZW
+--------------------------------------
+
+All quaternion functions in :mod:`isaaclab.utils.math` now expect and return
+quaternions in XYZW format:
+
+- :func:`~isaaclab.utils.math.quat_mul`
+- :func:`~isaaclab.utils.math.quat_apply`
+- :func:`~isaaclab.utils.math.quat_from_euler_xyz`
+- :func:`~isaaclab.utils.math.euler_xyz_from_quat`
+- :func:`~isaaclab.utils.math.quat_from_matrix`
+- :func:`~isaaclab.utils.math.matrix_from_quat`
+- And all other quaternion utilities
+
+
+Warp Backend for Asset and Sensor Data
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+All ``.data.*`` properties on asset and sensor classes now return ``wp.array`` instead of
+``torch.Tensor``. This change applies to all asset classes (:class:`~isaaclab.assets.Articulation`,
+:class:`~isaaclab.assets.RigidObject`, :class:`~isaaclab.assets.RigidObjectCollection`,
+:class:`~isaaclab_physx.assets.DeformableObject`) and all sensor classes
+(:class:`~isaaclab_physx.sensors.ContactSensor`, :class:`~isaaclab_physx.sensors.Imu`,
+:class:`~isaaclab_physx.sensors.Pva`, :class:`~isaaclab_physx.sensors.FrameTransformer`).
+
+To convert back to ``torch.Tensor`` for use with PyTorch operations, wrap the property
+access with ``wp.to_torch()``:
+
+.. code-block:: python
+
+ import warp as wp
+
+ # Before (Isaac Lab 2.x)
+ root_pos = robot.data.root_pos_w # torch.Tensor
+ joint_pos = robot.data.joint_pos # torch.Tensor
+ contact_forces = sensor.data.net_forces_w # torch.Tensor
+
+ # After (Isaac Lab 3.x)
+ root_pos = robot.data.root_pos_w # wp.array
+ joint_pos = robot.data.joint_pos # wp.array
+ contact_forces = sensor.data.net_forces_w # wp.array
+
+ # To use with torch operations, wrap with wp.to_torch()
+ root_pos_torch = wp.to_torch(robot.data.root_pos_w) # torch.Tensor
+ joint_pos_torch = wp.to_torch(robot.data.joint_pos) # torch.Tensor
+ contact_torch = wp.to_torch(sensor.data.net_forces_w) # torch.Tensor
+
+Common patterns that need updating:
+
+.. code-block:: python
+
+ # Cloning data
+ # Before:
+ pos = robot.data.root_pos_w.clone()
+ # After:
+ pos = wp.to_torch(robot.data.root_pos_w).clone()
+
+ # Creating zero tensors with matching shape
+ # Before:
+ zeros = torch.zeros_like(robot.data.root_pos_w)
+ # After:
+ zeros = torch.zeros_like(wp.to_torch(robot.data.root_pos_w))
+
+ # Assertions in tests
+ # Before:
+ torch.testing.assert_close(robot.data.root_pos_w, expected)
+ # After:
+ torch.testing.assert_close(wp.to_torch(robot.data.root_pos_w), expected)
+
+.. list-table:: Affected classes
+ :header-rows: 1
+ :widths: 40 60
+
+ * - Class
+ - Package
+ * - :class:`~isaaclab.assets.Articulation`
+ - ``isaaclab`` / ``isaaclab_physx``
+ * - :class:`~isaaclab.assets.RigidObject`
+ - ``isaaclab`` / ``isaaclab_physx``
+ * - :class:`~isaaclab.assets.RigidObjectCollection`
+ - ``isaaclab`` / ``isaaclab_physx``
+ * - :class:`~isaaclab_physx.assets.DeformableObject`
+ - ``isaaclab_physx``
+ * - :class:`~isaaclab_physx.sensors.ContactSensor`
+ - ``isaaclab_physx``
+ * - :class:`~isaaclab_physx.sensors.Imu`
+ - ``isaaclab_physx``
+ * - :class:`~isaaclab_physx.sensors.Pva`
+ - ``isaaclab_physx``
+ * - :class:`~isaaclab_physx.sensors.FrameTransformer`
+ - ``isaaclab_physx``
+ * - :class:`~isaaclab.sensors.RayCaster`
+ - ``isaaclab``
+ * - :class:`~isaaclab.sensors.RayCasterCamera`
+ - ``isaaclab``
+ * - :class:`~isaaclab.sensors.MultiMeshRayCaster`
+ - ``isaaclab``
+ * - :class:`~isaaclab.sensors.MultiMeshRayCasterCamera`
+ - ``isaaclab``
+
+.. note::
+
+ An automated migration tool is provided at ``scripts/tools/wrap_warp_to_torch.py``.
+ It scans Python files for ``.data.`` accesses and wraps them with
+ ``wp.to_torch()``. Usage:
+
+ .. code-block:: bash
+
+ # Dry run (preview changes)
+ python scripts/tools/wrap_warp_to_torch.py path/to/your/code --dry-run
+
+ # Apply changes in-place
+ python scripts/tools/wrap_warp_to_torch.py path/to/your/code
+
+ Always review the changes after running the tool, as some accesses (e.g., those
+ already passed to warp-native functions) should not be wrapped.
+
+
+Ray Caster Warp Backend
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The :class:`~isaaclab.sensors.RayCaster`, :class:`~isaaclab.sensors.RayCasterCamera`,
+:class:`~isaaclab.sensors.MultiMeshRayCaster`, and
+:class:`~isaaclab.sensors.MultiMeshRayCasterCamera` sensors have been transitioned from a
+PyTorch/USD-based backend to a native Warp kernel pipeline. This improves performance by
+eliminating per-step tensor allocations and torch-to-warp conversions, but introduces several
+breaking changes.
+
+
+RayCasterData Return Types
+--------------------------
+
+The :attr:`~isaaclab.sensors.RayCasterData.pos_w`,
+:attr:`~isaaclab.sensors.RayCasterData.quat_w`, and
+:attr:`~isaaclab.sensors.RayCasterData.ray_hits_w` properties now return ``wp.array`` instead of
+``torch.Tensor``. This follows the same pattern as the general warp backend migration described
+above.
+
+.. code-block:: python
+
+ import warp as wp
+
+ # Before (Isaac Lab 2.x)
+ ray_hits = ray_caster.data.ray_hits_w # torch.Tensor
+ sensor_pos = ray_caster.data.pos_w # torch.Tensor
+
+ # After (Isaac Lab 3.x)
+ ray_hits = ray_caster.data.ray_hits_w # wp.array
+ sensor_pos = ray_caster.data.pos_w # wp.array
+
+ # To use with torch operations, wrap with wp.to_torch()
+ ray_hits_torch = wp.to_torch(ray_caster.data.ray_hits_w)
+ sensor_pos_torch = wp.to_torch(ray_caster.data.pos_w)
+
+
+Ray Alignment Configuration
+----------------------------
+
+The ``attach_yaw_only`` boolean parameter on :class:`~isaaclab.sensors.RayCasterCfg` has been
+deprecated in favor of the new ``ray_alignment`` parameter, which accepts one of three string
+values:
+
+.. list-table::
+ :header-rows: 1
+ :widths: 30 30 40
+
+ * - Old (2.x)
+ - New (3.0)
+ - Behavior
+ * - ``attach_yaw_only=False``
+ - ``ray_alignment="base"``
+ - Rays follow the full sensor orientation.
+ * - ``attach_yaw_only=True``
+ - ``ray_alignment="yaw"``
+ - Rays follow only the yaw component of the sensor orientation.
+ * - *(not available)*
+ - ``ray_alignment="world"``
+ - Rays are always cast in the world frame (no rotation applied).
+
+.. code-block:: python
+
+ # Before (Isaac Lab 2.x)
+ cfg = RayCasterCfg(attach_yaw_only=True, ...)
+
+ # After (Isaac Lab 3.x)
+ cfg = RayCasterCfg(ray_alignment="yaw", ...)
+
+
+Raycasting Kernel Signature Change
+-----------------------------------
+
+The :func:`~isaaclab.utils.warp.kernels.raycast_dynamic_meshes_kernel` Warp kernel now requires
+an ``env_mask`` parameter as its first argument. This is a ``wp.array(dtype=wp.bool)`` that
+controls which environments are updated. The public Python wrapper
+:func:`~isaaclab.utils.warp.ops.raycast_dynamic_meshes` has been updated to inject an all-True
+mask automatically, so code using the wrapper is unaffected.
+
+If you call the kernel directly, update your launch call:
+
+.. code-block:: python
+
+ import warp as wp
+
+ # Before (Isaac Lab 2.x)
+ wp.launch(
+ raycast_dynamic_meshes_kernel,
+ dim=(num_meshes, num_envs, num_rays),
+ inputs=[ray_starts, ray_directions, mesh_ids, ...],
+ )
+
+ # After (Isaac Lab 3.x) -- env_mask is now the first input
+ env_mask = wp.ones(num_envs, dtype=wp.bool, device=device)
+ wp.launch(
+ raycast_dynamic_meshes_kernel,
+ dim=(num_meshes, num_envs, num_rays),
+ inputs=[env_mask, ray_starts, ray_directions, mesh_ids, ...],
+ )
+
+
+RayCaster.meshes Cache Key
+--------------------------
+
+The :attr:`~isaaclab.sensors.RayCaster.meshes` class variable, which caches warp meshes across
+all :class:`~isaaclab.sensors.RayCaster` instances, is now keyed by ``(prim_path, device)`` tuples
+instead of by ``prim_path`` alone. This prevents a mesh that was built on one device (e.g. CPU)
+from being reused by a sensor running on a different device (e.g. CUDA), which caused illegal
+memory accesses on systems without unified memory.
+
+Code that reads or writes this cache directly must update both the type annotation and the key:
+
+.. code-block:: python
+
+ # Before (Isaac Lab 2.x)
+ meshes: ClassVar[dict[str, wp.Mesh]] = {}
+ wp_mesh = RayCaster.meshes[prim_path]
+
+ # After (Isaac Lab 3.x)
+ meshes: ClassVar[dict[tuple[str, str], wp.Mesh]] = {}
+ wp_mesh = RayCaster.meshes[(prim_path, device)]
+
+
+Write Method Index/Mask Split
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+All asset write methods have been split into two explicit variants:
+
+- ``write_*_to_sim_index(data, env_ids)`` — accepts partial data for a sparse set of
+ environment indices. The ``data`` tensor has shape ``(len(env_ids), ...)``.
+- ``write_*_to_sim_mask(data, env_mask)`` — accepts full data for all environments with a
+ boolean mask selecting which environments to update. The ``data`` tensor has shape
+ ``(num_envs, ...)``.
+
+The previous ``write_*_to_sim(data, env_ids)`` methods have been removed.
+
+.. code-block:: python
+
+ # Before (Isaac Lab 2.x)
+ robot.write_root_pose_to_sim(pose_data, env_ids)
+
+ # After (Isaac Lab 3.x) — indexed variant (partial data)
+ robot.write_root_pose_to_sim_index(root_pose=pose_data, env_ids=env_ids)
+
+ # After (Isaac Lab 3.x) — mask variant (full data, boolean mask)
+ robot.write_root_pose_to_sim_mask(root_pose=pose_data, env_mask=env_mask)
+
+.. list-table:: Affected write methods (RigidObject / Articulation)
+ :header-rows: 1
+ :widths: 50 50
+
+ * - Old method
+ - New methods
+ * - ``write_root_pose_to_sim``
+ - ``write_root_pose_to_sim_index`` / ``write_root_pose_to_sim_mask``
+ * - ``write_root_link_pose_to_sim``
+ - ``write_root_link_pose_to_sim_index`` / ``write_root_link_pose_to_sim_mask``
+ * - ``write_root_com_pose_to_sim``
+ - ``write_root_com_pose_to_sim_index`` / ``write_root_com_pose_to_sim_mask``
+ * - ``write_root_velocity_to_sim``
+ - ``write_root_velocity_to_sim_index`` / ``write_root_velocity_to_sim_mask``
+ * - ``write_root_com_velocity_to_sim``
+ - ``write_root_com_velocity_to_sim_index`` / ``write_root_com_velocity_to_sim_mask``
+ * - ``write_root_link_velocity_to_sim``
+ - ``write_root_link_velocity_to_sim_index`` / ``write_root_link_velocity_to_sim_mask``
+
+.. list-table:: Additional Articulation-specific write methods
+ :header-rows: 1
+ :widths: 50 50
+
+ * - Old method
+ - New methods
+ * - ``write_joint_position_to_sim``
+ - ``write_joint_position_to_sim_index`` / ``write_joint_position_to_sim_mask``
+ * - ``write_joint_velocity_to_sim``
+ - ``write_joint_velocity_to_sim_index`` / ``write_joint_velocity_to_sim_mask``
+ * - ``write_joint_stiffness_to_sim``
+ - ``write_joint_stiffness_to_sim_index`` / ``write_joint_stiffness_to_sim_mask``
+ * - ``write_joint_damping_to_sim``
+ - ``write_joint_damping_to_sim_index`` / ``write_joint_damping_to_sim_mask``
+ * - ``write_joint_position_limit_to_sim``
+ - ``write_joint_position_limit_to_sim_index`` / ``write_joint_position_limit_to_sim_mask``
+ * - ``write_joint_velocity_limit_to_sim``
+ - ``write_joint_velocity_limit_to_sim_index`` / ``write_joint_velocity_limit_to_sim_mask``
+ * - ``write_joint_effort_limit_to_sim``
+ - ``write_joint_effort_limit_to_sim_index`` / ``write_joint_effort_limit_to_sim_mask``
+ * - ``write_joint_armature_to_sim``
+ - ``write_joint_armature_to_sim_index`` / ``write_joint_armature_to_sim_mask``
+ * - ``write_joint_friction_coefficient_to_sim``
+ - ``write_joint_friction_coefficient_to_sim_index`` / ``write_joint_friction_coefficient_to_sim_mask``
+
+.. list-table:: RigidObjectCollection write methods
+ :header-rows: 1
+ :widths: 50 50
+
+ * - Old method
+ - New methods
+ * - ``write_body_pose_to_sim``
+ - ``write_body_pose_to_sim_index`` / ``write_body_pose_to_sim_mask``
+ * - ``write_body_link_pose_to_sim``
+ - ``write_body_link_pose_to_sim_index`` / ``write_body_link_pose_to_sim_mask``
+ * - ``write_body_com_pose_to_sim``
+ - ``write_body_com_pose_to_sim_index`` / ``write_body_com_pose_to_sim_mask``
+ * - ``write_body_velocity_to_sim``
+ - ``write_body_velocity_to_sim_index`` / ``write_body_velocity_to_sim_mask``
+ * - ``write_body_com_velocity_to_sim``
+ - ``write_body_com_velocity_to_sim_index`` / ``write_body_com_velocity_to_sim_mask``
+ * - ``write_body_link_velocity_to_sim``
+ - ``write_body_link_velocity_to_sim_index`` / ``write_body_link_velocity_to_sim_mask``
+
+
+TimestampedBufferWarp
+~~~~~~~~~~~~~~~~~~~~~
+
+If you have custom asset or sensor data classes that subclass the Isaac Lab base data classes,
+note that internal buffers have changed from :class:`~isaaclab.utils.buffers.TimestampedBuffer`
+to :class:`~isaaclab.utils.buffers.TimestampedBufferWarp`. The new class takes ``(shape, device,
+wp_dtype)`` as constructor arguments instead of a ``torch.Tensor``:
+
+.. code-block:: python
+
+ import warp as wp
+ from isaaclab.utils.buffers import TimestampedBufferWarp
+
+ # Before (Isaac Lab 2.x)
+ self._data.root_pos_w = TimestampedBuffer(torch.zeros(num_envs, 3, device=device))
+
+ # After (Isaac Lab 3.x)
+ self._data.root_pos_w = TimestampedBufferWarp(
+ shape=(num_envs,), device=device, wp_dtype=wp.vec3f
+ )
+
+
+URDF Importer
+~~~~~~~~~~~~~
+
+The URDF importer in Isaac Sim was rewritten to version 3.0, using the ``urdf-usd-converter``
+library and the ``isaacsim.asset.transformer.rules`` extension to produce structured USD output.
+The old C++ binding-based API (using Kit commands ``URDFParseFile``/``URDFImportRobot`` and the
+``_urdf`` interface from ``acquire_urdf_interface()``) has been replaced with a new Python-based
+pipeline.
+
+The IsaacLab :class:`~sim.converters.UrdfConverter` has been updated to replicate the new
+``URDFImporter.import_urdf()`` pipeline, inserting IsaacLab-specific post-processing (fix base,
+joint drives, link density) on the intermediate USD stage before the asset transformer
+restructures the output.
+
+.. important::
+
+ The previous version-pinning mechanism that locked the URDF importer extension to
+ ``isaacsim.asset.importer.urdf-2.4.31`` has been removed. The converter now uses whichever
+ version of the extension is available in your Isaac Sim installation.
+
+
+Deprecated Settings
+-------------------
+
+The following :class:`~sim.converters.UrdfConverterCfg` settings are **deprecated** because
+the new URDF importer 3.0 no longer supports them. They are kept for backward compatibility
+but will log warnings if enabled:
+
++-----------------------------------------------------------+-----------------------------------------------------+
+| Setting | Notes |
++===========================================================+=====================================================+
+| ``convert_mimic_joints_to_normal_joints`` | No longer supported by the importer. |
++-----------------------------------------------------------+-----------------------------------------------------+
+| ``replace_cylinders_with_capsules`` | No longer supported by the importer. |
++-----------------------------------------------------------+-----------------------------------------------------+
+| ``root_link_name`` | No longer supported by the importer. |
++-----------------------------------------------------------+-----------------------------------------------------+
+
+.. note::
+
+ The ``merge_fixed_joints`` setting is **still supported**. It is now implemented as a URDF XML
+ pre-processing step that runs before the USD conversion. Fixed joints are removed and child
+ link elements (visual, collision, inertial) are merged into the parent link with correct
+ transform composition.
+
+Additionally, the :class:`~sim.converters.UrdfConverterCfg.JointDriveCfg.NaturalFrequencyGainsCfg`
+gains mode is **deprecated**. The ``compute_natural_stiffness`` function that it depended on has
+been removed from the importer. If ``NaturalFrequencyGainsCfg`` is used, a
+:exc:`DeprecationWarning` is emitted and joint drive gains are left at the values produced by the
+URDF importer. Use :class:`~sim.converters.UrdfConverterCfg.JointDriveCfg.PDGainsCfg` instead.
+
+The :attr:`~sim.converters.AssetConverterBaseCfg.make_instanceable` setting from the base class
+is also no longer supported and will be ignored. Assets will be made instanceable by default.
+
+
+Updated CLI Tool
+----------------
+
+The ``convert_urdf.py`` script has been updated. The ``usd_file_name`` is now determined
+automatically by the importer based on the robot name and cannot be overridden.
+
+**Before (Isaac Lab 2.x):**
+
+.. code-block:: bash
+
+ ./isaaclab.sh -p scripts/tools/convert_urdf.py \
+ robot.urdf \
+ /output/dir/robot.usd \
+ --fix-base \
+ --merge-joints
+
+**After (Isaac Lab 3.0):**
+
+.. code-block:: bash
+
+ ./isaaclab.sh -p scripts/tools/convert_urdf.py \
+ robot.urdf \
+ /output/dir \
+ --fix-base \
+ --joint-stiffness 100.0 \
+ --joint-damping 1.0
+
+.. note::
+
+ The ``--merge-joints`` flag is still accepted and correctly triggers the pre-processing
+ step to merge fixed joints.
+
+
+Updated Python API
+------------------
+
+If you use :class:`~sim.converters.UrdfConverter` or :class:`~sim.converters.UrdfConverterCfg`
+directly in your code, note the following changes:
+
+1. The ``usd_file_name`` is now set automatically by the converter based on the URDF file name.
+ The importer generates output at ``{usd_dir}/{robot_name}/{robot_name}.usda``.
+
+2. The ``make_instanceable`` setting is no longer supported. Assets will be made instanceable
+ by default.
+
+3. The ``merge_fixed_joints`` parameter is now implemented as a pre-processing step.
+
+**Before (Isaac Lab 2.x):**
+
+.. code-block:: python
+
+ from isaaclab.sim.converters import UrdfConverter, UrdfConverterCfg
+
+ cfg = UrdfConverterCfg(
+ asset_path="robot.urdf",
+ usd_dir="/output/dir",
+ usd_file_name="robot.usd",
+ fix_base=True,
+ merge_fixed_joints=True,
+ make_instanceable=True,
+ joint_drive=UrdfConverterCfg.JointDriveCfg(
+ gains=UrdfConverterCfg.JointDriveCfg.PDGainsCfg(
+ stiffness=None, # use URDF values
+ damping=None,
+ ),
+ ),
+ )
+
+**After (Isaac Lab 3.0):**
+
+.. code-block:: python
+
+ from isaaclab.sim.converters import UrdfConverter, UrdfConverterCfg
+
+ cfg = UrdfConverterCfg(
+ asset_path="robot.urdf",
+ usd_dir="/output/dir",
+ # usd_file_name is determined automatically from the robot name
+ fix_base=True,
+ merge_fixed_joints=True, # supported via pre-processing
+ joint_drive=UrdfConverterCfg.JointDriveCfg(
+ gains=UrdfConverterCfg.JointDriveCfg.PDGainsCfg(
+ stiffness=None, # use URDF values
+ damping=None,
+ ),
+ ),
+ )
+
+
+MJCF Importer
+~~~~~~~~~~~~~
+
+The MJCF importer in Isaac Sim was rewritten to use the ``mujoco-usd-converter`` library.
+The old C++ binding-based API (using Kit commands ``MJCFCreateAsset``/``MJCFCreateImportConfig``
+and the ``ImportConfig`` class) has been replaced with a new pure-Python ``MJCFImporter`` class
+and ``MJCFImporterConfig`` dataclass.
+
+.. important::
+
+ The new MJCF importer produces USD assets with **nested rigid bodies** (i.e., ``RigidBodyAPI``
+ is applied to each link prim individually) instead of a single articulation root with rigid
+ body applied only at the top level. This matches how MuJoCo represents bodies and is
+ physically more accurate, but it may affect code that assumes a flat rigid body hierarchy.
+ If you have downstream logic that traverses the USD structure of MJCF-imported assets,
+ verify that it handles nested rigid body prims correctly.
+
+Removed Settings
+----------------
+
+The following :class:`~sim.converters.MjcfConverterCfg` settings have been **removed** because
+the new converter handles them automatically based on the MJCF file content:
+
+- ``fix_base`` — base fixedness is now inferred from the MJCF ```` tag.
+- ``link_density`` — density is now read directly from the MJCF model.
+- ``import_inertia_tensor`` — inertia tensors are always imported.
+- ``import_sites`` — sites are always imported.
+
+The :attr:`~sim.converters.AssetConverterBaseCfg.make_instanceable` setting from the base class
+is also no longer supported and will be ignored.
+
+
+New Settings
+------------
+
+The following new settings were added to :class:`~sim.converters.MjcfConverterCfg`:
+
++-----------------------------------------------------------------+------------------------------------------------------+
+| Setting | Description |
++=================================================================+======================================================+
+| :attr:`~sim.converters.MjcfConverterCfg.merge_mesh` | Merge meshes where possible to optimize the model. |
++-----------------------------------------------------------------+------------------------------------------------------+
+| :attr:`~sim.converters.MjcfConverterCfg.collision_from_visuals` | Generate collision geometry from visuals. |
++-----------------------------------------------------------------+------------------------------------------------------+
+| :attr:`~sim.converters.MjcfConverterCfg.collision_type` | Type of collision geometry (e.g. ``"default"``, |
+| | ``"Convex Hull"``, ``"Convex Decomposition"``). |
++-----------------------------------------------------------------+------------------------------------------------------+
+
+
+Renamed Settings
+----------------
+
++------------------------------------------+------------------------------------------+
+| Old (2.x) | New (3.0) |
++==========================================+==========================================+
+| ``self_collision`` | ``self_collision`` (unchanged) |
++------------------------------------------+------------------------------------------+
+
+.. note::
+
+ The underlying Isaac Sim API renamed ``self_collision`` to ``allow_self_collision``.
+ The IsaacLab :class:`~sim.converters.MjcfConverterCfg` keeps using ``self_collision``
+ for backward compatibility and maps it to the new name internally.
+
+
+Updated CLI Tool
+----------------
+
+The ``convert_mjcf.py`` script has been updated to match the new importer settings.
+Old command-line flags (``--fix-base``, ``--make-instanceable``, ``--import-sites``)
+are no longer available.
+
+**Before (Isaac Lab 2.x):**
+
+.. code-block:: bash
+
+ ./isaaclab.sh -p scripts/tools/convert_mjcf.py \
+ ../mujoco_menagerie/unitree_h1/h1.xml \
+ source/isaaclab_assets/data/Robots/Unitree/h1.usd \
+ --import-sites \
+ --make-instanceable
+
+**After (Isaac Lab 3.0):**
+
+.. code-block:: bash
+
+ ./isaaclab.sh -p scripts/tools/convert_mjcf.py \
+ ../mujoco_menagerie/unitree_h1/h1.xml \
+ source/isaaclab_assets/data/Robots/Unitree/h1.usd \
+ --merge-mesh \
+ --self-collision
+
+New flags: ``--merge-mesh``, ``--collision-from-visuals``, ``--collision-type``, ``--self-collision``.
+
+
+Updated Python API
+------------------
+
+If you use :class:`~sim.converters.MjcfConverter` or :class:`~sim.converters.MjcfConverterCfg`
+directly in your code, update your configuration:
+
+**Before (Isaac Lab 2.x):**
+
+.. code-block:: python
+
+ from isaaclab.sim.converters import MjcfConverter, MjcfConverterCfg
+
+ cfg = MjcfConverterCfg(
+ asset_path="robot.xml",
+ usd_dir="/output/dir",
+ fix_base=True,
+ import_sites=True,
+ make_instanceable=True,
+ )
+
+**After (Isaac Lab 3.0):**
+
+.. code-block:: python
+
+ from isaaclab.sim.converters import MjcfConverter, MjcfConverterCfg
+
+ cfg = MjcfConverterCfg(
+ asset_path="robot.xml",
+ usd_dir="/output/dir",
+ merge_mesh=True,
+ collision_from_visuals=False,
+ self_collision=False,
+ )
+
+
+XR Teleoperation: Isaac Teleop Integration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The native XR teleoperation stack in ``isaaclab.devices.openxr`` has been deprecated and replaced
+by `Isaac Teleop `_, integrated via the ``isaaclab_teleop``
+extension. The ``isaac-teleop-device-plugins`` repository has also been deprecated; all device
+plugin support is now in Isaac Teleop.
+
+For full documentation on the new stack, see :ref:`isaac-teleop-feature`.
+
+
+Installation Requirement
+------------------------
+
+Isaac Teleop must now be installed in your Isaac Lab environment:
+
+.. code-block:: bash
+
+ pip install isaacteleop~=1.0 --extra-index-url https://pypi.nvidia.com
+
+See :ref:`install-isaac-teleop` for complete installation instructions.
+
+
+Import Changes
+--------------
+
+.. list-table::
+ :header-rows: 1
+ :widths: 50 50
+
+ * - Deprecated (2.x)
+ - New (3.0)
+ * - ``from isaaclab.devices.openxr import OpenXRDevice``
+ - ``from isaaclab_teleop import IsaacTeleopDevice``
+ * - ``from isaaclab.devices.openxr import OpenXRDeviceCfg``
+ - ``from isaaclab_teleop import IsaacTeleopCfg``
+ * - ``from isaaclab.devices.openxr import XrCfg``
+ - ``from isaaclab_teleop import XrCfg``
+ * - ``from isaaclab.devices.openxr import ManusVive``
+ - ``from isaaclab_teleop import IsaacTeleopDevice`` (with Manus plugin configured)
+ * - ``from isaaclab.devices import RetargeterBase``
+ - Use Isaac Teleop ``BaseRetargeter`` and pipeline builder pattern
+ * - ``from isaaclab.devices.openxr.retargeters import Se3AbsRetargeter``
+ - ``from isaacteleop.retargeters import Se3AbsRetargeter``
+
+
+Environment Configuration Changes
+----------------------------------
+
+The ``teleop_devices`` field with ``OpenXRDeviceCfg`` has been replaced by the ``isaac_teleop``
+field with ``IsaacTeleopCfg`` and a pipeline builder callable.
+
+**Before (Isaac Lab 2.x):**
+
+.. code-block:: python
+
+ from isaaclab.devices import DevicesCfg, OpenXRDeviceCfg
+ from isaaclab.devices.openxr import XrCfg
+ from isaaclab.devices.openxr.retargeters import Se3AbsRetargeterCfg, GripperRetargeterCfg
+
+ @configclass
+ class MyEnvCfg(ManagerBasedRLEnvCfg):
+
+ xr: XrCfg = XrCfg(anchor_pos=[0.0, 0.0, 0.0])
+
+ teleop_devices: DevicesCfg = field(default_factory=lambda: DevicesCfg(
+ handtracking=OpenXRDeviceCfg(
+ xr_cfg=None,
+ retargeters=[
+ Se3AbsRetargeterCfg(bound_hand=0, zero_out_xy_rotation=True),
+ GripperRetargeterCfg(bound_hand=0),
+ ]
+ ),
+ ))
+
+**After (Isaac Lab 3.0):**
+
+.. code-block:: python
+
+ from isaaclab_teleop import IsaacTeleopCfg, XrCfg
+
+ def _build_pipeline():
+ from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource, HandsSource
+ from isaacteleop.retargeting_engine.interface import OutputCombiner, ValueInput
+ from isaacteleop.retargeters import (
+ GripperRetargeter, GripperRetargeterConfig,
+ Se3AbsRetargeter, Se3RetargeterConfig, TensorReorderer,
+ )
+ from isaacteleop.retargeting_engine.tensor_types import TransformMatrix
+
+ controllers = ControllersSource(name="controllers")
+ hands = HandsSource(name="hands")
+ transform = ValueInput("world_T_anchor", TransformMatrix())
+ t_controllers = controllers.transformed(transform.output(ValueInput.VALUE))
+
+ se3 = Se3AbsRetargeter(Se3RetargeterConfig(input_device=ControllersSource.RIGHT), name="ee")
+ c_se3 = se3.connect({ControllersSource.RIGHT: t_controllers.output(ControllersSource.RIGHT)})
+
+ grip = GripperRetargeter(GripperRetargeterConfig(hand_side="right"), name="grip")
+ c_grip = grip.connect({
+ ControllersSource.RIGHT: t_controllers.output(ControllersSource.RIGHT),
+ HandsSource.RIGHT: hands.output(HandsSource.RIGHT),
+ })
+
+ reorder = TensorReorderer(
+ input_config={"ee": ["pos_x","pos_y","pos_z","quat_x","quat_y","quat_z","quat_w"],
+ "grip": ["gripper_value"]},
+ output_order=["pos_x","pos_y","pos_z","quat_x","quat_y","quat_z","quat_w","gripper_value"],
+ name="reorder", input_types={"ee": "array", "grip": "scalar"},
+ )
+ c_reorder = reorder.connect({"ee": c_se3.output("ee_pose"), "grip": c_grip.output("gripper_command")})
+ return OutputCombiner({"action": c_reorder.output("output")})
+
+ @configclass
+ class MyEnvCfg(ManagerBasedRLEnvCfg):
+
+ xr: XrCfg = XrCfg(anchor_pos=(0.0, 0.0, 0.0))
+
+ def __post_init__(self):
+ super().__post_init__()
+ self.isaac_teleop = IsaacTeleopCfg(
+ pipeline_builder=_build_pipeline,
+ sim_device=self.sim.device,
+ xr_cfg=self.xr,
+ )
+
+
+Backward Compatibility
+----------------------
+
+The old classes still exist and will issue ``DeprecationWarning`` when used:
+
+* ``isaaclab.devices.openxr.OpenXRDevice`` and ``OpenXRDeviceCfg``
+* ``isaaclab.devices.openxr.ManusVive`` and ``ManusViveCfg``
+* All retargeters under ``isaaclab.devices.openxr.retargeters``
+
+Deprecated retargeters have been moved to ``isaaclab_teleop.deprecated.openxr.retargeters`` for
+compatibility. These will be removed in a future release.
+
+
+PhysX Tensors API Module Path
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Recent Isaac Sim releases removed the internal ``impl`` submodule of
+``omni.physics.tensors`` and now expose the PhysX Tensor API types
+(``ArticulationView``, ``RigidBodyView``, ``SimulationView``, etc.) directly
+under ``omni.physics.tensors.api``. Importing from the old path raises
+``ModuleNotFoundError: No module named 'omni.physics.tensors.impl'`` at import
+time.
+
+Isaac Lab has been updated to import from the new path. Downstream code
+(custom assets, sensors, or scripts) that imported from the old path must be
+updated:
+
+.. code-block:: python
+
+ # Before (Isaac Lab 2.x / older Isaac Sim)
+ import omni.physics.tensors.impl.api as physx
+
+ # After (Isaac Lab 3.x / current Isaac Sim)
+ import omni.physics.tensors.api as physx
+
+The class identities are unchanged — only the module path moved. Type hints
+referencing the old path (``omni.physics.tensors.impl.api.ArticulationView``)
+should be similarly updated to ``omni.physics.tensors.api.ArticulationView``.
+
+
+Need Help?
+~~~~~~~~~~
+
+If you encounter issues during migration:
+
+1. Check the `IsaacLab GitHub Issues `_
+2. Review the `CHANGELOG `_
+3. Join the community on `Discord `_
diff --git a/docs/source/overview/core-concepts/index.rst b/docs/source/overview/core-concepts/index.rst
index 930ab665c18..10fdf8935fb 100644
--- a/docs/source/overview/core-concepts/index.rst
+++ b/docs/source/overview/core-concepts/index.rst
@@ -7,7 +7,10 @@ This section we introduce core concepts in Isaac Lab.
:maxdepth: 1
+ multi_backend_architecture
task_workflows
actuators
sensors/index.rst
+ renderers
motion_generators
+ scene_data_providers
diff --git a/docs/source/overview/core-concepts/multi_backend_architecture.rst b/docs/source/overview/core-concepts/multi_backend_architecture.rst
new file mode 100644
index 00000000000..c9821ebf4a6
--- /dev/null
+++ b/docs/source/overview/core-concepts/multi_backend_architecture.rst
@@ -0,0 +1,357 @@
+Multi-Backend Architecture
+==========================
+
+Isaac Lab 3.0 introduced a multi-backend architecture that enables running simulations with
+different physics engines (PhysX and Newton) while maintaining a unified API. This page explains
+how the backend system works and how to extend it.
+
+Overview
+--------
+
+Instead of hard-coding a single physics engine, Isaac Lab uses a **factory pattern** to
+dispatch object creation to backend-specific implementations at runtime. When you write:
+
+.. code-block:: python
+
+ from isaaclab.assets import Articulation
+
+ robot = Articulation(cfg)
+
+The ``Articulation`` class is a factory that automatically creates a
+:class:`~isaaclab_physx.assets.articulation.Articulation` or
+:class:`~isaaclab_newton.assets.articulation.Articulation` depending on which physics backend
+is active. Your code never needs to import backend-specific modules directly.
+
+This pattern applies to all simulation components:
+
+.. list-table::
+ :header-rows: 1
+
+ * - Component
+ - Core API (``isaaclab``)
+ - PhysX (``isaaclab_physx``)
+ - Newton (``isaaclab_newton``)
+ * - Physics Manager
+ - :class:`~isaaclab.physics.PhysicsManager`
+ - :class:`~isaaclab_physx.physics.PhysxManager`
+ - :class:`~isaaclab_newton.physics.NewtonManager`
+ * - Articulation
+ - :class:`~isaaclab.assets.Articulation`
+ - :class:`~isaaclab_physx.assets.Articulation`
+ - :class:`~isaaclab_newton.assets.Articulation`
+ * - Rigid Object
+ - :class:`~isaaclab.assets.RigidObject`
+ - :class:`~isaaclab_physx.assets.RigidObject`
+ - :class:`~isaaclab_newton.assets.RigidObject`
+ * - Contact Sensor
+ - :class:`~isaaclab.sensors.ContactSensor`
+ - :class:`~isaaclab_physx.sensors.ContactSensor`
+ - :class:`~isaaclab_newton.sensors.ContactSensor`
+ * - Renderer
+ - :class:`~isaaclab.renderers.Renderer`
+ - :class:`~isaaclab_physx.renderers.IsaacRtxRenderer`
+ - :class:`~isaaclab_newton.renderers.NewtonWarpRenderer`
+ * - Scene Data Provider
+ - :class:`~isaaclab.physics.SceneDataProvider`
+ - :class:`~isaaclab_physx.scene_data_providers.PhysxSceneDataProvider`
+ - :class:`~isaaclab_newton.scene_data_providers.NewtonSceneDataProvider`
+ * - Cloner
+ - :func:`~isaaclab.cloner.clone_from_template`
+ - :func:`~isaaclab_physx.cloner.physx_replicate`
+ - :func:`~isaaclab_newton.cloner.newton_physics_replicate`
+
+The Factory Pattern
+-------------------
+
+All factories inherit from :class:`~isaaclab.utils.backend_utils.FactoryBase`, which uses a
+**convention-over-configuration** approach to locate backend implementations:
+
+1. The active physics backend is determined by inspecting
+ ``SimulationContext.physics_manager``.
+2. The factory's module path is used to derive the backend module path by replacing ``isaaclab``
+ with ``isaaclab_{backend}``. For example, ``isaaclab.assets.articulation`` maps to
+ ``isaaclab_physx.assets.articulation`` or ``isaaclab_newton.assets.articulation``.
+3. The backend module is lazily imported and the implementation class is cached in a registry.
+
+.. code-block:: text
+
+ User code: Articulation(cfg)
+ │
+ ▼
+ FactoryBase.__new__()
+ │
+ ├─ _get_backend() → "physx" or "newton"
+ │ (reads SimulationContext.physics_manager)
+ │
+ ├─ _get_module_name() → "isaaclab_physx.assets.articulation"
+ │ (convention: isaaclab.X.Y → isaaclab_{backend}.X.Y)
+ │
+ ├─ importlib.import_module()
+ │ (lazy load — only on first use)
+ │
+ └─ Return backend-specific instance
+
+**Custom backend resolution:** Some factories override the default resolution. For example, the
+:class:`~isaaclab.renderers.Renderer` factory selects backends based on the renderer config type
+rather than the physics manager, because renderers and physics backends are independent:
+
+.. code-block:: python
+
+ class Renderer(FactoryBase, BaseRenderer):
+ _backend_class_names = {
+ "physx": "IsaacRtxRenderer",
+ "newton": "NewtonWarpRenderer",
+ "ov": "OVRTXRenderer",
+ }
+
+Similarly, visualizers select backends based on the ``visualizer_type`` field in their config,
+allowing any visualizer to work with any physics backend.
+
+Backend Selection
+-----------------
+
+The physics backend is selected via the ``physics`` field in
+:class:`~isaaclab.sim.SimulationCfg`:
+
+.. code-block:: python
+
+ from isaaclab.sim import SimulationCfg
+ from isaaclab_physx.physics import PhysxCfg
+ from isaaclab_newton.physics import NewtonCfg, MJWarpSolverCfg
+
+ # Use PhysX (default)
+ sim_cfg = SimulationCfg(physics=PhysxCfg())
+
+ # Use Newton with MuJoCo-Warp solver
+ sim_cfg = SimulationCfg(physics=NewtonCfg(
+ solver_cfg=MJWarpSolverCfg(),
+ num_substeps=4,
+ ))
+
+Once the :class:`~isaaclab.sim.SimulationContext` is initialized, all subsequent factory
+instantiations automatically use the selected backend.
+
+Multi-Backend Environments with Presets
+---------------------------------------
+
+Environments can support multiple backends simultaneously using the :doc:`preset system
+`. Each backend gets its own configuration variant:
+
+.. code-block:: python
+
+ from isaaclab.utils import configclass
+ from isaaclab_tasks.utils import PresetCfg
+ from isaaclab_physx.physics import PhysxCfg
+ from isaaclab_newton.physics import NewtonCfg, MJWarpSolverCfg
+
+ @configclass
+ class CartpolePhysicsCfg(PresetCfg):
+ default: PhysxCfg = PhysxCfg()
+ physx: PhysxCfg = PhysxCfg()
+ newton: NewtonCfg = NewtonCfg(
+ solver_cfg=MJWarpSolverCfg(njmax=5, nconmax=3)
+ )
+
+ @configclass
+ class CartpoleEnvCfg(ManagerBasedRLEnvCfg):
+ sim: SimulationCfg = SimulationCfg(physics=CartpolePhysicsCfg())
+
+Users then select a backend at the command line:
+
+.. code-block:: bash
+
+ # Default (PhysX)
+ python train.py --task Isaac-Cartpole-v0
+
+ # Newton
+ python train.py --task Isaac-Cartpole-v0 presets=newton
+
+The Physics Manager
+-------------------
+
+Each backend implements :class:`~isaaclab.physics.PhysicsManager`, the abstract base class
+that drives the simulation loop:
+
+.. code-block:: python
+
+ class PhysicsManager(ABC):
+ @classmethod
+ @abstractmethod
+ def initialize(cls, sim_context: SimulationContext) -> None: ...
+
+ @classmethod
+ @abstractmethod
+ def reset(cls, soft: bool = False) -> None: ...
+
+ @classmethod
+ @abstractmethod
+ def forward(cls) -> None: ...
+
+ @classmethod
+ @abstractmethod
+ def step(cls) -> None: ...
+
+ @classmethod
+ def close(cls) -> None: ... # concrete; dispatches STOP event
+
+The physics manager also provides a **callback system** via
+:class:`~isaaclab.physics.PhysicsEvent` for cross-backend event handling:
+
+.. code-block:: python
+
+ from isaaclab.physics import PhysicsManager, PhysicsEvent
+
+ handle = PhysicsManager.register_callback(
+ callback=my_setup_fn,
+ event=PhysicsEvent.PHYSICS_READY,
+ order=0,
+ name="my_callback",
+ )
+
+Available events: ``MODEL_INIT`` (during scene building), ``PHYSICS_READY`` (after physics
+initialization), and ``STOP`` (on simulation shutdown).
+
+Asset and Sensor Interfaces
+---------------------------
+
+Assets and sensors follow the same pattern. Each has:
+
+1. **A base class** in ``isaaclab`` defining the interface (e.g., ``BaseArticulation``,
+ ``BaseContactSensor``)
+2. **A factory class** that inherits from both ``FactoryBase`` and the base class
+3. **Backend implementations** in ``isaaclab_physx`` and ``isaaclab_newton``
+
+The base classes define the public API contract — properties, methods, and data accessors
+that all backends must provide. Both backends use ``wp.array`` (Warp arrays) as their
+primary data type for asset and sensor data.
+
+Data classes follow the same pattern with their own factories (e.g.,
+``ArticulationData(FactoryBase, BaseArticulationData)``).
+
+Adding a New Physics Backend
+----------------------------
+
+To add a new physics backend (e.g., ``mybackend``), create a new extension package following
+the established conventions:
+
+**1. Package structure:**
+
+.. code-block:: text
+
+ source/isaaclab_mybackend/
+ └── isaaclab_mybackend/
+ ├── __init__.py
+ ├── physics/
+ │ ├── __init__.py # lazy_export()
+ │ ├── __init__.pyi # public exports
+ │ ├── mybackend_manager.py
+ │ └── mybackend_manager_cfg.py
+ ├── assets/
+ │ ├── articulation/
+ │ │ ├── __init__.py
+ │ │ ├── __init__.pyi
+ │ │ ├── articulation.py
+ │ │ └── articulation_data.py
+ │ ├── rigid_object/
+ │ │ └── ...
+ │ └── rigid_object_collection/
+ │ └── ...
+ ├── sensors/
+ │ ├── contact_sensor/
+ │ └── ...
+ ├── renderers/
+ │ └── ...
+ ├── cloner/
+ │ └── ...
+ └── scene_data_providers/
+ └── ...
+
+**2. Implement the physics manager:**
+
+.. code-block:: python
+
+ # isaaclab_mybackend/physics/mybackend_manager.py
+ from isaaclab.physics import PhysicsManager
+
+ class MyBackendManager(PhysicsManager):
+ @classmethod
+ def initialize(cls, sim_context):
+ super().initialize(sim_context)
+ # Initialize your physics engine
+
+ @classmethod
+ def step(cls):
+ # Advance simulation by one timestep
+
+ @classmethod
+ def forward(cls):
+ # Update kinematics without stepping
+
+ @classmethod
+ def reset(cls, soft=False):
+ if not soft:
+ cls.dispatch_event(PhysicsEvent.PHYSICS_READY)
+ # Reset simulation state
+
+ @classmethod
+ def close(cls):
+ super().close()
+ # Clean up resources
+
+**3. Create the physics config:**
+
+.. code-block:: python
+
+ # isaaclab_mybackend/physics/mybackend_manager_cfg.py
+ from isaaclab.physics import PhysicsCfg
+ from isaaclab.utils import configclass
+
+ @configclass
+ class MyBackendCfg(PhysicsCfg):
+ class_type = "{DIR}.mybackend_manager:MyBackendManager"
+ # Backend-specific settings here
+
+**4. Implement assets and sensors:**
+
+Each asset/sensor must extend the corresponding base class from ``isaaclab``. The class name
+must match the factory's expected name (by convention, the same name as the factory class).
+Use ``lazy_export()`` in ``__init__.py`` files — no manual registration needed.
+
+.. code-block:: python
+
+ # isaaclab_mybackend/assets/articulation/articulation.py
+ from isaaclab.assets.articulation import BaseArticulation
+
+ class Articulation(BaseArticulation):
+ def __init__(self, cfg):
+ super().__init__(cfg)
+ # Set up backend-specific simulation structures
+
+**5. Module discovery is automatic.** The ``FactoryBase`` convention maps
+``isaaclab.assets.articulation`` to ``isaaclab_mybackend.assets.articulation`` based on the
+active physics manager name. As long as you follow the package structure above, your backend
+classes will be discovered automatically.
+
+Key Design Principles
+---------------------
+
+- **Lazy loading**: Backend modules are imported only when first instantiated, keeping startup
+ fast and avoiding hard dependencies on unused backends.
+- **Convention over configuration**: Module paths follow a strict pattern
+ (``isaaclab.X.Y`` → ``isaaclab_{backend}.X.Y``), so no manual registration is needed.
+- **Independent selection**: Physics backend, renderer, and visualizer are selected
+ independently — you can use any combination.
+- **Warp-native data types**: Both backends return ``wp.array`` for asset and sensor data.
+ Use ``wp.to_torch()`` when interoperating with PyTorch-based code.
+- **Zero runtime overhead**: Backend selection happens at instantiation time. There are no
+ if-statements or dispatch logic on the hot path.
+
+See Also
+--------
+
+- :doc:`/source/migration/migrating_to_isaaclab_3-0` — migration guide from Isaac Lab 2.x to the
+ multi-backend architecture
+- :doc:`/source/features/hydra` — preset system for multi-backend environment configurations
+- :doc:`/source/experimental-features/newton-physics-integration/index` — Newton physics integration
+ guide
+- :doc:`renderers` — renderer backend architecture
diff --git a/docs/source/overview/core-concepts/renderers.rst b/docs/source/overview/core-concepts/renderers.rst
new file mode 100644
index 00000000000..4971e7a1d0c
--- /dev/null
+++ b/docs/source/overview/core-concepts/renderers.rst
@@ -0,0 +1,117 @@
+.. _overview_renderers:
+
+Renderers
+=========
+
+Isaac Lab uses a pluggable renderer architecture to support different rendering backends for camera sensors.
+The :class:`~isaaclab.renderers.BaseRenderer` abstract base class defines the interface that all renderer
+implementations must follow.
+
+Isaac Lab supports two rendering backends:
+
+- **RTX renderer** (``IsaacRtxRendererCfg`` / ``OVRTXRendererCfg``) — NVIDIA's Omniverse RTX
+ rendering pipeline. Requires Isaac Sim. Best for photorealistic rendering, full camera sensor
+ support (RGB, depth, semantic segmentation, etc.), and production quality outputs.
+- **Newton Warp renderer** (``NewtonWarpRendererCfg``) — A lightweight GPU-accelerated renderer
+ built on NVIDIA Warp. Works with the Newton physics backend and does **not** require Isaac Sim
+ (kit-less mode). Ideal for training workflows where full RTX fidelity is not needed.
+
+Choosing a renderer backend
+----------------------------
+
++---------------------+-------------------------------+---------------------------------+
+| Backend | Requires Isaac Sim? | Best For |
++=====================+===============================+=================================+
+| Isaac RTX | Yes | Full sensor fidelity, RTX |
+| | | photorealism, PhysX backend |
++---------------------+-------------------------------+---------------------------------+
+| OVRTX | Yes (+ ``isaaclab_ov``) | Alternative RTX pipeline |
++---------------------+-------------------------------+---------------------------------+
+| Newton Warp | No (kit-less) | Newton backend, fast training |
++---------------------+-------------------------------+---------------------------------+
+
+Architecture Overview
+---------------------
+
+The renderer system consists of:
+
+1. **BaseRenderer** — Abstract base class defining the rendering lifecycle and interface
+2. **Renderer** — Factory that instantiates the appropriate backend based on renderer configuration class
+3. **RendererCfg** — Base configuration; each backend extends it with backend-specific options
+4. **Concrete implementations** — Backend-specific renderers in extension packages
+
+.. code-block:: python
+
+ from isaaclab.renderers import BaseRenderer, Renderer
+ from isaaclab_newton.renderers import NewtonWarpRendererCfg
+
+ # Create a Newton Warp renderer (no Isaac Sim required)
+ renderer: BaseRenderer = Renderer(NewtonWarpRendererCfg())
+ assert isinstance(renderer, BaseRenderer)
+
+For the RTX renderer (requires Isaac Sim):
+
+.. code-block:: python
+
+ from isaaclab.renderers import Renderer
+ from isaaclab.renderers import IsaacRtxRendererCfg # or OVRTXRendererCfg
+
+ # Create an RTX renderer
+ renderer: BaseRenderer = Renderer(IsaacRtxRendererCfg())
+
+For RTX renderer settings and presets (quality, balanced, performance), see
+:doc:`/source/how-to/configure_rendering`.
+
+Core concepts
+-------------
+
+- **Use the factory**: Always instantiate renderers via the factory with a renderer-specific config class
+ (e.g. ``Renderer(IsaacRtxRendererCfg())``). Do not import or instantiate concrete backend classes
+ (e.g. ``IsaacRtxRenderer``, ``OVRTXRenderer``) directly—their names and package locations are
+ implementation details and may change without notice.
+
+- **Lightweight config imports**: Importing a renderer configuration class does not pull in backend-specific
+ dependencies. The backend is lazily loaded when the renderer is instantiated, and instantiation may fail
+ if the backend is not installed.
+
+ .. code-block:: python
+
+ # Lightweight: does not import OVRTX backend dependencies
+ from isaaclab_ov.renderers import OVRTXRendererCfg
+
+ # Lazily loads ovrtx when instantiated; may fail if isaaclab_ov / ovrtx is not installed
+ renderer: BaseRenderer = Renderer(OVRTXRendererCfg())
+
+Installing the OVRTX renderer
+------------------------------
+
+The OVRTX renderer is provided by the ``isaaclab_ov`` extension and requires the
+`ovrtx `_ package (hosted on
+``pypi.nvidia.com``).
+
+Install via the Isaac Lab CLI:
+
+.. code-block:: bash
+
+ # Install isaaclab_ov (and its ovrtx dependency) alongside the core package
+ ./isaaclab.sh -i ov
+
+Or install manually with pip:
+
+.. code-block:: bash
+
+ pip install --extra-index-url https://pypi.nvidia.com -e source/isaaclab_ov
+
+- **Opaque render data**: The render data object returned by :meth:`~isaaclab.renderers.BaseRenderer.create_render_data` is passed to
+ subsequent renderer methods. It should be completely opaque to the caller: inspecting or modifying it
+ via get/set attributes is an anti-pattern and breaks the API contract.
+
+.. note::
+
+ The :class:`~isaaclab.renderers.BaseRenderer` class is under active development and may change without notice.
+
+See Also
+--------
+
+- :doc:`scene_data_providers` — how scene data flows from physics backends to renderers
+- :doc:`/source/features/visualization` — lightweight visualizer backends for interactive feedback
diff --git a/docs/source/overview/core-concepts/scene_data_providers.rst b/docs/source/overview/core-concepts/scene_data_providers.rst
new file mode 100644
index 00000000000..46244317ba2
--- /dev/null
+++ b/docs/source/overview/core-concepts/scene_data_providers.rst
@@ -0,0 +1,109 @@
+Scene Data Providers
+====================
+
+Scene Data Providers bridge physics simulation backends and visualization/rendering systems in
+Isaac Lab. They provide a unified interface for accessing scene data (transforms, velocities,
+Newton model/state) regardless of which physics backend is active.
+
+Overview
+--------
+
+Isaac Lab supports multiple physics backends (PhysX and Newton) and multiple visualizers
+(Omniverse Kit, Newton, Rerun, Viser). Each combination requires scene data to flow from the
+physics engine to the renderer. Scene Data Providers handle this translation automatically
+through a factory pattern.
+
+.. code-block:: python
+
+ from isaaclab.physics import SceneDataProvider
+
+ # Factory auto-selects the correct implementation based on active physics backend
+ provider = SceneDataProvider(stage, simulation_context)
+
+Architecture
+------------
+
+The system has three layers:
+
+1. **BaseSceneDataProvider** — abstract interface defining the contract:
+
+ - ``update(env_ids)`` — refresh cached scene data
+ - ``get_newton_model()`` — return Newton model handle (if available)
+ - ``get_newton_state(env_ids)`` — return Newton state handle (if available)
+ - ``get_usd_stage()`` — return USD stage handle (if available)
+ - ``get_transforms()`` — return body transforms
+ - ``get_velocities()`` — return body velocities
+ - ``get_contacts()`` — return contact data
+ - ``get_camera_transforms()`` — return per-camera, per-env transforms
+ - ``get_metadata()`` — return backend metadata (num_envs, gravity, etc.)
+
+2. **SceneDataProvider** — factory that auto-selects the backend-specific implementation
+ based on the active :class:`~isaaclab.physics.PhysicsManager`.
+
+3. **Backend implementations:**
+
+ - :class:`~isaaclab_physx.scene_data_providers.PhysxSceneDataProvider`
+ - :class:`~isaaclab_newton.scene_data_providers.NewtonSceneDataProvider`
+
+PhysX Scene Data Provider
+-------------------------
+
+When PhysX is the active physics backend, the provider **builds and maintains a Newton model
+from the USD stage**, then syncs PhysX transforms into it each frame. This is necessary because
+Newton-based visualizers (Newton, Rerun, Viser) require a Newton model/state to render.
+
+The sync pipeline:
+
+1. Reads transforms from PhysX ``RigidBodyView`` (fast tensor API)
+2. Falls back to :class:`~isaaclab.sim.views.FrameView` for bodies not covered by the rigid body view
+3. Converts and writes merged poses into the Newton state via Warp kernels
+
+Newton Scene Data Provider
+--------------------------
+
+When Newton is the active physics backend, the provider **delegates directly to the Newton
+manager** — no building or syncing required. Newton already owns the authoritative model and
+state.
+
+The only additional work is **optional USD sync**: when an Omniverse Kit visualizer is active,
+the provider syncs Newton transforms to the USD stage so Kit can render them. For Newton-only
+or Rerun/Viser visualizers, this sync is skipped.
+
+Data Requirements
+-----------------
+
+Visualizers and renderers declare their data needs, and the provider is configured accordingly:
+
+.. list-table::
+ :header-rows: 1
+
+ * - Component
+ - Requires Newton Model
+ - Requires USD Stage
+ * - Kit visualizer
+ - No
+ - Yes
+ * - Newton visualizer
+ - Yes
+ - No
+ * - Rerun visualizer
+ - Yes
+ - No
+ * - Viser visualizer
+ - Yes
+ - No
+ * - RTX renderer
+ - No
+ - Yes
+ * - Newton Warp renderer
+ - Yes
+ - No
+ * - OVRTX renderer
+ - Yes
+ - Yes
+
+See Also
+--------
+
+- :doc:`renderers` — renderer backends that consume scene data
+- :doc:`/source/features/visualization` — visualizer backends that consume scene data
diff --git a/docs/source/overview/core-concepts/sensors/camera.rst b/docs/source/overview/core-concepts/sensors/camera.rst
index 6a34c6fab30..917022b6cb4 100644
--- a/docs/source/overview/core-concepts/sensors/camera.rst
+++ b/docs/source/overview/core-concepts/sensors/camera.rst
@@ -5,26 +5,132 @@
Camera
======
-Camera sensors are uniquely defined by the use of the ``render_product``, a structure for managing data generated by the rendering pipeline (images). Isaac Lab provides the ability to fully control how these renderings are created through camera parameters like focal length, pose, type, etc... and what kind of data you want to render through the use of Annotators, allowing you to record not only RGB, but also Instance segmentation, object pose, object ID, etc...
+Camera sensors in Isaac Lab are renderer-backed sensors: each :class:`~sensors.TiledCamera` instance
+is coupled to a **renderer** that produces the image data. The renderer and camera are intentionally
+isolated from each other — the camera defines *what* to capture (pose, resolution, field of view,
+data types), while the renderer defines *how* to render it (RTX ray-tracing, Newton Warp rasterizer,
+etc.). This separation allows the same camera configuration to run across different physics and
+rendering backends without code changes.
+
+For an overview of the available renderer backends and how to choose between them, see
+:ref:`overview_renderers`.
+
+Rendered images are unique among supported sensor data types due to their large bandwidth requirements.
+A single 800 × 600 image with 32-bit color clocks in at just under 2 MB. At 60 fps across thousands
+of parallel environments, this grows quickly. Isaac Lab's tiled rendering API specifically addresses
+these scaling challenges by batching all cameras into a single render pass.
+
+
+Renderer Backends
+-----------------
+
+The renderer used by a camera is configured via the ``renderer_cfg`` field on
+:class:`~sensors.TiledCameraCfg`. The default is :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg`
+(NVIDIA RTX, requires Isaac Sim).
+
+.. list-table::
+ :header-rows: 1
+ :widths: 30 30 40
+
+ * - ``renderer_cfg``
+ - Requires Isaac Sim?
+ - Supported data types
+ * - ``IsaacRtxRendererCfg`` *(default)*
+ - Yes
+ - rgb, rgba, depth, normals, motion vectors, semantic/instance segmentation, and all other annotators
+ * - ``NewtonWarpRendererCfg``
+ - No (kit-less)
+ - ``rgb``, ``depth`` only
+ * - ``OVRTXRendererCfg``
+ - No (+ ``isaaclab_ov``)
+ - ``rgb``, ``depth`` only
+
+.. note::
+
+ The Newton Warp renderer currently supports only **``rgb``** and **``depth``** data types.
+ Annotators such as segmentation, normals, and motion vectors are Isaac RTX-specific features and
+ require :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg`.
-Rendered images are unique among the supported data types in Isaac Lab due to the inherently large bandwidth requirements for moving those data. A single 800 x 600 image with 32-bit color (a single float per pixel) clocks in at just under 2 MB. If we render at 60 fps and record every frame, that camera needs to move 120 MB/s. Multiply this by the number of cameras in an environment and environments in a simulation, and you can quickly see how scaling a naive vectorization of camera data could lead to bandwidth challenges. NVIDIA's Isaac Lab leverages our expertise in GPU hardware to provide an API that specifically addresses these scaling challenges in the rendering pipeline.
Tiled Rendering
~~~~~~~~~~~~~~~
.. note::
- This feature is only available from Isaac Sim version 4.2.0 onwards.
+ This feature is available from Isaac Sim version 4.2.0 onwards (for the RTX renderer).
+ The Newton Warp renderer supports tiled rendering in kit-less mode.
+
+ Tiled rendering in combination with image processing networks require heavy memory resources,
+ especially at larger resolutions. We recommend running 512 cameras on RTX 4090 GPUs or similar
+ when using the RTX renderer.
+
+The Tiled Rendering API provides a vectorized interface for collecting image data from all environment
+clones in a single batched render pass. Instead of one render call per camera, all copies of a camera
+are composited into a single large tiled image, dramatically reducing host-device transfer overhead.
+
+Isaac Lab provides tiled rendering through :class:`~sensors.TiledCamera`, configured via
+:class:`~sensors.TiledCameraCfg`. The ``renderer_cfg`` field selects the rendering backend.
+
+
+TiledCameraCfg with renderer_cfg
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- Tiled rendering in combination with image processing networks require heavy memory resources, especially
- at larger resolutions. We recommend running 512 cameras in the scene on RTX 4090 GPUs or similar.
+The renderer is specified via ``renderer_cfg`` on :class:`~sensors.TiledCameraCfg`. The camera and
+renderer configurations are fully decoupled: you can swap renderers without changing any other camera
+parameters.
+
+**Default (RTX, requires Isaac Sim):**
+
+.. code-block:: python
+
+ from isaaclab.sensors import TiledCameraCfg
+ import isaaclab.sim as sim_utils
+ # IsaacRtxRendererCfg is the default, no explicit import needed
+
+ tiled_camera: TiledCameraCfg = TiledCameraCfg(
+ prim_path="/World/envs/env_.*/Camera",
+ offset=TiledCameraCfg.OffsetCfg(pos=(-7.0, 0.0, 3.0), rot=(0.9945, 0.0, 0.1045, 0.0), convention="world"),
+ data_types=["rgb"],
+ spawn=sim_utils.PinholeCameraCfg(
+ focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(0.1, 20.0)
+ ),
+ width=80,
+ height=80,
+ # renderer_cfg defaults to IsaacRtxRendererCfg()
+ )
+
+**Newton Warp renderer (kit-less, no Isaac Sim required):**
+
+.. code-block:: python
+
+ from isaaclab.sensors import TiledCameraCfg
+ from isaaclab_newton.renderers import NewtonWarpRendererCfg
+ import isaaclab.sim as sim_utils
+
+ tiled_camera: TiledCameraCfg = TiledCameraCfg(
+ prim_path="/World/envs/env_.*/Camera",
+ offset=TiledCameraCfg.OffsetCfg(pos=(-7.0, 0.0, 3.0), rot=(0.9945, 0.0, 0.1045, 0.0), convention="world"),
+ data_types=["rgb", "depth"], # only rgb and depth supported with Newton renderer
+ spawn=sim_utils.PinholeCameraCfg(
+ focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(0.1, 20.0)
+ ),
+ width=80,
+ height=80,
+ renderer_cfg=NewtonWarpRendererCfg(),
+ )
-The Tiled Rendering APIs provide a vectorized interface for collecting data from camera sensors. This is useful for reinforcement learning environments where parallelization can be exploited to accelerate data collection and thus the training loop. Tiled rendering works by using a single ``render_product`` for **all** clones of a single camera in the scene. The desired dimensions of a single image and the number of environments are used to compute a much larger ``render_product``, consisting of the tiled individual renders from the separate clones of the camera. When all cameras have populated their buffers the render product is "completed" and can be moved around as a single, large image, dramatically reducing the overhead for moving the data from the host to the device, for example. Only a single call is used to synchronize the device data, instead of one call per camera, and this is a big part of what makes the Tiled Rendering API more efficient for working with vision data.
+**Multi-backend preset (switches renderer alongside physics backend):**
-Isaac Lab provides tiled rendering APIs for RGB, depth, along with other annotators through the :class:`~sensors.TiledCamera` class. Configurations for the tiled rendering APIs can be defined through the :class:`~sensors.TiledCameraCfg` class, specifying parameters such as the regex expression for all camera paths, the transform for the cameras, the desired data type, the type of cameras to add to the scene, and the camera resolution.
+For environments that need to support both backends, use
+:class:`~isaaclab_tasks.utils.presets.MultiBackendRendererCfg` together with the
+:ref:`PresetCfg pattern `:
.. code-block:: python
+ from isaaclab.sensors import TiledCameraCfg
+ from isaaclab_tasks.utils.presets import MultiBackendRendererCfg
+ import isaaclab.sim as sim_utils
+
tiled_camera: TiledCameraCfg = TiledCameraCfg(
prim_path="/World/envs/env_.*/Camera",
offset=TiledCameraCfg.OffsetCfg(pos=(-7.0, 0.0, 3.0), rot=(0.9945, 0.0, 0.1045, 0.0), convention="world"),
@@ -34,40 +140,64 @@ Isaac Lab provides tiled rendering APIs for RGB, depth, along with other annotat
),
width=80,
height=80,
+ renderer_cfg=MultiBackendRendererCfg(), # selects RTX or Newton Warp via presets= CLI arg
)
-To access the tiled rendering interface, a :class:`~sensors.TiledCamera` object can be created and used to retrieve data from the cameras.
+The active preset is selected at launch via the ``presets=`` CLI argument:
+
+.. code-block:: bash
+
+ # Use Newton Warp renderer
+ python train.py task=Isaac-Cartpole-RGB-Camera-Direct-v0 presets=newton_renderer
+
+ # Use OVRTX renderer
+ python train.py task=Isaac-Cartpole-RGB-Camera-Direct-v0 presets=ovrtx_renderer
+
+ # Use default (Isaac RTX)
+ python train.py task=Isaac-Cartpole-RGB-Camera-Direct-v0
+
+
+Accessing camera data
+~~~~~~~~~~~~~~~~~~~~~
.. code-block:: python
tiled_camera = TiledCamera(cfg.tiled_camera)
- data_type = "rgb"
- data = tiled_camera.data.output[data_type]
+ data = tiled_camera.data.output["rgb"] # shape: (num_cameras, H, W, 3), torch.uint8
-The returned data will be transformed into the shape (num_cameras, height, width, num_channels), which can be used directly as observation for reinforcement learning.
+The returned data has shape ``(num_cameras, height, width, num_channels)``, ready to use directly
+as an observation in RL training.
-When working with rendering, make sure to add the ``--enable_cameras`` argument when launching the environment. For example:
+When using the RTX renderer, add ``--enable_cameras`` when launching:
.. code-block:: shell
- python scripts/reinforcement_learning/rl_games/train.py --task=Isaac-Cartpole-RGB-Camera-Direct-v0 --headless --enable_cameras
+ python scripts/reinforcement_learning/rl_games/train.py \
+ --task=Isaac-Cartpole-RGB-Camera-Direct-v0 --headless --enable_cameras
+
+Annotators (RTX only)
+~~~~~~~~~~~~~~~~~~~~~
-Annotators
-~~~~~~~~~~
+.. note::
-Both :class:`~sensors.TiledCamera` and :class:`~sensors.Camera` classes provide APIs for retrieving various types annotator data from replicator:
+ Annotators are a feature of the **Isaac RTX renderer** (``IsaacRtxRendererCfg``).
+ They are **not** available with the Newton Warp renderer or ovrtx, which
+ support only ``rgb`` and ``depth``.
+
+:class:`~sensors.TiledCamera` exposes the following annotator
+data types when using the RTX renderer:
* ``"rgb"``: A 3-channel rendered color image.
* ``"rgba"``: A 4-channel rendered color image with alpha channel.
-* ``"distance_to_camera"``: An image containing the distance to camera optical center.
-* ``"distance_to_image_plane"``: An image containing distances of 3D points from camera plane along camera's z-axis.
-* ``"depth"``: The same as ``"distance_to_image_plane"``.
-* ``"normals"``: An image containing the local surface normal vectors at each pixel.
-* ``"motion_vectors"``: An image containing the motion vector data at each pixel.
-* ``"semantic_segmentation"``: The semantic segmentation data.
-* ``"instance_segmentation_fast"``: The instance segmentation data.
-* ``"instance_id_segmentation_fast"``: The instance id segmentation data.
+* ``"distance_to_camera"``: Distance to the camera optical center per pixel.
+* ``"distance_to_image_plane"``: Distance along the camera's Z-axis per pixel.
+* ``"depth"``: Alias for ``"distance_to_image_plane"``.
+* ``"normals"``: Local surface normal vectors at each pixel.
+* ``"motion_vectors"``: Per-pixel motion vectors in image space.
+* ``"semantic_segmentation"``: Semantic segmentation labels.
+* ``"instance_segmentation_fast"``: Instance segmentation data.
+* ``"instance_id_segmentation_fast"``: Instance ID segmentation data.
RGB and RGBA
~~~~~~~~~~~~
@@ -77,11 +207,11 @@ RGB and RGBA
:figwidth: 100%
:alt: A scene captured in RGB
-``rgb`` data type returns a 3-channel RGB colored image of type ``torch.uint8``, with dimension (B, H, W, 3).
+``rgb`` returns a 3-channel RGB image of type ``torch.uint8``, shape ``(B, H, W, 3)``.
-``rgba`` data type returns a 4-channel RGBA colored image of type ``torch.uint8``, with dimension (B, H, W, 4).
+``rgba`` returns a 4-channel RGBA image of type ``torch.uint8``, shape ``(B, H, W, 4)``.
-To convert the ``torch.uint8`` data to ``torch.float32``, divide the buffer by 255.0 to obtain a ``torch.float32`` buffer containing data from 0 to 1.
+To convert to ``torch.float32``, divide by 255.0.
Depth and Distances
~~~~~~~~~~~~~~~~~~~
@@ -89,13 +219,15 @@ Depth and Distances
.. figure:: ../../../_static/overview/sensors/camera_depth.jpg
:align: center
:figwidth: 100%
- :alt: A scene captured in RGB
+ :alt: A scene captured as depth
-``distance_to_camera`` returns a single-channel depth image with distance to the camera optical center. The dimension for this annotator is (B, H, W, 1) and has type ``torch.float32``.
+``distance_to_camera`` returns a single-channel depth image with distance to the camera optical
+center, shape ``(B, H, W, 1)``, type ``torch.float32``.
-``distance_to_image_plane`` returns a single-channel depth image with distances of 3D points from the camera plane along the camera's Z-axis. The dimension for this annotator is (B, H, W, 1) and has type ``torch.float32``.
+``distance_to_image_plane`` returns distances of 3D points from the camera plane along the Z-axis,
+shape ``(B, H, W, 1)``, type ``torch.float32``.
-``depth`` is provided as an alias for ``distance_to_image_plane`` and will return the same data as the ``distance_to_image_plane`` annotator, with dimension (B, H, W, 1) and type ``torch.float32``.
+``depth`` is an alias for ``distance_to_image_plane``.
Normals
~~~~~~~
@@ -103,14 +235,17 @@ Normals
.. figure:: ../../../_static/overview/sensors/camera_normals.jpg
:align: center
:figwidth: 100%
- :alt: A scene captured in RGB
+ :alt: A scene captured with surface normals
-``normals`` returns an image containing the local surface normal vectors at each pixel. The buffer has dimension (B, H, W, 3), containing the (x, y, z) information for each vector, and has data type ``torch.float32``.
+``normals`` returns local surface normal vectors at each pixel, shape ``(B, H, W, 3)`` containing
+``(x, y, z)``, type ``torch.float32``.
Motion Vectors
~~~~~~~~~~~~~~
-``motion_vectors`` returns the per-pixel motion vectors in image space, with a 2D array of motion vectors representing the relative motion of a pixel in the camera’s viewport between frames. The buffer has dimension (B, H, W, 2), representing x - the motion distance in the horizontal axis (image width) with movement to the left of the image being positive and movement to the right being negative and y - motion distance in the vertical axis (image height) with movement towards the top of the image being positive and movement to the bottom being negative. The data type is ``torch.float32``.
+``motion_vectors`` returns per-pixel motion vectors in image space between frames.
+Shape ``(B, H, W, 2)``: ``x`` is horizontal motion (positive = left), ``y`` is vertical motion
+(positive = up). Type ``torch.float32``.
Semantic Segmentation
~~~~~~~~~~~~~~~~~~~~~
@@ -118,13 +253,15 @@ Semantic Segmentation
.. figure:: ../../../_static/overview/sensors/camera_semantic.jpg
:align: center
:figwidth: 100%
- :alt: A scene captured in RGB
-
-``semantic_segmentation`` outputs semantic segmentation of each entity in the camera’s viewport that has semantic labels. In addition to the image buffer, an ``info`` dictionary can be retrieved with ``tiled_camera.data.info['semantic_segmentation']`` containing ID to labels information.
+ :alt: A scene with semantic segmentation
-- If ``colorize_semantic_segmentation=True`` in the camera config, a 4-channel RGBA image will be returned with dimension (B, H, W, 4) and type ``torch.uint8``. The info ``idToLabels`` dictionary will be the mapping from color to semantic labels.
+``semantic_segmentation`` outputs per-pixel semantic labels for entities with semantic annotations.
+An ``info`` dictionary is available via ``tiled_camera.data.info['semantic_segmentation']``.
-- If ``colorize_semantic_segmentation=False``, a buffer of dimension (B, H, W, 1) of type ``torch.int32`` will be returned, containing the semantic ID of each pixel. The info ``idToLabels`` dictionary will be the mapping from semantic ID to semantic labels.
+- If ``colorize_semantic_segmentation=True``: 4-channel RGBA image, shape ``(B, H, W, 4)``,
+ type ``torch.uint8``. The ``idToLabels`` dict maps color to semantic label.
+- If ``colorize_semantic_segmentation=False``: shape ``(B, H, W, 1)``, type ``torch.int32``,
+ containing semantic IDs. The ``idToLabels`` dict maps ID to label.
Instance ID Segmentation
~~~~~~~~~~~~~~~~~~~~~~~~
@@ -132,15 +269,15 @@ Instance ID Segmentation
.. figure:: ../../../_static/overview/sensors/camera_instanceID.jpg
:align: center
:figwidth: 100%
- :alt: A scene captured in RGB
-
-``instance_id_segmentation_fast`` outputs instance ID segmentation of each entity in the camera’s viewport. The instance ID is unique for each prim in the scene with different paths. In addition to the image buffer, an ``info`` dictionary can be retrieved with ``tiled_camera.data.info['instance_id_segmentation_fast']`` containing ID to labels information.
+ :alt: A scene with instance ID segmentation
-The main difference between ``instance_id_segmentation_fast`` and ``instance_segmentation_fast`` are that instance segmentation annotator goes down the hierarchy to the lowest level prim which has semantic labels, where instance ID segmentation always goes down to the leaf prim.
+``instance_id_segmentation_fast`` outputs per-pixel instance IDs, unique per USD prim path.
+An ``info`` dictionary is available via ``tiled_camera.data.info['instance_id_segmentation_fast']``.
-- If ``colorize_instance_id_segmentation=True`` in the camera config, a 4-channel RGBA image will be returned with dimension (B, H, W, 4) and type ``torch.uint8``. The info ``idToLabels`` dictionary will be the mapping from color to USD prim path of that entity.
-
-- If ``colorize_instance_id_segmentation=False``, a buffer of dimension (B, H, W, 1) of type ``torch.int32`` will be returned, containing the instance ID of each pixel. The info ``idToLabels`` dictionary will be the mapping from instance ID to USD prim path of that entity.
+- If ``colorize_instance_id_segmentation=True``: shape ``(B, H, W, 4)``, type ``torch.uint8``.
+ The ``idToLabels`` dict maps color to USD prim path.
+- If ``colorize_instance_id_segmentation=False``: shape ``(B, H, W, 1)``, type ``torch.int32``.
+ The ``idToLabels`` dict maps instance ID to USD prim path.
Instance Segmentation
"""""""""""""""""""""
@@ -148,12 +285,15 @@ Instance Segmentation
.. figure:: ../../../_static/overview/sensors/camera_instance.jpg
:align: center
:figwidth: 100%
- :alt: A scene captured in RGB
-
-``instance_segmentation_fast`` outputs instance segmentation of each entity in the camera’s viewport. In addition to the image buffer, an ``info`` dictionary can be retrieved with ``tiled_camera.data.info['instance_segmentation_fast']`` containing ID to labels and ID to semantic information.
+ :alt: A scene with instance segmentation
-- If ``colorize_instance_segmentation=True`` in the camera config, a 4-channel RGBA image will be returned with dimension (B, H, W, 4) and type ``torch.uint8``.
+``instance_segmentation_fast`` outputs instance segmentation, traversing down the prim hierarchy
+to the lowest level with semantic labels (unlike ``instance_id_segmentation_fast``, which always
+goes to the leaf prim).
+An ``info`` dictionary is available via ``tiled_camera.data.info['instance_segmentation_fast']``.
-- If ``colorize_instance_segmentation=False``, a buffer of dimension (B, H, W, 1) of type ``torch.int32`` will be returned, containing the instance ID of each pixel.
+- If ``colorize_instance_segmentation=True``: shape ``(B, H, W, 4)``, type ``torch.uint8``.
+- If ``colorize_instance_segmentation=False``: shape ``(B, H, W, 1)``, type ``torch.int32``.
-The info ``idToLabels`` dictionary will be the mapping from color to USD prim path of that semantic entity. The info ``idToSemantics`` dictionary will be the mapping from color to semantic labels of that semantic entity.
+The ``idToLabels`` dict maps color to USD prim path. The ``idToSemantics`` dict maps color to
+semantic label.
diff --git a/docs/source/overview/core-concepts/sensors/contact_sensor.rst b/docs/source/overview/core-concepts/sensors/contact_sensor.rst
index 4ddd2d10c07..ebd81e60a54 100644
--- a/docs/source/overview/core-concepts/sensors/contact_sensor.rst
+++ b/docs/source/overview/core-concepts/sensors/contact_sensor.rst
@@ -55,7 +55,7 @@ Here, we print both the net contact force and the filtered force matrix for each
-------------------------------
Contact sensor @ '/World/envs/env_.*/Robot/LF_FOOT':
- view type :
+ view type :
update period (s) : 0.0
number of bodies : 1
body names : ['LF_FOOT']
@@ -64,7 +64,7 @@ Here, we print both the net contact force and the filtered force matrix for each
Received contact force of: tensor([[[-1.3923e-05, 1.5727e-04, 1.1032e+02]]], device='cuda:0')
-------------------------------
Contact sensor @ '/World/envs/env_.*/Robot/RF_FOOT':
- view type :
+ view type :
update period (s) : 0.0
number of bodies : 1
body names : ['RF_FOOT']
@@ -85,7 +85,7 @@ Notice that even with filtering, both sensors report the net contact force actin
-------------------------------
Contact sensor @ '/World/envs/env_.*/Robot/.*H_FOOT':
- view type :
+ view type :
update period (s) : 0.0
number of bodies : 2
body names : ['LH_FOOT', 'RH_FOOT']
diff --git a/docs/source/overview/core-concepts/sensors/imu.rst b/docs/source/overview/core-concepts/sensors/imu.rst
index fe429f77a2b..5435bf227f5 100644
--- a/docs/source/overview/core-concepts/sensors/imu.rst
+++ b/docs/source/overview/core-concepts/sensors/imu.rst
@@ -61,7 +61,7 @@ The oscillations in the values reported by the sensor are a direct result of of
.. code-block:: bash
Imu sensor @ '/World/envs/env_.*/Robot/LF_FOOT':
- view type :
+ view type :
update period (s) : 0.0
number of sensors : 1
@@ -71,7 +71,7 @@ The oscillations in the values reported by the sensor are a direct result of of
Received angular acceleration: tensor([[-0.0389, -0.0262, -0.0045]], device='cuda:0')
-------------------------------
Imu sensor @ '/World/envs/env_.*/Robot/RF_FOOT':
- view type :
+ view type :
update period (s) : 0.0
number of sensors : 1
diff --git a/docs/source/overview/developer-guide/development.rst b/docs/source/overview/developer-guide/development.rst
index 48a5019609f..3d9ddbf014a 100644
--- a/docs/source/overview/developer-guide/development.rst
+++ b/docs/source/overview/developer-guide/development.rst
@@ -61,19 +61,19 @@ More specifically, when an extension is enabled, the python module specified in
While loading extensions into Omniverse happens automatically, using the python package
in standalone applications requires additional steps. To simplify the build process and
avoid the need to understand the `premake `__
-build system used by Omniverse, we directly use the `setuptools `__
+build system used by Omniverse, we directly use the `setuptools `__
python package to build the python module provided by the extensions. This is done by the
``setup.py`` file in the extension directory.
.. note::
The ``setup.py`` file is not required for extensions that are only loaded into Omniverse
- using the `Extension Manager `__.
+ using the `Extension Manager `__.
Lastly, the ``tests`` directory contains the unit tests for the extension. These are written
using the `unittest `__ framework. It is
important to note that Omniverse also provides a similar
-`testing framework `__.
+`testing framework `__.
However, it requires going through the build process and does not support testing of the python module in
standalone applications.
@@ -81,7 +81,7 @@ Custom Extension Dependency Management
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Certain extensions may have dependencies which require the installation of additional packages before the extension
-can be used. While Python dependencies are handled by the `setuptools `__
+can be used. While Python dependencies are handled by the `setuptools `__
package and specified in the ``setup.py`` file, non-Python dependencies such as `ROS `__
packages or `apt `__ packages are not handled by setuptools.
Handling these kinds of dependencies requires an additional procedure.
diff --git a/docs/source/overview/developer-guide/vs_code.rst b/docs/source/overview/developer-guide/vs_code.rst
index a19889d1bdb..11946a71c76 100644
--- a/docs/source/overview/developer-guide/vs_code.rst
+++ b/docs/source/overview/developer-guide/vs_code.rst
@@ -55,6 +55,25 @@ following links:
* `Isaac Sim VSCode support `__
+Attach to a Running ``debugpy`` Session
+---------------------------------------
+
+The generated ``.vscode/launch.json`` includes a ``Python: Debugger Attach``
+configuration that starts to listen on port ``localhost:3000`` for the debugpy session.
+
+To use it:
+
+1. Set your breakpoints.
+2. Run your code under debugpy like so:
+
+ .. code-block:: bash
+
+ ./isaaclab.sh -p -m debugpy --listen 3000 --wait-for-client -c "from isaaclab.cli import cli; cli()" [cli_args]
+
+3. In VS Code, select the ``Python: Debugger Attach`` configuration from the Run and Debug panel
+ and press the green play button or ``F5``. VS Code will connect to the debugpy server
+ running on ``localhost:3000``.
+
Configuring the python interpreter
----------------------------------
diff --git a/docs/source/overview/environments.rst b/docs/source/overview/environments.rst
index c1cfb7dcb2c..80c12abb522 100644
--- a/docs/source/overview/environments.rst
+++ b/docs/source/overview/environments.rst
@@ -43,54 +43,52 @@ Classic
Classic environments that are based on IsaacGymEnvs implementation of MuJoCo-style environments.
.. table::
- :widths: 33 37 30
-
- +------------------+-----------------------------+-------------------------------------------------------------------------+
- | World | Environment ID | Description |
- +==================+=============================+=========================================================================+
- | |humanoid| | |humanoid-link| | Move towards a direction with the MuJoCo humanoid robot |
- | | | |
- | | |humanoid-direct-link| | |
- +------------------+-----------------------------+-------------------------------------------------------------------------+
- | |ant| | |ant-link| | Move towards a direction with the MuJoCo ant robot |
- | | | |
- | | |ant-direct-link| | |
- +------------------+-----------------------------+-------------------------------------------------------------------------+
- | |cartpole| | |cartpole-link| | Move the cart to keep the pole upwards in the classic cartpole control |
- | | | |
- | | |cartpole-direct-link| | |
- +------------------+-----------------------------+-------------------------------------------------------------------------+
- | |cartpole| | |cartpole-rgb-link| | Move the cart to keep the pole upwards in the classic cartpole control |
- | | | and perceptive inputs. Requires running with ``--enable_cameras``. |
- | | |cartpole-depth-link| | |
- | | | |
- | | |cartpole-rgb-direct-link| | |
- | | | |
- | | |cartpole-depth-direct-link|| |
- +------------------+-----------------------------+-------------------------------------------------------------------------+
- | |cartpole| | |cartpole-resnet-link| | Move the cart to keep the pole upwards in the classic cartpole control |
- | | | based off of features extracted from perceptive inputs with pre-trained |
- | | |cartpole-theia-link| | frozen vision encoders. Requires running with ``--enable_cameras``. |
- +------------------+-----------------------------+-------------------------------------------------------------------------+
+ :widths: 25 30 25 20
+
+ +------------------+-----------------------------+-------------------------------------------------------------------------+-----------------------+
+ | World | Environment ID | Description | Presets |
+ +==================+=============================+=========================================================================+=======================+
+ | |humanoid| | |humanoid-link| | Move towards a direction with the MuJoCo humanoid robot | ``newton``, ``physx`` |
+ | | | | ``ovphysx`` |
+ | | |humanoid-direct-link| | | |
+ +------------------+-----------------------------+-------------------------------------------------------------------------+-----------------------+
+ | |ant| | |ant-link| | Move towards a direction with the MuJoCo ant robot | ``newton``, ``physx`` |
+ | | | | ``ovphysx`` |
+ | | |ant-direct-link| | | |
+ +------------------+-----------------------------+-------------------------------------------------------------------------+-----------------------+
+ | |cartpole| | |cartpole-link| | Move the cart to keep the pole upwards in the classic cartpole control | ``newton``, ``physx`` |
+ | | | | ``ovphysx`` |
+ | | |cartpole-direct-link| | | |
+ +------------------+-----------------------------+-------------------------------------------------------------------------+-----------------------+
+ | |cartpole| | |cartpole-camera-presets| | Move the cart to keep the pole upwards in the classic cartpole control | ``newton``, ``physx`` |
+ | | | and perceptive inputs. Select data type via ``presets=``. Requires | ``newton_renderer``, |
+ | | | running with ``--enable_cameras``. | ``ovrtx_renderer``, |
+ | | | | ``rgb``, ``depth``, |
+ | | | | ``albedo``, |
+ | | | | ``semantic_`` |
+ | | | | ``segmentation``, |
+ | | | | ``simple_shading_*`` |
+ +------------------+-----------------------------+-------------------------------------------------------------------------+-----------------------+
+ | |cartpole| | |cartpole-resnet-link| | Move the cart to keep the pole upwards in the classic cartpole control | ``newton``, ``physx`` |
+ | | | based off of features extracted from perceptive inputs with pre-trained | |
+ | | |cartpole-theia-link| | frozen vision encoders. Requires running with ``--enable_cameras``. | |
+ +------------------+-----------------------------+-------------------------------------------------------------------------+-----------------------+
.. |humanoid| image:: ../_static/tasks/classic/humanoid.jpg
.. |ant| image:: ../_static/tasks/classic/ant.jpg
.. |cartpole| image:: ../_static/tasks/classic/cartpole.jpg
-.. |humanoid-link| replace:: `Isaac-Humanoid-v0 `__
-.. |ant-link| replace:: `Isaac-Ant-v0 `__
-.. |cartpole-link| replace:: `Isaac-Cartpole-v0 `__
-.. |cartpole-rgb-link| replace:: `Isaac-Cartpole-RGB-v0 `__
-.. |cartpole-depth-link| replace:: `Isaac-Cartpole-Depth-v0 `__
-.. |cartpole-resnet-link| replace:: `Isaac-Cartpole-RGB-ResNet18-v0 `__
-.. |cartpole-theia-link| replace:: `Isaac-Cartpole-RGB-TheiaTiny-v0 `__
+.. |humanoid-link| replace:: `Isaac-Humanoid-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/classic/humanoid/humanoid_env_cfg.py>`__
+.. |ant-link| replace:: `Isaac-Ant-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/classic/ant/ant_env_cfg.py>`__
+.. |cartpole-link| replace:: `Isaac-Cartpole-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/classic/cartpole/cartpole_env_cfg.py>`__
+.. |cartpole-camera-presets| replace:: `Isaac-Cartpole-Camera-Presets-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/cartpole/cartpole_camera_presets_env_cfg.py>`__
+.. |cartpole-resnet-link| replace:: `Isaac-Cartpole-RGB-ResNet18-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/classic/cartpole/cartpole_camera_env_cfg.py>`__
+.. |cartpole-theia-link| replace:: `Isaac-Cartpole-RGB-TheiaTiny-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/classic/cartpole/cartpole_camera_env_cfg.py>`__
-.. |humanoid-direct-link| replace:: `Isaac-Humanoid-Direct-v0 `__
-.. |ant-direct-link| replace:: `Isaac-Ant-Direct-v0 `__
-.. |cartpole-direct-link| replace:: `Isaac-Cartpole-Direct-v0 `__
-.. |cartpole-rgb-direct-link| replace:: `Isaac-Cartpole-RGB-Camera-Direct-v0 `__
-.. |cartpole-depth-direct-link| replace:: `Isaac-Cartpole-Depth-Camera-Direct-v0 `__
+.. |humanoid-direct-link| replace:: `Isaac-Humanoid-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/humanoid/humanoid_env.py>`__
+.. |ant-direct-link| replace:: `Isaac-Ant-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/ant/ant_env.py>`__
+.. |cartpole-direct-link| replace:: `Isaac-Cartpole-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/cartpole/cartpole_env.py>`__
Manipulation
~~~~~~~~~~~~
@@ -105,79 +103,107 @@ for the lift-cube environment:
* |lift-cube-ik-rel-link|: Franka arm with relative IK control
.. table::
- :widths: 33 37 30
-
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | World | Environment ID | Description |
- +=========================+==============================+=============================================================================+
- | |reach-franka| | |reach-franka-link| | Move the end-effector to a sampled target pose with the Franka robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |reach-ur10| | |reach-ur10-link| | Move the end-effector to a sampled target pose with the UR10 robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |deploy-reach-ur10e| | |deploy-reach-ur10e-link| | Move the end-effector to a sampled target pose with the UR10e robot |
- | | | This policy has been deployed to a real robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |lift-cube| | |lift-cube-link| | Pick a cube and bring it to a sampled target position with the Franka robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |stack-cube| | |stack-cube-link| | Stack three cubes (bottom to top: blue, red, green) with the Franka robot. |
- | | | Blueprint env used for the NVIDIA Isaac GR00T blueprint for synthetic |
- | | |stack-cube-bp-link| | manipulation motion generation |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |surface-gripper| | |long-suction-link| | Stack three cubes (bottom to top: blue, red, green) |
- | | | with the UR10 arm and long surface gripper |
- | | |short-suction-link| | or short surface gripper. |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |cabi-franka| | |cabi-franka-link| | Grasp the handle of a cabinet's drawer and open it with the Franka robot |
- | | | |
- | | |franka-direct-link| | |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |cube-allegro| | |cube-allegro-link| | In-hand reorientation of a cube using Allegro hand |
- | | | |
- | | |allegro-direct-link| | |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |cube-shadow| | |cube-shadow-link| | In-hand reorientation of a cube using Shadow hand |
- | | | |
- | | |cube-shadow-ff-link| | |
- | | | |
- | | |cube-shadow-lstm-link| | |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |cube-shadow| | |cube-shadow-vis-link| | In-hand reorientation of a cube using Shadow hand using perceptive inputs. |
- | | | Requires running with ``--enable_cameras``. |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |gr1_pick_place| | |gr1_pick_place-link| | Pick up and place an object in a basket with a GR-1 humanoid robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |gr1_pp_waist| | |gr1_pp_waist-link| | Pick up and place an object in a basket with a GR-1 humanoid robot |
- | | | with waist degrees-of-freedom enables that provides a wider reach space. |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |g1_pick_place| | |g1_pick_place-link| | Pick up and place an object in a basket with a Unitree G1 humanoid robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |g1_pick_place_fixed| | |g1_pick_place_fixed-link| | Pick up and place an object in a basket with a Unitree G1 humanoid robot |
- | | | with three-fingered hands. Robot is set up with the base fixed in place. |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |g1_pick_place_lm| | |g1_pick_place_lm-link| | Pick up and place an object in a basket with a Unitree G1 humanoid robot |
- | | | with three-fingered hands and in-place locomanipulation capabilities |
- | | | enabled (i.e. Robot lower body balances in-place while upper body is |
- | | | controlled via Inverse Kinematics). |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |kuka-allegro-lift| | |kuka-allegro-lift-link| | Pick up a primitive shape on the table and lift it to target position |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |kuka-allegro-reorient| | |kuka-allegro-reorient-link| | Pick up a primitive shape on the table and orient it to target pose |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |galbot_stack| | |galbot_stack-link| | Stack three cubes (bottom to top: blue, red, green) with the left arm of |
- | | | a Galbot humanoid robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |agibot_place_mug| | |agibot_place_mug-link| | Pick up and place a mug upright with a Agibot A2D humanoid robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |agibot_place_toy| | |agibot_place_toy-link| | Pick up and place an object in a box with a Agibot A2D humanoid robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |reach_openarm_bi| | |reach_openarm_bi-link| | Move the end-effector to sampled target poses with the OpenArm robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |reach_openarm_uni| | |reach_openarm_uni-link| | Move the end-effector to a sampled target pose with the OpenArm robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |lift_openarm_uni| | |lift_openarm_uni-link| | Pick a cube and bring it to a sampled target position with the OpenArm robot|
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
- | |cabi_openarm_uni| | |cabi_openarm_uni-link| | Grasp the handle of a cabinet's drawer and open it with the OpenArm robot |
- +-------------------------+------------------------------+-----------------------------------------------------------------------------+
+ :widths: 25 30 25 20
+
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | World | Environment ID | Description | Presets |
+ +=========================+==============================+=============================================================================+=======================+
+ | |reach-franka| | |reach-franka-link| | Move the end-effector to a sampled target pose with the Franka robot | ``newton``, ``physx`` |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |reach-ur10| | |reach-ur10-link| | Move the end-effector to a sampled target pose with the UR10 robot | ``newton``, ``physx`` |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |deploy-reach-ur10e| | |deploy-reach-ur10e-link| | Move the end-effector to a sampled target pose with the UR10e robot | |
+ | | | This policy has been deployed to a real robot | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |lift-cube| | |lift-cube-link| | Pick a cube and bring it to a sampled target position with the Franka robot | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |stack-cube| | |stack-cube-link| | Stack three cubes (bottom to top: blue, red, green) with the Franka robot. | |
+ | | | Blueprint env used for the NVIDIA Isaac GR00T blueprint for synthetic | |
+ | | |stack-cube-bp-link| | manipulation motion generation | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |surface-gripper| | |long-suction-link| | Stack three cubes (bottom to top: blue, red, green) | |
+ | | | with the UR10 arm and long surface gripper | |
+ | | |short-suction-link| | or short surface gripper. | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |cabi-franka| | |cabi-franka-link| | Grasp the handle of a cabinet's drawer and open it with the Franka robot | ``newton``, ``physx`` |
+ | | | | |
+ | | |franka-direct-link| | | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |cube-allegro| | |cube-allegro-link| | In-hand reorientation of a cube using Allegro hand | ``newton``, ``physx`` |
+ | | | | |
+ | | |allegro-direct-link| | | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |cube-shadow| | |cube-shadow-link| | In-hand reorientation of a cube using Shadow hand | ``newton``, ``physx`` |
+ | | | | |
+ | | |cube-shadow-ff-link| | | |
+ | | | | |
+ | | |cube-shadow-lstm-link| | | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |cube-shadow| | |cube-shadow-vis-link| | In-hand reorientation of a cube using Shadow hand using perceptive inputs. | ``newton``, ``physx`` |
+ | | | Requires running with ``--enable_cameras``. | ``newton_renderer``, |
+ | | | | ``ovrtx_renderer``, |
+ | | | | ``rgb``, ``depth``, |
+ | | | | ``albedo``, ``full``, |
+ | | | | ``semantic_`` |
+ | | | | ``segmentation``, |
+ | | | | ``simple_shading_*`` |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |gr1_pick_place| | |gr1_pick_place-link| | Pick up and place an object in a basket with a GR-1 humanoid robot | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |gr1_pp_waist| | |gr1_pp_waist-link| | Pick up and place an object in a basket with a GR-1 humanoid robot | |
+ | | | with waist degrees-of-freedom enables that provides a wider reach space. | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |g1_pick_place| | |g1_pick_place-link| | Pick up and place an object in a basket with a Unitree G1 humanoid robot | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |g1_pick_place_fixed| | |g1_pick_place_fixed-link| | Pick up and place an object in a basket with a Unitree G1 humanoid robot | |
+ | | | with three-fingered hands. Robot is set up with the base fixed in place. | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |g1_pick_place_lm| | |g1_pick_place_lm-link| | Pick up and place an object in a basket with a Unitree G1 humanoid robot | |
+ | | | with three-fingered hands and in-place locomanipulation capabilities | |
+ | | | enabled (i.e. Robot lower body balances in-place while upper body is | |
+ | | | controlled via Inverse Kinematics). | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |kuka-allegro-lift| | |kuka-allegro-lift-link| | Pick up a primitive shape on the table and lift it to target position. | ``newton``, ``physx`` |
+ | | | Supports state, single-camera, and dual-camera observation modes via | ``single_camera``, |
+ | | | ``presets=single_camera`` / ``presets=duo_camera`` (see RL table below). | ``duo_camera``, |
+ | | | | ``newton_renderer``, |
+ | | | | ``ovrtx_renderer``, |
+ | | | | ``rgb{64,128,256}``, |
+ | | | | ``depth{..}``, |
+ | | | | ``albedo{..}``, |
+ | | | | ``semantic_`` |
+ | | | | ``segmentation{..}``, |
+ | | | | ``simple_shading_*`` |
+ | | | | ``{64,128,256}`` |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |kuka-allegro-reorient| | |kuka-allegro-reorient-link| | Pick up a primitive shape on the table and orient it to target pose. | ``newton``, ``physx`` |
+ | | | Supports state, single-camera, and dual-camera observation modes via | ``single_camera``, |
+ | | | ``presets=single_camera`` / ``presets=duo_camera`` (see RL table below). | ``duo_camera``, |
+ | | | | ``newton_renderer``, |
+ | | | | ``ovrtx_renderer``, |
+ | | | | ``rgb{64,128,256}``, |
+ | | | | ``depth{..}``, |
+ | | | | ``albedo{..}``, |
+ | | | | ``semantic_`` |
+ | | | | ``segmentation{..}``, |
+ | | | | ``simple_shading_*`` |
+ | | | | ``{64,128,256}`` |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |galbot_stack| | |galbot_stack-link| | Stack three cubes (bottom to top: blue, red, green) with the left arm of | |
+ | | | a Galbot humanoid robot | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |agibot_place_mug| | |agibot_place_mug-link| | Pick up and place a mug upright with a Agibot A2D humanoid robot | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |agibot_place_toy| | |agibot_place_toy-link| | Pick up and place an object in a box with a Agibot A2D humanoid robot | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |reach_openarm_bi| | |reach_openarm_bi-link| | Move the end-effector to sampled target poses with the OpenArm robot | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |reach_openarm_uni| | |reach_openarm_uni-link| | Move the end-effector to a sampled target pose with the OpenArm robot | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |lift_openarm_uni| | |lift_openarm_uni-link| | Pick a cube and bring it to a sampled target position with the OpenArm robot| |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |cabi_openarm_uni| | |cabi_openarm_uni-link| | Grasp the handle of a cabinet's drawer and open it with the OpenArm robot | |
+ +-------------------------+------------------------------+-----------------------------------------------------------------------------+-----------------------+
.. |reach-franka| image:: ../_static/tasks/manipulation/franka_reach.jpg
.. |reach-ur10| image:: ../_static/tasks/manipulation/ur10_reach.jpg
@@ -203,38 +229,38 @@ for the lift-cube environment:
.. |lift_openarm_uni| image:: ../_static/tasks/manipulation/openarm_uni_lift.jpg
.. |cabi_openarm_uni| image:: ../_static/tasks/manipulation/openarm_uni_open_drawer.jpg
-.. |reach-franka-link| replace:: `Isaac-Reach-Franka-v0 `__
-.. |reach-ur10-link| replace:: `Isaac-Reach-UR10-v0 `__
-.. |deploy-reach-ur10e-link| replace:: `Isaac-Deploy-Reach-UR10e-v0 `__
-.. |lift-cube-link| replace:: `Isaac-Lift-Cube-Franka-v0 `__
-.. |lift-cube-ik-abs-link| replace:: `Isaac-Lift-Cube-Franka-IK-Abs-v0 `__
-.. |lift-cube-ik-rel-link| replace:: `Isaac-Lift-Cube-Franka-IK-Rel-v0 `__
-.. |cabi-franka-link| replace:: `Isaac-Open-Drawer-Franka-v0 `__
-.. |franka-direct-link| replace:: `Isaac-Franka-Cabinet-Direct-v0 `__
-.. |cube-allegro-link| replace:: `Isaac-Repose-Cube-Allegro-v0 `__
-.. |allegro-direct-link| replace:: `Isaac-Repose-Cube-Allegro-Direct-v0 `__
-.. |stack-cube-link| replace:: `Isaac-Stack-Cube-Franka-v0 `__
-.. |stack-cube-bp-link| replace:: `Isaac-Stack-Cube-Franka-IK-Rel-Blueprint-v0 `__
-.. |gr1_pick_place-link| replace:: `Isaac-PickPlace-GR1T2-Abs-v0 `__
-.. |g1_pick_place-link| replace:: `Isaac-PickPlace-G1-InspireFTP-Abs-v0 `__
-.. |g1_pick_place_fixed-link| replace:: `Isaac-PickPlace-FixedBaseUpperBodyIK-G1-Abs-v0 `__
-.. |g1_pick_place_lm-link| replace:: `Isaac-PickPlace-Locomanipulation-G1-Abs-v0 `__
-.. |long-suction-link| replace:: `Isaac-Stack-Cube-UR10-Long-Suction-IK-Rel-v0 `__
-.. |short-suction-link| replace:: `Isaac-Stack-Cube-UR10-Short-Suction-IK-Rel-v0 `__
-.. |gr1_pp_waist-link| replace:: `Isaac-PickPlace-GR1T2-WaistEnabled-Abs-v0 `__
-.. |galbot_stack-link| replace:: `Isaac-Stack-Cube-Galbot-Left-Arm-Gripper-RmpFlow-v0 `__
-.. |kuka-allegro-lift-link| replace:: `Isaac-Dexsuite-Kuka-Allegro-Lift-v0 `__
-.. |kuka-allegro-reorient-link| replace:: `Isaac-Dexsuite-Kuka-Allegro-Reorient-v0 `__
-.. |cube-shadow-link| replace:: `Isaac-Repose-Cube-Shadow-Direct-v0 `__
-.. |cube-shadow-ff-link| replace:: `Isaac-Repose-Cube-Shadow-OpenAI-FF-Direct-v0 `__
-.. |cube-shadow-lstm-link| replace:: `Isaac-Repose-Cube-Shadow-OpenAI-LSTM-Direct-v0 `__
-.. |cube-shadow-vis-link| replace:: `Isaac-Repose-Cube-Shadow-Vision-Direct-v0 `__
-.. |agibot_place_mug-link| replace:: `Isaac-Place-Mug-Agibot-Left-Arm-RmpFlow-v0 `__
-.. |agibot_place_toy-link| replace:: `Isaac-Place-Toy2Box-Agibot-Right-Arm-RmpFlow-v0 `__
-.. |reach_openarm_bi-link| replace:: `Isaac-Reach-OpenArm-Bi-v0 `__
-.. |reach_openarm_uni-link| replace:: `Isaac-Reach-OpenArm-v0 `__
-.. |lift_openarm_uni-link| replace:: `Isaac-Lift-Cube-OpenArm-v0 `__
-.. |cabi_openarm_uni-link| replace:: `Isaac-Open-Drawer-OpenArm-v0 `__
+.. |reach-franka-link| replace:: `Isaac-Reach-Franka-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/reach/config/franka/joint_pos_env_cfg.py>`__
+.. |reach-ur10-link| replace:: `Isaac-Reach-UR10-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/reach/config/ur_10/joint_pos_env_cfg.py>`__
+.. |deploy-reach-ur10e-link| replace:: `Isaac-Deploy-Reach-UR10e-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/deploy/reach/config/ur_10e/joint_pos_env_cfg.py>`__
+.. |lift-cube-link| replace:: `Isaac-Lift-Cube-Franka-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift/config/franka/joint_pos_env_cfg.py>`__
+.. |lift-cube-ik-abs-link| replace:: `Isaac-Lift-Cube-Franka-IK-Abs-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift/config/franka/ik_abs_env_cfg.py>`__
+.. |lift-cube-ik-rel-link| replace:: `Isaac-Lift-Cube-Franka-IK-Rel-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift/config/franka/ik_rel_env_cfg.py>`__
+.. |cabi-franka-link| replace:: `Isaac-Open-Drawer-Franka-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/cabinet/config/franka/joint_pos_env_cfg.py>`__
+.. |franka-direct-link| replace:: `Isaac-Franka-Cabinet-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/franka_cabinet/franka_cabinet_env.py>`__
+.. |cube-allegro-link| replace:: `Isaac-Repose-Cube-Allegro-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/inhand/config/allegro_hand/allegro_env_cfg.py>`__
+.. |allegro-direct-link| replace:: `Isaac-Repose-Cube-Allegro-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/allegro_hand/allegro_hand_env_cfg.py>`__
+.. |stack-cube-link| replace:: `Isaac-Stack-Cube-Franka-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/stack/config/franka/stack_joint_pos_env_cfg.py>`__
+.. |stack-cube-bp-link| replace:: `Isaac-Stack-Cube-Franka-IK-Rel-Blueprint-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/stack/config/franka/stack_ik_rel_blueprint_env_cfg.py>`__
+.. |gr1_pick_place-link| replace:: `Isaac-PickPlace-GR1T2-Abs-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/pick_place/pickplace_gr1t2_env_cfg.py>`__
+.. |g1_pick_place-link| replace:: `Isaac-PickPlace-G1-InspireFTP-Abs-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/pick_place/pickplace_unitree_g1_inspire_hand_env_cfg.py>`__
+.. |g1_pick_place_fixed-link| replace:: `Isaac-PickPlace-FixedBaseUpperBodyIK-G1-Abs-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomanipulation/pick_place/fixed_base_upper_body_ik_g1_env_cfg.py>`__
+.. |g1_pick_place_lm-link| replace:: `Isaac-PickPlace-Locomanipulation-G1-Abs-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomanipulation/pick_place/locomanipulation_g1_env_cfg.py>`__
+.. |long-suction-link| replace:: `Isaac-Stack-Cube-UR10-Long-Suction-IK-Rel-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/stack/config/ur10_gripper/stack_ik_rel_env_cfg.py>`__
+.. |short-suction-link| replace:: `Isaac-Stack-Cube-UR10-Short-Suction-IK-Rel-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/stack/config/ur10_gripper/stack_ik_rel_env_cfg.py>`__
+.. |gr1_pp_waist-link| replace:: `Isaac-PickPlace-GR1T2-WaistEnabled-Abs-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py>`__
+.. |galbot_stack-link| replace:: `Isaac-Stack-Cube-Galbot-Left-Arm-Gripper-RmpFlow-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/stack/config/galbot/stack_rmp_rel_env_cfg.py>`__
+.. |kuka-allegro-lift-link| replace:: `Isaac-Dexsuite-Kuka-Allegro-Lift-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/dexsuite/config/kuka_allegro/dexsuite_kuka_allegro_env_cfg.py>`__
+.. |kuka-allegro-reorient-link| replace:: `Isaac-Dexsuite-Kuka-Allegro-Reorient-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/dexsuite/config/kuka_allegro/dexsuite_kuka_allegro_env_cfg.py>`__
+.. |cube-shadow-link| replace:: `Isaac-Repose-Cube-Shadow-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/shadow_hand/shadow_hand_env_cfg.py>`__
+.. |cube-shadow-ff-link| replace:: `Isaac-Repose-Cube-Shadow-OpenAI-FF-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/shadow_hand/shadow_hand_env_cfg.py>`__
+.. |cube-shadow-lstm-link| replace:: `Isaac-Repose-Cube-Shadow-OpenAI-LSTM-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/shadow_hand/shadow_hand_env_cfg.py>`__
+.. |cube-shadow-vis-link| replace:: `Isaac-Repose-Cube-Shadow-Vision-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/shadow_hand/shadow_hand_vision_env.py>`__
+.. |agibot_place_mug-link| replace:: `Isaac-Place-Mug-Agibot-Left-Arm-RmpFlow-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/place/config/agibot/place_upright_mug_rmp_rel_env_cfg.py>`__
+.. |agibot_place_toy-link| replace:: `Isaac-Place-Toy2Box-Agibot-Right-Arm-RmpFlow-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/place/config/agibot/place_toy2box_rmp_rel_env_cfg.py>`__
+.. |reach_openarm_bi-link| replace:: `Isaac-Reach-OpenArm-Bi-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/reach/config/openarm/bimanual/joint_pos_env_cfg.py>`__
+.. |reach_openarm_uni-link| replace:: `Isaac-Reach-OpenArm-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/reach/config/openarm/unimanual/joint_pos_env_cfg.py>`__
+.. |lift_openarm_uni-link| replace:: `Isaac-Lift-Cube-OpenArm-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/lift/config/openarm/joint_pos_env_cfg.py>`__
+.. |cabi_openarm_uni-link| replace:: `Isaac-Open-Drawer-OpenArm-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/cabinet/config/openarm/joint_pos_env_cfg.py>`__
Contact-rich Manipulation
@@ -250,25 +276,25 @@ For example:
* |factory-nut-link|: Nut-Bolt fastening with the Franka arm
.. table::
- :widths: 33 37 30
-
- +--------------------+-------------------------+-----------------------------------------------------------------------------+
- | World | Environment ID | Description |
- +====================+=========================+=============================================================================+
- | |factory-peg| | |factory-peg-link| | Insert peg into the socket with the Franka robot |
- +--------------------+-------------------------+-----------------------------------------------------------------------------+
- | |factory-gear| | |factory-gear-link| | Insert and mesh gear into the base with other gears, using the Franka robot |
- +--------------------+-------------------------+-----------------------------------------------------------------------------+
- | |factory-nut| | |factory-nut-link| | Thread the nut onto the first 2 threads of the bolt, using the Franka robot |
- +--------------------+-------------------------+-----------------------------------------------------------------------------+
+ :widths: 25 30 25 20
+
+ +--------------------+-------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | World | Environment ID | Description | Presets |
+ +====================+=========================+=============================================================================+=======================+
+ | |factory-peg| | |factory-peg-link| | Insert peg into the socket with the Franka robot | |
+ +--------------------+-------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |factory-gear| | |factory-gear-link| | Insert and mesh gear into the base with other gears, using the Franka robot | |
+ +--------------------+-------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |factory-nut| | |factory-nut-link| | Thread the nut onto the first 2 threads of the bolt, using the Franka robot | |
+ +--------------------+-------------------------+-----------------------------------------------------------------------------+-----------------------+
.. |factory-peg| image:: ../_static/tasks/factory/peg_insert.jpg
.. |factory-gear| image:: ../_static/tasks/factory/gear_mesh.jpg
.. |factory-nut| image:: ../_static/tasks/factory/nut_thread.jpg
-.. |factory-peg-link| replace:: `Isaac-Factory-PegInsert-Direct-v0 `__
-.. |factory-gear-link| replace:: `Isaac-Factory-GearMesh-Direct-v0 `__
-.. |factory-nut-link| replace:: `Isaac-Factory-NutThread-Direct-v0 `__
+.. |factory-peg-link| replace:: `Isaac-Factory-PegInsert-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/factory/factory_env_cfg.py>`__
+.. |factory-gear-link| replace:: `Isaac-Factory-GearMesh-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/factory/factory_env_cfg.py>`__
+.. |factory-nut-link| replace:: `Isaac-Factory-NutThread-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/factory/factory_env_cfg.py>`__
AutoMate
~~~~~~~~
@@ -283,7 +309,7 @@ We provide environments for both disassembly and assembly.
.. attention::
- CUDA is recommended for running the AutoMate environments with 570 drivers. If running with Nvidia driver 570 on Linux with architecture x86_64, we follow the below steps to install CUDA 12.8. This allows for computing rewards in AutoMate environments with CUDA. If you have a different operation system or architecture, please refer to the `CUDA installation page `_ for additional instruction.
+ CUDA is recommended for running the AutoMate environments. If running with Nvidia driver 570 on Linux with architecture x86_64, we follow the below steps to install CUDA 12.8. This allows for computing rewards in AutoMate environments with CUDA. If you have a different operation system or architecture, please refer to the `CUDA installation page `_ for additional instruction.
.. code-block:: bash
@@ -296,7 +322,14 @@ We provide environments for both disassembly and assembly.
conda install cudatoolkit
- With 580 drivers and CUDA 13, we are currently unable to enable CUDA for computing the rewards. The code automatically fallbacks to CPU, resulting in slightly slower performance.
+ With 580 drivers on Linux with architecture x86_64, we install CUDA 13 and additionally install several packages. Please ensure that the pytorch version is compatible with the CUDA version.
+
+ .. code-block:: bash
+
+ wget https://developer.download.nvidia.com/compute/cuda/13.0.2/local_installers/cuda_13.0.2_580.95.05_linux.run
+ sudo sh cuda_13.0.2_580.95.05_linux.run --toolkit
+ pip install numba-cuda[cu13] coverage==7.6.1
+
* |disassembly-link|: The plug starts inserted in the socket. A low-level controller lifts the plug out and moves it to a random position. This process is purely scripted and does not involve any learned policy. Therefore, it does not require policy training or evaluation. The resulting trajectories serve as demonstrations for the reverse process, i.e., learning to assemble. To run disassembly for a specific task: ``python source/isaaclab_tasks/isaaclab_tasks/direct/automate/run_disassembly_w_id.py --assembly_id=ASSEMBLY_ID --disassembly_dir=DISASSEMBLY_DIR``. All generated trajectories are saved to a local directory ``DISASSEMBLY_DIR``.
* |assembly-link|: The goal is to insert the plug into the socket. You can use this environment to train a policy via reinforcement learning or evaluate a pre-trained checkpoint.
@@ -305,21 +338,21 @@ We provide environments for both disassembly and assembly.
* To evaluate an assembly policy, we run the command ``python source/isaaclab_tasks/isaaclab_tasks/direct/automate/run_w_id.py --assembly_id=ASSEMBLY_ID --checkpoint=CHECKPOINT --log_eval``. The evaluation results are stored in ``evaluation_{ASSEMBLY_ID}.h5``.
.. table::
- :widths: 33 37 30
+ :widths: 25 30 25 20
- +--------------------+-------------------------+-----------------------------------------------------------------------------+
- | World | Environment ID | Description |
- +====================+=========================+=============================================================================+
- | |disassembly| | |disassembly-link| | Lift a plug out of the socket with the Franka robot |
- +--------------------+-------------------------+-----------------------------------------------------------------------------+
- | |assembly| | |assembly-link| | Insert a plug into its corresponding socket with the Franka robot |
- +--------------------+-------------------------+-----------------------------------------------------------------------------+
+ +--------------------+-------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | World | Environment ID | Description | Presets |
+ +====================+=========================+=============================================================================+=======================+
+ | |disassembly| | |disassembly-link| | Lift a plug out of the socket with the Franka robot | |
+ +--------------------+-------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |assembly| | |assembly-link| | Insert a plug into its corresponding socket with the Franka robot | |
+ +--------------------+-------------------------+-----------------------------------------------------------------------------+-----------------------+
.. |assembly| image:: ../_static/tasks/automate/00004.jpg
.. |disassembly| image:: ../_static/tasks/automate/01053_disassembly.jpg
-.. |assembly-link| replace:: `Isaac-AutoMate-Assembly-Direct-v0 `__
-.. |disassembly-link| replace:: `Isaac-AutoMate-Disassembly-Direct-v0 `__
+.. |assembly-link| replace:: `Isaac-AutoMate-Assembly-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/automate/assembly_env_cfg.py>`__
+.. |disassembly-link| replace:: `Isaac-AutoMate-Disassembly-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/automate/disassembly_env_cfg.py>`__
FORGE
~~~~~~~~
@@ -338,25 +371,25 @@ These tasks share the same task configurations and control options. You can swit
* |forge-nut-link|: Nut-Bolt fastening with the Franka arm
.. table::
- :widths: 33 37 30
-
- +--------------------+-------------------------+-----------------------------------------------------------------------------+
- | World | Environment ID | Description |
- +====================+=========================+=============================================================================+
- | |forge-peg| | |forge-peg-link| | Insert peg into the socket with the Franka robot |
- +--------------------+-------------------------+-----------------------------------------------------------------------------+
- | |forge-gear| | |forge-gear-link| | Insert and mesh gear into the base with other gears, using the Franka robot |
- +--------------------+-------------------------+-----------------------------------------------------------------------------+
- | |forge-nut| | |forge-nut-link| | Thread the nut onto the first 2 threads of the bolt, using the Franka robot |
- +--------------------+-------------------------+-----------------------------------------------------------------------------+
+ :widths: 25 30 25 20
+
+ +--------------------+-------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | World | Environment ID | Description | Presets |
+ +====================+=========================+=============================================================================+=======================+
+ | |forge-peg| | |forge-peg-link| | Insert peg into the socket with the Franka robot | |
+ +--------------------+-------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |forge-gear| | |forge-gear-link| | Insert and mesh gear into the base with other gears, using the Franka robot | |
+ +--------------------+-------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |forge-nut| | |forge-nut-link| | Thread the nut onto the first 2 threads of the bolt, using the Franka robot | |
+ +--------------------+-------------------------+-----------------------------------------------------------------------------+-----------------------+
.. |forge-peg| image:: ../_static/tasks/factory/peg_insert.jpg
.. |forge-gear| image:: ../_static/tasks/factory/gear_mesh.jpg
.. |forge-nut| image:: ../_static/tasks/factory/nut_thread.jpg
-.. |forge-peg-link| replace:: `Isaac-Forge-PegInsert-Direct-v0 `__
-.. |forge-gear-link| replace:: `Isaac-Forge-GearMesh-Direct-v0 `__
-.. |forge-nut-link| replace:: `Isaac-Forge-NutThread-Direct-v0 `__
+.. |forge-peg-link| replace:: `Isaac-Forge-PegInsert-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/forge/forge_env_cfg.py>`__
+.. |forge-gear-link| replace:: `Isaac-Forge-GearMesh-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/forge/forge_env_cfg.py>`__
+.. |forge-nut-link| replace:: `Isaac-Forge-NutThread-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/forge/forge_env_cfg.py>`__
Locomotion
@@ -365,88 +398,88 @@ Locomotion
Environments based on legged locomotion tasks.
.. table::
- :widths: 33 37 30
-
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | World | Environment ID | Description |
- +==============================+==============================================+==============================================================================+
- | |velocity-flat-anymal-b| | |velocity-flat-anymal-b-link| | Track a velocity command on flat terrain with the Anymal B robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-rough-anymal-b| | |velocity-rough-anymal-b-link| | Track a velocity command on rough terrain with the Anymal B robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-flat-anymal-c| | |velocity-flat-anymal-c-link| | Track a velocity command on flat terrain with the Anymal C robot |
- | | | |
- | | |velocity-flat-anymal-c-direct-link| | |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-rough-anymal-c| | |velocity-rough-anymal-c-link| | Track a velocity command on rough terrain with the Anymal C robot |
- | | | |
- | | |velocity-rough-anymal-c-direct-link| | |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-flat-anymal-d| | |velocity-flat-anymal-d-link| | Track a velocity command on flat terrain with the Anymal D robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-rough-anymal-d| | |velocity-rough-anymal-d-link| | Track a velocity command on rough terrain with the Anymal D robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-flat-unitree-a1| | |velocity-flat-unitree-a1-link| | Track a velocity command on flat terrain with the Unitree A1 robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-rough-unitree-a1| | |velocity-rough-unitree-a1-link| | Track a velocity command on rough terrain with the Unitree A1 robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-flat-unitree-go1| | |velocity-flat-unitree-go1-link| | Track a velocity command on flat terrain with the Unitree Go1 robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-rough-unitree-go1| | |velocity-rough-unitree-go1-link| | Track a velocity command on rough terrain with the Unitree Go1 robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-flat-unitree-go2| | |velocity-flat-unitree-go2-link| | Track a velocity command on flat terrain with the Unitree Go2 robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-rough-unitree-go2| | |velocity-rough-unitree-go2-link| | Track a velocity command on rough terrain with the Unitree Go2 robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-flat-spot| | |velocity-flat-spot-link| | Track a velocity command on flat terrain with the Boston Dynamics Spot robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-flat-h1| | |velocity-flat-h1-link| | Track a velocity command on flat terrain with the Unitree H1 robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-rough-h1| | |velocity-rough-h1-link| | Track a velocity command on rough terrain with the Unitree H1 robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-flat-g1| | |velocity-flat-g1-link| | Track a velocity command on flat terrain with the Unitree G1 robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-rough-g1| | |velocity-rough-g1-link| | Track a velocity command on rough terrain with the Unitree G1 robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-flat-digit| | |velocity-flat-digit-link| | Track a velocity command on flat terrain with the Agility Digit robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |velocity-rough-digit| | |velocity-rough-digit-link| | Track a velocity command on rough terrain with the Agility Digit robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
- | |tracking-loco-manip-digit| | |tracking-loco-manip-digit-link| | Track a root velocity and hand pose command with the Agility Digit robot |
- +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+
-
-.. |velocity-flat-anymal-b-link| replace:: `Isaac-Velocity-Flat-Anymal-B-v0 `__
-.. |velocity-rough-anymal-b-link| replace:: `Isaac-Velocity-Rough-Anymal-B-v0 `__
-
-.. |velocity-flat-anymal-c-link| replace:: `Isaac-Velocity-Flat-Anymal-C-v0 `__
-.. |velocity-rough-anymal-c-link| replace:: `Isaac-Velocity-Rough-Anymal-C-v0 `__
-
-.. |velocity-flat-anymal-c-direct-link| replace:: `Isaac-Velocity-Flat-Anymal-C-Direct-v0 `__
-.. |velocity-rough-anymal-c-direct-link| replace:: `Isaac-Velocity-Rough-Anymal-C-Direct-v0 `__
-
-.. |velocity-flat-anymal-d-link| replace:: `Isaac-Velocity-Flat-Anymal-D-v0 `__
-.. |velocity-rough-anymal-d-link| replace:: `Isaac-Velocity-Rough-Anymal-D-v0 `__
-
-.. |velocity-flat-unitree-a1-link| replace:: `Isaac-Velocity-Flat-Unitree-A1-v0 `__
-.. |velocity-rough-unitree-a1-link| replace:: `Isaac-Velocity-Rough-Unitree-A1-v0 `__
-
-.. |velocity-flat-unitree-go1-link| replace:: `Isaac-Velocity-Flat-Unitree-Go1-v0 `__
-.. |velocity-rough-unitree-go1-link| replace:: `Isaac-Velocity-Rough-Unitree-Go1-v0 `__
-
-.. |velocity-flat-unitree-go2-link| replace:: `Isaac-Velocity-Flat-Unitree-Go2-v0 `__
-.. |velocity-rough-unitree-go2-link| replace:: `Isaac-Velocity-Rough-Unitree-Go2-v0 `__
-
-.. |velocity-flat-spot-link| replace:: `Isaac-Velocity-Flat-Spot-v0 `__
-
-.. |velocity-flat-h1-link| replace:: `Isaac-Velocity-Flat-H1-v0 `__
-.. |velocity-rough-h1-link| replace:: `Isaac-Velocity-Rough-H1-v0 `__
-
-.. |velocity-flat-g1-link| replace:: `Isaac-Velocity-Flat-G1-v0 `__
-.. |velocity-rough-g1-link| replace:: `Isaac-Velocity-Rough-G1-v0 `__
-
-.. |velocity-flat-digit-link| replace:: `Isaac-Velocity-Flat-Digit-v0 `__
-.. |velocity-rough-digit-link| replace:: `Isaac-Velocity-Rough-Digit-v0 `__
-.. |tracking-loco-manip-digit-link| replace:: `Isaac-Tracking-LocoManip-Digit-v0 `__
+ :widths: 25 30 25 20
+
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | World | Environment ID | Description | Presets |
+ +==============================+==============================================+==============================================================================+=======================+
+ | |velocity-flat-anymal-b| | |velocity-flat-anymal-b-link| | Track a velocity command on flat terrain with the Anymal B robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-rough-anymal-b| | |velocity-rough-anymal-b-link| | Track a velocity command on rough terrain with the Anymal B robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-flat-anymal-c| | |velocity-flat-anymal-c-link| | Track a velocity command on flat terrain with the Anymal C robot | ``newton``, ``physx`` |
+ | | | | |
+ | | |velocity-flat-anymal-c-direct-link| | | |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-rough-anymal-c| | |velocity-rough-anymal-c-link| | Track a velocity command on rough terrain with the Anymal C robot | ``newton``, ``physx`` |
+ | | | | |
+ | | |velocity-rough-anymal-c-direct-link| | | |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-flat-anymal-d| | |velocity-flat-anymal-d-link| | Track a velocity command on flat terrain with the Anymal D robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-rough-anymal-d| | |velocity-rough-anymal-d-link| | Track a velocity command on rough terrain with the Anymal D robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-flat-unitree-a1| | |velocity-flat-unitree-a1-link| | Track a velocity command on flat terrain with the Unitree A1 robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-rough-unitree-a1| | |velocity-rough-unitree-a1-link| | Track a velocity command on rough terrain with the Unitree A1 robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-flat-unitree-go1| | |velocity-flat-unitree-go1-link| | Track a velocity command on flat terrain with the Unitree Go1 robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-rough-unitree-go1| | |velocity-rough-unitree-go1-link| | Track a velocity command on rough terrain with the Unitree Go1 robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-flat-unitree-go2| | |velocity-flat-unitree-go2-link| | Track a velocity command on flat terrain with the Unitree Go2 robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-rough-unitree-go2| | |velocity-rough-unitree-go2-link| | Track a velocity command on rough terrain with the Unitree Go2 robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-flat-spot| | |velocity-flat-spot-link| | Track a velocity command on flat terrain with the Boston Dynamics Spot robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-flat-h1| | |velocity-flat-h1-link| | Track a velocity command on flat terrain with the Unitree H1 robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-rough-h1| | |velocity-rough-h1-link| | Track a velocity command on rough terrain with the Unitree H1 robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-flat-g1| | |velocity-flat-g1-link| | Track a velocity command on flat terrain with the Unitree G1 robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-rough-g1| | |velocity-rough-g1-link| | Track a velocity command on rough terrain with the Unitree G1 robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-flat-digit| | |velocity-flat-digit-link| | Track a velocity command on flat terrain with the Agility Digit robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |velocity-rough-digit| | |velocity-rough-digit-link| | Track a velocity command on rough terrain with the Agility Digit robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+ | |tracking-loco-manip-digit| | |tracking-loco-manip-digit-link| | Track a root velocity and hand pose command with the Agility Digit robot | ``newton``, ``physx`` |
+ +------------------------------+----------------------------------------------+------------------------------------------------------------------------------+-----------------------+
+
+.. |velocity-flat-anymal-b-link| replace:: `Isaac-Velocity-Flat-Anymal-B-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/anymal_b/flat_env_cfg.py>`__
+.. |velocity-rough-anymal-b-link| replace:: `Isaac-Velocity-Rough-Anymal-B-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/anymal_b/rough_env_cfg.py>`__
+
+.. |velocity-flat-anymal-c-link| replace:: `Isaac-Velocity-Flat-Anymal-C-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/anymal_c/flat_env_cfg.py>`__
+.. |velocity-rough-anymal-c-link| replace:: `Isaac-Velocity-Rough-Anymal-C-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/anymal_c/rough_env_cfg.py>`__
+
+.. |velocity-flat-anymal-c-direct-link| replace:: `Isaac-Velocity-Flat-Anymal-C-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/anymal_c/anymal_c_env.py>`__
+.. |velocity-rough-anymal-c-direct-link| replace:: `Isaac-Velocity-Rough-Anymal-C-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/anymal_c/anymal_c_env.py>`__
+
+.. |velocity-flat-anymal-d-link| replace:: `Isaac-Velocity-Flat-Anymal-D-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/anymal_d/flat_env_cfg.py>`__
+.. |velocity-rough-anymal-d-link| replace:: `Isaac-Velocity-Rough-Anymal-D-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/anymal_d/rough_env_cfg.py>`__
+
+.. |velocity-flat-unitree-a1-link| replace:: `Isaac-Velocity-Flat-Unitree-A1-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/a1/flat_env_cfg.py>`__
+.. |velocity-rough-unitree-a1-link| replace:: `Isaac-Velocity-Rough-Unitree-A1-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/a1/rough_env_cfg.py>`__
+
+.. |velocity-flat-unitree-go1-link| replace:: `Isaac-Velocity-Flat-Unitree-Go1-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/go1/flat_env_cfg.py>`__
+.. |velocity-rough-unitree-go1-link| replace:: `Isaac-Velocity-Rough-Unitree-Go1-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/go1/rough_env_cfg.py>`__
+
+.. |velocity-flat-unitree-go2-link| replace:: `Isaac-Velocity-Flat-Unitree-Go2-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/go2/flat_env_cfg.py>`__
+.. |velocity-rough-unitree-go2-link| replace:: `Isaac-Velocity-Rough-Unitree-Go2-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/go2/rough_env_cfg.py>`__
+
+.. |velocity-flat-spot-link| replace:: `Isaac-Velocity-Flat-Spot-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/spot/flat_env_cfg.py>`__
+
+.. |velocity-flat-h1-link| replace:: `Isaac-Velocity-Flat-H1-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/h1/flat_env_cfg.py>`__
+.. |velocity-rough-h1-link| replace:: `Isaac-Velocity-Rough-H1-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/h1/rough_env_cfg.py>`__
+
+.. |velocity-flat-g1-link| replace:: `Isaac-Velocity-Flat-G1-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/g1/flat_env_cfg.py>`__
+.. |velocity-rough-g1-link| replace:: `Isaac-Velocity-Rough-G1-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/g1/rough_env_cfg.py>`__
+
+.. |velocity-flat-digit-link| replace:: `Isaac-Velocity-Flat-Digit-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/digit/flat_env_cfg.py>`__
+.. |velocity-rough-digit-link| replace:: `Isaac-Velocity-Rough-Digit-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomotion/velocity/config/digit/rough_env_cfg.py>`__
+.. |tracking-loco-manip-digit-link| replace:: `Isaac-Tracking-LocoManip-Digit-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/locomanipulation/tracking/config/digit/loco_manip_env_cfg.py>`__
.. |velocity-flat-anymal-b| image:: ../_static/tasks/locomotion/anymal_b_flat.jpg
.. |velocity-rough-anymal-b| image:: ../_static/tasks/locomotion/anymal_b_rough.jpg
@@ -473,15 +506,15 @@ Navigation
~~~~~~~~~~
.. table::
- :widths: 33 37 30
+ :widths: 25 30 25 20
- +----------------+---------------------+-----------------------------------------------------------------------------+
- | World | Environment ID | Description |
- +================+=====================+=============================================================================+
- | |anymal_c_nav| | |anymal_c_nav-link| | Navigate towards a target x-y position and heading with the ANYmal C robot. |
- +----------------+---------------------+-----------------------------------------------------------------------------+
+ +----------------+---------------------+-----------------------------------------------------------------------------+-----------------------+
+ | World | Environment ID | Description | Presets |
+ +================+=====================+=============================================================================+=======================+
+ | |anymal_c_nav| | |anymal_c_nav-link| | Navigate towards a target x-y position and heading with the ANYmal C robot. | ``newton``, ``physx`` |
+ +----------------+---------------------+-----------------------------------------------------------------------------+-----------------------+
-.. |anymal_c_nav-link| replace:: `Isaac-Navigation-Flat-Anymal-C-v0 `__
+.. |anymal_c_nav-link| replace:: `Isaac-Navigation-Flat-Anymal-C-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/navigation/config/anymal_c/navigation_env_cfg.py>`__
.. |anymal_c_nav| image:: ../_static/tasks/navigation/anymal_c_nav.jpg
@@ -494,18 +527,18 @@ Multirotor
See the `drone_arl` folder and the ARL robot config
(`ARL_ROBOT_1_CFG`) in the codebase for details.
-.. |arl_robot_track_position_state_based-link| replace:: `Isaac-TrackPositionNoObstacles-ARL-Robot-1-v0 `__
+.. |arl_robot_track_position_state_based-link| replace:: `Isaac-TrackPositionNoObstacles-ARL-Robot-1-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/manager_based/drone_arl/track_position_state_based/config/arl_robot_1/track_position_state_based_env_cfg.py>`__
.. |arl_robot_track_position_state_based| image:: ../_static/tasks/drone_arl/arl_robot_1_track_position_state_based.jpg
.. table::
- :widths: 33 37 30
+ :widths: 25 30 25 20
- +----------------------------------------+---------------------------------------------+----------------------------------------------------------------------------------------+
- | World | Environment ID | Description |
- +========================================+=============================================+========================================================================================+
- | |arl_robot_track_position_state_based| | |arl_robot_track_position_state_based-link| | Setpoint position control for the ARL robot using the track_position_state_based task. |
- +----------------------------------------+---------------------------------------------+----------------------------------------------------------------------------------------+
+ +----------------------------------------+---------------------------------------------+----------------------------------------------------------------------------------------+-----------------------+
+ | World | Environment ID | Description | Presets |
+ +========================================+=============================================+========================================================================================+=======================+
+ | |arl_robot_track_position_state_based| | |arl_robot_track_position_state_based-link| | Setpoint position control for the ARL robot using the track_position_state_based task. | |
+ +----------------------------------------+---------------------------------------------+----------------------------------------------------------------------------------------+-----------------------+
Others
@@ -521,24 +554,24 @@ Others
For evaluation, the play script's command line input ``--real-time`` allows the interaction loop between the environment and the agent to run in real time, if possible.
.. table::
- :widths: 33 37 30
-
- +----------------+---------------------------+-----------------------------------------------------------------------------+
- | World | Environment ID | Description |
- +================+===========================+=============================================================================+
- | |quadcopter| | |quadcopter-link| | Fly and hover the Crazyflie copter at a goal point by applying thrust. |
- +----------------+---------------------------+-----------------------------------------------------------------------------+
- | |humanoid_amp| | |humanoid_amp_dance-link| | Move a humanoid robot by imitating different pre-recorded human animations |
- | | | (Adversarial Motion Priors). |
- | | |humanoid_amp_run-link| | |
- | | | |
- | | |humanoid_amp_walk-link| | |
- +----------------+---------------------------+-----------------------------------------------------------------------------+
-
-.. |quadcopter-link| replace:: `Isaac-Quadcopter-Direct-v0 `__
-.. |humanoid_amp_dance-link| replace:: `Isaac-Humanoid-AMP-Dance-Direct-v0 `__
-.. |humanoid_amp_run-link| replace:: `Isaac-Humanoid-AMP-Run-Direct-v0 `__
-.. |humanoid_amp_walk-link| replace:: `Isaac-Humanoid-AMP-Walk-Direct-v0 `__
+ :widths: 25 30 25 20
+
+ +----------------+---------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | World | Environment ID | Description | Presets |
+ +================+===========================+=============================================================================+=======================+
+ | |quadcopter| | |quadcopter-link| | Fly and hover the Crazyflie copter at a goal point by applying thrust. | |
+ +----------------+---------------------------+-----------------------------------------------------------------------------+-----------------------+
+ | |humanoid_amp| | |humanoid_amp_dance-link| | Move a humanoid robot by imitating different pre-recorded human animations | |
+ | | | (Adversarial Motion Priors). | |
+ | | |humanoid_amp_run-link| | | |
+ | | | | |
+ | | |humanoid_amp_walk-link| | | |
+ +----------------+---------------------------+-----------------------------------------------------------------------------+-----------------------+
+
+.. |quadcopter-link| replace:: `Isaac-Quadcopter-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/quadcopter/quadcopter_env.py>`__
+.. |humanoid_amp_dance-link| replace:: `Isaac-Humanoid-AMP-Dance-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/humanoid_amp/humanoid_amp_env_cfg.py>`__
+.. |humanoid_amp_run-link| replace:: `Isaac-Humanoid-AMP-Run-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/humanoid_amp/humanoid_amp_env_cfg.py>`__
+.. |humanoid_amp_walk-link| replace:: `Isaac-Humanoid-AMP-Walk-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/humanoid_amp/humanoid_amp_env_cfg.py>`__
.. |quadcopter| image:: ../_static/tasks/others/quadcopter.jpg
.. |humanoid_amp| image:: ../_static/tasks/others/humanoid_amp.jpg
@@ -673,17 +706,17 @@ Classic
~~~~~~~
.. table::
- :widths: 33 37 30
+ :widths: 25 30 25 20
- +------------------------+------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
- | World | Environment ID | Description |
- +========================+====================================+=======================================================================================================================+
- | |cart-double-pendulum| | |cart-double-pendulum-direct-link| | Move the cart and the pendulum to keep the last one upwards in the classic inverted double pendulum on a cart control |
- +------------------------+------------------------------------+-----------------------------------------------------------------------------------------------------------------------+
+ +------------------------+------------------------------------+-----------------------------------------------------------------------------------------------------------------------+-----------------------+
+ | World | Environment ID | Description | Presets |
+ +========================+====================================+=======================================================================================================================+=======================+
+ | |cart-double-pendulum| | |cart-double-pendulum-direct-link| | Move the cart and the pendulum to keep the last one upwards in the classic inverted double pendulum on a cart control | |
+ +------------------------+------------------------------------+-----------------------------------------------------------------------------------------------------------------------+-----------------------+
.. |cart-double-pendulum| image:: ../_static/tasks/classic/cart_double_pendulum.jpg
-.. |cart-double-pendulum-direct-link| replace:: `Isaac-Cart-Double-Pendulum-Direct-v0 `__
+.. |cart-double-pendulum-direct-link| replace:: `Isaac-Cart-Double-Pendulum-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/cart_double_pendulum/cart_double_pendulum_env.py>`__
Manipulation
~~~~~~~~~~~~
@@ -691,17 +724,17 @@ Manipulation
Environments based on fixed-arm manipulation tasks.
.. table::
- :widths: 33 37 30
+ :widths: 25 30 25 20
- +----------------------+--------------------------------+--------------------------------------------------------+
- | World | Environment ID | Description |
- +======================+================================+========================================================+
- | |shadow-hand-over| | |shadow-hand-over-direct-link| | Passing an object from one hand over to the other hand |
- +----------------------+--------------------------------+--------------------------------------------------------+
+ +----------------------+--------------------------------+--------------------------------------------------------+-----------------------+
+ | World | Environment ID | Description | Presets |
+ +======================+================================+========================================================+=======================+
+ | |shadow-hand-over| | |shadow-hand-over-direct-link| | Passing an object from one hand over to the other hand | |
+ +----------------------+--------------------------------+--------------------------------------------------------+-----------------------+
.. |shadow-hand-over| image:: ../_static/tasks/manipulation/shadow_hand_over.jpg
-.. |shadow-hand-over-direct-link| replace:: `Isaac-Shadow-Hand-Over-Direct-v0 `__
+.. |shadow-hand-over-direct-link| replace:: `Isaac-Shadow-Hand-Over-Direct-v0 <../../../source/isaaclab_tasks/isaaclab_tasks/direct/shadow_hand_over/shadow_hand_over_env.py>`__
|
@@ -713,486 +746,610 @@ provided when running ``play.py`` or any inferencing workflows. These tasks prov
inferencing, including reading from an already trained checkpoint and disabling runtime perturbations used for training.
.. list-table::
- :widths: 33 25 19 25
+ :widths: 28 20 13 22 17
* - **Task Name**
- **Inference Task Name**
- **Workflow**
- **RL Library**
+ - **Presets**
* - Isaac-Ant-Direct-v0
-
- Direct
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Ant-v0
-
- Manager Based
- **rsl_rl** (PPO), **rl_games** (PPO), **skrl** (PPO), **sb3** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Cart-Double-Pendulum-Direct-v0
-
- Direct
- **rl_games** (PPO), **skrl** (IPPO, PPO, MAPPO)
+ -
* - Isaac-Cartpole-Camera-Showcase-Box-Box-Direct-v0 (Requires running with ``--enable_cameras``)
-
- Direct
- **skrl** (PPO)
+ - ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``
* - Isaac-Cartpole-Camera-Showcase-Box-Discrete-Direct-v0 (Requires running with ``--enable_cameras``)
-
- Direct
- **skrl** (PPO)
+ - ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``
* - Isaac-Cartpole-Camera-Showcase-Box-MultiDiscrete-Direct-v0 (Requires running with ``--enable_cameras``)
-
- Direct
- **skrl** (PPO)
+ - ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``
* - Isaac-Cartpole-Camera-Showcase-Dict-Box-Direct-v0 (Requires running with ``--enable_cameras``)
-
- Direct
- **skrl** (PPO)
+ - ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``
* - Isaac-Cartpole-Camera-Showcase-Dict-Discrete-Direct-v0 (Requires running with ``--enable_cameras``)
-
- Direct
- **skrl** (PPO)
+ - ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``
* - Isaac-Cartpole-Camera-Showcase-Dict-MultiDiscrete-Direct-v0 (Requires running with ``--enable_cameras``)
-
- Direct
- **skrl** (PPO)
+ - ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``
* - Isaac-Cartpole-Camera-Showcase-Tuple-Box-Direct-v0 (Requires running with ``--enable_cameras``)
-
- Direct
- **skrl** (PPO)
+ - ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``
* - Isaac-Cartpole-Camera-Showcase-Tuple-Discrete-Direct-v0 (Requires running with ``--enable_cameras``)
-
- Direct
- **skrl** (PPO)
+ - ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``
* - Isaac-Cartpole-Camera-Showcase-Tuple-MultiDiscrete-Direct-v0 (Requires running with ``--enable_cameras``)
-
- Direct
- **skrl** (PPO)
- * - Isaac-Cartpole-Depth-Camera-Direct-v0 (Requires running with ``--enable_cameras``)
+ - ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``
+ * - Isaac-Cartpole-Camera-Presets-Direct-v0 (Requires running with ``--enable_cameras``)
-
- Direct
- **rl_games** (PPO), **skrl** (PPO)
- * - Isaac-Cartpole-Depth-v0 (Requires running with ``--enable_cameras``)
- -
- - Manager Based
- - **rl_games** (PPO)
+ - ``newton``, ``physx``, ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``, ``rgb``, ``depth``, ``albedo``, ``semantic_segmentation``, ``simple_shading_constant_diffuse``, ``simple_shading_diffuse_mdl``, ``simple_shading_full_mdl``
* - Isaac-Cartpole-Direct-v0
-
- Direct
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO), **sb3** (PPO)
- * - Isaac-Cartpole-RGB-Camera-Direct-v0 (Requires running with ``--enable_cameras``)
- -
- - Direct
- - **rl_games** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-RGB-ResNet18-v0 (Requires running with ``--enable_cameras``)
-
- Manager Based
- **rl_games** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Cartpole-RGB-TheiaTiny-v0 (Requires running with ``--enable_cameras``)
-
- Manager Based
- **rl_games** (PPO)
- * - Isaac-Cartpole-RGB-v0 (Requires running with ``--enable_cameras``)
- -
- - Manager Based
- - **rl_games** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Cartpole-Showcase-Box-Box-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-Box-Discrete-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-Box-MultiDiscrete-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-Dict-Box-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-Dict-Discrete-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-Dict-MultiDiscrete-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-Discrete-Box-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-Discrete-Discrete-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-Discrete-MultiDiscrete-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-MultiDiscrete-Box-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-MultiDiscrete-Discrete-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-MultiDiscrete-MultiDiscrete-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-Tuple-Box-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-Tuple-Discrete-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-Showcase-Tuple-MultiDiscrete-Direct-v0
-
- Direct
- **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Cartpole-v0
-
- Manager Based
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO), **sb3** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Factory-GearMesh-Direct-v0
-
- Direct
- **rl_games** (PPO)
+ -
* - Isaac-Factory-NutThread-Direct-v0
-
- Direct
- **rl_games** (PPO)
+ -
* - Isaac-Factory-PegInsert-Direct-v0
-
- Direct
- **rl_games** (PPO)
+ -
* - Isaac-AutoMate-Assembly-Direct-v0
-
- Direct
- **rl_games** (PPO)
+ -
* - Isaac-AutoMate-Disassembly-Direct-v0
-
- Direct
-
+ -
* - Isaac-Forge-GearMesh-Direct-v0
-
- Direct
- **rl_games** (PPO)
+ -
* - Isaac-Forge-NutThread-Direct-v0
-
- Direct
- **rl_games** (PPO)
+ -
* - Isaac-Forge-PegInsert-Direct-v0
-
- Direct
- **rl_games** (PPO)
+ -
* - Isaac-Franka-Cabinet-Direct-v0
-
- Direct
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO)
+ -
* - Isaac-Humanoid-AMP-Dance-Direct-v0
-
- Direct
- **skrl** (AMP)
+ -
* - Isaac-Humanoid-AMP-Run-Direct-v0
-
- Direct
- **skrl** (AMP)
+ -
* - Isaac-Humanoid-AMP-Walk-Direct-v0
-
- Direct
- **skrl** (AMP)
+ -
* - Isaac-Humanoid-Direct-v0
-
- Direct
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``, ``ovphysx``
* - Isaac-Humanoid-v0
-
- Manager Based
- **rsl_rl** (PPO), **rl_games** (PPO), **skrl** (PPO), **sb3** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Lift-Cube-Franka-IK-Abs-v0
-
- Manager Based
-
+ -
* - Isaac-Lift-Cube-Franka-IK-Rel-v0
-
- Manager Based
-
+ -
* - Isaac-Lift-Cube-Franka-v0
- Isaac-Lift-Cube-Franka-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO), **rl_games** (PPO), **sb3** (PPO)
+ -
* - Isaac-Lift-Teddy-Bear-Franka-IK-Abs-v0
-
- Manager Based
-
+ -
* - Isaac-Tracking-LocoManip-Digit-v0
- Isaac-Tracking-LocoManip-Digit-Play-v0
- Manager Based
- **rsl_rl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Navigation-Flat-Anymal-C-v0
- Isaac-Navigation-Flat-Anymal-C-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Open-Drawer-Franka-IK-Abs-v0
-
- Manager Based
-
+ -
* - Isaac-Open-Drawer-Franka-IK-Rel-v0
-
- Manager Based
-
+ -
* - Isaac-Open-Drawer-Franka-v0
- Isaac-Open-Drawer-Franka-Play-v0
- Manager Based
- **rsl_rl** (PPO), **rl_games** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Quadcopter-Direct-v0
-
- Direct
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO)
+ -
* - Isaac-Reach-Franka-IK-Abs-v0
-
- Manager Based
-
+ -
* - Isaac-Reach-Franka-IK-Rel-v0
-
- Manager Based
-
+ -
* - Isaac-Reach-Franka-OSC-v0
- Isaac-Reach-Franka-OSC-Play-v0
- Manager Based
- **rsl_rl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Reach-Franka-v0
- Isaac-Reach-Franka-Play-v0
- Manager Based
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Reach-UR10-v0
- Isaac-Reach-UR10-Play-v0
- Manager Based
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Deploy-Reach-UR10e-v0
- Isaac-Deploy-Reach-UR10e-Play-v0
- Manager Based
- **rsl_rl** (PPO)
+ -
* - Isaac-Repose-Cube-Allegro-Direct-v0
-
- Direct
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Repose-Cube-Allegro-NoVelObs-v0
- Isaac-Repose-Cube-Allegro-NoVelObs-Play-v0
- Manager Based
- **rsl_rl** (PPO), **rl_games** (PPO), **skrl** (PPO)
+ -
* - Isaac-Repose-Cube-Allegro-v0
- Isaac-Repose-Cube-Allegro-Play-v0
- Manager Based
- **rsl_rl** (PPO), **rl_games** (PPO), **skrl** (PPO)
+ -
* - Isaac-Repose-Cube-Shadow-Direct-v0
-
- Direct
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Repose-Cube-Shadow-OpenAI-FF-Direct-v0
-
- Direct
- **rl_games** (FF), **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Repose-Cube-Shadow-OpenAI-LSTM-Direct-v0
-
- Direct
- **rl_games** (LSTM)
+ - ``newton``, ``physx``
* - Isaac-Repose-Cube-Shadow-Vision-Direct-v0 (Requires running with ``--enable_cameras``)
- Isaac-Repose-Cube-Shadow-Vision-Direct-Play-v0 (Requires running with ``--enable_cameras``)
- Direct
- **rsl_rl** (PPO), **rl_games** (VISION)
+ - ``newton``, ``physx``, ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``, ``rgb``, ``depth``, ``albedo``, ``full``, ``semantic_segmentation``, ``simple_shading_constant_diffuse``, ``simple_shading_diffuse_mdl``, ``simple_shading_full_mdl``
* - Isaac-Shadow-Hand-Over-Direct-v0
-
- Direct
- **rl_games** (PPO), **skrl** (IPPO, PPO, MAPPO)
+ -
* - Isaac-Stack-Cube-Franka-IK-Rel-v0
-
- Manager Based
-
+ -
* - Isaac-Dexsuite-Kuka-Allegro-Lift-v0
+
+ Camera variants (requires ``--enable_cameras``):
+
+ - single-camera: append ``presets=single_camera,isaacsim_rtx_renderer``
+ - dual-camera: append ``presets=duo_camera,isaacsim_rtx_renderer``
+
+ The same ``presets=`` flags must be passed to both the training and
+ play scripts. There is no separately registered
+ ``Isaac-Dexsuite-Kuka-Allegro-Lift-Single-Camera-v0`` environment;
+ all observation-mode variants share the base task name and are
+ selected via the preset system.
- Isaac-Dexsuite-Kuka-Allegro-Lift-Play-v0
- Manager Based
- **rl_games** (PPO), **rsl_rl** (PPO)
+ - ``newton``, ``physx``, ``single_camera``, ``duo_camera``, ``state``, ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``, ``rgb64``, ``rgb128``, ``rgb256``, ``depth64``, ``depth128``, ``depth256``, ``albedo64``, ``albedo128``, ``albedo256``, ``semantic_segmentation64``, ``semantic_segmentation128``, ``semantic_segmentation256``, ``simple_shading_constant_diffuse64``, ``simple_shading_constant_diffuse128``, ``simple_shading_constant_diffuse256``, ``simple_shading_diffuse_mdl64``, ``simple_shading_diffuse_mdl128``, ``simple_shading_diffuse_mdl256``, ``simple_shading_full_mdl64``, ``simple_shading_full_mdl128``, ``simple_shading_full_mdl256``
* - Isaac-Dexsuite-Kuka-Allegro-Reorient-v0
+
+ Camera variants (requires ``--enable_cameras``):
+
+ - single-camera: append ``presets=single_camera,isaacsim_rtx_renderer``
+ - dual-camera: append ``presets=duo_camera,isaacsim_rtx_renderer``
+
+ The same ``presets=`` flags must be passed to both the training and
+ play scripts.
- Isaac-Dexsuite-Kuka-Allegro-Reorient-Play-v0
- Manager Based
- **rl_games** (PPO), **rsl_rl** (PPO)
+ - ``newton``, ``physx``, ``single_camera``, ``duo_camera``, ``state``, ``newton_renderer``, ``ovrtx_renderer``, ``isaacsim_rtx_renderer``, ``rgb64``, ``rgb128``, ``rgb256``, ``depth64``, ``depth128``, ``depth256``, ``albedo64``, ``albedo128``, ``albedo256``, ``semantic_segmentation64``, ``semantic_segmentation128``, ``semantic_segmentation256``, ``simple_shading_constant_diffuse64``, ``simple_shading_constant_diffuse128``, ``simple_shading_constant_diffuse256``, ``simple_shading_diffuse_mdl64``, ``simple_shading_diffuse_mdl128``, ``simple_shading_diffuse_mdl256``, ``simple_shading_full_mdl64``, ``simple_shading_full_mdl128``, ``simple_shading_full_mdl256``
* - Isaac-Stack-Cube-Franka-v0
-
- Manager Based
-
+ -
* - Isaac-Stack-Cube-Instance-Randomize-Franka-IK-Rel-v0
-
- Manager Based
-
+ -
* - Isaac-Stack-Cube-Instance-Randomize-Franka-v0
-
- Manager Based
-
+ -
* - Isaac-PickPlace-G1-InspireFTP-Abs-v0
-
- Manager Based
-
+ -
* - Isaac-Stack-Cube-UR10-Long-Suction-IK-Rel-v0
-
- Manager Based
-
+ -
* - Isaac-Stack-Cube-UR10-Short-Suction-IK-Rel-v0
-
- Manager Based
-
+ -
* - Isaac-Stack-Cube-Galbot-Left-Arm-Gripper-RmpFlow-v0
-
- Manager Based
-
+ -
* - Isaac-Stack-Cube-Galbot-Right-Arm-Suction-RmpFlow-v0
-
- Manager Based
-
+ -
* - Isaac-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-v0
- Isaac-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-Play-v0
- Manager Based
-
+ -
* - Isaac-Place-Mug-Agibot-Left-Arm-RmpFlow-v0
-
- Manager Based
-
+ -
* - Isaac-Place-Toy2Box-Agibot-Right-Arm-RmpFlow-v0
-
- Manager Based
-
+ -
* - Isaac-Stack-Cube-Galbot-Left-Arm-Gripper-RmpFlow-v0
-
- Manager Based
-
+ -
* - Isaac-Stack-Cube-Galbot-Right-Arm-Suction-RmpFlow-v0
-
- Manager Based
-
+ -
* - Isaac-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-v0
- Isaac-Stack-Cube-Galbot-Left-Arm-Gripper-Visuomotor-Play-v0
- Manager Based
-
+ -
* - Isaac-Place-Mug-Agibot-Left-Arm-RmpFlow-v0
-
- Manager Based
-
+ -
* - Isaac-Place-Toy2Box-Agibot-Right-Arm-RmpFlow-v0
-
- Manager Based
-
+ -
* - Isaac-Velocity-Flat-Anymal-B-v0
- Isaac-Velocity-Flat-Anymal-B-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Flat-Anymal-C-Direct-v0
-
- Direct
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO)
+ -
* - Isaac-Velocity-Flat-Anymal-C-v0
- Isaac-Velocity-Flat-Anymal-C-Play-v0
- Manager Based
- **rsl_rl** (PPO), **rl_games** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Flat-Anymal-D-v0
- Isaac-Velocity-Flat-Anymal-D-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Flat-Cassie-v0
- Isaac-Velocity-Flat-Cassie-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ -
* - Isaac-Velocity-Flat-Digit-v0
- Isaac-Velocity-Flat-Digit-Play-v0
- Manager Based
- **rsl_rl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Flat-G1-v0
- Isaac-Velocity-Flat-G1-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Flat-H1-v0
- Isaac-Velocity-Flat-H1-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Flat-Spot-v0
- Isaac-Velocity-Flat-Spot-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Flat-Unitree-A1-v0
- Isaac-Velocity-Flat-Unitree-A1-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO), **sb3** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Flat-Unitree-Go1-v0
- Isaac-Velocity-Flat-Unitree-Go1-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Flat-Unitree-Go2-v0
- Isaac-Velocity-Flat-Unitree-Go2-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Rough-Anymal-B-v0
- Isaac-Velocity-Rough-Anymal-B-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Rough-Anymal-C-Direct-v0
-
- Direct
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO)
+ -
* - Isaac-Velocity-Rough-Anymal-C-v0
- Isaac-Velocity-Rough-Anymal-C-Play-v0
- Manager Based
- **rl_games** (PPO), **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Rough-Anymal-D-v0
- Isaac-Velocity-Rough-Anymal-D-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Rough-Cassie-v0
- Isaac-Velocity-Rough-Cassie-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ -
* - Isaac-Velocity-Rough-Digit-v0
- Isaac-Velocity-Rough-Digit-Play-v0
- Manager Based
- **rsl_rl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Rough-G1-v0
- Isaac-Velocity-Rough-G1-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Rough-H1-v0
- Isaac-Velocity-Rough-H1-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Rough-Unitree-A1-v0
- Isaac-Velocity-Rough-Unitree-A1-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO), **sb3** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Rough-Unitree-Go1-v0
- Isaac-Velocity-Rough-Unitree-Go1-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Velocity-Rough-Unitree-Go2-v0
- Isaac-Velocity-Rough-Unitree-Go2-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO)
+ - ``newton``, ``physx``
* - Isaac-Reach-OpenArm-Bi-v0
- Isaac-Reach-OpenArm-Bi-Play-v0
- Manager Based
- **rsl_rl** (PPO), **rl_games** (PPO)
+ -
* - Isaac-Reach-OpenArm-v0
- Isaac-Reach-OpenArm-Play-v0
- Manager Based
- **rsl_rl** (PPO), **skrl** (PPO), **rl_games** (PPO)
+ -
* - Isaac-Lift-Cube-OpenArm-v0
- Isaac-Lift-Cube-OpenArm-Play-v0
- Manager Based
- **rsl_rl** (PPO), **rl_games** (PPO)
+ -
* - Isaac-Open-Drawer-OpenArm-v0
- Isaac-Open-Drawer-OpenArm-Play-v0
- Manager Based
- **rsl_rl** (PPO), **rl_games** (PPO)
+ -
diff --git a/docs/source/overview/imitation-learning/augmented_imitation.rst b/docs/source/overview/imitation-learning/augmented_imitation.rst
index b3593f22e62..376f447f97a 100644
--- a/docs/source/overview/imitation-learning/augmented_imitation.rst
+++ b/docs/source/overview/imitation-learning/augmented_imitation.rst
@@ -373,8 +373,6 @@ Below is an explanation of the different settings used for evaluation:
- Maximum value of the action space normalization factor.
* - ``--disable_fabric``
- Whether to disable fabric and use USD I/O operations.
- * - ``--enable_pinocchio``
- - Whether to enable Pinocchio for IK controllers.
.. note::
The evaluation results will help you understand if the visual augmentation has improved the policy's performance and robustness. Compare these results with evaluations on the original dataset to measure the impact of augmentation.
diff --git a/docs/source/overview/imitation-learning/humanoids_imitation.rst b/docs/source/overview/imitation-learning/humanoids_imitation.rst
new file mode 100644
index 00000000000..d4782b14ab6
--- /dev/null
+++ b/docs/source/overview/imitation-learning/humanoids_imitation.rst
@@ -0,0 +1,777 @@
+.. _data-generation-imitation-learning-humanoids:
+
+Examples: Data Generation and Imitation Learning for Humanoids
+==============================================================
+
+This page covers data generation and imitation learning workflows for humanoid robots (GR-1, G1) with Isaac Lab Mimic:
+
+* **Demo 1:** Data generation and policy training for a humanoid robot (GR-1 pick and place)
+* **Demo 2:** Visuomotor policy for a humanoid robot (GR-1 nut pouring)
+* **Demo 3:** Data generation and policy training for humanoid robot locomanipulation (Unitree G1)
+
+.. important::
+
+ Complete the tutorial in :ref:`Teleoperation and Imitation Learning with Isaac Lab Mimic `
+ before proceeding with the following demonstrations to
+ understand the data collection, annotation, and generation steps of Isaac Lab Mimic.
+
+
+Demo 1: Data Generation and Policy Training for a Humanoid Robot
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/gr-1_steering_wheel_pick_place.gif
+ :width: 100%
+ :align: center
+ :alt: GR-1 humanoid robot performing a pick and place task
+ :figclass: align-center
+
+
+Isaac Lab Mimic supports data generation for robots with multiple end effectors. In the following demonstration, we will show how to generate data
+to train a Fourier GR-1 humanoid robot to perform a pick and place task.
+
+Optional: Collect and annotate demonstrations
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Collect human demonstrations
+""""""""""""""""""""""""""""
+.. note::
+
+ Data collection for the GR-1 humanoid robot environment requires use of an Apple Vision Pro headset. If you do not have access to
+ an Apple Vision Pro, you may skip this step and continue on to the next step: :ref:`Generate the dataset `.
+ A pre-recorded annotated dataset is provided in the next step.
+
+.. tip::
+ The GR1 scene utilizes the wrist poses from the Apple Vision Pro (AVP) as setpoints for a differential IK controller (Pink-IK).
+ The differential IK controller requires the user's wrist pose to be close to the robot's initial or current pose for optimal performance.
+ Rapid movements of the user's wrist may cause it to deviate significantly from the goal state, which could prevent the IK controller from finding the optimal solution.
+ This may result in a mismatch between the user's wrist and the robot's wrist.
+ You can increase the gain of all the `Pink-IK controller's FrameTasks `__ to track the AVP wrist poses with lower latency.
+ However, this may lead to more jerky motion.
+ Separately, the finger joints of the robot are retargeted to the user's finger joints using the `dex-retargeting `_ library.
+
+Set up the CloudXR Runtime and Apple Vision Pro for teleoperation by following the steps in :ref:`cloudxr-teleoperation`.
+CPU simulation is used in the following steps for better XR performance when running a single environment.
+
+Collect a set of human demonstrations.
+A success demo requires the object to be placed in the bin and for the robot's right arm to be retracted to the starting position.
+
+The Isaac Lab Mimic Env GR-1 humanoid robot is set up such that the left hand has a single subtask, while the right hand has two subtasks.
+The first subtask involves the right hand remaining idle while the left hand picks up and moves the object to the position where the right hand will grasp it.
+This setup allows Isaac Lab Mimic to interpolate the right hand's trajectory accurately by using the object's pose, especially when poses are randomized during data generation.
+Therefore, avoid moving the right hand while the left hand picks up the object and brings it to a stable position.
+
+
+.. |good_demo| image:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/gr-1_steering_wheel_pick_place_good_demo.gif
+ :width: 49%
+ :alt: GR-1 humanoid robot performing a good pick and place demonstration
+
+.. |bad_demo| image:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/gr-1_steering_wheel_pick_place_bad_demo.gif
+ :width: 49%
+ :alt: GR-1 humanoid robot performing a bad pick and place demonstration
+
+|good_demo| |bad_demo|
+
+.. centered:: Left: A good human demonstration with smooth and steady motion. Right: A bad demonstration with jerky and exaggerated motion.
+
+
+Collect five demonstrations by running the following command:
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/tools/record_demos.py \
+ --task Isaac-PickPlace-GR1T2-Abs-v0 \
+ --visualizer kit \
+ --xr \
+ --device cpu \
+ --xr \
+ --num_demos 5 \
+ --dataset_file ./datasets/dataset_gr1.hdf5
+
+
+.. note::
+ We also provide a GR-1 pick and place task with waist degrees-of-freedom enabled ``Isaac-PickPlace-GR1T2-WaistEnabled-Abs-v0`` (see :ref:`environments` for details on the available environments, including the GR1 Waist Enabled variant). The same command above applies but with the task name changed to ``Isaac-PickPlace-GR1T2-WaistEnabled-Abs-v0``.
+
+.. tip::
+ If a demo fails during data collection, the environment can be reset using the teleoperation controls panel in the XR teleop client
+ on the Apple Vision Pro or via voice control by saying "reset". See :ref:`teleoperate-apple-vision-pro` for more details.
+
+ The robot uses simplified collision meshes for physics calculations that differ from the detailed visual meshes displayed in the simulation. Due to this difference, you may occasionally observe visual artifacts where parts of the robot appear to penetrate other objects or itself, even though proper collision handling is occurring in the physics simulation.
+
+You can replay the collected demonstrations by running the following command:
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/tools/replay_demos.py \
+ --task Isaac-PickPlace-GR1T2-Abs-v0 \
+ --visualizer kit \
+ --device cpu \
+ --dataset_file ./datasets/dataset_gr1.hdf5
+
+.. note::
+ Non-determinism may be observed during replay as physics in IsaacLab are not determimnistically reproducible when using ``env.reset``.
+
+
+Annotate the demonstrations
+"""""""""""""""""""""""""""
+
+Unlike the :ref:`Franka stacking task `, the GR-1 pick and place task uses manual annotation to define subtasks.
+
+The pick and place task has one subtask for the left arm (pick) and two subtasks for the right arm (idle, place).
+Annotations denote the end of a subtask. For the pick and place task, this means there are no annotations for the left arm and one annotation for the right arm (the end of the final subtask is always implicit).
+
+Each demo requires a single annotation between the first and second subtask of the right arm. This annotation ("S" button press) should be done when the right robot arm finishes the "idle" subtask and begins to
+move towards the target object. An example of a correct annotation is shown below:
+
+.. figure:: ../../_static/tasks/manipulation/gr-1_pick_place_annotation.jpg
+ :width: 100%
+ :align: center
+
+Annotate the demonstrations by running the following command:
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/isaaclab_mimic/annotate_demos.py \
+ --task Isaac-PickPlace-GR1T2-Abs-Mimic-v0 \
+ --visualizer kit \
+ --device cpu \
+ --input_file ./datasets/dataset_gr1.hdf5 \
+ --output_file ./datasets/dataset_annotated_gr1.hdf5
+
+.. note::
+
+ The script prints the keyboard commands for manual annotation and the current subtask being annotated:
+
+ .. code:: text
+
+ Annotating episode #0 (demo_0)
+ Playing the episode for subtask annotations for eef "right".
+ Subtask signals to annotate:
+ - Termination: ['idle_right']
+
+ Press "N" to begin.
+ Press "B" to pause.
+ Press "S" to annotate subtask signals.
+ Press "Q" to skip the episode.
+
+.. tip::
+
+ If the object does not get placed in the bin during annotation, you can press "N" to replay the episode and annotate again. Or you can press "Q" to skip the episode and annotate the next one.
+
+.. _generate-the-dataset:
+
+Generate the dataset
+^^^^^^^^^^^^^^^^^^^^
+
+If you skipped the prior collection and annotation step, download the pre-recorded annotated dataset ``dataset_annotated_gr1.hdf5`` from
+here: `[Annotated GR1 Dataset] `_.
+Place the file under ``IsaacLab/datasets`` and run the following command to generate a new dataset with 1000 demonstrations.
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/isaaclab_mimic/generate_dataset.py \
+ --device cpu \
+ --headless \
+ --num_envs 20 \
+ --generation_num_trials 1000 \
+ --input_file ./datasets/dataset_annotated_gr1.hdf5 \
+ --output_file ./datasets/generated_dataset_gr1.hdf5
+
+Train a policy
+^^^^^^^^^^^^^^
+
+Use `Robomimic `__ to train a policy for the generated dataset.
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/robomimic/train.py \
+ --task Isaac-PickPlace-GR1T2-Abs-v0 \
+ --algo bc \
+ --normalize_training_actions \
+ --dataset ./datasets/generated_dataset_gr1.hdf5
+
+The training script will normalize the actions in the dataset to the range [-1, 1].
+The normalization parameters are saved in the model directory under ``PATH_TO_MODEL_DIRECTORY/logs/normalization_params.txt``.
+Record the normalization parameters for later use in the visualization step.
+
+.. note::
+ By default the trained models and logs will be saved to ``IssacLab/logs/robomimic``.
+
+Visualize the results
+^^^^^^^^^^^^^^^^^^^^^
+
+Visualize the results of the trained policy by running the following command, using the normalization parameters recorded in the prior training step:
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/robomimic/play.py \
+ --task Isaac-PickPlace-GR1T2-Abs-v0 \
+ --visualizer kit \
+ --device cpu \
+ --num_rollouts 50 \
+ --horizon 400 \
+ --norm_factor_min \
+ --norm_factor_max \
+ --checkpoint /PATH/TO/desired_model_checkpoint.pth
+
+.. note::
+ Change the ``NORM_FACTOR`` in the above command with the values generated in the training step.
+
+.. tip::
+
+ **If you don't see expected performance results:** It is critical to test policies from various checkpoint epochs.
+ Performance can vary significantly between epochs, and the best-performing checkpoint is often not the final one.
+
+.. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/gr-1_steering_wheel_pick_place_policy.gif
+ :width: 100%
+ :align: center
+ :alt: GR-1 humanoid robot performing a pick and place task
+ :figclass: align-center
+
+ The trained policy performing the pick and place task in Isaac Lab.
+
+.. note::
+
+ **Expected Success Rates and Timings for Pick and Place GR1T2 Task**
+
+ * Success rate for data generation depends on the quality of human demonstrations (how well the user performs them) and dataset annotation quality. Both data generation and downstream policy success are sensitive to these factors and can show high variance. See :ref:`Common Pitfalls when Generating Data ` for tips to improve your dataset.
+ * Data generation success for this task is typically 65-80% over 1000 demonstrations, taking 18-40 minutes depending on GPU hardware and success rate (19 minutes on a RTX ADA 6000 @ 80% success rate).
+ * Behavior Cloning (BC) policy success is typically 75-86% (evaluated on 50 rollouts) when trained on 1000 generated demonstrations for 2000 epochs (default), depending on demonstration quality. Training takes approximately 29 minutes on a RTX ADA 6000.
+ * **Recommendation:** Train for 2000 epochs with 1000 generated demonstrations, and **evaluate multiple checkpoints saved between the 1000th and 2000th epochs** to select the best-performing policy. Testing various epochs is essential for finding optimal performance.
+
+
+Demo 2: Visuomotor Policy for a Humanoid Robot
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/gr-1_nut_pouring_policy.gif
+ :width: 100%
+ :align: center
+ :alt: GR-1 humanoid robot performing a pouring task
+ :figclass: align-center
+
+Download the Dataset
+^^^^^^^^^^^^^^^^^^^^
+
+Download the pre-generated dataset from `here `__ and place it under ``IsaacLab/datasets/generated_dataset_gr1_nut_pouring.hdf5``
+(**Note: The dataset size is approximately 15GB**). The dataset contains 1000 demonstrations of a humanoid robot performing a pouring/placing task that was
+generated using Isaac Lab Mimic for the ``Isaac-NutPour-GR1T2-Pink-IK-Abs-Mimic-v0`` task.
+
+.. hint::
+
+ If desired, data collection, annotation, and generation can be done using the same commands as the prior examples.
+
+ The robot first picks up the red beaker and pours the contents into the yellow bowl.
+ Then, it drops the red beaker into the blue bin. Lastly, it places the yellow bowl onto the white scale.
+ See the video in the :ref:`visualize-results-demo-2` section below for a visual demonstration of the task.
+
+ **The success criteria for this task requires the red beaker to be placed in the blue bin, the green nut to be in the yellow bowl,
+ and the yellow bowl to be placed on top of the white scale.**
+
+ .. attention::
+ **The following commands are only for your reference and are not required for this demo.**
+
+ To collect demonstrations:
+
+ .. code:: bash
+
+ ./isaaclab.sh -p scripts/tools/record_demos.py \
+ --task Isaac-NutPour-GR1T2-Pink-IK-Abs-v0 \
+ --visualizer kit \
+ --device cpu \
+ --xr \
+ --num_demos 5 \
+ --dataset_file ./datasets/dataset_gr1_nut_pouring.hdf5
+
+ To annotate the demonstrations:
+
+ .. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/isaaclab_mimic/annotate_demos.py \
+ --task Isaac-NutPour-GR1T2-Pink-IK-Abs-Mimic-v0 \
+ --visualizer kit \
+ --enable_cameras \
+ --device cpu \
+ --input_file ./datasets/dataset_gr1_nut_pouring.hdf5 \
+ --output_file ./datasets/dataset_annotated_gr1_nut_pouring.hdf5
+
+ .. warning::
+ There are multiple right eef annotations for this task. Annotations for subtasks for the same eef cannot have the same action index.
+ Make sure to annotate the right eef subtasks with different action indices.
+
+
+ To generate the dataset:
+
+ .. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/isaaclab_mimic/generate_dataset.py \
+ --task Isaac-NutPour-GR1T2-Pink-IK-Abs-Mimic-v0 \
+ --visualizer kit \
+ --enable_cameras \
+ --device cpu \
+ --headless \
+ --generation_num_trials 1000 \
+ --num_envs 5 \
+ --input_file ./datasets/dataset_annotated_gr1_nut_pouring.hdf5 \
+ --output_file ./datasets/generated_dataset_gr1_nut_pouring.hdf5
+
+
+Train a policy
+^^^^^^^^^^^^^^
+
+Use `Robomimic `__ to train a visuomotor BC agent for the task.
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/robomimic/train.py \
+ --task Isaac-NutPour-GR1T2-Pink-IK-Abs-v0 \
+ --algo bc \
+ --normalize_training_actions \
+ --dataset ./datasets/generated_dataset_gr1_nut_pouring.hdf5
+
+The training script will normalize the actions in the dataset to the range [-1, 1].
+The normalization parameters are saved in the model directory under ``PATH_TO_MODEL_DIRECTORY/logs/normalization_params.txt``.
+Record the normalization parameters for later use in the visualization step.
+
+.. note::
+ By default the trained models and logs will be saved to ``IsaacLab/logs/robomimic``.
+
+You can also post-train a `GR00T `__ foundation model to deploy a Vision-Language-Action policy for the task.
+
+Please refer to the `IsaacLabEvalTasks `__ repository for more details.
+
+.. _visualize-results-demo-2:
+
+Visualize the results
+^^^^^^^^^^^^^^^^^^^^^
+
+Visualize the results of the trained policy by running the following command, using the normalization parameters recorded in the prior training step:
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/robomimic/play.py \
+ --task Isaac-NutPour-GR1T2-Pink-IK-Abs-v0 \
+ --visualizer kit \
+ --device cpu \
+ --enable_cameras \
+ --num_rollouts 50 \
+ --horizon 350 \
+ --norm_factor_min \
+ --norm_factor_max \
+ --checkpoint /PATH/TO/desired_model_checkpoint.pth
+
+.. note::
+ Change the ``NORM_FACTOR`` in the above command with the values generated in the training step.
+
+.. tip::
+
+ **If you don't see expected performance results:** Test policies from various checkpoint epochs, not just the final one.
+ Policy performance can vary substantially across training, and intermediate checkpoints often yield better results.
+
+.. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/gr-1_nut_pouring_policy.gif
+ :width: 100%
+ :align: center
+ :alt: GR-1 humanoid robot performing a pouring task
+ :figclass: align-center
+
+ The trained visuomotor policy performing the pouring task in Isaac Lab.
+
+.. note::
+
+ **Expected Success Rates and Timings for Visuomotor Nut Pour GR1T2 Task**
+
+ * Success rate for data generation depends on the quality of human demonstrations (how well the user performs them) and dataset annotation quality. Both data generation and downstream policy success are sensitive to these factors and can show high variance. See :ref:`Common Pitfalls when Generating Data ` for tips to improve your dataset.
+ * Data generation for 1000 demonstrations takes approximately 10 hours on a RTX ADA 6000.
+ * Behavior Cloning (BC) policy success is typically 50-60% (evaluated on 50 rollouts) when trained on 1000 generated demonstrations for 600 epochs (default). Training takes approximately 15 hours on a RTX ADA 6000.
+ * **Recommendation:** Train for 600 epochs with 1000 generated demonstrations, and **evaluate multiple checkpoints saved between the 300th and 600th epochs** to select the best-performing policy. Testing various epochs is critical for achieving optimal performance.
+
+
+Demo 3: Data Generation and Policy Training for Humanoid Robot Locomanipulation with Unitree G1
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In this demo, we showcase the integration of locomotion and manipulation capabilities within a single humanoid robot system.
+This locomanipulation environment enables data collection for complex tasks that combine navigation and object manipulation.
+The demonstration follows a multi-step process: first, it generates pick and place tasks similar to Demo 1, then introduces
+a navigation component that uses specialized scripts to generate scenes where the humanoid robot must move from point A to point B.
+The robot picks up an object at the initial location (point A) and places it at the target destination (point B).
+
+.. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/locomanipulation-g-1_steering_wheel_pick_place.gif
+ :width: 100%
+ :align: center
+ :alt: G1 humanoid robot with locomanipulation performing a pick and place task
+ :figclass: align-center
+
+.. note::
+ **Locomotion policy training**
+
+ The locomotion policy used in this integration example was trained using the `AGILE `__ framework.
+ AGILE is an officially supported humanoid control training pipeline that leverages the manager based environment in Isaac Lab. It will also be
+ seamlessly integrated with other evaluation and deployment tools across Isaac products. This allows teams to rely on a single, maintained stack
+ covering all necessary infrastructure and tooling for policy training, with easy export to real-world deployment. The AGILE repository contains
+ updated pre-trained policies with separate upper and lower body policies for flexibtility. They have been verified in the real world and can be
+ directly deployed. Users can also train their own locomotion or whole-body control policies using the AGILE framework.
+
+.. _generate-the-manipulation-dataset:
+
+Generate the manipulation dataset
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The same data generation and policy training steps from Demo 1 can be applied to the G1 humanoid robot with locomanipulation capabilities.
+This demonstration shows how to train a G1 robot to perform pick and place tasks with full-body locomotion and manipulation.
+
+The process follows the same workflow as Demo 1, but uses the ``Isaac-PickPlace-Locomanipulation-G1-Abs-v0`` task environment.
+
+Follow the same data collection, annotation, and generation process as demonstrated in Demo 1, but adapted for the G1 locomanipulation task.
+
+.. hint::
+
+ If desired, data collection and annotation can be done using the same commands as the prior examples for validation of the dataset.
+
+ The G1 robot with locomanipulation capabilities combines full-body locomotion with manipulation to perform pick and place tasks.
+
+ **Note that the following commands are only for your reference and dataset validation purposes - they are not required for this demo.**
+
+ To collect demonstrations:
+
+ .. code:: bash
+
+ ./isaaclab.sh -p scripts/tools/record_demos.py \
+ --device cpu \
+ --xr \
+ --visualizer kit \
+ --task Isaac-PickPlace-Locomanipulation-G1-Abs-v0 \
+ --dataset_file ./datasets/dataset_g1_locomanip.hdf5 \
+ --num_demos 5
+
+ .. note::
+
+ Depending on how the Apple Vision Pro app was initialized, the hands of the operator might be very far up or far down compared to the hands of the G1 robot. If this is the case, you can click **Stop AR** in the AR tab in Isaac Lab, and move the AR Anchor prim. Adjust it down to bring the hands of the operator lower, and up to bring them higher. Click **Start AR** to resume teleoperation session. Make sure to match the hands of the robot before clicking **Play** in the Apple Vision Pro, otherwise there will be an undesired large force generated initially.
+
+ You can replay the collected demonstrations by running:
+
+ .. code:: bash
+
+ ./isaaclab.sh -p scripts/tools/replay_demos.py \
+ --device cpu \
+ --visualizer kit \
+ --task Isaac-PickPlace-Locomanipulation-G1-Abs-v0 \
+ --dataset_file ./datasets/dataset_g1_locomanip.hdf5
+
+ To annotate the demonstrations:
+
+ .. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/isaaclab_mimic/annotate_demos.py \
+ --device cpu \
+ --visualizer kit \
+ --task Isaac-Locomanipulation-G1-Abs-Mimic-v0 \
+ --input_file ./datasets/dataset_g1_locomanip.hdf5 \
+ --output_file ./datasets/dataset_annotated_g1_locomanip.hdf5
+
+
+If you skipped the prior collection and annotation step, download the pre-recorded annotated dataset ``dataset_annotated_g1_locomanip.hdf5`` from
+here: `[Annotated G1 Dataset] `_.
+Place the file under ``IsaacLab/datasets`` and run the following command to generate a new dataset with 1000 demonstrations.
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/isaaclab_mimic/generate_dataset.py \
+ --device cpu --headless --num_envs 20 --generation_num_trials 1000 \
+ --input_file ./datasets/dataset_annotated_g1_locomanip.hdf5 --output_file ./datasets/generated_dataset_g1_locomanip.hdf5
+
+
+Train a manipulation-only policy
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+At this point you can train a policy that only performs manipulation tasks using the generated dataset:
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/robomimic/train.py \
+ --task Isaac-PickPlace-Locomanipulation-G1-Abs-v0 --algo bc \
+ --normalize_training_actions \
+ --dataset ./datasets/generated_dataset_g1_locomanip.hdf5
+
+Visualize the results
+^^^^^^^^^^^^^^^^^^^^^
+
+Visualize the trained policy performance:
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/robomimic/play.py \
+ --device cpu \
+ --visualizer kit \
+ --task Isaac-PickPlace-Locomanipulation-G1-Abs-v0 \
+ --num_rollouts 50 \
+ --horizon 400 \
+ --norm_factor_min \
+ --norm_factor_max \
+ --checkpoint /PATH/TO/desired_model_checkpoint.pth
+
+.. note::
+ Change the ``NORM_FACTOR`` in the above command with the values generated in the training step.
+
+.. tip::
+
+ **If you don't see expected performance results:** Always test policies from various checkpoint epochs.
+ Different epochs can produce significantly different results, so evaluate multiple checkpoints to find the optimal model.
+
+.. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/locomanipulation-g-1_steering_wheel_pick_place.gif
+ :width: 100%
+ :align: center
+ :alt: G1 humanoid robot performing a pick and place task
+ :figclass: align-center
+
+ The trained policy performing the pick and place task in Isaac Lab.
+
+.. note::
+
+ **Expected Success Rates and Timings for Locomanipulation Pick and Place Task**
+
+ * Success rate for data generation depends on the quality of human demonstrations (how well the user performs them) and dataset annotation quality. Both data generation and downstream policy success are sensitive to these factors and can show high variance. See :ref:`Common Pitfalls when Generating Data ` for tips to improve your dataset.
+ * Data generation success for this task is typically 65-82% over 1000 demonstrations, taking 18-40 minutes depending on GPU hardware and success rate (18 minutes on a RTX ADA 6000 @ 82% success rate).
+ * Behavior Cloning (BC) policy success is typically 75-85% (evaluated on 50 rollouts) when trained on 1000 generated demonstrations for 2000 epochs (default), depending on demonstration quality. Training takes approximately 40 minutes on a RTX ADA 6000.
+ * **Recommendation:** Train for 2000 epochs with 1000 generated demonstrations, and **evaluate multiple checkpoints saved between the 1000th and 2000th epochs** to select the best-performing policy. Testing various epochs is essential for finding optimal performance.
+
+Generate the dataset with manipulation and point-to-point navigation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To create a comprehensive locomanipulation dataset that combines both manipulation and navigation capabilities, you can generate a navigation dataset using the manipulation dataset from the previous step as input.
+
+.. list-table::
+ :widths: 50 50
+ :header-rows: 0
+
+ * - .. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/g1_locomanip_no_obstacles.gif
+ :height: 260px
+ :align: center
+ :alt: G1 locomanipulation data generation without obstacles
+
+ Default: no obstacles (open scene).
+ - .. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/g1_locomanip_with_obstacles.gif
+ :height: 260px
+ :align: center
+ :alt: G1 locomanipulation data generation with forklift obstacles
+
+ With 8 forklift obstacles and ``--randomize_placement`` enabled.
+
+The locomanipulation dataset generation process takes the previously generated manipulation dataset and creates scenarios where the robot must navigate from one location to another while performing manipulation tasks. This creates a more complex dataset that includes both locomotion and manipulation behaviors.
+
+To generate the locomanipulation dataset, use the following command:
+
+.. code:: bash
+
+ ./isaaclab.sh -p \
+ scripts/imitation_learning/locomanipulation_sdg/generate_data.py \
+ --device cpu \
+ --kit_args="--enable isaacsim.replicator.mobility_gen" \
+ --task="Isaac-G1-SteeringWheel-Locomanipulation" \
+ --dataset ./datasets/generated_dataset_g1_locomanip.hdf5 \
+ --num_runs 1 \
+ --lift_step 60 \
+ --navigate_step 130 \
+ --output_file ./datasets/generated_dataset_g1_locomanipulation_sdg.hdf5 \
+ --enable_cameras \
+ --randomize_placement \
+ --visualizer kit
+
+.. note::
+
+ The input dataset (``--dataset``) should be the manipulation dataset generated in the previous step. You can specify any output filename using the ``--output_file_name`` parameter.
+
+The key parameters for locomanipulation dataset generation are:
+
+* ``--lift_step 60``: Number of steps for the lifting phase of the manipulation task. This should mark the point immediately after the robot has grasped the object.
+* ``--navigate_step 130``: Number of steps for the navigation phase between locations. This should make the point where the robot has lifted the object and is ready to walk.
+* ``--output_file``: Name of the output dataset file
+
+.. note::
+
+ You can change the number of obstacles (forklifts and boxes) in the scene by editing the locomanipulation SDG environment configuration. In ``source/isaaclab_mimic/isaaclab_mimic/locomanipulation_sdg/envs/g1_locomanipulation_sdg_env.py``, set the module-level constants ``NUM_FORKLIFTS`` and ``NUM_BOXES`` to the desired counts. Use ``--randomize_placement`` when running :file:`generate_data.py` to randomize obstacle and fixture positions each run.
+
+This process creates a dataset where the robot performs the manipulation task at different locations, requiring it to navigate between points while maintaining the learned manipulation behaviors. The resulting dataset can be used to train policies that combine both locomotion and manipulation capabilities.
+
+.. note::
+
+ You can visualize the robot trajectory results with the following script command:
+
+ .. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/locomanipulation_sdg/plot_navigation_trajectory.py --input_file datasets/generated_dataset_g1_locomanipulation_sdg.hdf5 --output_dir /PATH/TO/DESIRED_OUTPUT_DIR
+
+The data generated from this locomanipulation pipeline can also be used to finetune an imitation learning policy using GR00T N1.5.
+The following steps describe how to install GR00T, convert the dataset to LeRobot format, finetune the policy, and run rollouts in Isaac Lab.
+
+Finetune GR00T N1.5 policy for locomanipulation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+**Prerequisites:** Generate the locomanipulation dataset using the command in the previous section (e.g. ``generated_dataset_g1_locomanipulation_sdg.hdf5``).
+You may place one or more such HDF5 files in a single directory for the conversion step.
+
+Install GR00T with Isaac Lab (uv)
+"""""""""""""""""""""""""""""""""
+
+Clone the Isaac-GR00T repository and install GR00T N1.5 in the same uv environment used for Isaac Lab. From a parent directory that contains both repositories (or adjust paths accordingly), run:
+
+.. code:: bash
+
+ git clone -b n1.5-release https://github.com/NVIDIA/Isaac-GR00T
+
+Copy the G1 locomanipulation data config from Isaac Lab into the GR00T experiment data config:
+
+.. code:: bash
+
+ cp IsaacLab/scripts/imitation_learning/locomanipulation_sdg/gr00t/data_config.py Isaac-GR00T/gr00t/experiment/data_config.py
+
+Then, from the **Isaac-GR00T** directory, install GR00T N1.5 and its dependencies:
+
+.. code:: bash
+
+ cd Isaac-GR00T
+ uv pip install -e .
+ uv pip install wheel
+ MAX_JOBS=4 uv pip install --no-build-isolation flash-attn==2.7.1.post4
+ MAX_JOBS=4 uv pip install --no-build-isolation pytorch3d
+ uv pip install diffusers decord zmq
+
+Convert dataset to LeRobot format
+"""""""""""""""""""""""""""""""""
+
+GR00T N1.5 expects data in LeRobot format. From the **IsaacLab** repository root, run the conversion script. ```` is a directory containing one or more ``.hdf5`` files (e.g. the output directory where you saved files from ``generate_data.py``). ```` is the LeRobot-format output directory (e.g. ``./datasets/datasets_train_200_lerobot``). Episodes with very low object displacement are skipped.
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/locomanipulation_sdg/gr00t/convert_dataset.py
+
+Example:
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/locomanipulation_sdg/gr00t/convert_dataset.py ./datasets ./datasets/datasets_train_200_lerobot
+
+Finetune the policy
+"""""""""""""""""""
+
+Run finetuning from the **Isaac-GR00T** repository root. Use the LeRobot-format output path from the previous step as ``--dataset-path`` and choose an ``--output-dir`` for checkpoints. The ``--data-config g1_locomanipulation_sdg`` and ``--embodiment-tag new_embodiment`` options are required for the G1 locomanipulation task.
+
+.. code:: bash
+
+ cd Isaac-GR00T
+ python scripts/gr00t_finetune.py \
+ --dataset-path \
+ --output-dir \
+ --data-config g1_locomanipulation_sdg \
+ --embodiment-tag new_embodiment \
+ --num-gpus 1 \
+ --max-steps 10000 \
+ --save-steps 1000 \
+ --video-backend decord \
+ --report-to tensorboard
+
+See the GR00T N1.5 repository documentation for additional training options.
+
+Rollout the policy in Isaac Lab
+"""""""""""""""""""""""""""""""
+
+From the **IsaacLab** repository root, run the rollout script with the path to your finetuned checkpoint, the static manipulation dataset (used for scene/demo setup), and the task name:
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/locomanipulation_sdg/gr00t/rollout_policy.py \
+ --model_path \
+ --embodiment_tag new_embodiment \
+ --dataset ./datasets/generated_dataset_g1_locomanip.hdf5 \
+ --demo demo_0 \
+ --output_file ./datasets/rollout_output.hdf5 \
+ --task Isaac-G1-SteeringWheel-Locomanipulation \
+ --device cpu \
+ --enable_cameras \
+ --visualizer kit
+
+Optional arguments include ``--randomize_placement`` and ``--policy_quat_format wxyz`` (use if your checkpoint was trained with wxyz quaternion format).
+
+.. figure:: https://download.isaacsim.omniverse.nvidia.com/isaaclab/images/locomanipulation_sdg_disjoint_nav_groot_policy_4x.gif
+ :width: 100%
+ :align: center
+ :alt: Simulation rollout of GR00T N1.5 policy finetuned for locomanipulation
+ :figclass: align-center
+
+ Simulation rollout of GR00T N1.5 policy finetuned for locomanipulation.
+
+The policy shown above uses the camera image, hand poses, hand joint positions, object pose, and base goal pose as inputs.
+The output of the model is the target base velocity, hand poses, and hand joint positions for the next several timesteps.
+
+Use NuRec Background in Locomanipulation SDG
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+**Prerequisites:** Generate a manipulation dataset or download a pre-recorded annotated dataset from :ref:`Generate the manipulation dataset `.
+
+The `NuRec assets `__
+are neural volumes reconstructed from real-world captures. When integrated into the locomanipulation SDG workflow, these
+assets allow you to generate synthetic data in photorealistic environments that mirror real-world.
+
+Custom NuRec Asset Requirements
+"""""""""""""""""""""""""""""""
+
+To load a custom USD asset, ensure it meets the following specifications:
+
+- Neural Rendering: Include neural reconstruction for rendering.
+- Navigation: Include a pre-computed occupancy map for path planning and navigation. You can use the `Occupancy Map Generator `_ to generate the occupancy map.
+- Orientation: Transform the asset so that the ground aligns with the z=0 plane.
+- Collision Mesh (optional): If a collision mesh is included, set it to invisible.
+
+Using Pre-constructed Assets
+""""""""""""""""""""""""""""
+
+Pre-constructed assets are available via the `PhysicalAI Robotics NuRec `__
+dataset. Some of them are captured from a humanoid-viewpoint to match the camera view of the humanoid robot.
+
+For example, when using the asset ``hand_hold-voyager-babyboom``, the relevant files are:
+
+- `stage.usdz `__: a USDZ archive that bundles 3D Gaussian splatting (``volume.nurec``), a collision mesh (``mesh.usd``), etc.
+- `occupancy_map.yaml `__ and `occupancy_map.png `__: occupancy map for path planning and navigation.
+
+Download the files and place them under ````, then run the following command to generate a new dataset with background:
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/imitation_learning/locomanipulation_sdg/generate_data.py \
+ --device cpu \
+ --kit_args="--enable isaacsim.replicator.mobility_gen" \
+ --task="Isaac-G1-SteeringWheel-Locomanipulation" \
+ --dataset /dataset_annotated_g1_locomanip.hdf5 \
+ --num_runs 1 \
+ --lift_step 60 \
+ --navigate_step 130 \
+ --output_file /generated_dataset_g1_locomanipulation_sdg_with_background.hdf5 \
+ --enable_cameras \
+ --visualizer kit \
+ --background_usd_path /stage.usdz \
+ --background_occupancy_yaml_file /occupancy_map.yaml \
+ --randomize_placement \
+ --high_res_video
+
+The key parameters are:
+
+- ``--background_usd_path``: Path to the NuRec USD asset.
+- ``--background_occupancy_yaml_file``: Path to the occupancy map file.
+- ``--high_res_video``: Generate a higher resolution video (540x960) for the ego-centric camera view.
+- ``--sensor_camera_view``: Optionally set the Sim GUI viewport to the ``robot_pov_cam`` sensor view.
+
+On successful task completion, an HDF5 dataset is generated containing camera observations. You can convert
+the ego-centric camera view to MP4.
+
+.. code:: bash
+
+ ./isaaclab.sh -p scripts/tools/hdf5_to_mp4.py \
+ --input_file /generated_dataset_g1_locomanipulation_sdg_with_background.hdf5 \
+ --output_dir / \
+ --input_keys robot_pov_cam \
+ --video_width 960 \
+ --video_height 540
+
+To play the generated MP4 video on Ubuntu, install the following multimedia packages:
+
+.. code:: bash
+
+ sudo apt update
+ sudo apt install libavcodec-extra gstreamer1.0-libav gstreamer1.0-plugins-ugly
diff --git a/docs/source/overview/imitation-learning/index.rst b/docs/source/overview/imitation-learning/index.rst
index f8f77d031fb..769b259185f 100644
--- a/docs/source/overview/imitation-learning/index.rst
+++ b/docs/source/overview/imitation-learning/index.rst
@@ -8,5 +8,6 @@ with Isaac Lab.
:maxdepth: 1
teleop_imitation
+ humanoids_imitation
augmented_imitation
skillgen
diff --git a/docs/source/overview/imitation-learning/skillgen.rst b/docs/source/overview/imitation-learning/skillgen.rst
index b577f82e13a..be8382b5c5e 100644
--- a/docs/source/overview/imitation-learning/skillgen.rst
+++ b/docs/source/overview/imitation-learning/skillgen.rst
@@ -135,7 +135,7 @@ The dataset contains:
Download and Setup
^^^^^^^^^^^^^^^^^^
-1. Download the pre-annotated dataset by clicking `here `__.
+1. Download the pre-annotated dataset by clicking `here `__.
2. Prepare the datasets directory and move the downloaded file:
@@ -173,7 +173,8 @@ Download and Setup
--task Isaac-Stack-Cube-Franka-IK-Rel-Skillgen-v0 \
--teleop_device spacemouse \
--dataset_file ./datasets/dataset_skillgen.hdf5 \
- --num_demos 10
+ --num_demos 10 \
+ --visualizer kit
**Annotate demonstrations for SkillGen** (writes both term and start boundaries):
@@ -184,7 +185,8 @@ Download and Setup
--task Isaac-Stack-Cube-Franka-IK-Rel-Skillgen-v0 \
--input_file ./datasets/dataset_skillgen.hdf5 \
--output_file ./datasets/annotated_dataset_skillgen.hdf5 \
- --annotate_subtask_start_signals
+ --annotate_subtask_start_signals \
+ --visualizer kit
Understanding Dataset Annotation
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -283,7 +285,7 @@ Start with a small dataset to verify everything works:
--input_file ./datasets/annotated_dataset_skillgen.hdf5 \
--output_file ./datasets/generated_dataset_small_skillgen_cube_stack.hdf5 \
--task Isaac-Stack-Cube-Franka-IK-Rel-Skillgen-v0 \
- --use_skillgen
+ --use_skillgen --visualizer kit
Full-Scale Generation
^^^^^^^^^^^^^^^^^^^^^
@@ -337,7 +339,7 @@ Test the adaptive stacking setup:
--input_file ./datasets/annotated_dataset_skillgen.hdf5 \
--output_file ./datasets/generated_dataset_small_skillgen_bin_cube_stack.hdf5 \
--task Isaac-Stack-Cube-Bin-Franka-IK-Rel-Mimic-v0 \
- --use_skillgen
+ --use_skillgen --visualizer kit
Full-Scale Generation
^^^^^^^^^^^^^^^^^^^^^
@@ -419,7 +421,8 @@ Test your trained policies:
--device cpu \
--task Isaac-Stack-Cube-Franka-IK-Rel-Skillgen-v0 \
--num_rollouts 50 \
- --checkpoint /path/to/model_checkpoint.pth
+ --checkpoint /path/to/model_checkpoint.pth \
+ --visualizer kit
.. code:: bash
@@ -428,7 +431,8 @@ Test your trained policies:
--device cpu \
--task Isaac-Stack-Cube-Bin-Franka-IK-Rel-Mimic-v0 \
--num_rollouts 50 \
- --checkpoint /path/to/model_checkpoint.pth
+ --checkpoint /path/to/model_checkpoint.pth \
+ --visualizer kit
.. note::
diff --git a/docs/source/overview/imitation-learning/teleop_imitation.rst b/docs/source/overview/imitation-learning/teleop_imitation.rst
index 14017e65b5d..2511b6a57b7 100644
--- a/docs/source/overview/imitation-learning/teleop_imitation.rst
+++ b/docs/source/overview/imitation-learning/teleop_imitation.rst
@@ -47,21 +47,20 @@ For smoother operation and off-axis operation, we recommend using a SpaceMouse a
where ``<#>`` is the device index of the connected SpaceMouse.
- If you are using the IsaacLab + CloudXR container deployment (:ref:`cloudxr-teleoperation`), you can add the ``devices`` attribute under the ``services -> isaac-lab-base`` section of the
- ``docker/docker-compose.cloudxr-runtime.patch.yaml`` file.
-
Isaac Lab is only compatible with the SpaceMouse Wireless and SpaceMouse Compact models from 3Dconnexion.
-For tasks that benefit from the use of an extended reality (XR) device with hand tracking, Isaac Lab supports using NVIDIA CloudXR to immersively stream the scene to compatible XR devices for teleoperation. Note that when using hand tracking we recommend using the absolute variant of the task (``Isaac-Stack-Cube-Franka-IK-Abs-v0``), which requires the ``handtracking`` device:
+For tasks that benefit from the use of an extended reality (XR) device with hand tracking, Isaac Lab supports using `Isaac Teleop `_ with NVIDIA CloudXR to immersively stream the scene to compatible XR devices for teleoperation. Note that when using hand tracking we recommend using the absolute variant of the task (``Isaac-Stack-Cube-Franka-IK-Abs-v0``):
.. code:: bash
- ./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py --task Isaac-Stack-Cube-Franka-IK-Abs-v0 --teleop_device handtracking --device cpu
+ ./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py --task Isaac-Stack-Cube-Franka-IK-Abs-v0 --visualizer kit --xr
.. note::
- See :ref:`cloudxr-teleoperation` to learn how to use CloudXR and experience teleoperation with Isaac Lab.
+ See :ref:`cloudxr-teleoperation` to learn how to install Isaac Teleop and set up CloudXR for
+ teleoperation. For architecture details, retargeting pipelines, and control scheme
+ recommendations, see :ref:`isaac-teleop-feature`.
The script prints the teleoperation events configured. For keyboard,
@@ -96,8 +95,50 @@ The next section describes how teleoperation devices can be used for data collec
Imitation Learning with Isaac Lab Mimic
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Using the teleoperation devices, it is also possible to collect data for
-learning from demonstrations (LfD). For this, we provide scripts to collect data into the open HDF5 format.
+.. figure:: ../../_static/mimic/franka_mimic_imitation_learning.jpg
+ :width: 100%
+ :align: center
+ :alt: Franka robot performing a stacking task
+ :figclass: align-center
+
+What is Isaac Lab Mimic?
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Isaac Lab Mimic (Mimic) is a trajectory data generation tool that can be used to
+augment human demonstrations by generating new synthetic data. Given a set of human demonstrations,
+Mimic can automatically generate new demonstrations involving the same task but with different spatial configurations.
+The generated data can be used to train imitation learning policies that are more robust to spatial variations
+even if just a handful of manual demonstrations are available.
+
+Mimic works by taking a set of human demonstrations and splitting each demonstration into a sequence of subtasks.
+Subtasks are defined based on reference objects that dictate the motion of the robot's end-effectors (eefs). Each subtask
+is a contiguous segment of the demonstration where the eef's motion is dictated by a single reference object. A new subtask begins
+when the reference object changes. Annotations mark points in the demonstration where a subtask is completed.
+
+During data generation, Mimic takes the human demonstration subtask segments and applies rigid body transformations to the robot's actions
+to transform them into new demonstrations involving the same task but with different spatial configurations.
+The new demonstrations are evaluated to determine if they are successful, and if so, are added to the output dataset.
+
+Mimic is compatible with a variety of embodiments including single-eef (e.g. manipulator robots) and multi-eef (e.g. humanoid robots).
+The use of rigid body transformations requires that the embodiment's action space is defined in **task space**. If
+the embodiment's action is in joint space, then the action must be converted to task space using forward kinematics.
+
+In the following section, we will show how to collect a small batch of human demonstrations for a stacking task
+with the Franka robot and then increase the size of the dataset by generating new synthetic data using Isaac Lab Mimic.
+
+
+Pre-recorded demonstrations
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+We provide a pre-recorded ``dataset.hdf5`` containing 10 human demonstrations for ``Isaac-Stack-Cube-Franka-IK-Rel-v0``
+here: `[Franka Dataset] `__.
+This dataset may be downloaded and used in the remaining tutorial steps if you do not wish to collect your own demonstrations.
+
+If using the pre-recorded dataset, skip the next section and proceed directly to the :ref:`Generating additional demonstrations