Add dVRK needle-pass environment to contrib

.github/actions/run-package-tests/action.yml

@@ -83,6 +83,10 @@ inputs:
     description: 'Space-separated pip packages to install inside the Docker container before pytest starts'
     default: ''
     required: false
+  install-isaacteleop-pr769:
+    description: 'Build and install the immutable NVIDIA/IsaacTeleop PR 769 test pin before pytest'
+    default: 'false'
+    required: false
   wheelhouse-resource:
     description: 'Optional NGC resource containing wheelhouse/ and manifest.json for offline pip installs'
     default: ''
@@ -302,6 +306,7 @@ runs:
         test-node-ids-key: ${{ inputs.test-node-ids-key }}
         volume-mount-source: ${{ github.workspace }}
         extra-pip-packages: ${{ inputs.extra-pip-packages }}
+        install-isaacteleop-pr769: ${{ inputs.install-isaacteleop-pr769 }}
         wheelhouse-host-dir: ${{ steps.extract-wheelhouse.outputs.wheelhouse_host_dir }}
         wheelhouse-packages: ${{ inputs.wheelhouse-packages }}
         omni-github-test-type: ${{ inputs.omni-github-test-type }}

.github/actions/run-tests/action.yml

@@ -85,6 +85,10 @@ inputs:
     description: 'Space-separated pip packages to install inside the Docker container before pytest starts'
     default: ''
     required: false
+  install-isaacteleop-pr769:
+    description: 'Build and install the immutable NVIDIA/IsaacTeleop PR 769 test pin before pytest'
+    default: 'false'
+    required: false
   wheelhouse-host-dir:
     description: 'Host directory containing wheelhouse/ and manifest.json for offline pip installs'
     default: ''
@@ -137,10 +141,13 @@ runs:
           local wheelhouse_packages="${19}"
           local test_k_expr="${20}"
           local ci_marker="${21}"
+          local install_isaacteleop_pr769="${22}"
           local logs_pid=""
           local wait_pid=""
           local docker_wait_file="/tmp/.docker_exit_${container_name}"
           local docker_runtime_dir=""
+          local test_run_uid="1000"
+          local test_run_gid="1000"
 
           # Kill the container immediately if the runner is cancelled.
           # The GitHub Actions runner can deliver HUP, INT, or TERM on cancellation
@@ -169,6 +176,12 @@ runs:
           if [ -n "$wheelhouse_packages" ]; then
             echo "With wheelhouse packages: $wheelhouse_packages"
           fi
+          if [ "$install_isaacteleop_pr769" = "true" ]; then
+            echo "With NVIDIA/IsaacTeleop PR 769 test pin"
+          elif [ "$install_isaacteleop_pr769" != "false" ]; then
+            echo "install-isaacteleop-pr769 must be 'true' or 'false'"
+            return 1
+          fi
           if [ -n "$filter_pattern" ]; then
             echo "With filter pattern: $filter_pattern"
           fi
@@ -299,6 +312,8 @@ runs:
             host_uid="$(id -u)"
             host_gid="$(id -g)"
             host_user="$(id -un)"
+            test_run_uid="$host_uid"
+            test_run_gid="$host_gid"
             # Kit writes generated cache, config, data, and log files outside
             # the Isaac Lab source tree. Provide writable runtime storage for
             # host-uid test runs, mirroring the compose/singularity mounts.
@@ -339,6 +354,17 @@ runs:
             echo "🔵 Running volume-mounted container as host uid:gid ${host_uid}:${host_gid} (${host_user})"
           fi
 
+          if [ "$install_isaacteleop_pr769" = "true" ]; then
+            # The pinned source build may need to install libx11-dev. Start as
+            # root for that bootstrap, then drop to the normal test uid:gid
+            # before creating any reports or cache files in the bind mount.
+            docker_user_args="--user 0:0"
+            docker_env_vars="$docker_env_vars \
+              -e TEST_INSTALL_ISAACTELEOP_PR769=true \
+              -e TEST_RUN_UID=${test_run_uid} \
+              -e TEST_RUN_GID=${test_run_gid}"
+          fi
+
           if [ -n "$wheelhouse_host_dir" ]; then
             if [ -z "$wheelhouse_packages" ]; then
               echo "::error::wheelhouse-host-dir was provided but wheelhouse-packages is empty"
@@ -389,46 +415,63 @@ runs:
             -c "
               set -e
               cd /workspace/isaaclab
-              mkdir -p tests
-              rm _isaac_sim || true
-              ln -s /isaac-sim _isaac_sim
-              # Allow OmniHub to start in the test container. Some base images
-              # set this detect-only flag, which makes cold asset downloads
-              # fall back to slow repeated retries.
-              unset HUB__ARGS__DETECT_ONLY
-              ./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky \"coverage>=7.6.1\"
-              if [ -n \"\${TEST_WHEELHOUSE_PACKAGES:-}\" ]; then
-                if [ ! -d \"\${TEST_WHEELHOUSE_PATH:-}\" ]; then
-                  echo \"Wheelhouse path is missing: \${TEST_WHEELHOUSE_PATH:-}\"
-                  exit 1
+              if [ \"\${TEST_INSTALL_ISAACTELEOP_PR769:-}\" = \"true\" ]; then
+                # Keep root's source-build caches out of the runtime home that
+                # is owned by the non-root test user selected below.
+                HOME=/root \
+                  XDG_CACHE_HOME=/root/.cache \
+                  XDG_DATA_HOME=/root/.local/share \
+                  bash scripts/tools/install_isaacteleop_pr769_for_tests.sh
+              fi
+
+              run_test_body() {
+                mkdir -p tests
+                rm _isaac_sim || true
+                ln -s /isaac-sim _isaac_sim
+                # Allow OmniHub to start in the test container. Some base images
+                # set this detect-only flag, which makes cold asset downloads
+                # fall back to slow repeated retries.
+                unset HUB__ARGS__DETECT_ONLY
+                ./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky \"coverage>=7.6.1\"
+                if [ -n \"\${TEST_WHEELHOUSE_PACKAGES:-}\" ]; then
+                  if [ ! -d \"\${TEST_WHEELHOUSE_PATH:-}\" ]; then
+                    echo \"Wheelhouse path is missing: \${TEST_WHEELHOUSE_PATH:-}\"
+                    exit 1
+                  fi
+                  if [ ! -f \"\${TEST_WHEELHOUSE_MANIFEST:-}\" ]; then
+                    echo \"Wheelhouse manifest is missing: \${TEST_WHEELHOUSE_MANIFEST:-}\"
+                    exit 1
+                  fi
+
+                  echo \"Installing wheelhouse packages offline: \${TEST_WHEELHOUSE_PACKAGES}\"
+                  ./isaaclab.sh -p -m pip uninstall -y \${TEST_WHEELHOUSE_PACKAGES} || true
+                  PIP_NO_INDEX=1 ./isaaclab.sh -p -m pip install --no-index --find-links=\"\${TEST_WHEELHOUSE_PATH}\" --upgrade --force-reinstall \${TEST_WHEELHOUSE_PACKAGES}
+
+                  case \" \${TEST_WHEELHOUSE_PACKAGES} \" in
+                    *\" ovphysx \"*)
+                      ./isaaclab.sh -p -c \"import importlib.metadata,json,os,pathlib; from packaging.version import Version; manifest=json.loads(pathlib.Path(os.environ['TEST_WHEELHOUSE_MANIFEST']).read_text(encoding='utf-8')); expected=manifest.get('ovphysx_version'); actual=importlib.metadata.version('ovphysx'); print(f'Resolved ovphysx package version: {actual}'); print(f'Wheelhouse manifest ovphysx version: {expected}'); import ovphysx; runtime=getattr(ovphysx, '__version__', actual); print(f'Imported ovphysx runtime version: {runtime}'); raise SystemExit(0 if Version(actual) == Version(expected) and Version(runtime) == Version(expected) else f'ovphysx version mismatch: installed {actual}, import {runtime}, manifest {expected}')\"
+                      ;;
+                  esac
                 fi
-                if [ ! -f \"\${TEST_WHEELHOUSE_MANIFEST:-}\" ]; then
-                  echo \"Wheelhouse manifest is missing: \${TEST_WHEELHOUSE_MANIFEST:-}\"
-                  exit 1
+                if [ -n \"\${TEST_EXTRA_PIP_PACKAGES:-}\" ]; then
+                  echo \"Installing extra pip packages: \${TEST_EXTRA_PIP_PACKAGES}\"
+                  ./isaaclab.sh -p -m pip install \${TEST_EXTRA_PIP_PACKAGES}
+                  case \" \${TEST_EXTRA_PIP_PACKAGES} \" in
+                    *\" leapp\"*)
+                      echo \"Resolved LEAPP package:\"
+                      ./isaaclab.sh -p -m pip show leapp || true
+                      ;;
+                  esac
                 fi
+                echo 'Starting pytest with path: $test_path'
+                ./isaaclab.sh -p -m pytest --ignore=tools/conftest.py $test_path $pytest_options -v --junitxml=tests/$result_file
+              }
 
-                echo \"Installing wheelhouse packages offline: \${TEST_WHEELHOUSE_PACKAGES}\"
-                ./isaaclab.sh -p -m pip uninstall -y \${TEST_WHEELHOUSE_PACKAGES} || true
-                PIP_NO_INDEX=1 ./isaaclab.sh -p -m pip install --no-index --find-links=\"\${TEST_WHEELHOUSE_PATH}\" --upgrade --force-reinstall \${TEST_WHEELHOUSE_PACKAGES}
-
-                case \" \${TEST_WHEELHOUSE_PACKAGES} \" in
-                  *\" ovphysx \"*)
-                    ./isaaclab.sh -p -c \"import importlib.metadata,json,os,pathlib; from packaging.version import Version; manifest=json.loads(pathlib.Path(os.environ['TEST_WHEELHOUSE_MANIFEST']).read_text(encoding='utf-8')); expected=manifest.get('ovphysx_version'); actual=importlib.metadata.version('ovphysx'); print(f'Resolved ovphysx package version: {actual}'); print(f'Wheelhouse manifest ovphysx version: {expected}'); import ovphysx; runtime=getattr(ovphysx, '__version__', actual); print(f'Imported ovphysx runtime version: {runtime}'); raise SystemExit(0 if Version(actual) == Version(expected) and Version(runtime) == Version(expected) else f'ovphysx version mismatch: installed {actual}, import {runtime}, manifest {expected}')\"
-                    ;;
-                esac
-              fi
-              if [ -n \"\${TEST_EXTRA_PIP_PACKAGES:-}\" ]; then
-                echo \"Installing extra pip packages: \${TEST_EXTRA_PIP_PACKAGES}\"
-                ./isaaclab.sh -p -m pip install \${TEST_EXTRA_PIP_PACKAGES}
-                case \" \${TEST_EXTRA_PIP_PACKAGES} \" in
-                  *\" leapp\"*)
-                    echo \"Resolved LEAPP package:\"
-                    ./isaaclab.sh -p -m pip show leapp || true
-                    ;;
-                esac
+              if [ \"\${TEST_INSTALL_ISAACTELEOP_PR769:-}\" = \"true\" ]; then
+                export -f run_test_body
+                exec setpriv --reuid=\"\${TEST_RUN_UID}\" --regid=\"\${TEST_RUN_GID}\" --clear-groups bash -e -c run_test_body
               fi
-              echo 'Starting pytest with path: $test_path'
-              ./isaaclab.sh -p -m pytest --ignore=tools/conftest.py $test_path $pytest_options -v --junitxml=tests/$result_file
+              run_test_body
             "
 
           echo "::group::Following Docker container logs"
@@ -531,7 +574,7 @@ runs:
         }
 
         # Call the function with provided parameters
-        run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" "$CI_MARKER_INPUT"
+        run_tests "${{ inputs.test-path }}" "${{ inputs.result-file }}" "${{ inputs.container-name }}" "${{ inputs.image-tag }}" "${{ inputs.reports-dir }}" "$PYTEST_OPTIONS" "${{ inputs.filter-pattern }}" "${{ inputs.exclude-pattern }}" "${{ inputs.curobo-only }}" "${{ inputs.include-files }}" "${{ inputs.quarantined-only }}" "${{ inputs.shard-index }}" "${{ inputs.shard-count }}" "${{ inputs.volume-mount-source }}" "${{ inputs.extra-pip-packages }}" "${{ inputs.test-node-ids-file }}" "${{ inputs.test-node-ids-key }}" "${{ inputs.wheelhouse-host-dir }}" "${{ inputs.wheelhouse-packages }}" "$TEST_K_EXPR_INPUT" "$CI_MARKER_INPUT" "${{ inputs.install-isaacteleop-pr769 }}"
 
     - name: Kill container on cancellation
       if: cancelled()

.github/workflows/build.yaml

@@ -412,6 +412,29 @@ jobs:
         shard-count: "3"
         container-name: isaac-lab-tasks-3-test
 
+  test-dvrk-needle-pass-teleop:
+    name: dVRK needle-pass teleop
+    runs-on: [self-hosted, gpu]
+    timeout-minutes: 180
+    needs: [build, config]
+    if: >-
+      github.event_name != 'push' &&
+      needs.build.result == 'success'
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 1
+        lfs: true
+    - uses: ./.github/actions/run-package-tests
+      with:
+        image-tag: ${{ needs.config.outputs.ci_image_tag }}
+        isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }}
+        isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }}
+        filter-pattern: "isaaclab_tasks"
+        include-files: "test_dvrk_needle_pass_teleop_pipeline.py"
+        install-isaacteleop-pr769: "true"
+        container-name: isaac-lab-dvrk-needle-pass-teleop-test
+
   test-isaaclab-core:
     name: isaaclab (core) [1/3]
     runs-on: [self-hosted, gpu]

CONTRIBUTORS.md

@@ -73,6 +73,7 @@ Guidelines for modifications:
 * Cathy Y. Li
 * Cheng-Rong Lai
 * Chenyu Yang
+* Chris von Csefalvay
 * Connor Smith
 * CY (Chien-Ying) Chen
 * David Leon

docs/source/features/isaac_teleop.rst

@@ -75,6 +75,11 @@ starting point, then see the detailed pipeline examples below.
      - ``Se3AbsRetargeter`` + ``GripperRetargeter``
      - 8
      - ``stack_ik_abs_env_cfg.py``
+   * - Bimanual surgical manipulation (dVRK)
+     - Motion controllers
+     - Bimanual ``DVRKPSMClutchRetargeter`` + ``DVRKPSMGripperRetargeter``
+     - 18
+     - ``ik_abs_env_cfg.py``
    * - Bimanual dex + locomotion (e.g. G1 TriHand)
      - Motion controllers
      - Bimanual ``Se3AbsRetargeter`` + ``TriHandMotionControllerRetargeter`` + ``LocomotionRootCmdRetargeter``
@@ -261,6 +266,19 @@ retargeters not listed here -- refer to the
    Outputs a single float (-1.0 closed, 1.0 open). Uses controller trigger (priority) or
    thumb-index pinch distance from hand tracking.
 
+.. dropdown:: DVRKPSMClutchRetargeter / DVRKPSMGripperRetargeter
+
+   Provides the paired motion-controller mapping for a da Vinci Research Kit (dVRK) Patient
+   Side Manipulator (PSM). ``DVRKPSMClutchRetargeter`` emits a workspace-bounded 7D absolute
+   tool-tip pose. The controller squeeze is a deadman clutch: the first valid squeezed frame
+   captures the controller origin, and releasing squeeze holds the last target before the next
+   engagement captures a fresh origin.
+
+   ``DVRKPSMGripperRetargeter`` maps relative analogue-trigger motion to the two ordered PSM jaw
+   targets. Closing intent is applied immediately, while opening intent must cross a deadband
+   and persist for a configured duration. Tracking loss, an inactive session, or a released
+   squeeze holds the last pose and jaw targets for that PSM.
+
 .. dropdown:: DexHandRetargeter / DexBiManualRetargeter
 
    Retargets full hand tracking (26 joints) to robot-specific hand joint angles using the
@@ -305,6 +323,8 @@ The built-in Isaac Lab environments use these retargeters as follows:
      - ``Se3AbsRetargeter``, ``TriHandMotionControllerRetargeter``, ``TensorReorderer``
    * - G1 loco-manipulation
      - ``Se3AbsRetargeter``, ``TriHandMotionControllerRetargeter``, ``LocomotionRootCmdRetargeter``, ``TensorReorderer``
+   * - dVRK PSM needle pass
+     - ``DVRKPSMClutchRetargeter``, ``DVRKPSMGripperRetargeter``, ``TensorReorderer``
 
 
 .. _isaac-teleop-env-control-reference:
@@ -334,6 +354,11 @@ These environments use the Isaac Teleop XR pipeline with motion controllers or h
      - Right
      - **Arm:** right controller grip pose drives end-effector.
        **Gripper:** right trigger.
+   * - ``IsaacContrib-NeedlePass-dVRK-IK-Abs``
+     - Controllers
+     - Both
+     - **Arms:** left/right controller grip pose drives the corresponding PSM while squeeze is held.
+       **Jaws:** relative left/right trigger motion commands the corresponding paired jaws.
    * - ``IsaacContrib-PickPlace-GR1T2-Abs``
      - Hand tracking
      - Both

docs/source/how-to/cloudxr_teleoperation.rst

@@ -217,10 +217,9 @@ choose the tab that matches your hardware.
 
          .. note::
 
-            The web client URL is versioned. The ``release-1.3.x`` path corresponds to the
-            Isaac Teleop version Isaac Lab is pinned to (``isaacteleop~=1.3.0`` in the
-            ``teleop`` extra of the root ``pyproject.toml``). When Isaac Lab bumps its Isaac
-            Teleop pin, update this link to the matching client release.
+            The web client URL is versioned. This URL targets ``release-1.3.x``. The ``teleop``
+            extra accepts ``isaacteleop>=1.3.0,<2.0.0``; when using a newer 1.x release, select
+            the matching client release instead.
 
          .. tip::
 
@@ -385,6 +384,72 @@ choose the tab that matches your hardware.
       #. Click **Disconnect** when finished.
 
 
+.. _teleoperate-dvrk-needle-pass:
+
+Teleoperate the dVRK needle-pass task
+-------------------------------------
+
+The ``IsaacContrib-NeedlePass-dVRK-IK-Abs`` environment uses paired motion controllers to
+operate two da Vinci Research Kit (dVRK) Patient Side Manipulators. It references revisioned
+public assets from the Isaac for Healthcare ``0.6.0`` catalogue. Verify the downloaded bytes
+against the pinned SHA-256 digests before the first run:
+
+.. code-block:: bash
+
+   ./isaaclab.sh -p scripts/tools/preflight_dvrk_needle_pass_assets.py
+
+.. note::
+
+   The dVRK retargeter API in
+   `NVIDIA/IsaacTeleop PR #769 <https://github.com/NVIDIA/IsaacTeleop/pull/769>`__ is not yet
+   released. Until it is included in a release satisfying Isaac Lab's normal version constraint,
+   the dVRK retargeters are a temporary source-pinned prerequisite. From the Isaac Lab repository
+   root, install that immutable revision for local validation with:
+
+   .. code-block:: bash
+
+      bash scripts/tools/install_isaacteleop_pr769_for_tests.sh
+
+   The helper installs ``git`` and ``libx11-dev`` through ``sudo`` when they are missing, pins
+   its ``uv`` build tool in the selected Python environment, builds the exact source revision
+   against that Python version, and force-reinstalls the resulting ``isaacteleop`` wheel without
+   changing its runtime dependencies. Set ``ISAACLAB_PYTHON`` to an executable interpreter or
+   launcher to override the automatic environment selection.
+
+   Use the normal Isaac Lab installation again once a released ``isaacteleop`` package contains
+   the dVRK retargeters.
+
+Launch the task through its unified Isaac Teleop pipeline. Do not pass ``--teleop_device``;
+that option selects the legacy native-device path.
+
+.. code-block:: bash
+
+   ./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py \
+       --task IsaacContrib-NeedlePass-dVRK-IK-Abs \
+       --device cuda:0 \
+       --visualizer kit \
+       --xr
+
+The controller mapping is:
+
+* the left controller commands the left PSM, and the right controller commands the right PSM;
+* squeeze acts as the deadman clutch for the corresponding PSM;
+* the controller grip pose commands the tool-tip pose while squeeze is held; and
+* relative index-trigger motion commands the two corresponding jaw targets.
+
+The first valid squeezed frame captures a controller origin without moving the tool. Releasing
+squeeze or losing valid grip tracking holds that PSM's last pose and jaw targets, while the other
+side continues independently. The next valid squeeze re-clutches at the held target. ``RESET``
+returns the donor to its needle-holding state and opens the receiver jaws.
+
+The resulting action has 18 values in this order:
+
+.. code-block:: text
+
+   [left position xyz, left quaternion xyzw, left jaw_1, left jaw_2,
+    right position xyz, right quaternion xyzw, right jaw_1, right jaw_2]
+
+
 .. _manus-vive-handtracking:
 
 Manus Gloves