From eb12217f7abe5dc71a8e7e74a26ddc9b0610a98c Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Thu, 30 Jul 2026 19:57:19 -0400 Subject: [PATCH 1/3] Add XR camera picture-in-picture feedback Signed-off-by: Hougant Chen --- .../source/api/lab_teleop/isaaclab_teleop.rst | 11 + docs/source/features/isaac_teleop.rst | 157 ++++- .../teleoperation/teleop_se3_agent.py | 23 +- scripts/tools/record_demos.py | 27 +- .../changelog.d/xr-camera-feedback.minor.rst | 5 + .../isaaclab/renderers/base_renderer.py | 15 + .../isaaclab/sensors/camera/camera.py | 12 +- .../test_simulation_render_context.py | 15 + .../changelog.d/xr-camera-dlss-ghosting.rst | 7 + .../renderers/isaac_rtx_renderer.py | 60 +- .../renderers/isaac_rtx_renderer_cfg.py | 52 ++ .../test_isaac_rtx_render_product_settings.py | 113 ++++ .../test_isaac_rtx_renderer_contract.py | 169 ++++++ .../changelog.d/xr-camera-feedback.minor.rst | 12 + .../locomanipulation_g1_env_cfg.py | 57 +- .../exhaustpipe_gr1t2_base_env_cfg.py | 2 +- .../exhaustpipe_gr1t2_pink_ik_env_cfg.py | 10 +- .../nutpour_gr1t2_pink_ik_env_cfg.py | 10 +- .../pick_place/pickplace_gr1t2_env_cfg.py | 69 ++- .../changelog.d/xr-camera-feedback.minor.rst | 19 + source/isaaclab_teleop/docs/README.md | 32 +- .../isaaclab_teleop/__init__.pyi | 11 +- .../isaaclab_teleop/camera_feed.py | 463 ++++++++++++++ .../camera_feed_kit_scene_ui.py | 426 +++++++++++++ .../isaaclab_teleop/isaac_teleop_cfg.py | 89 ++- .../teleop_message_processor.py | 16 +- .../test/test_control_events.py | 9 + .../test/test_xr_camera_feed.py | 441 ++++++++++++++ .../test/test_xr_camera_feed_kit_scene_ui.py | 570 ++++++++++++++++++ 29 files changed, 2870 insertions(+), 32 deletions(-) create mode 100644 source/isaaclab/changelog.d/xr-camera-feedback.minor.rst create mode 100644 source/isaaclab_physx/changelog.d/xr-camera-dlss-ghosting.rst create mode 100644 source/isaaclab_physx/test/renderers/test_isaac_rtx_render_product_settings.py create mode 100644 source/isaaclab_tasks/changelog.d/xr-camera-feedback.minor.rst create mode 100644 source/isaaclab_teleop/changelog.d/xr-camera-feedback.minor.rst create mode 100644 source/isaaclab_teleop/isaaclab_teleop/camera_feed.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/camera_feed_kit_scene_ui.py create mode 100644 source/isaaclab_teleop/test/test_xr_camera_feed.py create mode 100644 source/isaaclab_teleop/test/test_xr_camera_feed_kit_scene_ui.py diff --git a/docs/source/api/lab_teleop/isaaclab_teleop.rst b/docs/source/api/lab_teleop/isaaclab_teleop.rst index 8ca4ab9de1a5..e516aaf86bbd 100644 --- a/docs/source/api/lab_teleop/isaaclab_teleop.rst +++ b/docs/source/api/lab_teleop/isaaclab_teleop.rst @@ -11,6 +11,8 @@ isaaclab_teleop IsaacTeleopCfg IsaacTeleopDevice + XrCameraFeedCfg + XrCameraFeedLayoutCfg HapticFeedbackCfg ControllerHapticFeedbackCfg GloveHapticFeedbackCfg @@ -40,6 +42,15 @@ Configuration .. autoclass:: XrAnchorRotationMode :members: +XR Camera Feedback +------------------ + +.. autoclass:: XrCameraFeedCfg + :members: + +.. autoclass:: XrCameraFeedLayoutCfg + :members: + Device ------ diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 0ca42aaae6ef..44aaac091de6 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -949,6 +949,10 @@ Key ``IsaacTeleopCfg`` fields: * ``pipeline_builder`` -- callable that returns an ``OutputCombiner`` with an ``"action"`` output. * ``retargeters_to_tune`` -- optional callable returning retargeters to expose in the live tuning UI. * ``xr_cfg`` -- :class:`~isaaclab_teleop.XrCfg` for anchor configuration (see below). +* ``xr_camera_feeds`` -- ordered selection and per-feed panel settings for existing task cameras. + The list is empty by default, so tasks opt in to PiP explicitly. +* ``xr_camera_feed_layout`` -- viewer reference, fixed world pose, and manual, horizontal, vertical, + or grid panel packing. * ``plugins`` -- list of Isaac Teleop plugin configurations (e.g. Manus). * ``sim_device`` -- torch device string (default ``"cuda:0"``). * ``retargeting_execution`` -- IsaacTeleop retargeting execution settings. @@ -1152,8 +1156,149 @@ XR device's view. .. tip:: - When using XR, call :func:`~isaaclab_teleop.remove_camera_configs` on your env config to strip - camera sensors. Additional cameras cause GPU contention and degrade XR performance. + Camera sensors add GPU cost. Strip them with ``--disable_external_cameras`` when the workflow + needs neither image observations nor XR camera feedback. That flag also skips configured PiP + feeds. + + +XR Camera Feedback +------------------ + +An ordered list of :class:`~isaaclab_teleop.XrCameraFeedCfg` objects selects existing task scene +cameras. The manager publishes each new RGBA frame after rendering, while +:class:`~isaaclab_teleop.XrCameraFeedLayoutCfg` places the panels manually or in horizontal, +vertical, and grid layouts. ``IsaacContrib-PickPlace-GR1T2-Abs`` and +``IsaacContrib-PickPlace-Locomanipulation-G1-Abs`` are the primary reference examples. + +``teleop_se3_agent.py`` and ``record_demos.py`` show every enabled feed when an IsaacTeleop-enabled +environment runs with ``--xr``. PiP is absent unless the task explicitly selects an existing +``CameraCfg`` through ``xr_camera_feeds``. In the reference examples, the selected +``robot_pov_cam`` is also a policy image observation, so the normal demonstration recorder stores +the same view shown to the operator. GR1T2 demonstrates a fixed task camera; Locomanipulation uses +the same fixed task-camera placement pattern for G1. The NutPour and ExhaustPipe GR1T2 +teleoperation tasks also present their existing recorded ``robot_pov_cam``: + +.. code-block:: bash + + uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ + --task IsaacContrib-PickPlace-GR1T2-Abs \ + --xr --device cpu + + uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ + --task IsaacContrib-PickPlace-Locomanipulation-G1-Abs \ + --xr --device cpu + +XR camera PiP currently supports exactly one environment. When a task has enabled PiP feeds, +startup rejects ``--num_envs`` values other than ``1``; IsaacTeleop XR behavior without PiP is +unchanged. + +On Isaac Sim 6.1 and newer, camera-feed setup enables responsive DLSS Ray Reconstruction denoising +automatically. Earlier runtimes fall back to classic DLSS for PiP cameras to avoid the temporal +ghosting in their Ray Reconstruction implementation. + +Camera selection +~~~~~~~~~~~~~~~~ + +Tasks declare their default selection through ``IsaacTeleopCfg.xr_camera_feeds``: + +.. code-block:: python + + from isaaclab_teleop import IsaacTeleopCfg, XrCameraFeedCfg + + self.isaac_teleop = IsaacTeleopCfg( + pipeline_builder=_build_my_pipeline, + xr_camera_feeds=[ + XrCameraFeedCfg(camera_name="left_wrist_camera"), + XrCameraFeedCfg(camera_name="overview_camera", enabled=False), + ], + ) + +For a recorded training view, define the named ``CameraCfg`` in the task scene and a matching +``mdp.image`` term in ``observations.policy``. The normal recorder then stores that observation, +while the PiP declaration above only selects it for presentation. Enabled task-declared entries +control camera selection, panel count, display order, and automatic-layout order without mutating +the reusable task configuration. + +Placement +~~~~~~~~~ + +The default :attr:`~isaaclab_teleop.XrCameraFeedLayoutCfg.placement` is ``"viewer_start"``. The +presenter waits for the first valid headset pose, captures its eye position and upright yaw, then +places the layout 0.8 m ahead at eye height. That pose remains fixed in the world while the user +moves. Panels hide when the XR display disconnects and capture a new starting pose after reconnect; +resetting the environment does not recenter them. + +Three placement references are available: + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Placement + - Behavior + * - ``viewer_start`` + - Capture the first valid eye position and upright yaw, then keep the layout world-static. + This is the default. + * - ``head_locked`` + - Follow the current headset position and orientation with the configured offset and distance. + * - ``world`` + - Use ``world_position_m`` and ``world_orientation_xyzw`` as a fixed pose in the Isaac Lab USD + stage world. ``distance_m`` is unused. + +.. code-block:: python + + from isaaclab_teleop import XrCameraFeedLayoutCfg + + # Default: fixed in the world at the user's starting eye pose. + self.isaac_teleop.xr_camera_feed_layout = XrCameraFeedLayoutCfg() + + # Follow the headset and pack feeds left to right. + self.isaac_teleop.xr_camera_feed_layout = XrCameraFeedLayoutCfg( + placement="head_locked", + mode="horizontal", + distance_m=0.8, + ) + + # Place a grid in an Isaac Lab Z-up world. This example assumes an eye at + # (0, 0, 1.6) looking along world +Y. + self.isaac_teleop.xr_camera_feed_layout = XrCameraFeedLayoutCfg( + placement="world", + mode="grid", + world_position_m=(0.0, 0.8, 1.6), + world_orientation_xyzw=(0.7071067812, 0.0, 0.0, 0.7071067812), + max_columns=2, + ) + +``manual`` mode preserves each feed's ``offset_m`` and ``distance_m``. ``horizontal``, ``vertical``, +and ``grid`` modes pack enabled feeds around ``center_offset_m`` using ``panel_gap_m``; grid mode +also honors ``max_columns``. Offsets are measured in the selected layout plane. With +``viewer_start`` and ``head_locked``, ``distance_m`` places that plane in front of the viewer +reference. In manual mode, feeds with identical offsets overlap; select an automatic mode or assign +distinct feed offsets when showing multiple panels. + +With ``world``, the explicit pose is measured in the Isaac Lab USD stage world in meters, not +OpenXR physical space. Isaac Lab stages are Z-up. ``world_orientation_xyzw`` maps panel-local +coordinates into that world: local +X is image right, local +Y is image up, and local +Z points +from the readable face toward the viewer. Feed and automatic-layout offsets are applied in the +panel's local XY plane. + +Disable cameras and PiP +~~~~~~~~~~~~~~~~~~~~~~~ + +Set :attr:`~isaaclab_teleop.XrCameraFeedCfg.enabled` to ``False`` to suppress one feed, or set +``xr_camera_feeds=[]`` to suppress all PiP. The list is empty by default. These choices do not +remove camera sensors independently owned by the task. + +``--disable_external_cameras`` is the master camera-rendering switch for ``teleop_se3_agent.py`` and +``record_demos.py``. It strips camera sensors and suppresses every configured PiP feed. + +Kit Scene UI presentation +~~~~~~~~~~~~~~~~~~~~~~~~~ + +PiP presentation uses Kit Scene UI and ``SpatialSource`` placement. Kit imports are deferred until +an enabled feed is requested. If Scene UI is unavailable, the scripts log a warning and continue +without PiP; task-owned cameras and recording observations remain unchanged. This keeps the camera +selection configuration usable when a future kitless entry point no longer provides Scene UI. .. _isaac-teleop-haptics: @@ -1557,8 +1702,10 @@ Optimize XR Performance The flag strips the environment's camera sensors (equivalent to calling :func:`~isaaclab_teleop.remove_camera_configs` on the env config) and selects a lighter Kit - experience. Omit it to keep cameras enabled (the default) -- required when recording camera - observations, or when you want ``teleop_replay_agent.py`` to mirror the production render load. + experience. For ``teleop_se3_agent.py`` it is also the master PiP gate: task-default and + task-configured feeds are ignored and no camera panel is created. Omit it to keep cameras enabled + (the default) -- required when recording camera observations, showing XR camera feedback, or when + you want ``teleop_replay_agent.py`` to mirror the production render load. .. dropdown:: Run headless (skip the local viewport) :open: @@ -1683,6 +1830,8 @@ See the :ref:`isaaclab_teleop-api` for full class and function documentation: * :class:`~isaaclab_teleop.HapticFeedbackReceiver` * :class:`~isaaclab_teleop.HapticFeedbackDriver` * :func:`~isaaclab_teleop.create_haptic_feedback_driver` +* :class:`~isaaclab_teleop.XrCameraFeedCfg` +* :class:`~isaaclab_teleop.XrCameraFeedLayoutCfg` * :class:`~isaaclab_teleop.XrCfg` * :class:`~isaaclab_teleop.XrAnchorRotationMode` diff --git a/scripts/environments/teleoperation/teleop_se3_agent.py b/scripts/environments/teleoperation/teleop_se3_agent.py index 66ab57fc4205..8d79f26ce5fa 100644 --- a/scripts/environments/teleoperation/teleop_se3_agent.py +++ b/scripts/environments/teleoperation/teleop_se3_agent.py @@ -272,6 +272,14 @@ def main() -> None: # noqa: C901 not teleop_device_explicitly_set and hasattr(env_cfg, "isaac_teleop") and env_cfg.isaac_teleop is not None ) + from isaaclab_teleop.camera_feed import _XrCameraFeedSession + + camera_feed_session = _XrCameraFeedSession.prepare( + env_cfg, + enabled=args_cli.xr and use_isaac_teleop, + camera_rendering_enabled=not args_cli.disable_external_cameras, + ) + # XR-rendering setup (camera removal + DLSS) is only needed for the Kit XR # path. Without --xr, IsaacTeleop runs standalone (I/O only) and renders # normally, so gate on --xr alone. @@ -286,7 +294,14 @@ def main() -> None: # noqa: C901 if _rtx_rendering_requested(args_cli): _ensure_replicator_loaded() apply_isaac_rtx_global_settings( - IsaacRtxRendererGlobalSettingsCfg(antialiasing_mode="DLSS"), + IsaacRtxRendererGlobalSettingsCfg( + antialiasing_mode="DLSS", + carb_settings=( + {"/rtx/dldenoiser/responsiveDenoising": True} + if camera_feed_session.requires_responsive_denoising + else None + ), + ), ) try: @@ -493,6 +508,7 @@ def run_loop(): if should_reset_recording_instance: env.reset() teleop_interface.reset() + camera_feed_session.refresh() should_reset_recording_instance = False print("Environment reset complete") except Exception as e: @@ -502,10 +518,11 @@ def run_loop(): # Run the teleoperation loop # IsaacTeleop requires a context manager, native devices don't if use_isaac_teleop: - with teleop_interface: + with teleop_interface, camera_feed_session.bind(env): run_loop() else: - run_loop() + with camera_feed_session.bind(env): + run_loop() # close the simulator env.close() diff --git a/scripts/tools/record_demos.py b/scripts/tools/record_demos.py index 000f66bb91f3..21919e8aa47e 100644 --- a/scripts/tools/record_demos.py +++ b/scripts/tools/record_demos.py @@ -34,11 +34,15 @@ # Standard library imports import argparse import contextlib +from typing import TYPE_CHECKING # Isaac Lab AppLauncher from isaaclab.app import AppLauncher from isaaclab.utils.string import list_intersection, string_to_callable +if TYPE_CHECKING: + from isaaclab_teleop.camera_feed import _XrCameraFeedSession + # add argparse arguments parser = argparse.ArgumentParser(description="Record demonstrations for Isaac Lab environments.") parser.add_argument("--task", type=str, required=True, help="Name of the task.") @@ -575,6 +579,7 @@ def run_simulation_loop( # noqa: C901 teleop_interface: object | None, success_term: object | None, rate_limiter: RateLimiter | None, + camera_feed_session: "_XrCameraFeedSession", use_isaac_teleop: bool = False, ) -> int: """Run the main simulation loop for collecting demonstrations. @@ -588,6 +593,7 @@ def run_simulation_loop( # noqa: C901 teleop_interface: Optional teleop interface (will be created if None) success_term: The success termination object or None if not available rate_limiter: Optional rate limiter to control simulation speed + camera_feed_session: Shared XR camera-feed lifecycle use_isaac_teleop: Whether to use IsaacTeleop stack Returns: @@ -694,7 +700,7 @@ def inner_loop(): if use_isaac_teleop: from isaaclab_teleop import poll_control_events - with contextlib.suppress(KeyboardInterrupt), torch.inference_mode(): + with contextlib.suppress(KeyboardInterrupt), torch.inference_mode(), camera_feed_session.bind(env): while simulation_app.is_running(): # Get teleop command (may be None while waiting for session start) action = teleop_interface.advance() @@ -764,6 +770,7 @@ def inner_loop(): success_step_count = handle_reset( env, success_step_count, instruction_display, label_text, teleop_interface ) + camera_feed_session.refresh() should_reset_recording_instance = False # Check if simulation is stopped @@ -804,6 +811,20 @@ def main() -> None: global env_cfg # Make env_cfg available to setup_teleop_device env_cfg, success_term, use_isaac_teleop = create_environment_config(output_dir, output_file_name) + from isaaclab_teleop.camera_feed import _XrCameraFeedSession + + camera_feed_session = _XrCameraFeedSession.prepare( + env_cfg, + enabled=args_cli.xr and use_isaac_teleop, + camera_rendering_enabled=not args_cli.disable_external_cameras, + ) + if camera_feed_session.requires_responsive_denoising: + apply_isaac_rtx_global_settings( + IsaacRtxRendererGlobalSettingsCfg( + carb_settings={"/rtx/dldenoiser/responsiveDenoising": True}, + ) + ) + # With --xr, rate limiting is achieved via OpenXR and the XR visualization # manager is installed. Without --xr (including standalone IsaacTeleop I/O), # fall back to the software rate limiter and skip the XR viz stack. @@ -820,7 +841,9 @@ def main() -> None: env = create_environment(env_cfg) # Run simulation loop - current_recorded_demo_count = run_simulation_loop(env, None, success_term, rate_limiter, use_isaac_teleop) + current_recorded_demo_count = run_simulation_loop( + env, None, success_term, rate_limiter, camera_feed_session, use_isaac_teleop + ) # Clean up env.close() diff --git a/source/isaaclab/changelog.d/xr-camera-feedback.minor.rst b/source/isaaclab/changelog.d/xr-camera-feedback.minor.rst new file mode 100644 index 000000000000..2c38716e6b9d --- /dev/null +++ b/source/isaaclab/changelog.d/xr-camera-feedback.minor.rst @@ -0,0 +1,5 @@ +Added +^^^^^ + +* Allowed renderers to select a camera pixel-output device independently from camera pose and + simulation state. diff --git a/source/isaaclab/isaaclab/renderers/base_renderer.py b/source/isaaclab/isaaclab/renderers/base_renderer.py index 5b571e3a6936..95170934a0b4 100644 --- a/source/isaaclab/isaaclab/renderers/base_renderer.py +++ b/source/isaaclab/isaaclab/renderers/base_renderer.py @@ -44,6 +44,21 @@ def initialize(self) -> None: """Post-physics one-time initialization hook. Called only once.""" return + def resolve_camera_output_device(self, simulation_device: str) -> str: + """Resolve the device used for persistent camera pixel outputs. + + The default keeps outputs on the simulation device. A renderer may + override this when its pixel production is independent of physics and + camera pose storage. + + Args: + simulation_device: Device used by simulation and camera state. + + Returns: + Device used by the render specification and image output buffers. + """ + return simulation_device + def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: """Pre-render per-camera setup the backend needs. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index d88fee4e6b55..26509f0cad8e 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -215,6 +215,7 @@ def __init__(self, cfg: CameraCfg): # Renderer and render data — assigned in _initialize_impl. self._renderer: BaseRenderer | None = None self._render_data = None + self._camera_output_device: str | None = None def __del__(self): """Unsubscribes from callbacks and cleans up renderer resources.""" @@ -523,9 +524,10 @@ def _initialize_impl(self): cam_paths[0].removeprefix(env_0_prefix) if cam_paths and cam_paths[0].startswith(env_0_prefix) else "" ) device_str = self._device if isinstance(self._device, str) else str(self._device) + self._camera_output_device = self._renderer.resolve_camera_output_device(device_str) render_spec = CameraRenderSpec( cfg=self.cfg, - device=device_str, + device=self._camera_output_device, num_instances=self._num_envs, camera_prim_paths=cam_paths, view_count=self._num_envs, @@ -647,18 +649,20 @@ def _create_buffers(self): type(self._renderer).__name__, unsupported, ) - device_str = self._device if isinstance(self._device, str) else str(self._device) + state_device = self._device if isinstance(self._device, str) else str(self._device) + if self._camera_output_device is None: + raise RuntimeError("Camera output device was not resolved before buffer allocation.") self._data = CameraData.allocate( data_types=known, height=self.cfg.height, width=self.cfg.width, num_views=self._view.count, - device=self._device, + device=self._camera_output_device, supported_specs=specs, ) # Camera-frame state (pose / intrinsics) is owned by the camera, not # the renderer: allocate warp buffers and populate them. - self._data.create_buffers(self._view.count, device_str) + self._data.create_buffers(self._view.count, state_device) self._update_intrinsic_matrices() self._update_poses() self._renderer.set_outputs(self._render_data, self._data.output) diff --git a/source/isaaclab/test/renderers/test_simulation_render_context.py b/source/isaaclab/test/renderers/test_simulation_render_context.py index 0eb7fbc343b5..f3f8beca26b9 100644 --- a/source/isaaclab/test/renderers/test_simulation_render_context.py +++ b/source/isaaclab/test/renderers/test_simulation_render_context.py @@ -111,6 +111,21 @@ def test_get_renderer_returns_equal_cfg_singleton(): assert r1 is r2 +def test_get_renderer_isolates_per_camera_dlss_settings(): + """Camera-local RTX settings create a distinct backend while equal settings reuse one.""" + ctx = RenderContext() + + default_renderer = ctx.get_renderer(IsaacRtxRendererCfg()) + tuned_cfg = IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=False, dlss_exec_mode="quality") + tuned_renderer = ctx.get_renderer(tuned_cfg) + tuned_renderer_again = ctx.get_renderer( + IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=False, dlss_exec_mode="quality") + ) + + assert tuned_renderer is not default_renderer + assert tuned_renderer_again is tuned_renderer + + def test_get_renderer_two_different_concrete_types_coexist(): """Different renderer_cfg concrete classes register distinct backends (no error).""" diff --git a/source/isaaclab_physx/changelog.d/xr-camera-dlss-ghosting.rst b/source/isaaclab_physx/changelog.d/xr-camera-dlss-ghosting.rst new file mode 100644 index 000000000000..4e14e48c57dc --- /dev/null +++ b/source/isaaclab_physx/changelog.d/xr-camera-dlss-ghosting.rst @@ -0,0 +1,7 @@ +Added +^^^^^ + +* Added per-camera DLSS Ray Reconstruction enable and execution-mode overrides to + :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg`, plus a process-global responsive-denoising setting. +* Added an optional RTX camera output-device override so CUDA pixels can be produced with CPU + physics and CPU camera pose state. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 04da134d6035..2bfebd11e673 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -18,7 +18,7 @@ import warp as wp from packaging import version -from pxr import Sdf, Usd, UsdGeom +from pxr import Sdf, Usd, UsdGeom, UsdRender from isaaclab.app.settings_manager import get_settings_manager from isaaclab.renderers import BaseRenderer, RenderBufferKind, RenderBufferSpec @@ -82,6 +82,10 @@ def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: "simple_shading_full_mdl": 3, } SIMPLE_SHADING_MODE_SETTING = "/rtx/minimal/mode" +DLSS_RAY_RECONSTRUCTION_API_SCHEMA = "OmniRtxDebugSettingsAPI_1" +DLSS_RAY_RECONSTRUCTION_ATTR = "omni:rtx:newDenoiser:enabled" +DLSS_EXEC_MODE_API_SCHEMA = "OmniRtxSettingsRtAPI_1" +DLSS_EXEC_MODE_ATTR = "omni:rtx:post:dlss:execMode" def _camera_semantic_filter_predicate(semantic_filter: str | list[str]) -> str: @@ -134,6 +138,10 @@ def __init__(self, cfg: IsaacRtxRendererCfg): ensure_rtx_hydra_engine_attached() # ``/isaaclab/render/rtx_sensors`` is owned by ``Camera.__init__`` (must be set pre-``sim.reset()``). + def resolve_camera_output_device(self, simulation_device: str) -> str: + """Use the optional RTX pixel-output override without moving camera state.""" + return self.cfg.camera_output_device or simulation_device + def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: """Resolve the camera's PPISP cfg and apply RTX-specific USD overrides. @@ -422,6 +430,10 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: for annotator in annotators.values(): annotator.attach([rp.path]) + # HydraTexture attachment synchronizes legacy global settings onto the render product. + # Author camera-local overrides only after every annotator has finished attaching. + self._apply_render_product_settings(stage, [rp.path]) + ppisp_pipeline = None if spec.cfg.isp_cfg is not None: try: @@ -438,6 +450,52 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: ppisp_pipeline=ppisp_pipeline, ) + def _apply_render_product_settings(self, stage: Usd.Stage, render_product_paths: list[str]) -> None: + """Author optional renderer settings that must remain local to one camera.""" + ray_reconstruction = self.cfg.enable_dlss_ray_reconstruction + dlss_exec_mode = self.cfg.dlss_exec_mode + if ray_reconstruction is None and dlss_exec_mode is None: + return + for render_product_path in render_product_paths: + render_product = stage.GetPrimAtPath(render_product_path) + if not render_product.IsValid(): + raise RuntimeError(f"Render product '{render_product_path}' was not materialized on the USD stage.") + if not render_product.IsA(UsdRender.Product): + raise RuntimeError(f"Prim '{render_product_path}' is not a RenderProduct.") + if ray_reconstruction is not None: + self._set_render_product_schema_attribute( + render_product, + DLSS_RAY_RECONSTRUCTION_API_SCHEMA, + DLSS_RAY_RECONSTRUCTION_ATTR, + ray_reconstruction, + ) + if dlss_exec_mode is not None: + self._set_render_product_schema_attribute( + render_product, + DLSS_EXEC_MODE_API_SCHEMA, + DLSS_EXEC_MODE_ATTR, + dlss_exec_mode, + ) + + @staticmethod + def _set_render_product_schema_attribute( + render_product: Usd.Prim, + api_schema: str, + attribute_name: str, + value: bool | str, + ) -> None: + """Apply one RTX API schema and author its validated render-product attribute.""" + if not render_product.ApplyAPI(api_schema): + raise RuntimeError(f"Failed to apply RTX API schema '{api_schema}' to '{render_product.GetPath()}'.") + attribute = render_product.GetAttribute(attribute_name) + if not attribute.IsValid(): + raise RuntimeError( + f"RTX API schema '{api_schema}' does not provide attribute '{attribute_name}' " + f"on '{render_product.GetPath()}'." + ) + if not attribute.Set(value): + raise RuntimeError(f"Failed to set RTX attribute '{attribute_name}' on '{render_product.GetPath()}'.") + def _resolve_simple_shading_mode(self, spec: CameraRenderSpec) -> int | None: """Resolve the requested simple shading mode from data types.""" requested = [dt for dt in spec.cfg.data_types if dt in SIMPLE_SHADING_MODES] diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py index 9bd1a7e1ad88..503443723782 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py @@ -10,6 +10,8 @@ from isaaclab.renderers.renderer_cfg import RendererCfg from isaaclab.utils.configclass import configclass +_DLSS_EXEC_MODES = {"performance", "balanced", "quality", "auto", "rtxaa", "manual"} + @configclass class IsaacRtxRendererGlobalSettingsCfg: @@ -105,6 +107,31 @@ class IsaacRtxRendererCfg(RendererCfg): global_settings: IsaacRtxRendererGlobalSettingsCfg = IsaacRtxRendererGlobalSettingsCfg() """Global Kit/RTX quality settings applied before RTX Hydra attach.""" + camera_output_device: str | None = None + """Optional device for Replicator annotators and persistent camera pixel outputs. + + This leaves physics and camera pose state on the simulation device. For + example, set ``"cuda:0"`` with CPU physics to avoid an RTX GPU-to-CPU + readback when a GPU consumer uses the camera image. ``None`` preserves the + simulation-device behavior. + """ + + enable_dlss_ray_reconstruction: bool | None = None + """Enable DLSS Ray Reconstruction for this renderer's render products. + + Set to ``False`` to use classic DLSS while leaving other render products, + such as XR headset views, on their process-global setting. ``None`` + preserves Kit's render-product default. + """ + + dlss_exec_mode: Literal["performance", "balanced", "quality", "auto", "rtxaa", "manual"] | None = None + """Optional render-product-local DLSS execution mode. + + ``None`` preserves Kit's render-product default. Unlike + :attr:`global_settings`, this setting affects only cameras using this + renderer configuration. + """ + semantic_filter: str | list[str] = "*:*" """A string or a list specifying a semantic filter predicate. Defaults to ``"*:*"``. @@ -172,3 +199,28 @@ class IsaacRtxRendererCfg(RendererCfg): - ``"zero"``: Values are clipped to zero. - ``"none"``: No clipping is applied. Values will be returned as ``inf``. """ + + def __post_init__(self) -> None: + """Validate render-product settings before Kit silently ignores them.""" + if self.camera_output_device is not None and ( + not isinstance(self.camera_output_device, str) + or not ( + self.camera_output_device == "cpu" + or self.camera_output_device == "cuda" + or ( + self.camera_output_device.startswith("cuda:") + and self.camera_output_device.removeprefix("cuda:").isdigit() + ) + ) + ): + raise ValueError( + "camera_output_device must be 'cpu', 'cuda', 'cuda:', or None, " + f"got {self.camera_output_device!r}." + ) + ray_reconstruction = self.enable_dlss_ray_reconstruction + if ray_reconstruction is not None and type(ray_reconstruction) is not bool: + raise TypeError("enable_dlss_ray_reconstruction must be a bool or None.") + if self.dlss_exec_mode is not None and self.dlss_exec_mode not in _DLSS_EXEC_MODES: + raise ValueError( + f"dlss_exec_mode must be one of {sorted(_DLSS_EXEC_MODES)} or None, got {self.dlss_exec_mode!r}." + ) diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_render_product_settings.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_render_product_settings.py new file mode 100644 index 000000000000..8700a86cab79 --- /dev/null +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_render_product_settings.py @@ -0,0 +1,113 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Isaac RTX RenderProduct settings integration tests.""" + +from isaaclab.app import AppLauncher + +# RenderProduct materialization and HydraTexture attachment require a camera-enabled Kit app. +simulation_app = AppLauncher(headless=True, enable_cameras=True, device="cpu").app + +"""Rest everything follows.""" + +import pytest +from isaaclab_physx.renderers import IsaacRtxRendererCfg, IsaacRtxRendererGlobalSettingsCfg + +import omni.replicator.core as rep +import usdrt.Usd as UsdRtUsd +from pxr import UsdUtils + +import isaaclab.sim as sim_utils +from isaaclab.app.settings_manager import get_settings_manager +from isaaclab.sensors.camera import Camera, CameraCfg + +pytestmark = [pytest.mark.integration, pytest.mark.rendering, pytest.mark.isaacsim_ci] + + +def _read_dlss_settings(prim) -> tuple[str, bool]: + """Read the schema-backed settings from a USD or Fabric RenderProduct prim.""" + exec_mode = prim.GetAttribute("omni:rtx:post:dlss:execMode").Get() + ray_reconstruction = prim.GetAttribute("omni:rtx:newDenoiser:enabled").Get() + return str(exec_mode), bool(ray_reconstruction) + + +def test_camera_local_dlss_settings_survive_annotator_attachment(): + """Per-camera settings override conflicting globals in both USD and Fabric after attachment.""" + sim_utils.create_new_stage() + sim = sim_utils.SimulationContext(sim_utils.SimulationCfg(device="cpu", dt=1.0 / 60.0)) + camera = Camera( + CameraCfg( + prim_path="/World/Camera", + height=96, + width=96, + data_types=["rgb"], + spawn=sim_utils.PinholeCameraCfg(), + renderer_cfg=IsaacRtxRendererCfg( + global_settings=IsaacRtxRendererGlobalSettingsCfg( + antialiasing_mode="DLSS", + dlss_mode=0, + enable_dl_denoiser=True, + carb_settings={"/rtx/dldenoiser/responsiveDenoising": True}, + ), + enable_dlss_ray_reconstruction=False, + dlss_exec_mode="quality", + ), + ) + ) + + try: + sim.reset() + for _ in range(2): + sim.step() + camera.update(sim.cfg.dt) + + stage = sim_utils.get_current_stage() + render_product_path = camera._render_data.render_product.path + usd_prim = stage.GetPrimAtPath(render_product_path) + + stage_id = UsdUtils.StageCache.Get().GetId(stage).ToLongInt() + fabric_stage = UsdRtUsd.Stage.Attach(stage_id) + fabric_prim = fabric_stage.GetPrimAtPath(render_product_path) + + expected = ("quality", False) + assert _read_dlss_settings(usd_prim) == expected + assert _read_dlss_settings(fabric_prim) == expected + assert get_settings_manager().get("/rtx/dldenoiser/responsiveDenoising") is True + finally: + camera._invalidate_initialize_callback(None) + rep.vp_manager.destroy_hydra_textures("Replicator") + sim.stop() + sim.clear_instance() + + +def test_cpu_simulation_camera_can_publish_cuda_pixels(): + """Keep camera pose state on CPU while Isaac RTX publishes its image on CUDA.""" + sim_utils.create_new_stage() + sim = sim_utils.SimulationContext(sim_utils.SimulationCfg(device="cpu", dt=1.0 / 60.0)) + camera = Camera( + CameraCfg( + prim_path="/World/Camera", + height=64, + width=64, + data_types=["rgb"], + spawn=sim_utils.PinholeCameraCfg(), + renderer_cfg=IsaacRtxRendererCfg(camera_output_device="cuda:0"), + ) + ) + + try: + sim.reset() + for _ in range(2): + sim.step() + camera.update(sim.cfg.dt) + + assert camera.data.pos_w.torch.device.type == "cpu" + assert camera.data.output["rgb"].torch.device.type == "cuda" + assert camera.data.output["rgb"].torch.device.index == 0 + finally: + camera._invalidate_initialize_callback(None) + rep.vp_manager.destroy_hydra_textures("Replicator") + sim.stop() + sim.clear_instance() diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index d25b5c762b72..85d1a5afb8b1 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -155,6 +155,175 @@ def test_render_product_uuid_name_format_is_sdf_safe(): assert Sdf.Path.IsValidPathString(f"/Render/{name}") +def test_camera_output_device_override_is_rtx_local(monkeypatch): + """RTX may put pixels on CUDA while the base renderer keeps simulation placement.""" + _install_omni_stubs(monkeypatch) + from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + from isaaclab.renderers.base_renderer import BaseRenderer + + renderer = IsaacRtxRenderer.__new__(IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg(camera_output_device="cuda:0") + + assert renderer.resolve_camera_output_device("cpu") == "cuda:0" + assert BaseRenderer.resolve_camera_output_device(renderer, "cpu") == "cpu" + + +@pytest.mark.parametrize("value", ["gpu", "cuda:x", "", 0]) +def test_renderer_cfg_rejects_invalid_camera_output_device(value): + """Invalid output devices fail during configuration rather than Replicator startup.""" + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + with pytest.raises(ValueError, match="camera_output_device"): + IsaacRtxRendererCfg(camera_output_device=value) + + +def test_dlss_settings_are_authored_per_render_product(monkeypatch): + """Camera-local DLSS settings use their schema-backed RenderProduct attributes.""" + _install_omni_stubs(monkeypatch) + from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + renderer = IsaacRtxRenderer.__new__(IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=False, dlss_exec_mode="quality") + stage = MagicMock() + prims = {path: MagicMock() for path in ("/Render/RP_0", "/Render/RP_1")} + stage.GetPrimAtPath.side_effect = prims.__getitem__ + for prim in prims.values(): + prim.IsValid.return_value = True + prim.IsA.return_value = True + prim.ApplyAPI.return_value = True + attributes = { + "omni:rtx:newDenoiser:enabled": MagicMock(), + "omni:rtx:post:dlss:execMode": MagicMock(), + } + for attribute in attributes.values(): + attribute.IsValid.return_value = True + attribute.Set.return_value = True + prim.GetAttribute.side_effect = attributes.__getitem__ + prim._test_attributes = attributes + + renderer._apply_render_product_settings(stage, list(prims)) + + for prim in prims.values(): + assert prim.ApplyAPI.call_args_list == [ + call("OmniRtxDebugSettingsAPI_1"), + call("OmniRtxSettingsRtAPI_1"), + ] + assert prim.GetAttribute.call_args_list == [ + call("omni:rtx:newDenoiser:enabled"), + call("omni:rtx:post:dlss:execMode"), + ] + prim._test_attributes["omni:rtx:newDenoiser:enabled"].Set.assert_called_once_with(False) + prim._test_attributes["omni:rtx:post:dlss:execMode"].Set.assert_called_once_with("quality") + + +def test_default_renderer_cfg_does_not_author_render_product_settings(monkeypatch): + """The default config preserves Kit's render-product settings.""" + _install_omni_stubs(monkeypatch) + from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + renderer = IsaacRtxRenderer.__new__(IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg() + stage = MagicMock() + + renderer._apply_render_product_settings(stage, ["/Render/RP"]) + + stage.GetPrimAtPath.assert_not_called() + + +def test_dlss_settings_are_authored_after_all_annotators_attach(monkeypatch): + """HydraTexture attachment must finish before camera-local DLSS settings are authored.""" + replicator_module, syntheticdata_module = _install_omni_stubs(monkeypatch) + monkeypatch.setattr(syntheticdata_module, "SyntheticData", MagicMock(), raising=False) + + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + import isaaclab.sim.utils.stage as stage_utils + + events = [] + color_annotator = MagicMock() + depth_annotator = MagicMock() + color_annotator.attach.side_effect = lambda paths: events.append(("attach", "rgba", paths)) + depth_annotator.attach.side_effect = lambda paths: events.append(("attach", "depth", paths)) + + replicator_module.create = SimpleNamespace( + render_product_tiled=MagicMock(return_value=SimpleNamespace(path="/Render/RP")) + ) + replicator_module.AnnotatorRegistry = MagicMock() + replicator_module.AnnotatorRegistry.get_annotator.side_effect = [color_annotator, depth_annotator] + + stage = MagicMock() + stage.GetPrimAtPath.return_value.IsA.return_value = True + spec = SimpleNamespace( + camera_prim_paths=("/World/Camera",), + cfg=SimpleNamespace( + data_types=["rgba", "distance_to_image_plane"], + width=64, + height=48, + isp_cfg=None, + ), + device="cpu", + ) + renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=False, dlss_exec_mode="quality") + renderer._apply_render_product_settings = MagicMock( + side_effect=lambda authored_stage, paths: events.append(("author", authored_stage, paths)) + ) + + with ( + patch.object(rtx_renderer, "get_settings_manager", return_value=MagicMock()), + patch.object(rtx_renderer, "get_isaac_sim_version", return_value=version.parse("5.1")), + patch.object(stage_utils, "get_current_stage", return_value=stage), + ): + renderer.create_render_data(spec) + + assert events == [ + ("attach", "rgba", ["/Render/RP"]), + ("attach", "depth", ["/Render/RP"]), + ("author", stage, ["/Render/RP"]), + ] + + +@pytest.mark.parametrize("value", ["", "Quality", "ultra", 2]) +def test_renderer_cfg_rejects_invalid_dlss_exec_mode(value): + """Invalid schema tokens fail before Kit can silently ignore them.""" + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + with pytest.raises(ValueError, match="dlss_exec_mode"): + IsaacRtxRendererCfg(dlss_exec_mode=value) + + +@pytest.mark.parametrize("value", [0, 1, "false"]) +def test_renderer_cfg_rejects_non_bool_dlss_ray_reconstruction(value): + """The RenderProduct boolean must not accept truthy values of another type.""" + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + with pytest.raises(TypeError, match="enable_dlss_ray_reconstruction"): + IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=value) + + +def test_render_product_schema_failure_is_reported(monkeypatch): + """A missing RTX schema fails at camera setup instead of creating an inert custom attribute.""" + _install_omni_stubs(monkeypatch) + from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + renderer = IsaacRtxRenderer.__new__(IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=False) + stage = MagicMock() + render_product = stage.GetPrimAtPath.return_value + render_product.IsValid.return_value = True + render_product.IsA.return_value = True + render_product.ApplyAPI.return_value = False + + with pytest.raises(RuntimeError, match="OmniRtxDebugSettingsAPI_1"): + renderer._apply_render_product_settings(stage, ["/Render/RP"]) + + @pytest.mark.parametrize( ("has_gui", "expected_disable_color_render"), [ diff --git a/source/isaaclab_tasks/changelog.d/xr-camera-feedback.minor.rst b/source/isaaclab_tasks/changelog.d/xr-camera-feedback.minor.rst new file mode 100644 index 000000000000..078b652f723f --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/xr-camera-feedback.minor.rst @@ -0,0 +1,12 @@ +Added +^^^^^ + +* Added recorded ``robot_pov_cam`` policy observations to the GR1T2 pick-place and G1 + locomanipulation tasks for XR camera feedback. +* Configured the recorded cameras to produce their RTX pixel output on ``cuda:0`` while preserving + CPU physics. + +Fixed +^^^^^ + +* Fixed the ExhaustPipe ``robot_pov_cam`` rotation after the WXYZ-to-XYZW quaternion migration. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py index 1be577481714..d627c5d7b0d6 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py @@ -3,7 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab_teleop import ControllerHapticFeedbackCfg, IsaacTeleopCfg, XrAnchorRotationMode, XrCfg +from isaaclab_teleop import ( + ControllerHapticFeedbackCfg, + IsaacTeleopCfg, + XrAnchorRotationMode, + XrCameraFeedCfg, + XrCfg, +) import isaaclab.envs.mdp as base_mdp import isaaclab.sim as sim_utils @@ -14,7 +20,7 @@ from isaaclab.managers import SceneEntityCfg from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg -from isaaclab.sensors import ContactSensorCfg +from isaaclab.sensors import CameraCfg, ContactSensorCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR from isaaclab.utils.configclass import configclass @@ -31,6 +37,18 @@ from isaaclab_tasks.contrib.locomanip_pick_place.configs.pink_controller_cfg import ( # isort: skip G1_UPPER_BODY_IK_ACTION_CFG, ) +from isaaclab_physx.renderers import IsaacRtxRendererCfg # isort: skip +from isaaclab_tasks.utils.presets import MultiBackendRendererCfg # isort: skip + + +@configclass +class _RobotPovCameraRendererCfg(MultiBackendRendererCfg): + default: IsaacRtxRendererCfg = IsaacRtxRendererCfg( + camera_output_device="cuda:0", + enable_dlss_ray_reconstruction=True, + dlss_exec_mode="quality", + ) + isaacsim_rtx = default def _build_g1_locomanipulation_pipeline(): @@ -290,6 +308,22 @@ class LocomanipulationG1SceneCfg(InteractiveSceneCfg): # Humanoid robot w/ arms higher robot: ArticulationCfg = G1_29DOF_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot") + # Fixed task camera matching the GR1T2 training-camera placement. + robot_pov_cam = CameraCfg( + prim_path="{ENV_REGEX_NS}/RobotPOVCam", + update_period=0.0, + height=450, + width=720, + data_types=["rgb"], + renderer_cfg=_RobotPovCameraRendererCfg(), + spawn=sim_utils.PinholeCameraCfg(focal_length=18.15, clipping_range=(0.1, 2.0)), + offset=CameraCfg.OffsetCfg( + pos=(0.0, 0.12, 1.67675), + rot=(0.9801, 0.0, 0.0, -0.19848), + convention="ros", + ), + ) + # Per-hand contact sensors over all finger links, used to drive controller # haptics (see HapticFeedbackCfg below). Requires activate_contact_sensors # on the robot spawn, enabled in the env __post_init__. @@ -369,6 +403,16 @@ class PolicyCfg(ObsGroup): params={"left_eef_link_name": "left_wrist_yaw_link", "right_eef_link_name": "right_wrist_yaw_link"}, ) + robot_pov_cam = ObsTerm( + func=base_mdp.image, + params={ + "sensor_cfg": SceneEntityCfg("robot_pov_cam"), + "data_type": "rgb", + "normalize": False, + "clone": False, + }, + ) + def __post_init__(self): self.enable_corruption = False self.concatenate_terms = False @@ -456,7 +500,16 @@ def __post_init__(self): pipeline_builder=_build_g1_locomanipulation_pipeline, sim_device=self.sim.device, xr_cfg=self.xr, + xr_camera_feeds=[ + XrCameraFeedCfg( + camera_name="robot_pov_cam", + # Keep the 0.30 m-tall panel below the eye-level controls. + offset_m=(0.0, -0.15), + max_update_hz=0.0, + ) + ], ) + self.image_obs_list = ["robot_pov_cam"] # Enable contact reporting on the robot so the per-hand ContactSensors # report finger forces, and drive controller haptics from them. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_base_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_base_env_cfg.py index 7ab5fa883440..738230f45b8b 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_base_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_base_env_cfg.py @@ -150,7 +150,7 @@ class ObjectTableSceneCfg(InteractiveSceneCfg): width=256, data_types=["rgb"], spawn=sim_utils.PinholeCameraCfg(focal_length=18.15, clipping_range=(0.1, 2)), - offset=CameraCfg.OffsetCfg(pos=(0.0, 0.12, 1.85418), rot=(0.0, 0.98502, 0.0, -0.17246), convention="ros"), + offset=CameraCfg.OffsetCfg(pos=(0.0, 0.12, 1.85418), rot=(0.98502, 0.0, 0.0, -0.17246), convention="ros"), ) # Ground plane diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_pink_ik_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_pink_ik_env_cfg.py index 8cab1b33fce8..3fe6bdbbe150 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_pink_ik_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_pink_ik_env_cfg.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg +from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg, XrCameraFeedCfg from isaaclab.controllers.pink_ik import DampingTaskCfg, FrameTaskCfg, NullSpacePostureTaskCfg, PinkIKControllerCfg from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg @@ -132,4 +132,12 @@ def __post_init__(self): pipeline_builder=lambda: _build_gr1t2_pickplace_pipeline()[0], sim_device=self.sim.device, xr_cfg=self.xr, + xr_camera_feeds=[ + XrCameraFeedCfg( + camera_name="robot_pov_cam", + # Keep the 0.30 m-tall panel below the eye-level controls. + offset_m=(0.0, -0.15), + max_update_hz=0.0, + ) + ], ) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/nutpour_gr1t2_pink_ik_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/nutpour_gr1t2_pink_ik_env_cfg.py index b5c2e3a7d6c7..0ee984b7d4d2 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/nutpour_gr1t2_pink_ik_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/nutpour_gr1t2_pink_ik_env_cfg.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg +from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg, XrCameraFeedCfg from isaaclab.controllers.pink_ik import DampingTaskCfg, FrameTaskCfg, NullSpacePostureTaskCfg, PinkIKControllerCfg from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg @@ -130,4 +130,12 @@ def __post_init__(self): pipeline_builder=lambda: _build_gr1t2_pickplace_pipeline()[0], sim_device=self.sim.device, xr_cfg=self.xr, + xr_camera_feeds=[ + XrCameraFeedCfg( + camera_name="robot_pov_cam", + # Keep the 0.30 m-tall panel below the eye-level controls. + offset_m=(0.0, -0.15), + max_update_hz=0.0, + ) + ], ) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py index a0220b2464c5..260b0ccdfba2 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py @@ -18,7 +18,7 @@ from isaaclab.managers import SceneEntityCfg from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg -from isaaclab.sensors import ContactSensorCfg +from isaaclab.sensors import CameraCfg, ContactSensorCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR, retrieve_file_path from isaaclab.utils.configclass import configclass @@ -27,8 +27,20 @@ from isaaclab_assets.robots.fourier import GR1T2_HIGH_PD_CFG # isort: skip from isaaclab_teleop.haptic_feedback import GloveHapticFeedbackCfg # isort: skip -from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg # isort: skip +from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg, XrCameraFeedCfg # isort: skip from isaaclab_teleop.xr_cfg import XrCfg # isort: skip +from isaaclab_physx.renderers import IsaacRtxRendererCfg # isort: skip +from isaaclab_tasks.utils.presets import MultiBackendRendererCfg # isort: skip + + +@configclass +class _RobotPovCameraRendererCfg(MultiBackendRendererCfg): + default: IsaacRtxRendererCfg = IsaacRtxRendererCfg( + camera_output_device="cuda:0", + enable_dlss_ray_reconstruction=True, + dlss_exec_mode="quality", + ) + isaacsim_rtx = default def _build_gr1t2_pickplace_pipeline(): @@ -363,6 +375,26 @@ class ObjectTableSceneCfg(InteractiveSceneCfg): ) +@configclass +class PickPlaceGR1T2SceneCfg(ObjectTableSceneCfg): + """GR1T2 pick-place scene with the camera observation shown in XR PiP.""" + + robot_pov_cam = CameraCfg( + prim_path="{ENV_REGEX_NS}/RobotPOVCam", + update_period=0.0, + height=450, + width=720, + data_types=["rgb"], + renderer_cfg=_RobotPovCameraRendererCfg(), + spawn=sim_utils.PinholeCameraCfg(focal_length=18.15, clipping_range=(0.1, 2.0)), + offset=CameraCfg.OffsetCfg( + pos=(0.0, 0.12, 1.67675), + rot=(0.9801, 0.0, 0.0, -0.19848), + convention="ros", + ), + ) + + ## # MDP settings ## @@ -515,6 +547,25 @@ def __post_init__(self): policy: PolicyCfg = PolicyCfg() +@configclass +class PickPlaceGR1T2ObservationsCfg(ObservationsCfg): + """GR1T2 pick-place observations including the camera shown in XR PiP.""" + + @configclass + class PolicyCfg(ObservationsCfg.PolicyCfg): + robot_pov_cam = ObsTerm( + func=base_mdp.image, + params={ + "sensor_cfg": SceneEntityCfg("robot_pov_cam"), + "data_type": "rgb", + "normalize": False, + "clone": False, + }, + ) + + policy: PolicyCfg = PolicyCfg() + + @configclass class TerminationsCfg: """Termination terms for the MDP.""" @@ -553,9 +604,9 @@ class PickPlaceGR1T2EnvCfg(ManagerBasedRLEnvCfg): """Configuration for the GR1T2 environment.""" # Scene settings - scene: ObjectTableSceneCfg = ObjectTableSceneCfg(num_envs=1, env_spacing=2.5, replicate_physics=True) + scene: PickPlaceGR1T2SceneCfg = PickPlaceGR1T2SceneCfg(num_envs=1, env_spacing=2.5, replicate_physics=True) # Basic settings - observations: ObservationsCfg = ObservationsCfg() + observations: PickPlaceGR1T2ObservationsCfg = PickPlaceGR1T2ObservationsCfg() actions: ActionsCfg = ActionsCfg() # MDP settings terminations: TerminationsCfg = TerminationsCfg() @@ -633,7 +684,17 @@ def __post_init__(self): pipeline_builder=lambda: _build_gr1t2_pickplace_pipeline()[0], sim_device=self.sim.device, xr_cfg=self.xr, + xr_camera_feeds=[ + XrCameraFeedCfg( + camera_name="robot_pov_cam", + # The 0.48 m-wide 720x450 image is 0.30 m tall; lower its + # center so its top edge starts at viewer eye height. + offset_m=(0.0, -0.15), + max_update_hz=0.0, + ) + ], ) + self.image_obs_list = ["robot_pov_cam"] # Per-finger haptic glove feedback: vibrate each finger of the operator's # glove in proportion to how tightly it grips the object. The session diff --git a/source/isaaclab_teleop/changelog.d/xr-camera-feedback.minor.rst b/source/isaaclab_teleop/changelog.d/xr-camera-feedback.minor.rst new file mode 100644 index 000000000000..cadf30ff54bd --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/xr-camera-feedback.minor.rst @@ -0,0 +1,19 @@ +Added +^^^^^ + +* Added XR camera feedback to ``teleop_se3_agent.py`` and ``record_demos.py``, with task-configured + existing-camera selection, declarative layouts, and viewer-start, head-locked, or explicit-world + placement. +* Added direct and staged CUDA image presentation paths so camera feedback works with CPU physics. + +Changed +^^^^^^^ + +* Changed Kit Scene UI loading to be lazy so kitless teleoperation warns and continues without PiP. +* Changed ``--disable_external_cameras`` into the master camera-rendering and PiP gate for + ``teleop_se3_agent.py`` and ``record_demos.py``. + +Fixed +^^^^^ + +* Fixed repeated IWER reset messages and refreshed camera feedback immediately after resets. diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index dff9aa5d41a2..79ef0d3a469f 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -19,6 +19,8 @@ XR anchor synchronization, retargeting pipelines, and action-tensor generation. modes for controlling how the anchor orientation tracks the reference prim. - **Retargeting tuning UI** -- optional ImGui window for real-time adjustment of retargeter parameters when `retargeters_to_tune` is provided. +- **XR camera feedback** -- selected scene cameras can be shown through Kit Scene UI using their + existing RGBA allocation. ## Architecture @@ -37,7 +39,11 @@ XR anchor synchronization, retargeting pipelines, and action-tensor generation. Add an `isaac_teleop` attribute to your environment config: ```python -from isaaclab_teleop import IsaacTeleopCfg, XrCfg +from isaaclab_teleop import ( + IsaacTeleopCfg, + XrCameraFeedCfg, + XrCfg, +) @configclass class MyEnvCfg(ManagerBasedRLEnvCfg): @@ -52,6 +58,9 @@ class MyEnvCfg(ManagerBasedRLEnvCfg): ), pipeline_builder=lambda: pipeline, retargeters_to_tune=lambda: retargeters, + xr_camera_feeds=[ + XrCameraFeedCfg(camera_name="robot_pov_cam"), + ], ) ``` @@ -86,9 +95,21 @@ The existing teleop scripts automatically detect `isaac_teleop` in the environme ```bash uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ - --task My-IsaacTeleop-Env-v0 + --task My-IsaacTeleop-Env-v0 --xr +``` + +`IsaacContrib-PickPlace-GR1T2-Abs` and +`IsaacContrib-PickPlace-Locomanipulation-G1-Abs` are camera-feedback reference tasks. Both show +their recorded `robot_pov_cam` observation in PiP: + +```bash +./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py \ + --task IsaacContrib-PickPlace-GR1T2-Abs --xr --device cpu ``` +See [XR Camera Feedback](../../../docs/source/features/isaac_teleop.rst#xr-camera-feedback) for +camera selection, layout, placement, renderer, disable, and kitless behavior. + ### 4. Programmatic Usage `IsaacTeleopDevice` supports Python's context-manager protocol: @@ -116,6 +137,8 @@ rendering without blocking. | Field | Type | Default | Description | |---|---|---|---| | `xr_cfg` | `XrCfg` | `XrCfg()` | XR anchor position, rotation, and dynamic-anchoring settings | +| `xr_camera_feeds` | `list[XrCameraFeedCfg]` | `[]` | Existing task scene cameras shown as image panels | +| `xr_camera_feed_layout` | `XrCameraFeedLayoutCfg` | viewer-start manual layout | Placement reference and ordered multi-feed packing | | `pipeline_builder` | `Callable[[], OutputCombiner]` | *required* | Builds the retargeting pipeline | | `retargeters_to_tune` | `Callable[[], list[BaseRetargeter]] \| None` | `None` | Retargeters to expose in the tuning UI | | `plugins` | `list[PluginConfig]` | `[]` | IsaacTeleop plugin configurations | @@ -143,9 +166,8 @@ contend for the GIL at the start of the step. ## Utilities -- **`remove_camera_configs(env_cfg)`** -- strips camera sensors and their associated observation - terms from an environment config. XR does not support additional cameras as they cause rendering - conflicts. +- **`remove_camera_configs(env_cfg)`** -- strips camera sensors and their associated policy + observation terms. ## Run with Docker diff --git a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi index f6838f1dde45..1e699694b4a6 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi +++ b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi @@ -19,6 +19,8 @@ __all__ = [ "SystemCheckItem", "SystemCheckResult", "TELEOP_CONTROL_CHANNEL_UUID", + "XrCameraFeedCfg", + "XrCameraFeedLayoutCfg", "XrAnchorRotationMode", "XrAnchorSynchronizer", "XrCfg", @@ -38,7 +40,14 @@ from .haptic_feedback import ( HapticFeedbackReceiver, create_haptic_feedback_driver, ) -from .isaac_teleop_cfg import CLOUDXR_AVP_ENV, CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV, IsaacTeleopCfg +from .isaac_teleop_cfg import ( + CLOUDXR_AVP_ENV, + CLOUDXR_JS_ENV, + CLOUDXR_STANDALONE_ENV, + IsaacTeleopCfg, + XrCameraFeedCfg, + XrCameraFeedLayoutCfg, +) from .isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device from .system_check import SystemCheckItem, SystemCheckResult, check_system_requirements from .xr_anchor_utils import XrAnchorSynchronizer diff --git a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py new file mode 100644 index 000000000000..6f80618bd3a3 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py @@ -0,0 +1,463 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Lifecycle management for camera image panels in XR.""" + +from __future__ import annotations + +import importlib +import logging +import math +import time +from collections import Counter +from copy import deepcopy +from dataclasses import dataclass +from functools import lru_cache +from typing import TYPE_CHECKING, Any + +import torch + +from .isaac_teleop_cfg import XrCameraFeedCfg, XrCameraFeedLayoutCfg + +if TYPE_CHECKING: + from isaaclab.sensors import Camera + +logger = logging.getLogger(__name__) + +_RESPONSIVE_DENOISING_MIN_ISAAC_SIM_VERSION = (6, 1) + + +@lru_cache(maxsize=1) +def _camera_type() -> type[Camera]: + """Load the concrete camera type only when feeds are presented.""" + from isaaclab.sensors import Camera + + return Camera + + +def _load_kit_scene_ui_presenter() -> Any | None: + """Load the optional Kit SceneUI presenter without making Kit a package dependency.""" + try: + module = importlib.import_module(".camera_feed_kit_scene_ui", __package__) + except (ImportError, ModuleNotFoundError) as exc: + logger.warning( + "XR camera PiP is unavailable because Kit SceneUI could not be loaded (%s: %s). Continuing without PiP.", + type(exc).__name__, + exc, + ) + return None + return module._KitSceneUiCameraFeedPresenter() + + +def _configure_ray_reconstruction_compatibility(camera_cfg: Any, camera_name: str) -> None: + """Fall back to classic DLSS on runtimes without responsive denoising.""" + renderer_cfg = getattr(camera_cfg, "renderer_cfg", None) + if getattr(renderer_cfg, "enable_dlss_ray_reconstruction", None) is not True: + return + + from isaaclab.utils.version import get_isaac_sim_version + + isaac_sim_version = get_isaac_sim_version() + if (isaac_sim_version.major, isaac_sim_version.minor) < _RESPONSIVE_DENOISING_MIN_ISAAC_SIM_VERSION: + renderer_cfg.enable_dlss_ray_reconstruction = False + logger.warning( + "XR camera feed %r requested DLSS Ray Reconstruction, but Isaac Sim %s predates responsive " + "denoising. Falling back to classic DLSS.", + camera_name, + isaac_sim_version, + ) + + +def _prepare_camera_feed_cfgs(env_cfg: Any, cfgs: list[XrCameraFeedCfg]) -> list[XrCameraFeedCfg]: + """Validate selected scene cameras.""" + from isaaclab.sensors import CameraCfg + + prepared: list[XrCameraFeedCfg] = [] + enabled_cfgs = deepcopy(cfgs) + duplicates = sorted(name for name, count in Counter(cfg.camera_name for cfg in enabled_cfgs).items() if count > 1) + if duplicates: + raise ValueError(f"XR camera feeds must have unique camera names. Duplicates: {duplicates}.") + + scene = getattr(env_cfg, "scene", None) + if scene is None: + raise ValueError("XR camera feeds require an environment configuration with a scene.") + for cfg in enabled_cfgs: + camera_cfg = getattr(scene, cfg.camera_name, None) + if camera_cfg is None: + raise ValueError(f"XR camera feed {cfg.camera_name!r} is not present in the scene.") + if not isinstance(camera_cfg, CameraCfg): + raise TypeError(f"XR camera feed {cfg.camera_name!r} does not reference a CameraCfg.") + if not any(data_type in {"rgb", "rgba"} for data_type in camera_cfg.data_types): + raise ValueError(f"XR camera feed {cfg.camera_name!r} camera must provide RGB or RGBA output.") + _configure_ray_reconstruction_compatibility(camera_cfg, cfg.camera_name) + prepared.append(cfg) + return prepared + + +def _feeds_require_responsive_denoising(env_cfg: Any, cfgs: list[XrCameraFeedCfg]) -> bool: + scene = env_cfg.scene + return any( + getattr(getattr(getattr(scene, cfg.camera_name), "renderer_cfg", None), "enable_dlss_ray_reconstruction", None) + is True + for cfg in cfgs + ) + + +class _XrCameraFeedSession: + """Internal two-phase lifecycle shared by the teleoperation entry points.""" + + def __init__( + self, + cfgs: list[XrCameraFeedCfg], + layout_cfg: XrCameraFeedLayoutCfg | None, + presenter: Any | None, + *, + requires_responsive_denoising: bool, + ): + self._cfgs = cfgs + self._layout_cfg = layout_cfg + self._presenter = presenter + self._requires_responsive_denoising = requires_responsive_denoising + self._env = None + self._manager = None + self._bound = False + + @classmethod + def prepare( + cls, + env_cfg: Any, + *, + enabled: bool, + camera_rendering_enabled: bool, + ) -> _XrCameraFeedSession: + """Prepare task-configured camera feeds before constructing the environment.""" + if type(enabled) is not bool or type(camera_rendering_enabled) is not bool: + raise TypeError("enabled and camera_rendering_enabled must be bool values.") + teleop_cfg = getattr(env_cfg, "isaac_teleop", None) + if not enabled or teleop_cfg is None: + return cls([], None, None, requires_responsive_denoising=False) + + requested = [cfg for cfg in teleop_cfg.xr_camera_feeds if cfg.enabled] + if not camera_rendering_enabled: + if requested: + logger.warning("XR camera PiP is disabled because external camera rendering is disabled.") + return cls([], None, None, requires_responsive_denoising=False) + if not requested: + return cls([], teleop_cfg.xr_camera_feed_layout, None, requires_responsive_denoising=False) + + _validate_layout_cfg(teleop_cfg.xr_camera_feed_layout) + presenter = _load_kit_scene_ui_presenter() + if presenter is None: + return cls([], teleop_cfg.xr_camera_feed_layout, None, requires_responsive_denoising=False) + if int(env_cfg.scene.num_envs) != 1: + raise ValueError("XR camera PiP supports exactly one environment; set --num_envs 1 or disable PiP feeds.") + cfgs = _prepare_camera_feed_cfgs(env_cfg, requested) + return cls( + cfgs, + teleop_cfg.xr_camera_feed_layout, + presenter, + requires_responsive_denoising=_feeds_require_responsive_denoising(env_cfg, cfgs), + ) + + @property + def enabled(self) -> bool: + return bool(self._cfgs) + + @property + def requires_responsive_denoising(self) -> bool: + return self._requires_responsive_denoising + + def bind(self, env: Any) -> _XrCameraFeedSession: + if self._bound: + raise RuntimeError("XR camera feed session is already bound.") + self._env = env + self._bound = True + try: + if self.enabled: + self._manager = _XrCameraFeedManager(env, self._cfgs, self._layout_cfg, self._presenter) + except Exception: + self.close() + raise + return self + + def refresh(self) -> None: + if not self._bound: + raise RuntimeError("XR camera feed session must be bound before refresh.") + if self._manager is not None: + # Request post-reset camera output before each render. Temporal + # RTX cameras need several frames to replace their pre-reset + # annotator contents, matching Isaac Lab's reset rerender path. + for _ in range(3): + self._manager.refresh(publish=False) + self._env.sim.render() + self._manager.refresh() + + def close(self) -> None: + if self._manager is not None: + self._manager.close() + self._manager = None + self._env = None + self._bound = False + + def __enter__(self) -> _XrCameraFeedSession: + if not self._bound: + raise RuntimeError("Call bind(env) before entering an XR camera feed session.") + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.close() + + +def _panel_size_m(cfg: XrCameraFeedCfg, image_size: tuple[int, int]) -> tuple[float, float]: + width, height = image_size + if width <= 0 or height <= 0: + raise ValueError(f"XR camera feed image dimensions must be positive, got {image_size}.") + return cfg.panel_width_m, cfg.panel_width_m * height / width + (0.04 if cfg.label else 0.0) + + +def _centered_positions(sizes: list[float], gap: float) -> list[float]: + cursor = -0.5 * (sum(sizes) + gap * max(0, len(sizes) - 1)) + positions = [] + for size in sizes: + positions.append(cursor + 0.5 * size) + cursor += size + gap + return positions + + +def _validate_layout_cfg(layout_cfg: XrCameraFeedLayoutCfg) -> None: + if layout_cfg.mode not in {"manual", "horizontal", "vertical", "grid"}: + raise ValueError(f"Unknown XR camera feed layout mode {layout_cfg.mode!r}.") + if layout_cfg.placement not in {"viewer_start", "head_locked", "world"}: + raise ValueError(f"Unknown XR camera feed placement {layout_cfg.placement!r}.") + if layout_cfg.placement != "world" and (not math.isfinite(layout_cfg.distance_m) or layout_cfg.distance_m <= 0.0): + raise ValueError("XR camera feed layout distance_m must be finite and positive.") + if not math.isfinite(layout_cfg.panel_gap_m) or layout_cfg.panel_gap_m < 0.0: + raise ValueError("XR camera feed layout panel_gap_m must be finite and non-negative.") + if len(layout_cfg.center_offset_m) != 2 or not all(math.isfinite(value) for value in layout_cfg.center_offset_m): + raise ValueError("XR camera feed layout center_offset_m must contain two finite values.") + if type(layout_cfg.max_columns) is not int or layout_cfg.max_columns <= 0: + raise ValueError("XR camera feed layout max_columns must be positive.") + if layout_cfg.placement == "world": + position = layout_cfg.world_position_m + if position is None or len(position) != 3 or not all(math.isfinite(value) for value in position): + raise ValueError("XR camera feed layout world_position_m must contain three finite values.") + orientation = layout_cfg.world_orientation_xyzw + if len(orientation) != 4 or not all(math.isfinite(value) for value in orientation): + raise ValueError("XR camera feed layout world_orientation_xyzw must contain four finite values.") + if math.sqrt(sum(value * value for value in orientation)) <= 1.0e-8: + raise ValueError("XR camera feed layout world_orientation_xyzw must be non-zero.") + + +def _layout_feed_cfgs( + cfgs: list[XrCameraFeedCfg], + image_sizes: list[tuple[int, int]], + layout_cfg: XrCameraFeedLayoutCfg, +) -> list[XrCameraFeedCfg]: + if len(cfgs) != len(image_sizes): + raise ValueError("XR camera feed configs and image sizes must have the same length.") + _validate_layout_cfg(layout_cfg) + resolved = deepcopy(cfgs) + if layout_cfg.mode == "manual" or not resolved: + return resolved + + panel_sizes = [_panel_size_m(cfg, size) for cfg, size in zip(resolved, image_sizes, strict=True)] + center_x, center_y = layout_cfg.center_offset_m + if layout_cfg.mode == "horizontal": + xs = _centered_positions([width for width, _ in panel_sizes], layout_cfg.panel_gap_m) + offsets = [(center_x + x, center_y) for x in xs] + elif layout_cfg.mode == "vertical": + ys = _centered_positions([height for _, height in panel_sizes], layout_cfg.panel_gap_m) + offsets = [(center_x, center_y - y) for y in ys] + else: + columns = min(layout_cfg.max_columns, len(resolved)) + rows = [list(range(start, min(start + columns, len(resolved)))) for start in range(0, len(resolved), columns)] + row_heights = [max(panel_sizes[index][1] for index in row) for row in rows] + row_ys = _centered_positions(row_heights, layout_cfg.panel_gap_m) + offsets = [(0.0, 0.0)] * len(resolved) + for row, row_y in zip(rows, row_ys, strict=True): + row_xs = _centered_positions([panel_sizes[index][0] for index in row], layout_cfg.panel_gap_m) + for index, row_x in zip(row, row_xs, strict=True): + offsets[index] = (center_x + row_x, center_y - row_y) + + for cfg, offset in zip(resolved, offsets, strict=True): + cfg.offset_m = offset + cfg.distance_m = layout_cfg.distance_m + return resolved + + +@dataclass(frozen=True) +class _PanelDescriptor: + label: str | None + width_m: float + offset_m: tuple[float, float] + distance_m: float + placement: str + world_position_m: tuple[float, float, float] | None + world_orientation_xyzw: tuple[float, float, float, float] + + +def _panel_descriptor(cfg: XrCameraFeedCfg, layout_cfg: XrCameraFeedLayoutCfg) -> _PanelDescriptor: + return _PanelDescriptor( + label=cfg.label, + width_m=cfg.panel_width_m, + offset_m=tuple(cfg.offset_m), + distance_m=cfg.distance_m, + placement=layout_cfg.placement, + world_position_m=None if layout_cfg.world_position_m is None else tuple(layout_cfg.world_position_m), + world_orientation_xyzw=tuple(layout_cfg.world_orientation_xyzw), + ) + + +@dataclass +class _ActiveFeed: + cfg: XrCameraFeedCfg + camera: Camera + image: torch.Tensor + upload_image: torch.Tensor + panel: Any + next_update_time: float = 0.0 + + +class _XrCameraFeedManager: + """Bind camera RGBA buffers to persistent Kit SceneUI panels.""" + + def __init__( + self, + env: Any, + cfgs: list[XrCameraFeedCfg], + layout_cfg: XrCameraFeedLayoutCfg | None, + presenter: Any, + ): + self._env = env + self._feeds: list[_ActiveFeed] = [] + self._presenter = presenter + self._layout_cfg = deepcopy(layout_cfg or XrCameraFeedLayoutCfg()) + self._frame_subscription = None + _validate_layout_cfg(self._layout_cfg) + + try: + bound_feeds = [] + for cfg in cfgs: + self._validate_cfg(cfg, self._layout_cfg.placement) + camera, image = self._bind_image(cfg) + upload_image = self._presenter.prepare_upload_image(cfg.camera_name, image) + bound_feeds.append((camera, image, upload_image)) + image_sizes = [(int(image.shape[1]), int(image.shape[0])) for _, image, _ in bound_feeds] + resolved_cfgs = _layout_feed_cfgs(cfgs, image_sizes, self._layout_cfg) + for cfg, (camera, image, upload_image) in zip(resolved_cfgs, bound_feeds, strict=True): + panel = self._presenter.create_panel( + _panel_descriptor(cfg, self._layout_cfg), + width=int(image.shape[1]), + height=int(image.shape[0]), + ) + self._feeds.append(_ActiveFeed(cfg, camera, image, upload_image, panel)) + self._frame_subscription = self._presenter.subscribe_to_frame_updates(self._on_frame) + except Exception: + self.close() + raise + + def _on_frame(self, _event: Any) -> None: + self.update() + + @staticmethod + def _validate_cfg(cfg: XrCameraFeedCfg, placement: str) -> None: + if not math.isfinite(cfg.panel_width_m) or cfg.panel_width_m <= 0.0: + raise ValueError(f"panel_width_m for XR camera feed {cfg.camera_name!r} must be positive.") + if len(cfg.offset_m) != 2 or not all(math.isfinite(value) for value in cfg.offset_m): + raise ValueError(f"offset_m for XR camera feed {cfg.camera_name!r} must contain two finite values.") + if placement != "world" and (not math.isfinite(cfg.distance_m) or cfg.distance_m <= 0.0): + raise ValueError(f"distance_m for XR camera feed {cfg.camera_name!r} must be positive.") + if not math.isfinite(cfg.max_update_hz) or cfg.max_update_hz < 0.0: + raise ValueError(f"max_update_hz for XR camera feed {cfg.camera_name!r} must be finite and non-negative.") + + def _bind_image(self, cfg: XrCameraFeedCfg) -> tuple[Camera, torch.Tensor]: + sensors = getattr(getattr(self._env, "scene", None), "sensors", {}) + if cfg.camera_name not in sensors: + raise ValueError( + f"XR camera feed {cfg.camera_name!r} is not present in the interactive scene. " + f"Available sensors: {sorted(sensors.keys())}." + ) + camera = sensors[cfg.camera_name] + if not isinstance(camera, _camera_type()): + raise TypeError(f"XR camera feed {cfg.camera_name!r} did not resolve to an Isaac Lab Camera.") + return camera, self._image_from_output(cfg, camera.data.output) + + def _image_from_output(self, cfg: XrCameraFeedCfg, output: Any) -> torch.Tensor: + if output is None or "rgba" not in output: + available = [] if output is None else sorted(output.keys()) + raise ValueError(f"Camera {cfg.camera_name!r} has no RGBA buffer. Available outputs: {available}.") + batch = output["rgba"].torch + if batch.ndim != 4 or int(batch.shape[-1]) != 4: + raise ValueError(f"Camera {cfg.camera_name!r} RGBA output must have shape (N, H, W, 4).") + image = batch[0] + if image.dtype != torch.uint8: + raise TypeError(f"Camera {cfg.camera_name!r} RGBA buffer must be uint8, got {image.dtype}.") + return image + + @staticmethod + def _same_allocation(first: torch.Tensor, second: torch.Tensor) -> bool: + return ( + tuple(first.shape) == tuple(second.shape) + and first.device == second.device + and first.data_ptr() == second.data_ptr() + ) + + def _rebind_feed(self, feed: _ActiveFeed, camera: Camera, image: torch.Tensor) -> None: + if self._same_allocation(feed.image, image): + feed.camera = camera + return + old_shape = tuple(feed.image.shape) + feed.upload_image = self._presenter.prepare_upload_image( + feed.cfg.camera_name, + image, + previous_source=feed.image, + previous_upload=feed.upload_image, + ) + if tuple(image.shape) != old_shape: + replacement = self._presenter.create_panel( + _panel_descriptor(feed.cfg, self._layout_cfg), + width=int(image.shape[1]), + height=int(image.shape[0]), + ) + old_panel = feed.panel + feed.panel = replacement + old_panel.close() + feed.camera = camera + feed.image = image + + def update(self) -> None: + now = time.monotonic() + for feed in self._feeds: + if now < feed.next_update_time: + continue + image = self._image_from_output(feed.cfg, feed.camera.data.output) + self._rebind_feed(feed, feed.camera, image) + self._publish_feed(feed) + period = 0.0 if feed.cfg.max_update_hz == 0.0 else 1.0 / feed.cfg.max_update_hz + feed.next_update_time = now + period + + def _publish_feed(self, feed: _ActiveFeed) -> None: + self._presenter.stage_upload_image(feed.image, feed.upload_image) + feed.panel.upload(feed.upload_image) + + def refresh(self, *, publish: bool = True) -> None: + for feed in self._feeds: + feed.camera.update(0.0, force_recompute=True) + camera, image = self._bind_image(feed.cfg) + self._rebind_feed(feed, camera, image) + if publish: + self._publish_feed(feed) + feed.next_update_time = 0.0 + + def close(self) -> None: + if self._frame_subscription is not None: + self._frame_subscription.close() + self._frame_subscription = None + for feed in reversed(self._feeds): + try: + feed.panel.close() + except Exception: + logger.exception("Failed to close XR camera feed %r.", feed.cfg.camera_name) + self._feeds.clear() diff --git a/source/isaaclab_teleop/isaaclab_teleop/camera_feed_kit_scene_ui.py b/source/isaaclab_teleop/isaaclab_teleop/camera_feed_kit_scene_ui.py new file mode 100644 index 000000000000..c581b12ec384 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/camera_feed_kit_scene_ui.py @@ -0,0 +1,426 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Kit Scene UI presentation for one XR camera feed.""" + +from __future__ import annotations + +import logging +import math +from collections.abc import Callable +from contextlib import suppress +from typing import Any + +import torch + +import omni.gpu_foundation_factory as gf +import omni.ui as ui +from omni.kit.scene_view.xr import XRSceneView +from omni.kit.scene_view.xr_utils import SpatialSource, UiContainer, UpdatePolicy, WidgetComponent +from omni.kit.xr.core import XRCore, XRCoreEventType, XRPoseValidityFlags +from pxr import Gf + +logger = logging.getLogger(__name__) + + +def _meters_per_unit(coordinate_system: Any, name: str) -> float: + """Return a validated coordinate-system scale.""" + value = float(coordinate_system.meters_per_unit) + if not math.isfinite(value) or value <= 0.0: + raise RuntimeError(f"{name} meters_per_unit must be finite and positive, got {value}.") + return value + + +class KitSceneUiViewerStartAnchor: + """Share one frozen viewer-start pose across Kit Scene UI panels.""" + + def __init__(self, xr_core: Any | None = None): + self._xr_core = xr_core or XRCore.get_singleton() + self._registrations: dict[int, tuple[Any, tuple[float, float], float, Callable[[bool], None]]] = {} + self._upright_pose = None + self._post_sync_subscription = None + self._display_disabled_subscription = None + + @property + def captured(self) -> bool: + """Whether a valid viewer-start pose has been frozen.""" + return self._upright_pose is not None + + @property + def xr_core(self) -> Any: + """XR core used to resolve this anchor.""" + return self._xr_core + + def register( + self, + source: Any, + offset_m: tuple[float, float], + distance_m: float, + readiness_callback: Callable[[bool], None], + ) -> int: + """Register a panel source and return its lifecycle token.""" + token = id(source) + self._registrations[token] = (source, offset_m, distance_m, readiness_callback) + try: + self._ensure_subscriptions() + if self._upright_pose is not None: + self._apply_registration(self._registrations[token]) + readiness_callback(True) + else: + readiness_callback(False) + except Exception: + self.unregister(token) + raise + return token + + def unregister(self, token: int) -> None: + """Release one panel and subscriptions after the final panel closes.""" + self._registrations.pop(token, None) + if self._registrations: + return + self._post_sync_subscription = None + self._display_disabled_subscription = None + self._upright_pose = None + + def _ensure_subscriptions(self) -> None: + if self._post_sync_subscription is not None: + return + message_bus = self._xr_core.get_message_bus() + self._post_sync_subscription = message_bus.create_subscription_to_pop_by_type( + XRCoreEventType.post_sync_update, + self._on_post_sync_update, + name="Isaac Lab XR camera PiP viewer-start capture", + ) + self._display_disabled_subscription = message_bus.create_subscription_to_pop_by_type( + XRCoreEventType.xr_display_disabled, + self._on_display_disabled, + name="Isaac Lab XR camera PiP viewer-start reset", + ) + + def _on_post_sync_update(self, _event: Any) -> None: + self._try_capture() + + def _on_display_disabled(self, _event: Any) -> None: + """Hide panels and capture a new starting pose after reconnect.""" + self._upright_pose = None + for _, _, _, callback in tuple(self._registrations.values()): + callback(False) + + def _try_capture(self) -> bool: + if self._upright_pose is not None or not self._registrations: + return self._upright_pose is not None + if not self._xr_core.is_xr_display_enabled(): + return False + input_device = self._xr_core.get_input_device("displayDevice") + if input_device is None: + input_device = self._xr_core.get_input_device("/user/head") + if input_device is None: + return False + pose_desc = input_device.get_virtual_world_pose_desc("") + if pose_desc is None: + return False + required_flags = XRPoseValidityFlags.POSITION_VALID | XRPoseValidityFlags.ORIENTATION_VALID + if pose_desc.validity_flags & required_flags != required_flags: + return False + + coordinate_system = self._xr_core.get_coordinate_system() + self._upright_pose = self._xr_core.reorient_transform_matrix_up_right( + pose_desc.pose_matrix, + coordinate_system.up_axis == "y", + ) + for registration in tuple(self._registrations.values()): + self._apply_registration(registration) + registration[3](True) + logger.info( + "XR camera PiP viewer-start anchor captured at %s (validity_flags=%s).", + tuple(float(value) for value in self._upright_pose.ExtractTranslation()), + pose_desc.validity_flags, + ) + return True + + def _apply_registration( + self, + registration: tuple[Any, tuple[float, float], float, Callable[[bool], None]], + ) -> None: + source, offset_m, distance_m, _ = registration + coordinate_system = self._xr_core.get_coordinate_system() + meters_per_unit = _meters_per_unit(coordinate_system, "XR coordinate-system") + # XR poses use local +X right, +Y up, and -Z forward. Apply the + # panel offset in that local frame before the frozen pose maps it into + # the stage's Y-up or Z-up world. + offset = Gf.Vec3d(offset_m[0], offset_m[1], -distance_m) / meters_per_unit + panel_world = Gf.Matrix4d().SetTranslate(offset) * self._upright_pose + source.source = SpatialSource.new_transform_matrix_source(panel_world).source + + +def _world_panel_matrix( + descriptor: Any, + meters_per_unit: float, +) -> Gf.Matrix4d: + """Build one fixed panel transform from the explicit world layout pose.""" + if not math.isfinite(meters_per_unit) or meters_per_unit <= 0.0: + raise RuntimeError(f"Stage meters_per_unit must be finite and positive, got {meters_per_unit}.") + if descriptor.world_position_m is None: + raise ValueError("world_position_m is required for world XR camera-feed placement.") + x, y, z, w = descriptor.world_orientation_xyzw + norm = math.sqrt(x * x + y * y + z * z + w * w) + if not math.isfinite(norm) or norm <= 1.0e-8: + raise ValueError("world_orientation_xyzw must be a finite, non-zero quaternion.") + rotation = Gf.Quatd(w / norm, Gf.Vec3d(x / norm, y / norm, z / norm)) + anchor = Gf.Matrix4d(1.0).SetRotate(rotation) + anchor.SetTranslateOnly(Gf.Vec3d(*(value / meters_per_unit for value in descriptor.world_position_m))) + local_offset = Gf.Matrix4d().SetTranslate( + Gf.Vec3d( + descriptor.offset_m[0] / meters_per_unit, + descriptor.offset_m[1] / meters_per_unit, + 0.0, + ) + ) + return local_offset * anchor + + +class _CameraImageWidget(ui.Widget): + """Omni UI image surface backed by a Kit byte-image provider.""" + + def __init__(self, provider: ui.ByteImageProvider, label: str | None = None, **kwargs): + super().__init__(**kwargs) + with ui.ZStack(): + ui.Rectangle( + style={ + "Rectangle": { + "background_color": 0xFF101214, + "border_color": 0xFF73787E, + "border_width": 1, + "border_radius": 2, + } + } + ) + with ui.VStack(spacing=0): + if label: + ui.Label( + label, + height=22, + alignment=ui.Alignment.CENTER, + style={"Label": {"color": 0xFFE8EAED, "font_size": 14}}, + ) + ui.ImageWithProvider( + provider, + fill_policy=ui.IwpFillPolicy.IWP_PRESERVE_ASPECT_FIT, + ) + + +class KitSceneUiCameraFeedPanel: + """Persistent Kit Scene UI panel for one RGBA camera.""" + + def __init__( + self, + descriptor: Any, + image_width: int, + image_height: int, + viewer_start_anchor: KitSceneUiViewerStartAnchor | None = None, + ): + """Create the image provider and attach its panel to the XR scene.""" + self._closed = False + self._provider = None + self._component = None + self._container = None + self._viewer_start_anchor = viewer_start_anchor + self._viewer_start_registration = None + + if descriptor.placement == "viewer_start": + if viewer_start_anchor is None: + raise ValueError("viewer_start placement requires a shared viewer-start anchor.") + xr_core = viewer_start_anchor.xr_core + coordinate_system = xr_core.get_coordinate_system() + coordinate_system_name = "XR coordinate-system" + else: + xr_core = XRCore.get_singleton() + coordinate_system = ( + xr_core.get_stage_coordinate_system() + if descriptor.placement == "world" + else xr_core.get_coordinate_system() + ) + coordinate_system_name = "Stage" if descriptor.placement == "world" else "XR coordinate-system" + meters_per_unit = _meters_per_unit(coordinate_system, coordinate_system_name) + image_height_m = descriptor.width_m * image_height / image_width + label_height_m = 0.04 if descriptor.label else 0.0 + panel_width_units = descriptor.width_m / meters_per_unit + panel_height_units = (image_height_m + label_height_m) / meters_per_unit + resolution_scale = max(image_width / descriptor.width_m, image_height / image_height_m) + if descriptor.placement == "viewer_start": + anchor_source = SpatialSource.new_transform_matrix_source(Gf.Matrix4d(1.0)) + space_stack = [anchor_source] + elif descriptor.placement == "head_locked": + anchor_source = None + space_stack = [ + SpatialSource.new_prim_path_source("/_xr/stage/xrCamera"), + SpatialSource.new_translation_source( + Gf.Vec3f( + float(descriptor.offset_m[0] / meters_per_unit), + float(descriptor.offset_m[1] / meters_per_unit), + -float(descriptor.distance_m / meters_per_unit), + ) + ), + SpatialSource.new_look_at_camera_source(), + ] + elif descriptor.placement == "world": + anchor_source = None + space_stack = [ + SpatialSource.new_transform_matrix_source( + _world_panel_matrix(descriptor, meters_per_unit), + ) + ] + else: + raise ValueError(f"Unknown XR camera-feed placement {descriptor.placement!r}.") + try: + self._provider = ui.ByteImageProvider() + self._component = WidgetComponent( + _CameraImageWidget, + width=panel_width_units, + height=panel_height_units, + resolution_scale=resolution_scale, + unit_to_pixel_scale=meters_per_unit, + update_policy=UpdatePolicy.ALWAYS, + widget_args=[self._provider, descriptor.label], + ) + self._container = UiContainer( + XRSceneView, + self._component, + space_stack=space_stack, + ) + if descriptor.placement == "viewer_start": + self._container.hide() + self._viewer_start_registration = viewer_start_anchor.register( + anchor_source, + descriptor.offset_m, + descriptor.distance_m, + self._on_viewer_start_readiness_changed, + ) + except Exception: + with suppress(Exception): + self.close() + raise + + def _on_viewer_start_readiness_changed(self, ready: bool) -> None: + if self._container is None: + return + if ready: + self._container.show() + else: + self._container.hide() + + def upload(self, image: torch.Tensor) -> None: + """Upload a contiguous cuda:0 RGBA tensor through Kit's GPU copy path.""" + if self._closed: + return + self._provider.set_bytes_data_from_gpu( + int(image.data_ptr()), + [int(image.shape[1]), int(image.shape[0])], + gf.TextureFormat.RGBA8_UNORM, + ) + + def close(self) -> None: + """Release the XR scene hierarchy and provider references.""" + if self._closed: + return + self._closed = True + if self._viewer_start_anchor is not None and self._viewer_start_registration is not None: + self._viewer_start_anchor.unregister(self._viewer_start_registration) + self._viewer_start_registration = None + self._viewer_start_anchor = None + container = self._container + self._component = None + self._container = None + self._provider = None + if container is not None: + try: + container.hide() + finally: + container.root.clear() + + +class _KitSceneUiCameraFeedPresenter: + """Private adapter from camera buffers to Kit SceneUI panels.""" + + def __init__(self): + self._viewer_start_anchor = None + + @staticmethod + def _validate_image(camera_name: str, image: torch.Tensor) -> None: + if image.device.type not in {"cpu", "cuda"}: + raise ValueError(f"Camera {camera_name!r} RGBA image must reside on CPU or cuda:0, got {image.device}.") + if image.device.type == "cuda" and image.device.index != 0: + raise ValueError(f"Camera {camera_name!r} RGBA image must reside on cuda:0, got {image.device}.") + if not image.is_contiguous(): + raise ValueError(f"Camera {camera_name!r} RGBA image must be contiguous.") + + def prepare_upload_image( + self, + camera_name: str, + image: torch.Tensor, + previous_source: torch.Tensor | None = None, + previous_upload: torch.Tensor | None = None, + ) -> torch.Tensor: + """Return direct CUDA storage or a persistent cuda:0 staging buffer.""" + self._validate_image(camera_name, image) + if image.device.type == "cuda": + return image + if not torch.cuda.is_available(): + raise RuntimeError(f"Camera {camera_name!r} uses CPU pixels, but cuda:0 is unavailable for XR upload.") + if ( + previous_source is not None + and previous_upload is not None + and previous_upload is not previous_source + and tuple(previous_upload.shape) == tuple(image.shape) + and previous_upload.dtype == image.dtype + and previous_upload.device == torch.device("cuda:0") + ): + return previous_upload + logger.warning( + "XR camera feed %r is using a CPU buffer and requires a CPU-to-GPU staging copy each frame.", + camera_name, + ) + return torch.empty_like(image, device="cuda:0", memory_format=torch.contiguous_format) + + def create_panel(self, descriptor: Any, width: int, height: int) -> KitSceneUiCameraFeedPanel: + viewer_start_anchor = None + if descriptor.placement == "viewer_start": + if self._viewer_start_anchor is None: + self._viewer_start_anchor = KitSceneUiViewerStartAnchor() + viewer_start_anchor = self._viewer_start_anchor + return KitSceneUiCameraFeedPanel( + descriptor=descriptor, + image_width=width, + image_height=height, + viewer_start_anchor=viewer_start_anchor, + ) + + @staticmethod + def stage_upload_image(image: torch.Tensor, upload_image: torch.Tensor) -> None: + if upload_image is not image: + upload_image.copy_(image, non_blocking=False) + + @staticmethod + def subscribe_to_frame_updates(callback: Callable[[Any], None]) -> _KitFrameSubscription: + import carb.eventdispatcher + import omni.kit.app + + observer = carb.eventdispatcher.get_eventdispatcher().observe_event( + order=omni.kit.app.POST_UPDATE_ORDER_PYTHON_EXEC, + event_name=omni.kit.app.GLOBAL_EVENT_POST_UPDATE, + on_event=callback, + observer_name="Isaac Lab XR camera PiP frame update", + ) + return _KitFrameSubscription(observer) + + +class _KitFrameSubscription: + def __init__(self, observer: Any): + self._observer = observer + + def close(self) -> None: + if self._observer is not None: + self._observer.reset() + self._observer = None diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py index 92618f312515..5bd963075925 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py @@ -10,7 +10,7 @@ from collections.abc import Callable from dataclasses import MISSING, field from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal from isaaclab.utils.configclass import configclass @@ -39,6 +39,84 @@ from isaacteleop.teleop_session_manager import PluginConfig, RetargetingExecutionConfig +@configclass +class XrCameraFeedCfg: + """Configuration for one camera image panel shown in XR.""" + + camera_name: str = MISSING + """Name of the :class:`~isaaclab.sensors.Camera` in the interactive scene.""" + + enabled: bool = True + """Whether to create and update this feed.""" + + panel_width_m: float = 0.48 + """Physical panel width [m].""" + + distance_m: float = 0.8 + """Distance in front of the viewer anchor [m]. + + This value is unused when :attr:`XrCameraFeedLayoutCfg.placement` is + ``"world"``. + """ + + offset_m: tuple[float, float] = (0.0, 0.0) + """Horizontal and vertical panel offset in the selected placement frame [m].""" + + max_update_hz: float = 30.0 + """Maximum provider upload rate [Hz]. Set to zero to update after every rendered frame.""" + + label: str | None = None + """Optional short label shown above the image.""" + + +@configclass +class XrCameraFeedLayoutCfg: + """Declarative placement and packing for enabled XR camera feeds.""" + + mode: Literal["manual", "horizontal", "vertical", "grid"] = "manual" + """Layout mode. Manual preserves each feed's offset and distance.""" + + placement: Literal["viewer_start", "head_locked", "world"] = "viewer_start" + """Reference frame used to place the panels. + + ``"viewer_start"`` captures the first valid viewer eye position and yaw, + then leaves the panels fixed in the world. ``"head_locked"`` follows the + viewer with full pose. ``"world"`` uses :attr:`world_position_m` and + :attr:`world_orientation_xyzw` as a fixed pose in the Isaac Lab USD stage + world. + """ + + center_offset_m: tuple[float, float] = (0.0, 0.0) + """Horizontal and vertical center of an automatic layout [m].""" + + distance_m: float = 0.8 + """Distance of every automatically placed panel from the viewer anchor [m]. + + This value is unused when :attr:`placement` is ``"world"``. + """ + + panel_gap_m: float = 0.04 + """Edge-to-edge gap between automatically placed panels [m].""" + + max_columns: int = 2 + """Maximum number of columns in grid mode.""" + + world_position_m: tuple[float, float, float] | None = None + """Layout-plane center in the Isaac Lab USD stage world [m]. + + Isaac Lab stages are Z-up. This value is required when :attr:`placement` + is ``"world"``. + """ + + world_orientation_xyzw: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 1.0) + """Panel-local-to-world orientation as a quaternion in ``xyzw`` order. + + Panel local +X is image right, +Y is image up, and +Z points from the + panel's readable side toward the viewer. Feed and layout offsets are + applied in the resulting local XY plane. + """ + + @configclass class IsaacTeleopCfg: """Configuration for IsaacTeleop-based teleoperation. @@ -82,6 +160,15 @@ def build_pipeline(): to follow a prim (e.g., robot base) during locomotion tasks. """ + xr_camera_feeds: list[XrCameraFeedCfg] = field(default_factory=list) + """Existing task camera outputs to show as XR image panels. + + The default empty list disables PiP. + """ + + xr_camera_feed_layout: XrCameraFeedLayoutCfg = field(default_factory=XrCameraFeedLayoutCfg) + """Placement and packing applied to the ordered enabled camera feeds.""" + pipeline_builder: Callable[[], OutputCombiner] = MISSING """Callable that builds the IsaacTeleop retargeting pipeline. diff --git a/source/isaaclab_teleop/isaaclab_teleop/teleop_message_processor.py b/source/isaaclab_teleop/isaaclab_teleop/teleop_message_processor.py index 15bd7936a08e..18bbc6893597 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/teleop_message_processor.py +++ b/source/isaaclab_teleop/isaaclab_teleop/teleop_message_processor.py @@ -106,6 +106,7 @@ def __init__(self, name: str) -> None: self._run_toggle_queue: list[bool] = [] self._prev_toggle_output = False self._pending_commands: list[str] = [] + self._channel_reset_latched = False super().__init__(name=name) def inject_reset(self, pause: bool = False) -> None: @@ -208,9 +209,12 @@ def _compute_fn( self._inject_reset_pending = False self._inject_reset_pause = False - # Parse incoming messages and enqueue toggle sequences. + # Parse incoming messages and enqueue toggle sequences. Some clients + # repeat a reset command while its control is held, so treat a + # continuous run of reset-bearing frames as one operator action. messages_tracked = inputs[self.INPUT_MESSAGES][0] data = getattr(messages_tracked, "data", None) + channel_reset_seen = False if data: for message in data: payload = getattr(message, "payload", None) @@ -225,9 +229,17 @@ def _compute_fn( if command is None: continue - if self._apply_command_kind(_classify_command(command)): + command_kind = _classify_command(command) + if command_kind == "reset": + channel_reset_seen = True + if self._channel_reset_latched: + continue + self._channel_reset_latched = True + if self._apply_command_kind(command_kind): reset = True pause_on_reset = True + if not channel_reset_seen: + self._channel_reset_latched = False # Apply locally injected commands (e.g. keyboard bindings) identically to # channel messages, so a host can start/stop/reset without an XR client. diff --git a/source/isaaclab_teleop/test/test_control_events.py b/source/isaaclab_teleop/test/test_control_events.py index dc0ddfac699c..3002beea3dd9 100644 --- a/source/isaaclab_teleop/test/test_control_events.py +++ b/source/isaaclab_teleop/test/test_control_events.py @@ -220,6 +220,15 @@ def test_reset_clears_on_next_step(self): result = _step(proc, _empty_tracked()) assert result["reset"] is False + def test_repeated_reset_frames_produce_one_pulse_until_released(self): + proc = TeleopMessageProcessor(name="test") + + assert _step(proc, _tracked(b"reset"))["reset"] is True + assert _step(proc, _tracked(b"reset"))["reset"] is False + assert _step(proc, _tracked(b"reset"))["reset"] is False + assert _step(proc, _empty_tracked())["reset"] is False + assert _step(proc, _tracked(b"reset"))["reset"] is True + class TestKillAlwaysFalse: def test_kill_is_always_false(self): diff --git a/source/isaaclab_teleop/test/test_xr_camera_feed.py b/source/isaaclab_teleop/test/test_xr_camera_feed.py new file mode 100644 index 000000000000..e5ceeba11972 --- /dev/null +++ b/source/isaaclab_teleop/test/test_xr_camera_feed.py @@ -0,0 +1,441 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import Mock + +import isaaclab_teleop.camera_feed as camera_feed +import pytest +import torch +from isaaclab_physx.renderers import IsaacRtxRendererCfg +from isaaclab_teleop import IsaacTeleopCfg, XrCameraFeedCfg, XrCameraFeedLayoutCfg +from packaging.version import Version + +from isaaclab.sensors import CameraCfg + +from isaaclab_tasks.utils.hydra import resolve_presets +from isaaclab_tasks.utils.presets import MultiBackendRendererCfg + + +class _FakeImage: + def __init__(self, height=8, width=12, *, device="cuda:0", data_ptr=100): + self.dtype = torch.uint8 + self.ndim = 3 + self.shape = (height, width, 4) + self.device = torch.device(device) + self._data_ptr = data_ptr + + def data_ptr(self): + return self._data_ptr + + +class _FakeBatch: + def __init__(self, image): + self.image = image + self.ndim = 4 + self.shape = (1, *image.shape) + + def __getitem__(self, index): + assert index == 0 + return self.image + + +class _FakeCamera: + def __init__(self, image): + self.output = {"rgba": SimpleNamespace(torch=_FakeBatch(image))} + self.update_calls = [] + self.image_on_update = None + + @property + def data(self): + return SimpleNamespace(output=self.output) + + def update(self, dt, force_recompute=False): + self.update_calls.append((dt, force_recompute)) + if self.image_on_update is not None: + self.output["rgba"].torch = _FakeBatch(self.image_on_update) + + +class _FakePanel: + def __init__(self, descriptor, width, height): + self.descriptor = descriptor + self.width = width + self.height = height + self.uploads = [] + self.closed = False + + def upload(self, image): + self.uploads.append(image) + + def close(self): + self.closed = True + + +class _FakeSubscription: + def __init__(self, callback): + self.callback = callback + self.closed = False + + def publish(self): + if not self.closed: + self.callback(None) + + def close(self): + self.closed = True + + +class _FakePresenter: + def __init__(self): + self.panels = [] + self.subscription = None + self.staged = [] + + def prepare_upload_image(self, _name, image, previous_source=None, previous_upload=None): + del previous_source, previous_upload + return image + + def create_panel(self, descriptor, width, height): + panel = _FakePanel(descriptor, width, height) + self.panels.append(panel) + return panel + + def stage_upload_image(self, image, upload_image): + self.staged.append((image, upload_image)) + + def subscribe_to_frame_updates(self, callback): + self.subscription = _FakeSubscription(callback) + return self.subscription + + +def _camera_cfg(renderer_cfg=None): + return CameraCfg( + prim_path="{ENV_REGEX_NS}/Camera", + height=8, + width=12, + data_types=["rgb"], + renderer_cfg=renderer_cfg, + ) + + +def _teleop_env_cfg( + feeds, + *, + num_envs=1, + camera=None, + layout=None, +): + scene = SimpleNamespace(num_envs=num_envs) + if camera is not None: + scene.robot_pov_cam = camera + return SimpleNamespace( + scene=scene, + isaac_teleop=SimpleNamespace( + xr_camera_feeds=feeds, + xr_camera_feed_layout=layout or XrCameraFeedLayoutCfg(), + ), + ) + + +def _manager(monkeypatch, cfgs, images, layout=None): + cameras = {name: _FakeCamera(image) for name, image in images.items()} + monkeypatch.setattr(camera_feed, "_camera_type", lambda: _FakeCamera) + presenter = _FakePresenter() + env = SimpleNamespace(scene=SimpleNamespace(sensors=cameras)) + manager = camera_feed._XrCameraFeedManager(env, cfgs, layout or XrCameraFeedLayoutCfg(), presenter) + return manager, presenter, cameras + + +def test_pip_rejects_multiple_environments_before_camera_creation(monkeypatch): + env_cfg = _teleop_env_cfg([XrCameraFeedCfg(camera_name="robot_pov_cam")], num_envs=2, camera=_camera_cfg()) + load_presenter = Mock(return_value=_FakePresenter()) + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", load_presenter) + + with pytest.raises(ValueError, match="exactly one environment"): + camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=True) + + load_presenter.assert_called_once_with() + + +def test_xr_without_pip_preserves_multiple_environments(monkeypatch): + env_cfg = _teleop_env_cfg([], num_envs=2) + load_presenter = Mock() + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", load_presenter) + + session = camera_feed._XrCameraFeedSession.prepare( + env_cfg, + enabled=True, + camera_rendering_enabled=True, + ) + + assert not session.enabled + load_presenter.assert_not_called() + assert env_cfg.scene.num_envs == 2 + + +def test_kitless_xr_with_configured_pip_preserves_multiple_environments(monkeypatch): + env_cfg = _teleop_env_cfg( + [XrCameraFeedCfg(camera_name="robot_pov_cam")], + num_envs=2, + camera=_camera_cfg(), + ) + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", lambda: None) + + session = camera_feed._XrCameraFeedSession.prepare( + env_cfg, + enabled=True, + camera_rendering_enabled=True, + ) + + assert not session.enabled + assert env_cfg.scene.num_envs == 2 + + +def test_empty_camera_feed_selection_skips_pip(monkeypatch): + env_cfg = _teleop_env_cfg([]) + load_presenter = Mock() + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", load_presenter) + + session = camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=True) + + assert not session.enabled + load_presenter.assert_not_called() + assert vars(env_cfg.scene) == {"num_envs": 1} + + +def test_existing_camera_is_selected_without_replacement(monkeypatch): + selected = _camera_cfg(IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=True)) + env_cfg = _teleop_env_cfg([XrCameraFeedCfg(camera_name="robot_pov_cam")], camera=selected) + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", _FakePresenter) + monkeypatch.setattr("isaaclab.utils.version.get_isaac_sim_version", lambda: Version("6.1.0")) + + session = camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=True) + + assert session.enabled + assert env_cfg.scene.robot_pov_cam is selected + assert session.requires_responsive_denoising + + +def test_session_refresh_renders_reset_state_before_publishing(): + events = [] + session = camera_feed._XrCameraFeedSession([], None, None, requires_responsive_denoising=False) + session._env = SimpleNamespace(sim=SimpleNamespace(render=lambda: events.append("render"))) + + class _Manager: + def refresh(self, *, publish=True): + events.append("publish" if publish else "request") + + session._manager = _Manager() + session._bound = True + + session.refresh() + + assert events == ["request", "render"] * 3 + ["publish"] + + +def test_pre_61_runtime_falls_back_to_classic_dlss(monkeypatch): + selected = _camera_cfg(IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=True)) + env_cfg = _teleop_env_cfg([XrCameraFeedCfg(camera_name="robot_pov_cam")], camera=selected) + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", _FakePresenter) + monkeypatch.setattr("isaaclab.utils.version.get_isaac_sim_version", lambda: Version("6.0.0")) + + session = camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=True) + + assert session.enabled + assert selected.renderer_cfg.enable_dlss_ray_reconstruction is False + assert not session.requires_responsive_denoising + + +def test_camera_rendering_switch_disables_pip_before_presenter_load(monkeypatch): + env_cfg = _teleop_env_cfg( + [XrCameraFeedCfg(camera_name="robot_pov_cam")], + num_envs=2, + camera=_camera_cfg(), + ) + load_presenter = Mock() + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", load_presenter) + + session = camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=False) + + assert not session.enabled + load_presenter.assert_not_called() + + +def test_feed_selection_requires_existing_rgb_camera(monkeypatch): + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", _FakePresenter) + missing = _teleop_env_cfg([XrCameraFeedCfg(camera_name="missing")]) + with pytest.raises(ValueError, match="not present"): + camera_feed._XrCameraFeedSession.prepare(missing, enabled=True, camera_rendering_enabled=True) + + depth = _camera_cfg() + depth.data_types = ["distance_to_image_plane"] + env_cfg = _teleop_env_cfg([XrCameraFeedCfg(camera_name="robot_pov_cam")], camera=depth) + with pytest.raises(ValueError, match="RGB or RGBA"): + camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=True) + + +def test_manual_layout_preserves_per_feed_transforms(): + cfgs = [ + XrCameraFeedCfg(camera_name="left", offset_m=(-0.4, 0.1), distance_m=0.7), + XrCameraFeedCfg(camera_name="right", offset_m=(0.5, -0.2), distance_m=1.1), + ] + + resolved = camera_feed._layout_feed_cfgs(cfgs, [(100, 50), (100, 50)], XrCameraFeedLayoutCfg()) + + assert [(cfg.offset_m, cfg.distance_m) for cfg in resolved] == [ + ((-0.4, 0.1), 0.7), + ((0.5, -0.2), 1.1), + ] + assert resolved is not cfgs + + +@pytest.mark.parametrize( + ("layout", "expected"), + [ + ( + XrCameraFeedLayoutCfg(mode="horizontal", panel_gap_m=0.1), + [(-0.25, 0.0), (0.25, 0.0)], + ), + ( + XrCameraFeedLayoutCfg(mode="vertical", panel_gap_m=0.1), + [(0.0, 0.15), (0.0, -0.15)], + ), + ( + XrCameraFeedLayoutCfg(mode="grid", panel_gap_m=0.1, max_columns=2), + [(-0.25, 0.15), (0.25, 0.15), (0.0, -0.15)], + ), + ], +) +def test_automatic_layouts_preserve_order(layout, expected): + count = len(expected) + cfgs = [XrCameraFeedCfg(camera_name=str(index), panel_width_m=0.4) for index in range(count)] + + resolved = camera_feed._layout_feed_cfgs(cfgs, [(100, 50)] * count, layout) + + for cfg, expected_offset in zip(resolved, expected): + assert cfg.offset_m == pytest.approx(expected_offset) + + +@pytest.mark.parametrize( + "layout", + [ + XrCameraFeedLayoutCfg(mode="diagonal"), + XrCameraFeedLayoutCfg(distance_m=0.0), + XrCameraFeedLayoutCfg(placement="world"), + XrCameraFeedLayoutCfg(placement="world", world_position_m=(0.0, 0.0, 0.0), world_orientation_xyzw=(0, 0, 0, 0)), + ], +) +def test_invalid_layouts_fail_before_panel_creation(layout): + with pytest.raises(ValueError): + camera_feed._validate_layout_cfg(layout) + + +def test_manager_publishes_on_kit_frame_and_closes(monkeypatch): + cfg = XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=0.0) + image = _FakeImage() + manager, presenter, _ = _manager(monkeypatch, [cfg], {"robot_pov_cam": image}) + + presenter.subscription.publish() + manager.close() + + assert presenter.panels[0].uploads == [image] + assert presenter.subscription.closed + assert presenter.panels[0].closed + + +def test_manager_refresh_rebinds_reset_camera_output(monkeypatch): + cfg = XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=0.0) + before = _FakeImage(data_ptr=100) + after = _FakeImage(data_ptr=200) + manager, presenter, cameras = _manager(monkeypatch, [cfg], {"robot_pov_cam": before}) + cameras["robot_pov_cam"].image_on_update = after + + manager.refresh() + + assert cameras["robot_pov_cam"].update_calls == [(0.0, True)] + assert presenter.panels[0].uploads == [after] + + +def test_manager_can_refresh_reset_camera_without_publishing(monkeypatch): + cfg = XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=0.0) + before = _FakeImage(data_ptr=100) + after = _FakeImage(data_ptr=200) + manager, presenter, cameras = _manager(monkeypatch, [cfg], {"robot_pov_cam": before}) + cameras["robot_pov_cam"].image_on_update = after + + manager.refresh(publish=False) + + assert cameras["robot_pov_cam"].update_calls == [(0.0, True)] + assert presenter.panels[0].uploads == [] + + +def test_manager_recreates_panel_when_resolution_changes(monkeypatch): + cfg = XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=0.0) + before = _FakeImage(height=8, width=12) + after = _FakeImage(height=10, width=16, data_ptr=200) + manager, presenter, cameras = _manager(monkeypatch, [cfg], {"robot_pov_cam": before}) + cameras["robot_pov_cam"].output["rgba"].torch = _FakeBatch(after) + + manager.update() + + assert len(presenter.panels) == 2 + assert presenter.panels[0].closed + assert presenter.panels[1].uploads == [after] + + +@pytest.mark.parametrize("value", [-1.0, float("nan"), float("inf")]) +def test_manager_rejects_invalid_update_rate(monkeypatch, value): + cfg = XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=value) + with pytest.raises(ValueError, match="max_update_hz"): + _manager(monkeypatch, [cfg], {"robot_pov_cam": _FakeImage()}) + + +def test_reference_tasks_select_their_recorded_camera(): + from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import LocomanipulationG1EnvCfg + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg + + for cfg in (PickPlaceGR1T2EnvCfg(), LocomanipulationG1EnvCfg()): + assert cfg.isaac_teleop.xr_camera_feeds[0].camera_name == "robot_pov_cam" + assert hasattr(cfg.observations.policy, "robot_pov_cam") + assert isinstance(cfg.scene.robot_pov_cam.renderer_cfg, MultiBackendRendererCfg) + + +def test_reference_camera_renderer_resolves_to_isaac_rtx_and_ovrtx(): + from isaaclab_ov.renderers import OVRTXRendererCfg + + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg + + default = resolve_presets(PickPlaceGR1T2EnvCfg().scene.robot_pov_cam.renderer_cfg) + isaacsim_rtx = resolve_presets(PickPlaceGR1T2EnvCfg().scene.robot_pov_cam.renderer_cfg, selected=("isaacsim_rtx",)) + ovrtx = resolve_presets(PickPlaceGR1T2EnvCfg().scene.robot_pov_cam.renderer_cfg, selected=("ovrtx",)) + + assert isinstance(default, IsaacRtxRendererCfg) + assert default.camera_output_device == "cuda:0" + assert default.enable_dlss_ray_reconstruction + assert isinstance(isaacsim_rtx, IsaacRtxRendererCfg) + assert isaacsim_rtx.camera_output_device == "cuda:0" + assert isaacsim_rtx.enable_dlss_ray_reconstruction + assert isinstance(ovrtx, OVRTXRendererCfg) + + +def test_public_api_only_exports_declarative_camera_feed_types(): + import isaaclab_teleop + + assert isaaclab_teleop.XrCameraFeedCfg is XrCameraFeedCfg + assert isaaclab_teleop.XrCameraFeedLayoutCfg is XrCameraFeedLayoutCfg + for removed_name in ( + "XrCameraFeedManager", + "XrCameraFeedSession", + "XrCameraFeedPresentationBackend", + "XrCameraFeedPresentationCfg", + ): + assert not hasattr(isaaclab_teleop, removed_name) + + +def test_isaac_teleop_default_has_no_camera_feeds(): + cfg = IsaacTeleopCfg(pipeline_builder=lambda: None) + + assert cfg.xr_camera_feeds == [] diff --git a/source/isaaclab_teleop/test/test_xr_camera_feed_kit_scene_ui.py b/source/isaaclab_teleop/test/test_xr_camera_feed_kit_scene_ui.py new file mode 100644 index 000000000000..dbcc8d716e16 --- /dev/null +++ b/source/isaaclab_teleop/test/test_xr_camera_feed_kit_scene_ui.py @@ -0,0 +1,570 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import gc +import importlib.util +import sys +from enum import IntFlag +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any +from unittest.mock import Mock + +import numpy as np +import pytest + +from pxr import Gf + + +class _PoseValidityFlags(IntFlag): + POSITION_VALID = 1 + ORIENTATION_VALID = 2 + POSITION_TRACKED = 4 + ORIENTATION_TRACKED = 8 + + +class _EventType: + post_sync_update = "post_sync_update" + xr_display_disabled = "xr_display_disabled" + + +class _TransformSource: + def __init__(self, matrix: Gf.Matrix4d): + self.source = matrix + + +class _SpatialSource: + @staticmethod + def new_transform_matrix_source(matrix: Gf.Matrix4d) -> _TransformSource: + return _TransformSource(matrix) + + @staticmethod + def new_prim_path_source(path: str): + return SimpleNamespace(kind="prim_path", value=path) + + @staticmethod + def new_translation_source(value): + return SimpleNamespace(kind="translation", value=value) + + @staticmethod + def new_look_at_camera_source(): + return SimpleNamespace(kind="look_at_camera") + + +class _Subscription: + def __init__(self, bus: _MessageBus, token: int): + self._bus = bus + self._token = token + + def __del__(self): + self._bus.release(self._token) + + +class _MessageBus: + def __init__(self): + self._next_token = 0 + self._listeners: dict[int, tuple[Any, Any]] = {} + self.released_event_types: list[Any] = [] + + @property + def listener_count(self) -> int: + return len(self._listeners) + + def create_subscription_to_pop_by_type(self, event_type, callback, *, name): + del name + token = self._next_token + self._next_token += 1 + self._listeners[token] = (event_type, callback) + return _Subscription(self, token) + + def release(self, token: int) -> None: + listener = self._listeners.pop(token, None) + if listener is not None: + self.released_event_types.append(listener[0]) + + def emit(self, event_type) -> None: + for subscribed_type, callback in tuple(self._listeners.values()): + if subscribed_type == event_type: + callback(SimpleNamespace(type=event_type)) + + +class _InputDevice: + def __init__(self, pose_desc): + self.pose_desc = pose_desc + + def get_virtual_world_pose_desc(self, path: str): + assert path == "" + return self.pose_desc + + +class _StageCoordinateSystem: + def __init__( + self, + *, + right: tuple[float, float, float] = (1.0, 0.0, 0.0), + up: tuple[float, float, float] = (0.0, 1.0, 0.0), + forward: tuple[float, float, float] = (0.0, 0.0, -1.0), + meters_per_unit: float = 1.0, + up_axis: str = "y", + ): + self._right = right + self._up = up + self._forward = forward + self.meters_per_unit = meters_per_unit + self.up_axis = up_axis + + def get_right_vector(self): + return self._right + + def get_up_vector(self): + return self._up + + def get_forward_vector(self): + return self._forward + + +class _XrCore: + def __init__( + self, + *, + display_enabled: bool = True, + input_device: _InputDevice | None = None, + head_device: _InputDevice | None = None, + meters_per_unit: float = 1.0, + up_axis: str = "y", + stage_coordinate_system: _StageCoordinateSystem | None = None, + ): + self.display_enabled = display_enabled + self.input_device = input_device + self.head_device = head_device + self.coordinate_system = SimpleNamespace(meters_per_unit=meters_per_unit, up_axis=up_axis) + self.stage_coordinate_system = stage_coordinate_system or _StageCoordinateSystem() + self.message_bus = _MessageBus() + self.reorient_calls: list[tuple[Gf.Matrix4d, bool]] = [] + + def get_message_bus(self): + return self.message_bus + + def is_xr_display_enabled(self): + return self.display_enabled + + def get_input_device(self, path: str): + if path == "displayDevice": + return self.input_device + if path == "/user/head": + return self.head_device + return None + + def get_coordinate_system(self): + return self.coordinate_system + + def get_stage_coordinate_system(self): + return self.stage_coordinate_system + + def reorient_transform_matrix_up_right(self, matrix: Gf.Matrix4d, y_up: bool): + self.reorient_calls.append((matrix, y_up)) + return matrix + + +def _pose(flags: _PoseValidityFlags, translation=(0.0, 0.0, 0.0)): + matrix = Gf.Matrix4d(1.0).SetTranslate(Gf.Vec3d(*translation)) + return SimpleNamespace(validity_flags=flags, pose_matrix=matrix) + + +def _matrix_values(matrix: Gf.Matrix4d) -> np.ndarray: + return np.asarray([[float(matrix[row][column]) for column in range(4)] for row in range(4)]) + + +def _translation(matrix: Gf.Matrix4d) -> tuple[float, float, float]: + return tuple(float(value) for value in matrix.ExtractTranslation()) + + +def _module(name: str) -> ModuleType: + module = ModuleType(name) + module.__path__ = [] + return module + + +@pytest.fixture +def scene_ui_module(monkeypatch): + """Load the Scene UI implementation against fake Kit and OpenXR modules.""" + omni = _module("omni") + gpu_foundation = _module("omni.gpu_foundation_factory") + ui = _module("omni.ui") + kit = _module("omni.kit") + scene_view = _module("omni.kit.scene_view") + scene_view_xr = _module("omni.kit.scene_view.xr") + scene_view_xr_utils = _module("omni.kit.scene_view.xr_utils") + kit_xr = _module("omni.kit.xr") + kit_xr_core = _module("omni.kit.xr.core") + + class _Widget: + pass + + class _XrCoreType: + @staticmethod + def get_singleton(): + raise AssertionError("Tests must inject their XRCore instance.") + + gpu_foundation.TextureFormat = SimpleNamespace(RGBA8_UNORM="rgba8") + ui.Widget = _Widget + scene_view_xr.XRSceneView = object() + scene_view_xr_utils.SpatialSource = _SpatialSource + scene_view_xr_utils.UiContainer = object + scene_view_xr_utils.UpdatePolicy = SimpleNamespace(ALWAYS="always") + scene_view_xr_utils.WidgetComponent = object + kit_xr_core.XRCore = _XrCoreType + kit_xr_core.XRCoreEventType = _EventType + kit_xr_core.XRPoseValidityFlags = _PoseValidityFlags + + omni.gpu_foundation_factory = gpu_foundation + omni.ui = ui + omni.kit = kit + kit.scene_view = scene_view + kit.xr = kit_xr + scene_view.xr = scene_view_xr + scene_view.xr_utils = scene_view_xr_utils + kit_xr.core = kit_xr_core + + fake_modules = { + "omni": omni, + "omni.gpu_foundation_factory": gpu_foundation, + "omni.ui": ui, + "omni.kit": kit, + "omni.kit.scene_view": scene_view, + "omni.kit.scene_view.xr": scene_view_xr, + "omni.kit.scene_view.xr_utils": scene_view_xr_utils, + "omni.kit.xr": kit_xr, + "omni.kit.xr.core": kit_xr_core, + } + for name, module in fake_modules.items(): + monkeypatch.setitem(sys.modules, name, module) + + module_path = Path(__file__).parents[1] / "isaaclab_teleop" / "camera_feed_kit_scene_ui.py" + module_name = "_isaaclab_teleop_camera_feed_kit_scene_ui_under_test" + spec = importlib.util.spec_from_file_location(module_name, module_path) + assert spec is not None and spec.loader is not None + loaded_module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, module_name, loaded_module) + spec.loader.exec_module(loaded_module) + return loaded_module + + +@pytest.mark.parametrize( + "input_device", + [ + None, + _InputDevice(_pose(_PoseValidityFlags.POSITION_VALID)), + ], +) +def test_viewer_start_invalid_or_missing_pose_stays_hidden(scene_ui_module, input_device): + core = _XrCore(input_device=input_device) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + source = _TransformSource(Gf.Matrix4d(1.0)) + visibility = [] + + token = anchor.register(source, (0.0, 0.0), 0.8, visibility.append) + + assert visibility == [False] + assert not anchor.captured + assert _translation(source.source) == pytest.approx((0.0, 0.0, 0.0)) + anchor.unregister(token) + + +def test_viewer_start_accepts_valid_pose_without_tracked_flags(scene_ui_module): + flags = _PoseValidityFlags.POSITION_VALID | _PoseValidityFlags.ORIENTATION_VALID + pose_desc = _pose(flags, translation=(1.0, 2.0, 3.0)) + core = _XrCore(input_device=_InputDevice(pose_desc)) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + source = _TransformSource(Gf.Matrix4d(1.0)) + visibility = [] + + token = anchor.register(source, (0.0, 0.0), 0.0, visibility.append) + + assert visibility == [False] + assert not anchor.captured + + core.message_bus.emit(_EventType.post_sync_update) + + assert visibility == [False, True] + assert anchor.captured + assert core.reorient_calls == [(pose_desc.pose_matrix, True)] + assert _translation(source.source) == pytest.approx((1.0, 2.0, 3.0)) + anchor.unregister(token) + + +def test_viewer_start_freezes_first_valid_pose_across_later_motion(scene_ui_module): + flags = _PoseValidityFlags.POSITION_VALID | _PoseValidityFlags.ORIENTATION_VALID + device = _InputDevice(_pose(flags, translation=(1.0, 2.0, 3.0))) + core = _XrCore(input_device=device) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + source = _TransformSource(Gf.Matrix4d(1.0)) + + token = anchor.register(source, (0.1, -0.2), 0.5, lambda _ready: None) + core.message_bus.emit(_EventType.post_sync_update) + frozen_matrix = _matrix_values(source.source).copy() + device.pose_desc = _pose(flags, translation=(9.0, 8.0, 7.0)) + + core.message_bus.emit(_EventType.post_sync_update) + + np.testing.assert_allclose(_matrix_values(source.source), frozen_matrix) + assert len(core.reorient_calls) == 1 + anchor.unregister(token) + + +def test_viewer_start_panels_share_anchor_and_convert_offsets_to_stage_units(scene_ui_module): + flags = _PoseValidityFlags.POSITION_VALID | _PoseValidityFlags.ORIENTATION_VALID + pose_desc = _pose(flags) + core = _XrCore( + display_enabled=False, + input_device=_InputDevice(pose_desc), + meters_per_unit=0.01, + ) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + first_source = _TransformSource(Gf.Matrix4d(1.0)) + second_source = _TransformSource(Gf.Matrix4d(1.0)) + first_visibility = [] + second_visibility = [] + + first_token = anchor.register(first_source, (0.1, 0.2), 0.5, first_visibility.append) + second_token = anchor.register(second_source, (-0.3, 0.4), 0.8, second_visibility.append) + core.display_enabled = True + core.message_bus.emit(_EventType.post_sync_update) + + assert anchor._upright_pose is pose_desc.pose_matrix + assert len(core.reorient_calls) == 1 + assert first_visibility == [False, True] + assert second_visibility == [False, True] + assert _translation(first_source.source) == pytest.approx((10.0, 20.0, -50.0)) + assert _translation(second_source.source) == pytest.approx((-30.0, 40.0, -80.0)) + np.testing.assert_allclose(_matrix_values(first_source.source)[:3, :3], np.eye(3)) + np.testing.assert_allclose(_matrix_values(second_source.source)[:3, :3], np.eye(3)) + + anchor.unregister(first_token) + anchor.unregister(second_token) + + +def test_viewer_start_uses_pose_local_axes_in_z_up_stage(scene_ui_module): + flags = _PoseValidityFlags.POSITION_VALID | _PoseValidityFlags.ORIENTATION_VALID + # Local +X/+Y/+Z map to stage +X/+Z/-Y. The viewer therefore looks along + # stage +Y from an eye height of 1.6 m. + pose_matrix = Gf.Matrix4d( + (1.0, 0.0, 0.0, 0.0), + (0.0, 0.0, 1.0, 0.0), + (0.0, -1.0, 0.0, 0.0), + (0.0, 0.0, 1.6, 1.0), + ) + core = _XrCore( + input_device=_InputDevice(SimpleNamespace(validity_flags=flags, pose_matrix=pose_matrix)), + up_axis="z", + stage_coordinate_system=_StageCoordinateSystem( + up=(0.0, 0.0, 1.0), + forward=(0.0, -1.0, 0.0), + up_axis="z", + ), + ) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + source = _TransformSource(Gf.Matrix4d(1.0)) + + token = anchor.register(source, (0.1, 0.2), 0.8, lambda _ready: None) + core.message_bus.emit(_EventType.post_sync_update) + + assert _translation(source.source) == pytest.approx((0.1, 0.8, 1.8)) + np.testing.assert_allclose(_matrix_values(source.source)[:3, :3], _matrix_values(pose_matrix)[:3, :3]) + anchor.unregister(token) + + +def test_viewer_start_disconnect_hides_resets_and_recaptures(scene_ui_module): + flags = _PoseValidityFlags.POSITION_VALID | _PoseValidityFlags.ORIENTATION_VALID + device = _InputDevice(_pose(flags, translation=(1.0, 2.0, 3.0))) + core = _XrCore(input_device=device) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + source = _TransformSource(Gf.Matrix4d(1.0)) + visibility = [] + + token = anchor.register(source, (0.0, 0.0), 0.0, visibility.append) + core.message_bus.emit(_EventType.post_sync_update) + first_matrix = _matrix_values(source.source).copy() + core.display_enabled = False + core.message_bus.emit(_EventType.xr_display_disabled) + + assert not anchor.captured + assert visibility[-1] is False + np.testing.assert_allclose(_matrix_values(source.source), first_matrix) + + core.message_bus.emit(_EventType.post_sync_update) + assert visibility[-1] is False + assert len(core.reorient_calls) == 1 + + device.pose_desc = _pose(flags, translation=(4.0, 5.0, 6.0)) + core.display_enabled = True + core.message_bus.emit(_EventType.post_sync_update) + + assert anchor.captured + assert visibility == [False, True, False, True] + assert len(core.reorient_calls) == 2 + assert _translation(source.source) == pytest.approx((4.0, 5.0, 6.0)) + anchor.unregister(token) + + +def test_viewer_start_unregister_releases_listeners_after_last_panel(scene_ui_module): + core = _XrCore(display_enabled=False) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + first_source = _TransformSource(Gf.Matrix4d(1.0)) + second_source = _TransformSource(Gf.Matrix4d(1.0)) + + first_token = anchor.register(first_source, (0.0, 0.0), 0.8, lambda _ready: None) + second_token = anchor.register(second_source, (0.1, 0.1), 0.8, lambda _ready: None) + assert core.message_bus.listener_count == 2 + + anchor.unregister(first_token) + assert core.message_bus.listener_count == 2 + + anchor.unregister(second_token) + gc.collect() + + assert anchor._post_sync_subscription is None + assert anchor._display_disabled_subscription is None + assert core.message_bus.listener_count == 0 + assert set(core.message_bus.released_event_types) == { + _EventType.post_sync_update, + _EventType.xr_display_disabled, + } + + +def test_world_panel_matrix_is_fixed_and_normalizes_quaternion(scene_ui_module): + scaled_descriptor = SimpleNamespace( + world_position_m=(1.0, 2.0, 3.0), + world_orientation_xyzw=(0.0, 0.0, 2.0, 2.0), + offset_m=(0.5, -0.25), + ) + unit = np.sqrt(0.5) + unit_descriptor = SimpleNamespace( + world_position_m=(1.0, 2.0, 3.0), + world_orientation_xyzw=(0.0, 0.0, unit, unit), + offset_m=(0.5, -0.25), + ) + + scaled_matrix = scene_ui_module._world_panel_matrix(scaled_descriptor, 0.01) + repeated_matrix = scene_ui_module._world_panel_matrix(scaled_descriptor, 0.01) + unit_matrix = scene_ui_module._world_panel_matrix(unit_descriptor, 0.01) + + np.testing.assert_allclose(_matrix_values(scaled_matrix), _matrix_values(repeated_matrix), atol=1.0e-12) + np.testing.assert_allclose(_matrix_values(scaled_matrix), _matrix_values(unit_matrix), atol=1.0e-12) + np.testing.assert_allclose( + _matrix_values(scaled_matrix)[:3, :3], + np.asarray( + [ + [0.0, 1.0, 0.0], + [-1.0, 0.0, 0.0], + [0.0, 0.0, 1.0], + ] + ), + atol=1.0e-12, + ) + assert _translation(scaled_matrix) == pytest.approx((125.0, 250.0, 300.0)) + + +@pytest.mark.parametrize( + ("position", "orientation"), + [ + (None, (0.0, 0.0, 0.0, 1.0)), + ((0.0, 0.0, 0.0), (0.0, 0.0, 0.0, 0.0)), + ((0.0, 0.0, 0.0), (0.0, 0.0, float("nan"), 1.0)), + ], +) +def test_world_panel_matrix_rejects_missing_position_or_invalid_quaternion( + scene_ui_module, + position, + orientation, +): + descriptor = SimpleNamespace( + world_position_m=position, + world_orientation_xyzw=orientation, + offset_m=(0.0, 0.0), + ) + + with pytest.raises(ValueError): + scene_ui_module._world_panel_matrix(descriptor, 1.0) + + +@pytest.mark.parametrize("meters_per_unit", [0.0, -1.0, float("nan")]) +def test_world_panel_matrix_rejects_invalid_stage_scale(scene_ui_module, meters_per_unit): + descriptor = SimpleNamespace( + world_position_m=(0.0, 0.0, 0.0), + world_orientation_xyzw=(0.0, 0.0, 0.0, 1.0), + offset_m=(0.0, 0.0), + ) + + with pytest.raises(RuntimeError, match="meters_per_unit"): + scene_ui_module._world_panel_matrix(descriptor, meters_per_unit) + + +@pytest.mark.parametrize( + ("placement", "expected_meters_per_unit", "expected_width", "expected_height"), + [ + ("viewer_start", 0.02, 24.0, 16.0), + ("head_locked", 0.02, 24.0, 16.0), + ("world", 0.5, 0.96, 0.64), + ], +) +def test_panel_converts_metric_geometry_in_selected_coordinate_system( + scene_ui_module, + monkeypatch, + placement, + expected_meters_per_unit, + expected_width, + expected_height, +): + core = _XrCore( + display_enabled=False, + meters_per_unit=0.02, + stage_coordinate_system=_StageCoordinateSystem(meters_per_unit=0.5, up_axis="z"), + ) + monkeypatch.setattr(scene_ui_module, "XRCore", SimpleNamespace(get_singleton=lambda: core)) + component = SimpleNamespace() + component_type = Mock(return_value=component) + monkeypatch.setattr(scene_ui_module, "WidgetComponent", component_type) + monkeypatch.setattr(scene_ui_module.ui, "ByteImageProvider", Mock(return_value=object()), raising=False) + + class _Container: + def __init__(self, _view, _component, *, space_stack): + self.space_stack = space_stack + self.root = SimpleNamespace(clear=Mock()) + self.show = Mock() + self.hide = Mock() + + monkeypatch.setattr(scene_ui_module, "UiContainer", _Container) + descriptor = SimpleNamespace( + label=None, + width_m=0.48, + offset_m=(0.1, -0.2), + distance_m=0.8, + placement=placement, + world_position_m=(1.0, 2.0, 3.0) if placement == "world" else None, + world_orientation_xyzw=(0.0, 0.0, 0.0, 1.0), + ) + viewer_start_anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) if placement == "viewer_start" else None + + panel = scene_ui_module.KitSceneUiCameraFeedPanel( + descriptor, + image_width=720, + image_height=480, + viewer_start_anchor=viewer_start_anchor, + ) + + kwargs = component_type.call_args.kwargs + assert kwargs["width"] == pytest.approx(expected_width) + assert kwargs["height"] == pytest.approx(expected_height) + assert kwargs["resolution_scale"] == pytest.approx(1500.0) + assert kwargs["unit_to_pixel_scale"] == pytest.approx(expected_meters_per_unit) + if placement == "head_locked": + translation = panel._container.space_stack[1].value + assert tuple(translation) == pytest.approx((5.0, -10.0, -40.0)) + elif placement == "world": + assert _translation(panel._container.space_stack[0].source) == pytest.approx((2.2, 3.6, 6.0)) + panel.close() From ff179ee2775088d61cc7449b3b11aab43232bde8 Mon Sep 17 00:00:00 2001 From: hougantc-nvda <127865892+hougantc-nvda@users.noreply.github.com> Date: Fri, 31 Jul 2026 16:07:56 +0000 Subject: [PATCH 2/3] Address XR camera PiP review feedback Move the pre-6.1 Ray Reconstruction fallback into the Isaac RTX renderer so every camera consumer gets compatible behavior. Enable camera extensions in the Pink IK test now that reference tasks include image observations. Align the documentation with supported invocation and error contracts. Signed-off-by: Hougant Chen --- docs/source/features/isaac_teleop.rst | 11 ++++--- .../isaaclab/test/controllers/test_pink_ik.py | 4 +-- .../changelog.d/xr-camera-dlss-ghosting.rst | 6 ++++ .../renderers/isaac_rtx_renderer.py | 11 +++++++ .../renderers/isaac_rtx_renderer_cfg.py | 3 +- .../test_isaac_rtx_renderer_contract.py | 33 +++++++++++++++++++ source/isaaclab_teleop/docs/README.md | 2 +- .../isaaclab_teleop/camera_feed.py | 22 ------------- .../test/test_xr_camera_feed.py | 15 --------- 9 files changed, 61 insertions(+), 46 deletions(-) diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 44aaac091de6..4ef6bfddc000 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -1193,8 +1193,8 @@ startup rejects ``--num_envs`` values other than ``1``; IsaacTeleop XR behavior unchanged. On Isaac Sim 6.1 and newer, camera-feed setup enables responsive DLSS Ray Reconstruction denoising -automatically. Earlier runtimes fall back to classic DLSS for PiP cameras to avoid the temporal -ghosting in their Ray Reconstruction implementation. +automatically. For every camera consumer, the Isaac RTX renderer falls back to classic DLSS on +earlier runtimes to avoid the temporal ghosting in their Ray Reconstruction implementation. Camera selection ~~~~~~~~~~~~~~~~ @@ -1296,9 +1296,10 @@ Kit Scene UI presentation ~~~~~~~~~~~~~~~~~~~~~~~~~ PiP presentation uses Kit Scene UI and ``SpatialSource`` placement. Kit imports are deferred until -an enabled feed is requested. If Scene UI is unavailable, the scripts log a warning and continue -without PiP; task-owned cameras and recording observations remain unchanged. This keeps the camera -selection configuration usable when a future kitless entry point no longer provides Scene UI. +an enabled feed is requested. If the Scene UI modules cannot be imported, the scripts log a warning +and continue without PiP; task-owned cameras and recording observations remain unchanged. This keeps +the camera selection configuration usable when a future kitless entry point no longer provides +Scene UI. Configuration, camera-buffer, and panel-initialization errors still fail during startup. .. _isaac-teleop-haptics: diff --git a/source/isaaclab/test/controllers/test_pink_ik.py b/source/isaaclab/test/controllers/test_pink_ik.py index a7935875fa7b..f51b8239bb82 100644 --- a/source/isaaclab/test/controllers/test_pink_ik.py +++ b/source/isaaclab/test/controllers/test_pink_ik.py @@ -7,8 +7,8 @@ from isaaclab.app import AppLauncher -# launch omniverse app -simulation_app = AppLauncher(headless=True).app +# Camera-enabled tasks under test require the rendering experience. +simulation_app = AppLauncher(headless=True, enable_cameras=True).app """Rest everything follows.""" diff --git a/source/isaaclab_physx/changelog.d/xr-camera-dlss-ghosting.rst b/source/isaaclab_physx/changelog.d/xr-camera-dlss-ghosting.rst index 4e14e48c57dc..ba2618e52ff7 100644 --- a/source/isaaclab_physx/changelog.d/xr-camera-dlss-ghosting.rst +++ b/source/isaaclab_physx/changelog.d/xr-camera-dlss-ghosting.rst @@ -5,3 +5,9 @@ Added :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg`, plus a process-global responsive-denoising setting. * Added an optional RTX camera output-device override so CUDA pixels can be produced with CPU physics and CPU camera pose state. + +Fixed +^^^^^ + +* Fixed the pre-6.1 Ray Reconstruction compatibility fallback so it applies to every Isaac RTX + camera consumer rather than only cameras selected for XR PiP. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 2bfebd11e673..bb57d0808f3a 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -86,6 +86,7 @@ def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: DLSS_RAY_RECONSTRUCTION_ATTR = "omni:rtx:newDenoiser:enabled" DLSS_EXEC_MODE_API_SCHEMA = "OmniRtxSettingsRtAPI_1" DLSS_EXEC_MODE_ATTR = "omni:rtx:post:dlss:execMode" +_DLSS_RAY_RECONSTRUCTION_MIN_ISAAC_SIM_VERSION = (6, 1) def _camera_semantic_filter_predicate(semantic_filter: str | list[str]) -> str: @@ -456,6 +457,16 @@ def _apply_render_product_settings(self, stage: Usd.Stage, render_product_paths: dlss_exec_mode = self.cfg.dlss_exec_mode if ray_reconstruction is None and dlss_exec_mode is None: return + if ray_reconstruction is True: + isaac_sim_version = get_isaac_sim_version() + isaac_sim_major_minor = (isaac_sim_version.major, isaac_sim_version.minor) + if isaac_sim_major_minor < _DLSS_RAY_RECONSTRUCTION_MIN_ISAAC_SIM_VERSION: + ray_reconstruction = False + logger.warning( + "DLSS Ray Reconstruction was requested, but Isaac Sim %s predates responsive denoising. " + "Falling back to classic DLSS for this render product.", + isaac_sim_version, + ) for render_product_path in render_product_paths: render_product = stage.GetPrimAtPath(render_product_path) if not render_product.IsValid(): diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py index 503443723782..43a5f8ed37b1 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py @@ -121,7 +121,8 @@ class IsaacRtxRendererCfg(RendererCfg): Set to ``False`` to use classic DLSS while leaving other render products, such as XR headset views, on their process-global setting. ``None`` - preserves Kit's render-product default. + preserves Kit's render-product default. On Isaac Sim versions before 6.1, + ``True`` falls back to classic DLSS because responsive denoising is unavailable. """ dlss_exec_mode: Literal["performance", "balanced", "quality", "auto", "rtxaa", "manual"] | None = None diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index 85d1a5afb8b1..85c4d455e60d 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -219,6 +219,39 @@ def test_dlss_settings_are_authored_per_render_product(monkeypatch): prim._test_attributes["omni:rtx:post:dlss:execMode"].Set.assert_called_once_with("quality") +@pytest.mark.parametrize( + ("isaac_sim_version", "expected_ray_reconstruction"), + [ + pytest.param("6.0.0", False, id="pre-responsive-denoising"), + pytest.param("6.1.0", True, id="responsive-denoising"), + ], +) +def test_ray_reconstruction_uses_renderer_owned_version_fallback( + monkeypatch, caplog, isaac_sim_version, expected_ray_reconstruction +): + """All Isaac RTX camera consumers share the pre-6.1 compatibility fallback.""" + _install_omni_stubs(monkeypatch) + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=True) + stage = MagicMock() + render_product = stage.GetPrimAtPath.return_value + render_product.IsValid.return_value = True + render_product.IsA.return_value = True + render_product.ApplyAPI.return_value = True + attribute = render_product.GetAttribute.return_value + attribute.IsValid.return_value = True + attribute.Set.return_value = True + + with patch.object(rtx_renderer, "get_isaac_sim_version", return_value=version.parse(isaac_sim_version)): + renderer._apply_render_product_settings(stage, ["/Render/RP"]) + + attribute.Set.assert_called_once_with(expected_ray_reconstruction) + assert ("Falling back to classic DLSS" in caplog.text) == (not expected_ray_reconstruction) + + def test_default_renderer_cfg_does_not_author_render_product_settings(monkeypatch): """The default config preserves Kit's render-product settings.""" _install_omni_stubs(monkeypatch) diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index 79ef0d3a469f..8d6a25c5dda0 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -103,7 +103,7 @@ uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ their recorded `robot_pov_cam` observation in PiP: ```bash -./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py \ +uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ --task IsaacContrib-PickPlace-GR1T2-Abs --xr --device cpu ``` diff --git a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py index 6f80618bd3a3..706da8d9bb5f 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py +++ b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py @@ -26,8 +26,6 @@ logger = logging.getLogger(__name__) -_RESPONSIVE_DENOISING_MIN_ISAAC_SIM_VERSION = (6, 1) - @lru_cache(maxsize=1) def _camera_type() -> type[Camera]: @@ -51,25 +49,6 @@ def _load_kit_scene_ui_presenter() -> Any | None: return module._KitSceneUiCameraFeedPresenter() -def _configure_ray_reconstruction_compatibility(camera_cfg: Any, camera_name: str) -> None: - """Fall back to classic DLSS on runtimes without responsive denoising.""" - renderer_cfg = getattr(camera_cfg, "renderer_cfg", None) - if getattr(renderer_cfg, "enable_dlss_ray_reconstruction", None) is not True: - return - - from isaaclab.utils.version import get_isaac_sim_version - - isaac_sim_version = get_isaac_sim_version() - if (isaac_sim_version.major, isaac_sim_version.minor) < _RESPONSIVE_DENOISING_MIN_ISAAC_SIM_VERSION: - renderer_cfg.enable_dlss_ray_reconstruction = False - logger.warning( - "XR camera feed %r requested DLSS Ray Reconstruction, but Isaac Sim %s predates responsive " - "denoising. Falling back to classic DLSS.", - camera_name, - isaac_sim_version, - ) - - def _prepare_camera_feed_cfgs(env_cfg: Any, cfgs: list[XrCameraFeedCfg]) -> list[XrCameraFeedCfg]: """Validate selected scene cameras.""" from isaaclab.sensors import CameraCfg @@ -91,7 +70,6 @@ def _prepare_camera_feed_cfgs(env_cfg: Any, cfgs: list[XrCameraFeedCfg]) -> list raise TypeError(f"XR camera feed {cfg.camera_name!r} does not reference a CameraCfg.") if not any(data_type in {"rgb", "rgba"} for data_type in camera_cfg.data_types): raise ValueError(f"XR camera feed {cfg.camera_name!r} camera must provide RGB or RGBA output.") - _configure_ray_reconstruction_compatibility(camera_cfg, cfg.camera_name) prepared.append(cfg) return prepared diff --git a/source/isaaclab_teleop/test/test_xr_camera_feed.py b/source/isaaclab_teleop/test/test_xr_camera_feed.py index e5ceeba11972..16eb3672a876 100644 --- a/source/isaaclab_teleop/test/test_xr_camera_feed.py +++ b/source/isaaclab_teleop/test/test_xr_camera_feed.py @@ -13,7 +13,6 @@ import torch from isaaclab_physx.renderers import IsaacRtxRendererCfg from isaaclab_teleop import IsaacTeleopCfg, XrCameraFeedCfg, XrCameraFeedLayoutCfg -from packaging.version import Version from isaaclab.sensors import CameraCfg @@ -210,7 +209,6 @@ def test_existing_camera_is_selected_without_replacement(monkeypatch): selected = _camera_cfg(IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=True)) env_cfg = _teleop_env_cfg([XrCameraFeedCfg(camera_name="robot_pov_cam")], camera=selected) monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", _FakePresenter) - monkeypatch.setattr("isaaclab.utils.version.get_isaac_sim_version", lambda: Version("6.1.0")) session = camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=True) @@ -236,19 +234,6 @@ def refresh(self, *, publish=True): assert events == ["request", "render"] * 3 + ["publish"] -def test_pre_61_runtime_falls_back_to_classic_dlss(monkeypatch): - selected = _camera_cfg(IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=True)) - env_cfg = _teleop_env_cfg([XrCameraFeedCfg(camera_name="robot_pov_cam")], camera=selected) - monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", _FakePresenter) - monkeypatch.setattr("isaaclab.utils.version.get_isaac_sim_version", lambda: Version("6.0.0")) - - session = camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=True) - - assert session.enabled - assert selected.renderer_cfg.enable_dlss_ray_reconstruction is False - assert not session.requires_responsive_denoising - - def test_camera_rendering_switch_disables_pip_before_presenter_load(monkeypatch): env_cfg = _teleop_env_cfg( [XrCameraFeedCfg(camera_name="robot_pov_cam")], From 30d17335caabe3e5b5d09e5fa8326ef390b27e25 Mon Sep 17 00:00:00 2001 From: hougantc Date: Mon, 3 Aug 2026 15:57:16 +0000 Subject: [PATCH 3/3] Fix XR camera PiP review findings Signed-off-by: hougantc --- .../isaaclab/test/controllers/test_pink_ik.py | 7 +-- .../renderers/isaac_rtx_renderer.py | 4 +- .../test_isaac_rtx_render_product_settings.py | 23 +++++++++- .../test_isaac_rtx_renderer_contract.py | 26 +++++++++-- .../locomanipulation_g1_env_cfg.py | 31 ++----------- .../pick_place/pickplace_gr1t2_env_cfg.py | 31 ++----------- .../contrib/robot_pov_camera_cfg.py | 44 +++++++++++++++++++ .../test/core/test_xr_camera_feed_presets.py | 43 ++++++++++++++++++ .../changelog.d/xr-camera-feedback.minor.rst | 5 ++- source/isaaclab_teleop/docs/README.md | 11 +---- .../isaaclab_teleop/camera_feed.py | 6 --- .../test/test_xr_camera_feed.py | 36 +-------------- ...xr_camera_feed_kit_scene_ui_integration.py | 44 +++++++++++++++++++ 13 files changed, 196 insertions(+), 115 deletions(-) create mode 100644 source/isaaclab_tasks/isaaclab_tasks/contrib/robot_pov_camera_cfg.py create mode 100644 source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py create mode 100644 source/isaaclab_teleop/test/test_xr_camera_feed_kit_scene_ui_integration.py diff --git a/source/isaaclab/test/controllers/test_pink_ik.py b/source/isaaclab/test/controllers/test_pink_ik.py index f51b8239bb82..5af72dadbbbd 100644 --- a/source/isaaclab/test/controllers/test_pink_ik.py +++ b/source/isaaclab/test/controllers/test_pink_ik.py @@ -7,8 +7,8 @@ from isaaclab.app import AppLauncher -# Camera-enabled tasks under test require the rendering experience. -simulation_app = AppLauncher(headless=True, enable_cameras=True).app +# Pink IK tests strip task cameras before environment construction. +simulation_app = AppLauncher(headless=True).app """Rest everything follows.""" @@ -21,6 +21,7 @@ import numpy as np import pytest import torch +from isaaclab_teleop import remove_camera_configs from pink.configuration import Configuration from pink.tasks import FrameTask @@ -66,7 +67,7 @@ def create_test_env(env_name, num_envs): sim_utils.create_new_stage() try: - env_cfg = parse_env_cfg(env_name, device=device, num_envs=num_envs) + env_cfg = remove_camera_configs(parse_env_cfg(env_name, device=device, num_envs=num_envs)) # Deterministic seed so IK convergence residual is reproducible across runs / machines. env_cfg.seed = 42 # Modify scene config to not spawn the packing table to avoid collision with the robot diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index bb57d0808f3a..51672c48acbf 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -18,7 +18,7 @@ import warp as wp from packaging import version -from pxr import Sdf, Usd, UsdGeom, UsdRender +from pxr import Sdf, Usd, UsdGeom from isaaclab.app.settings_manager import get_settings_manager from isaaclab.renderers import BaseRenderer, RenderBufferKind, RenderBufferSpec @@ -471,7 +471,7 @@ def _apply_render_product_settings(self, stage: Usd.Stage, render_product_paths: render_product = stage.GetPrimAtPath(render_product_path) if not render_product.IsValid(): raise RuntimeError(f"Render product '{render_product_path}' was not materialized on the USD stage.") - if not render_product.IsA(UsdRender.Product): + if render_product.GetTypeName() != "RenderProduct": raise RuntimeError(f"Prim '{render_product_path}' is not a RenderProduct.") if ray_reconstruction is not None: self._set_render_product_schema_attribute( diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_render_product_settings.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_render_product_settings.py index 8700a86cab79..f9da18d475fd 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_render_product_settings.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_render_product_settings.py @@ -15,6 +15,7 @@ import pytest from isaaclab_physx.renderers import IsaacRtxRendererCfg, IsaacRtxRendererGlobalSettingsCfg +import carb import omni.replicator.core as rep import usdrt.Usd as UsdRtUsd from pxr import UsdUtils @@ -25,6 +26,27 @@ pytestmark = [pytest.mark.integration, pytest.mark.rendering, pytest.mark.isaacsim_ci] +_GLOBAL_RTX_SETTINGS = ( + "/rtx/post/dlss/execMode", + "/rtx-transient/dldenoiser/enabled", + "/rtx/dldenoiser/responsiveDenoising", +) + + +@pytest.fixture(autouse=True) +def restore_global_rtx_settings(): + """Restore every process-global RTX setting changed by these tests.""" + settings = carb.settings.get_settings() + previous = {path: settings.get(path) for path in _GLOBAL_RTX_SETTINGS} + try: + yield + finally: + for path, value in previous.items(): + if value is None: + settings.destroy_item(path) + else: + settings.set(path, value) + def _read_dlss_settings(prim) -> tuple[str, bool]: """Read the schema-backed settings from a USD or Fabric RenderProduct prim.""" @@ -46,7 +68,6 @@ def test_camera_local_dlss_settings_survive_annotator_attachment(): spawn=sim_utils.PinholeCameraCfg(), renderer_cfg=IsaacRtxRendererCfg( global_settings=IsaacRtxRendererGlobalSettingsCfg( - antialiasing_mode="DLSS", dlss_mode=0, enable_dl_denoiser=True, carb_settings={"/rtx/dldenoiser/responsiveDenoising": True}, diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index 85c4d455e60d..aaebc57f809a 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -192,7 +192,7 @@ def test_dlss_settings_are_authored_per_render_product(monkeypatch): stage.GetPrimAtPath.side_effect = prims.__getitem__ for prim in prims.values(): prim.IsValid.return_value = True - prim.IsA.return_value = True + prim.GetTypeName.return_value = "RenderProduct" prim.ApplyAPI.return_value = True attributes = { "omni:rtx:newDenoiser:enabled": MagicMock(), @@ -239,7 +239,7 @@ def test_ray_reconstruction_uses_renderer_owned_version_fallback( stage = MagicMock() render_product = stage.GetPrimAtPath.return_value render_product.IsValid.return_value = True - render_product.IsA.return_value = True + render_product.GetTypeName.return_value = "RenderProduct" render_product.ApplyAPI.return_value = True attribute = render_product.GetAttribute.return_value attribute.IsValid.return_value = True @@ -350,13 +350,33 @@ def test_render_product_schema_failure_is_reported(monkeypatch): stage = MagicMock() render_product = stage.GetPrimAtPath.return_value render_product.IsValid.return_value = True - render_product.IsA.return_value = True + render_product.GetTypeName.return_value = "RenderProduct" render_product.ApplyAPI.return_value = False with pytest.raises(RuntimeError, match="OmniRtxDebugSettingsAPI_1"): renderer._apply_render_product_settings(stage, ["/Render/RP"]) +def test_render_product_settings_reject_non_render_product(monkeypatch): + """The schema authoring path rejects a valid prim with the wrong USD type.""" + _install_omni_stubs(monkeypatch) + from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + renderer = IsaacRtxRenderer.__new__(IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=False) + render_product = MagicMock() + render_product.IsValid.return_value = True + render_product.GetTypeName.return_value = "Camera" + stage = MagicMock() + stage.GetPrimAtPath.return_value = render_product + + with pytest.raises(RuntimeError, match="not a RenderProduct"): + renderer._apply_render_product_settings(stage, ["/Render/NotRenderProduct"]) + + render_product.ApplyAPI.assert_not_called() + + @pytest.mark.parametrize( ("has_gui", "expected_disable_color_render"), [ diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py index d627c5d7b0d6..8f2ca1de5c0b 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py @@ -20,7 +20,7 @@ from isaaclab.managers import SceneEntityCfg from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg -from isaaclab.sensors import CameraCfg, ContactSensorCfg +from isaaclab.sensors import ContactSensorCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR from isaaclab.utils.configclass import configclass @@ -37,18 +37,7 @@ from isaaclab_tasks.contrib.locomanip_pick_place.configs.pink_controller_cfg import ( # isort: skip G1_UPPER_BODY_IK_ACTION_CFG, ) -from isaaclab_physx.renderers import IsaacRtxRendererCfg # isort: skip -from isaaclab_tasks.utils.presets import MultiBackendRendererCfg # isort: skip - - -@configclass -class _RobotPovCameraRendererCfg(MultiBackendRendererCfg): - default: IsaacRtxRendererCfg = IsaacRtxRendererCfg( - camera_output_device="cuda:0", - enable_dlss_ray_reconstruction=True, - dlss_exec_mode="quality", - ) - isaacsim_rtx = default +from isaaclab_tasks.contrib.robot_pov_camera_cfg import robot_pov_camera_cfg # isort: skip def _build_g1_locomanipulation_pipeline(): @@ -309,20 +298,7 @@ class LocomanipulationG1SceneCfg(InteractiveSceneCfg): robot: ArticulationCfg = G1_29DOF_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot") # Fixed task camera matching the GR1T2 training-camera placement. - robot_pov_cam = CameraCfg( - prim_path="{ENV_REGEX_NS}/RobotPOVCam", - update_period=0.0, - height=450, - width=720, - data_types=["rgb"], - renderer_cfg=_RobotPovCameraRendererCfg(), - spawn=sim_utils.PinholeCameraCfg(focal_length=18.15, clipping_range=(0.1, 2.0)), - offset=CameraCfg.OffsetCfg( - pos=(0.0, 0.12, 1.67675), - rot=(0.9801, 0.0, 0.0, -0.19848), - convention="ros", - ), - ) + robot_pov_cam = robot_pov_camera_cfg() # Per-hand contact sensors over all finger links, used to drive controller # haptics (see HapticFeedbackCfg below). Requires activate_contact_sensors @@ -484,6 +460,7 @@ def __post_init__(self): # simulation settings self.sim.dt = 1 / 200 # 200Hz self.sim.render_interval = 2 + self.num_rerenders_on_reset = 3 # Set the URDF path for the IK controller. Path resolution (Nucleus → local) happens at runtime. self.actions.upper_body_ik.controller.urdf_path = f"{ISAACLAB_NUCLEUS_DIR}/Controllers/LocomanipulationAssets/unitree_g1_kinematics_asset/g1_29dof_with_hand_only_kinematics.urdf" # noqa: E501 diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py index 260b0ccdfba2..3166b5f598a5 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py @@ -18,7 +18,7 @@ from isaaclab.managers import SceneEntityCfg from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg -from isaaclab.sensors import CameraCfg, ContactSensorCfg +from isaaclab.sensors import ContactSensorCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR, retrieve_file_path from isaaclab.utils.configclass import configclass @@ -29,18 +29,7 @@ from isaaclab_teleop.haptic_feedback import GloveHapticFeedbackCfg # isort: skip from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg, XrCameraFeedCfg # isort: skip from isaaclab_teleop.xr_cfg import XrCfg # isort: skip -from isaaclab_physx.renderers import IsaacRtxRendererCfg # isort: skip -from isaaclab_tasks.utils.presets import MultiBackendRendererCfg # isort: skip - - -@configclass -class _RobotPovCameraRendererCfg(MultiBackendRendererCfg): - default: IsaacRtxRendererCfg = IsaacRtxRendererCfg( - camera_output_device="cuda:0", - enable_dlss_ray_reconstruction=True, - dlss_exec_mode="quality", - ) - isaacsim_rtx = default +from isaaclab_tasks.contrib.robot_pov_camera_cfg import robot_pov_camera_cfg # isort: skip def _build_gr1t2_pickplace_pipeline(): @@ -379,20 +368,7 @@ class ObjectTableSceneCfg(InteractiveSceneCfg): class PickPlaceGR1T2SceneCfg(ObjectTableSceneCfg): """GR1T2 pick-place scene with the camera observation shown in XR PiP.""" - robot_pov_cam = CameraCfg( - prim_path="{ENV_REGEX_NS}/RobotPOVCam", - update_period=0.0, - height=450, - width=720, - data_types=["rgb"], - renderer_cfg=_RobotPovCameraRendererCfg(), - spawn=sim_utils.PinholeCameraCfg(focal_length=18.15, clipping_range=(0.1, 2.0)), - offset=CameraCfg.OffsetCfg( - pos=(0.0, 0.12, 1.67675), - rot=(0.9801, 0.0, 0.0, -0.19848), - convention="ros", - ), - ) + robot_pov_cam = robot_pov_camera_cfg() ## @@ -670,6 +646,7 @@ def __post_init__(self): # simulation settings self.sim.dt = 1 / 120 # 120Hz self.sim.render_interval = 2 + self.num_rerenders_on_reset = 3 # Defer USD→URDF conversion to controller initialization (requires Isaac Sim at runtime). self.actions.upper_body_ik.controller.usd_path = self.scene.robot.spawn.usd_path diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/robot_pov_camera_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/robot_pov_camera_cfg.py new file mode 100644 index 000000000000..542828860a44 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/robot_pov_camera_cfg.py @@ -0,0 +1,44 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared recorded robot-PoV camera configuration for contributed tasks.""" + +from isaaclab_physx.renderers import IsaacRtxRendererCfg + +import isaaclab.sim as sim_utils +from isaaclab.sensors import CameraCfg +from isaaclab.utils.configclass import configclass + +from isaaclab_tasks.utils.presets import MultiBackendRendererCfg + + +@configclass +class _RobotPovCameraRendererCfg(MultiBackendRendererCfg): + """Renderer presets shared by the recorded robot-PoV cameras.""" + + default: IsaacRtxRendererCfg = IsaacRtxRendererCfg( + camera_output_device="cuda:0", + enable_dlss_ray_reconstruction=True, + dlss_exec_mode="quality", + ) + isaacsim_rtx = default + + +def robot_pov_camera_cfg() -> CameraCfg: + """Return the shared recorded robot-PoV camera configuration.""" + return CameraCfg( + prim_path="{ENV_REGEX_NS}/RobotPOVCam", + update_period=0.0, + height=450, + width=720, + data_types=["rgb"], + renderer_cfg=_RobotPovCameraRendererCfg(), + spawn=sim_utils.PinholeCameraCfg(focal_length=18.15, clipping_range=(0.1, 2.0)), + offset=CameraCfg.OffsetCfg( + pos=(0.0, 0.12, 1.67675), + rot=(0.9801, 0.0, 0.0, -0.19848), + convention="ros", + ), + ) diff --git a/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py new file mode 100644 index 000000000000..aed625f783d3 --- /dev/null +++ b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py @@ -0,0 +1,43 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest +from isaaclab_ov.renderers import OVRTXRendererCfg +from isaaclab_physx.renderers import IsaacRtxRendererCfg + +from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import LocomanipulationG1EnvCfg +from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg +from isaaclab_tasks.utils.hydra import resolve_presets +from isaaclab_tasks.utils.presets import MultiBackendRendererCfg + + +@pytest.mark.parametrize("env_cfg_type", [PickPlaceGR1T2EnvCfg, LocomanipulationG1EnvCfg]) +def test_xr_camera_reference_tasks_select_recorded_camera(env_cfg_type): + """Reference tasks record and present the same camera after a temporal reset refresh.""" + cfg = env_cfg_type() + + assert cfg.isaac_teleop.xr_camera_feeds[0].camera_name == "robot_pov_cam" + assert hasattr(cfg.observations.policy, "robot_pov_cam") + assert isinstance(cfg.scene.robot_pov_cam.renderer_cfg, MultiBackendRendererCfg) + assert cfg.num_rerenders_on_reset == 3 + + +@pytest.mark.parametrize("env_cfg_type", [PickPlaceGR1T2EnvCfg, LocomanipulationG1EnvCfg]) +def test_xr_camera_reference_renderer_resolves_for_supported_backends(env_cfg_type): + """Reference cameras retain Isaac RTX defaults and OVRTX compatibility.""" + default = resolve_presets(env_cfg_type().scene.robot_pov_cam.renderer_cfg) + isaacsim_rtx = resolve_presets( + env_cfg_type().scene.robot_pov_cam.renderer_cfg, + selected=("isaacsim_rtx",), + ) + ovrtx = resolve_presets(env_cfg_type().scene.robot_pov_cam.renderer_cfg, selected=("ovrtx",)) + + assert isinstance(default, IsaacRtxRendererCfg) + assert default.camera_output_device == "cuda:0" + assert default.enable_dlss_ray_reconstruction + assert isinstance(isaacsim_rtx, IsaacRtxRendererCfg) + assert isaacsim_rtx.camera_output_device == "cuda:0" + assert isaacsim_rtx.enable_dlss_ray_reconstruction + assert isinstance(ovrtx, OVRTXRendererCfg) diff --git a/source/isaaclab_teleop/changelog.d/xr-camera-feedback.minor.rst b/source/isaaclab_teleop/changelog.d/xr-camera-feedback.minor.rst index cadf30ff54bd..2c681e78e470 100644 --- a/source/isaaclab_teleop/changelog.d/xr-camera-feedback.minor.rst +++ b/source/isaaclab_teleop/changelog.d/xr-camera-feedback.minor.rst @@ -5,13 +5,14 @@ Added existing-camera selection, declarative layouts, and viewer-start, head-locked, or explicit-world placement. * Added direct and staged CUDA image presentation paths so camera feedback works with CPU physics. +* Added lazy Kit Scene UI loading so kitless teleoperation warns and continues without PiP. Changed ^^^^^^^ -* Changed Kit Scene UI loading to be lazy so kitless teleoperation warns and continues without PiP. * Changed ``--disable_external_cameras`` into the master camera-rendering and PiP gate for - ``teleop_se3_agent.py`` and ``record_demos.py``. + ``teleop_se3_agent.py`` and ``record_demos.py``. To keep task cameras enabled without PiP, leave + this flag unset and configure ``xr_camera_feeds`` as an empty list or with every feed disabled. Fixed ^^^^^ diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index 8d6a25c5dda0..16b8e093adff 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -98,17 +98,8 @@ uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ --task My-IsaacTeleop-Env-v0 --xr ``` -`IsaacContrib-PickPlace-GR1T2-Abs` and -`IsaacContrib-PickPlace-Locomanipulation-G1-Abs` are camera-feedback reference tasks. Both show -their recorded `robot_pov_cam` observation in PiP: - -```bash -uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ - --task IsaacContrib-PickPlace-GR1T2-Abs --xr --device cpu -``` - See [XR Camera Feedback](../../../docs/source/features/isaac_teleop.rst#xr-camera-feedback) for -camera selection, layout, placement, renderer, disable, and kitless behavior. +reference tasks, camera selection, layout, placement, renderer, disable, and kitless behavior. ### 4. Programmatic Usage diff --git a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py index 706da8d9bb5f..dcdafff22862 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py +++ b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py @@ -164,12 +164,6 @@ def refresh(self) -> None: if not self._bound: raise RuntimeError("XR camera feed session must be bound before refresh.") if self._manager is not None: - # Request post-reset camera output before each render. Temporal - # RTX cameras need several frames to replace their pre-reset - # annotator contents, matching Isaac Lab's reset rerender path. - for _ in range(3): - self._manager.refresh(publish=False) - self._env.sim.render() self._manager.refresh() def close(self) -> None: diff --git a/source/isaaclab_teleop/test/test_xr_camera_feed.py b/source/isaaclab_teleop/test/test_xr_camera_feed.py index 16eb3672a876..2e7847fc157e 100644 --- a/source/isaaclab_teleop/test/test_xr_camera_feed.py +++ b/source/isaaclab_teleop/test/test_xr_camera_feed.py @@ -16,9 +16,6 @@ from isaaclab.sensors import CameraCfg -from isaaclab_tasks.utils.hydra import resolve_presets -from isaaclab_tasks.utils.presets import MultiBackendRendererCfg - class _FakeImage: def __init__(self, height=8, width=12, *, device="cuda:0", data_ptr=100): @@ -217,10 +214,9 @@ def test_existing_camera_is_selected_without_replacement(monkeypatch): assert session.requires_responsive_denoising -def test_session_refresh_renders_reset_state_before_publishing(): +def test_session_refresh_publishes_buffer_refreshed_by_env_reset(): events = [] session = camera_feed._XrCameraFeedSession([], None, None, requires_responsive_denoising=False) - session._env = SimpleNamespace(sim=SimpleNamespace(render=lambda: events.append("render"))) class _Manager: def refresh(self, *, publish=True): @@ -231,7 +227,7 @@ def refresh(self, *, publish=True): session.refresh() - assert events == ["request", "render"] * 3 + ["publish"] + assert events == ["publish"] def test_camera_rendering_switch_disables_pip_before_presenter_load(monkeypatch): @@ -378,34 +374,6 @@ def test_manager_rejects_invalid_update_rate(monkeypatch, value): _manager(monkeypatch, [cfg], {"robot_pov_cam": _FakeImage()}) -def test_reference_tasks_select_their_recorded_camera(): - from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import LocomanipulationG1EnvCfg - from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg - - for cfg in (PickPlaceGR1T2EnvCfg(), LocomanipulationG1EnvCfg()): - assert cfg.isaac_teleop.xr_camera_feeds[0].camera_name == "robot_pov_cam" - assert hasattr(cfg.observations.policy, "robot_pov_cam") - assert isinstance(cfg.scene.robot_pov_cam.renderer_cfg, MultiBackendRendererCfg) - - -def test_reference_camera_renderer_resolves_to_isaac_rtx_and_ovrtx(): - from isaaclab_ov.renderers import OVRTXRendererCfg - - from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg - - default = resolve_presets(PickPlaceGR1T2EnvCfg().scene.robot_pov_cam.renderer_cfg) - isaacsim_rtx = resolve_presets(PickPlaceGR1T2EnvCfg().scene.robot_pov_cam.renderer_cfg, selected=("isaacsim_rtx",)) - ovrtx = resolve_presets(PickPlaceGR1T2EnvCfg().scene.robot_pov_cam.renderer_cfg, selected=("ovrtx",)) - - assert isinstance(default, IsaacRtxRendererCfg) - assert default.camera_output_device == "cuda:0" - assert default.enable_dlss_ray_reconstruction - assert isinstance(isaacsim_rtx, IsaacRtxRendererCfg) - assert isaacsim_rtx.camera_output_device == "cuda:0" - assert isaacsim_rtx.enable_dlss_ray_reconstruction - assert isinstance(ovrtx, OVRTXRendererCfg) - - def test_public_api_only_exports_declarative_camera_feed_types(): import isaaclab_teleop diff --git a/source/isaaclab_teleop/test/test_xr_camera_feed_kit_scene_ui_integration.py b/source/isaaclab_teleop/test/test_xr_camera_feed_kit_scene_ui_integration.py new file mode 100644 index 000000000000..2e1f4c378ec3 --- /dev/null +++ b/source/isaaclab_teleop/test/test_xr_camera_feed_kit_scene_ui_integration.py @@ -0,0 +1,44 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Launch the XR Kit experience before importing Scene UI.""" + +from isaaclab.app import AppLauncher + +simulation_app = AppLauncher(headless=True, xr=True, device="cpu").app + +"""Rest everything follows.""" + +import pytest +from isaaclab_teleop.camera_feed import _PanelDescriptor +from isaaclab_teleop.camera_feed_kit_scene_ui import _KitSceneUiCameraFeedPresenter + +import isaaclab.sim as sim_utils + +pytestmark = [pytest.mark.integration, pytest.mark.isaacsim_ci] + + +def test_real_scene_ui_imports_and_constructs_world_panel(): + """The XR experience provides the real Scene UI signatures used by PiP.""" + sim_utils.create_new_stage() + presenter = _KitSceneUiCameraFeedPresenter() + descriptor = _PanelDescriptor( + label="Camera", + width_m=0.48, + offset_m=(0.0, 0.0), + distance_m=0.8, + placement="world", + world_position_m=(0.0, 0.8, 1.6), + world_orientation_xyzw=(0.0, 0.0, 0.0, 1.0), + ) + + panel = presenter.create_panel(descriptor, width=720, height=450) + try: + assert panel._container is not None + assert panel._component is not None + finally: + panel.close() + + assert panel._closed