diff --git a/docs/source/api/lab_teleop/isaaclab_teleop.rst b/docs/source/api/lab_teleop/isaaclab_teleop.rst index 8ca4ab9de1a5..e516aaf86bbd 100644 --- a/docs/source/api/lab_teleop/isaaclab_teleop.rst +++ b/docs/source/api/lab_teleop/isaaclab_teleop.rst @@ -11,6 +11,8 @@ isaaclab_teleop IsaacTeleopCfg IsaacTeleopDevice + XrCameraFeedCfg + XrCameraFeedLayoutCfg HapticFeedbackCfg ControllerHapticFeedbackCfg GloveHapticFeedbackCfg @@ -40,6 +42,15 @@ Configuration .. autoclass:: XrAnchorRotationMode :members: +XR Camera Feedback +------------------ + +.. autoclass:: XrCameraFeedCfg + :members: + +.. autoclass:: XrCameraFeedLayoutCfg + :members: + Device ------ diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 0ca42aaae6ef..4ef6bfddc000 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -949,6 +949,10 @@ Key ``IsaacTeleopCfg`` fields: * ``pipeline_builder`` -- callable that returns an ``OutputCombiner`` with an ``"action"`` output. * ``retargeters_to_tune`` -- optional callable returning retargeters to expose in the live tuning UI. * ``xr_cfg`` -- :class:`~isaaclab_teleop.XrCfg` for anchor configuration (see below). +* ``xr_camera_feeds`` -- ordered selection and per-feed panel settings for existing task cameras. + The list is empty by default, so tasks opt in to PiP explicitly. +* ``xr_camera_feed_layout`` -- viewer reference, fixed world pose, and manual, horizontal, vertical, + or grid panel packing. * ``plugins`` -- list of Isaac Teleop plugin configurations (e.g. Manus). * ``sim_device`` -- torch device string (default ``"cuda:0"``). * ``retargeting_execution`` -- IsaacTeleop retargeting execution settings. @@ -1152,8 +1156,150 @@ XR device's view. .. tip:: - When using XR, call :func:`~isaaclab_teleop.remove_camera_configs` on your env config to strip - camera sensors. Additional cameras cause GPU contention and degrade XR performance. + Camera sensors add GPU cost. Strip them with ``--disable_external_cameras`` when the workflow + needs neither image observations nor XR camera feedback. That flag also skips configured PiP + feeds. + + +XR Camera Feedback +------------------ + +An ordered list of :class:`~isaaclab_teleop.XrCameraFeedCfg` objects selects existing task scene +cameras. The manager publishes each new RGBA frame after rendering, while +:class:`~isaaclab_teleop.XrCameraFeedLayoutCfg` places the panels manually or in horizontal, +vertical, and grid layouts. ``IsaacContrib-PickPlace-GR1T2-Abs`` and +``IsaacContrib-PickPlace-Locomanipulation-G1-Abs`` are the primary reference examples. + +``teleop_se3_agent.py`` and ``record_demos.py`` show every enabled feed when an IsaacTeleop-enabled +environment runs with ``--xr``. PiP is absent unless the task explicitly selects an existing +``CameraCfg`` through ``xr_camera_feeds``. In the reference examples, the selected +``robot_pov_cam`` is also a policy image observation, so the normal demonstration recorder stores +the same view shown to the operator. GR1T2 demonstrates a fixed task camera; Locomanipulation uses +the same fixed task-camera placement pattern for G1. The NutPour and ExhaustPipe GR1T2 +teleoperation tasks also present their existing recorded ``robot_pov_cam``: + +.. code-block:: bash + + uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ + --task IsaacContrib-PickPlace-GR1T2-Abs \ + --xr --device cpu + + uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ + --task IsaacContrib-PickPlace-Locomanipulation-G1-Abs \ + --xr --device cpu + +XR camera PiP currently supports exactly one environment. When a task has enabled PiP feeds, +startup rejects ``--num_envs`` values other than ``1``; IsaacTeleop XR behavior without PiP is +unchanged. + +On Isaac Sim 6.1 and newer, camera-feed setup enables responsive DLSS Ray Reconstruction denoising +automatically. For every camera consumer, the Isaac RTX renderer falls back to classic DLSS on +earlier runtimes to avoid the temporal ghosting in their Ray Reconstruction implementation. + +Camera selection +~~~~~~~~~~~~~~~~ + +Tasks declare their default selection through ``IsaacTeleopCfg.xr_camera_feeds``: + +.. code-block:: python + + from isaaclab_teleop import IsaacTeleopCfg, XrCameraFeedCfg + + self.isaac_teleop = IsaacTeleopCfg( + pipeline_builder=_build_my_pipeline, + xr_camera_feeds=[ + XrCameraFeedCfg(camera_name="left_wrist_camera"), + XrCameraFeedCfg(camera_name="overview_camera", enabled=False), + ], + ) + +For a recorded training view, define the named ``CameraCfg`` in the task scene and a matching +``mdp.image`` term in ``observations.policy``. The normal recorder then stores that observation, +while the PiP declaration above only selects it for presentation. Enabled task-declared entries +control camera selection, panel count, display order, and automatic-layout order without mutating +the reusable task configuration. + +Placement +~~~~~~~~~ + +The default :attr:`~isaaclab_teleop.XrCameraFeedLayoutCfg.placement` is ``"viewer_start"``. The +presenter waits for the first valid headset pose, captures its eye position and upright yaw, then +places the layout 0.8 m ahead at eye height. That pose remains fixed in the world while the user +moves. Panels hide when the XR display disconnects and capture a new starting pose after reconnect; +resetting the environment does not recenter them. + +Three placement references are available: + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Placement + - Behavior + * - ``viewer_start`` + - Capture the first valid eye position and upright yaw, then keep the layout world-static. + This is the default. + * - ``head_locked`` + - Follow the current headset position and orientation with the configured offset and distance. + * - ``world`` + - Use ``world_position_m`` and ``world_orientation_xyzw`` as a fixed pose in the Isaac Lab USD + stage world. ``distance_m`` is unused. + +.. code-block:: python + + from isaaclab_teleop import XrCameraFeedLayoutCfg + + # Default: fixed in the world at the user's starting eye pose. + self.isaac_teleop.xr_camera_feed_layout = XrCameraFeedLayoutCfg() + + # Follow the headset and pack feeds left to right. + self.isaac_teleop.xr_camera_feed_layout = XrCameraFeedLayoutCfg( + placement="head_locked", + mode="horizontal", + distance_m=0.8, + ) + + # Place a grid in an Isaac Lab Z-up world. This example assumes an eye at + # (0, 0, 1.6) looking along world +Y. + self.isaac_teleop.xr_camera_feed_layout = XrCameraFeedLayoutCfg( + placement="world", + mode="grid", + world_position_m=(0.0, 0.8, 1.6), + world_orientation_xyzw=(0.7071067812, 0.0, 0.0, 0.7071067812), + max_columns=2, + ) + +``manual`` mode preserves each feed's ``offset_m`` and ``distance_m``. ``horizontal``, ``vertical``, +and ``grid`` modes pack enabled feeds around ``center_offset_m`` using ``panel_gap_m``; grid mode +also honors ``max_columns``. Offsets are measured in the selected layout plane. With +``viewer_start`` and ``head_locked``, ``distance_m`` places that plane in front of the viewer +reference. In manual mode, feeds with identical offsets overlap; select an automatic mode or assign +distinct feed offsets when showing multiple panels. + +With ``world``, the explicit pose is measured in the Isaac Lab USD stage world in meters, not +OpenXR physical space. Isaac Lab stages are Z-up. ``world_orientation_xyzw`` maps panel-local +coordinates into that world: local +X is image right, local +Y is image up, and local +Z points +from the readable face toward the viewer. Feed and automatic-layout offsets are applied in the +panel's local XY plane. + +Disable cameras and PiP +~~~~~~~~~~~~~~~~~~~~~~~ + +Set :attr:`~isaaclab_teleop.XrCameraFeedCfg.enabled` to ``False`` to suppress one feed, or set +``xr_camera_feeds=[]`` to suppress all PiP. The list is empty by default. These choices do not +remove camera sensors independently owned by the task. + +``--disable_external_cameras`` is the master camera-rendering switch for ``teleop_se3_agent.py`` and +``record_demos.py``. It strips camera sensors and suppresses every configured PiP feed. + +Kit Scene UI presentation +~~~~~~~~~~~~~~~~~~~~~~~~~ + +PiP presentation uses Kit Scene UI and ``SpatialSource`` placement. Kit imports are deferred until +an enabled feed is requested. If the Scene UI modules cannot be imported, the scripts log a warning +and continue without PiP; task-owned cameras and recording observations remain unchanged. This keeps +the camera selection configuration usable when a future kitless entry point no longer provides +Scene UI. Configuration, camera-buffer, and panel-initialization errors still fail during startup. .. _isaac-teleop-haptics: @@ -1557,8 +1703,10 @@ Optimize XR Performance The flag strips the environment's camera sensors (equivalent to calling :func:`~isaaclab_teleop.remove_camera_configs` on the env config) and selects a lighter Kit - experience. Omit it to keep cameras enabled (the default) -- required when recording camera - observations, or when you want ``teleop_replay_agent.py`` to mirror the production render load. + experience. For ``teleop_se3_agent.py`` it is also the master PiP gate: task-default and + task-configured feeds are ignored and no camera panel is created. Omit it to keep cameras enabled + (the default) -- required when recording camera observations, showing XR camera feedback, or when + you want ``teleop_replay_agent.py`` to mirror the production render load. .. dropdown:: Run headless (skip the local viewport) :open: @@ -1683,6 +1831,8 @@ See the :ref:`isaaclab_teleop-api` for full class and function documentation: * :class:`~isaaclab_teleop.HapticFeedbackReceiver` * :class:`~isaaclab_teleop.HapticFeedbackDriver` * :func:`~isaaclab_teleop.create_haptic_feedback_driver` +* :class:`~isaaclab_teleop.XrCameraFeedCfg` +* :class:`~isaaclab_teleop.XrCameraFeedLayoutCfg` * :class:`~isaaclab_teleop.XrCfg` * :class:`~isaaclab_teleop.XrAnchorRotationMode` diff --git a/scripts/environments/teleoperation/teleop_se3_agent.py b/scripts/environments/teleoperation/teleop_se3_agent.py index 66ab57fc4205..8d79f26ce5fa 100644 --- a/scripts/environments/teleoperation/teleop_se3_agent.py +++ b/scripts/environments/teleoperation/teleop_se3_agent.py @@ -272,6 +272,14 @@ def main() -> None: # noqa: C901 not teleop_device_explicitly_set and hasattr(env_cfg, "isaac_teleop") and env_cfg.isaac_teleop is not None ) + from isaaclab_teleop.camera_feed import _XrCameraFeedSession + + camera_feed_session = _XrCameraFeedSession.prepare( + env_cfg, + enabled=args_cli.xr and use_isaac_teleop, + camera_rendering_enabled=not args_cli.disable_external_cameras, + ) + # XR-rendering setup (camera removal + DLSS) is only needed for the Kit XR # path. Without --xr, IsaacTeleop runs standalone (I/O only) and renders # normally, so gate on --xr alone. @@ -286,7 +294,14 @@ def main() -> None: # noqa: C901 if _rtx_rendering_requested(args_cli): _ensure_replicator_loaded() apply_isaac_rtx_global_settings( - IsaacRtxRendererGlobalSettingsCfg(antialiasing_mode="DLSS"), + IsaacRtxRendererGlobalSettingsCfg( + antialiasing_mode="DLSS", + carb_settings=( + {"/rtx/dldenoiser/responsiveDenoising": True} + if camera_feed_session.requires_responsive_denoising + else None + ), + ), ) try: @@ -493,6 +508,7 @@ def run_loop(): if should_reset_recording_instance: env.reset() teleop_interface.reset() + camera_feed_session.refresh() should_reset_recording_instance = False print("Environment reset complete") except Exception as e: @@ -502,10 +518,11 @@ def run_loop(): # Run the teleoperation loop # IsaacTeleop requires a context manager, native devices don't if use_isaac_teleop: - with teleop_interface: + with teleop_interface, camera_feed_session.bind(env): run_loop() else: - run_loop() + with camera_feed_session.bind(env): + run_loop() # close the simulator env.close() diff --git a/scripts/tools/record_demos.py b/scripts/tools/record_demos.py index 000f66bb91f3..21919e8aa47e 100644 --- a/scripts/tools/record_demos.py +++ b/scripts/tools/record_demos.py @@ -34,11 +34,15 @@ # Standard library imports import argparse import contextlib +from typing import TYPE_CHECKING # Isaac Lab AppLauncher from isaaclab.app import AppLauncher from isaaclab.utils.string import list_intersection, string_to_callable +if TYPE_CHECKING: + from isaaclab_teleop.camera_feed import _XrCameraFeedSession + # add argparse arguments parser = argparse.ArgumentParser(description="Record demonstrations for Isaac Lab environments.") parser.add_argument("--task", type=str, required=True, help="Name of the task.") @@ -575,6 +579,7 @@ def run_simulation_loop( # noqa: C901 teleop_interface: object | None, success_term: object | None, rate_limiter: RateLimiter | None, + camera_feed_session: "_XrCameraFeedSession", use_isaac_teleop: bool = False, ) -> int: """Run the main simulation loop for collecting demonstrations. @@ -588,6 +593,7 @@ def run_simulation_loop( # noqa: C901 teleop_interface: Optional teleop interface (will be created if None) success_term: The success termination object or None if not available rate_limiter: Optional rate limiter to control simulation speed + camera_feed_session: Shared XR camera-feed lifecycle use_isaac_teleop: Whether to use IsaacTeleop stack Returns: @@ -694,7 +700,7 @@ def inner_loop(): if use_isaac_teleop: from isaaclab_teleop import poll_control_events - with contextlib.suppress(KeyboardInterrupt), torch.inference_mode(): + with contextlib.suppress(KeyboardInterrupt), torch.inference_mode(), camera_feed_session.bind(env): while simulation_app.is_running(): # Get teleop command (may be None while waiting for session start) action = teleop_interface.advance() @@ -764,6 +770,7 @@ def inner_loop(): success_step_count = handle_reset( env, success_step_count, instruction_display, label_text, teleop_interface ) + camera_feed_session.refresh() should_reset_recording_instance = False # Check if simulation is stopped @@ -804,6 +811,20 @@ def main() -> None: global env_cfg # Make env_cfg available to setup_teleop_device env_cfg, success_term, use_isaac_teleop = create_environment_config(output_dir, output_file_name) + from isaaclab_teleop.camera_feed import _XrCameraFeedSession + + camera_feed_session = _XrCameraFeedSession.prepare( + env_cfg, + enabled=args_cli.xr and use_isaac_teleop, + camera_rendering_enabled=not args_cli.disable_external_cameras, + ) + if camera_feed_session.requires_responsive_denoising: + apply_isaac_rtx_global_settings( + IsaacRtxRendererGlobalSettingsCfg( + carb_settings={"/rtx/dldenoiser/responsiveDenoising": True}, + ) + ) + # With --xr, rate limiting is achieved via OpenXR and the XR visualization # manager is installed. Without --xr (including standalone IsaacTeleop I/O), # fall back to the software rate limiter and skip the XR viz stack. @@ -820,7 +841,9 @@ def main() -> None: env = create_environment(env_cfg) # Run simulation loop - current_recorded_demo_count = run_simulation_loop(env, None, success_term, rate_limiter, use_isaac_teleop) + current_recorded_demo_count = run_simulation_loop( + env, None, success_term, rate_limiter, camera_feed_session, use_isaac_teleop + ) # Clean up env.close() diff --git a/source/isaaclab/changelog.d/xr-camera-feedback.minor.rst b/source/isaaclab/changelog.d/xr-camera-feedback.minor.rst new file mode 100644 index 000000000000..2c38716e6b9d --- /dev/null +++ b/source/isaaclab/changelog.d/xr-camera-feedback.minor.rst @@ -0,0 +1,5 @@ +Added +^^^^^ + +* Allowed renderers to select a camera pixel-output device independently from camera pose and + simulation state. diff --git a/source/isaaclab/isaaclab/renderers/base_renderer.py b/source/isaaclab/isaaclab/renderers/base_renderer.py index 5b571e3a6936..95170934a0b4 100644 --- a/source/isaaclab/isaaclab/renderers/base_renderer.py +++ b/source/isaaclab/isaaclab/renderers/base_renderer.py @@ -44,6 +44,21 @@ def initialize(self) -> None: """Post-physics one-time initialization hook. Called only once.""" return + def resolve_camera_output_device(self, simulation_device: str) -> str: + """Resolve the device used for persistent camera pixel outputs. + + The default keeps outputs on the simulation device. A renderer may + override this when its pixel production is independent of physics and + camera pose storage. + + Args: + simulation_device: Device used by simulation and camera state. + + Returns: + Device used by the render specification and image output buffers. + """ + return simulation_device + def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: """Pre-render per-camera setup the backend needs. diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index 660065996cf6..4fcf5da0ad04 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -208,6 +208,7 @@ def __init__(self, cfg: CameraCfg): # Renderer and render data — assigned in _initialize_impl. self._renderer: BaseRenderer | None = None self._render_data = None + self._camera_output_device: str | None = None def __del__(self): """Unsubscribes from callbacks and cleans up renderer resources.""" @@ -516,9 +517,10 @@ def _initialize_impl(self): cam_paths[0].removeprefix(env_0_prefix) if cam_paths and cam_paths[0].startswith(env_0_prefix) else "" ) device_str = self._device if isinstance(self._device, str) else str(self._device) + self._camera_output_device = self._renderer.resolve_camera_output_device(device_str) render_spec = CameraRenderSpec( cfg=self.cfg, - device=device_str, + device=self._camera_output_device, num_instances=self._num_envs, camera_prim_paths=cam_paths, view_count=self._num_envs, @@ -640,18 +642,20 @@ def _create_buffers(self): type(self._renderer).__name__, unsupported, ) - device_str = self._device if isinstance(self._device, str) else str(self._device) + state_device = self._device if isinstance(self._device, str) else str(self._device) + if self._camera_output_device is None: + raise RuntimeError("Camera output device was not resolved before buffer allocation.") self._data = CameraData.allocate( data_types=known, height=self.cfg.height, width=self.cfg.width, num_views=self._view.count, - device=self._device, + device=self._camera_output_device, supported_specs=specs, ) # Camera-frame state (pose / intrinsics) is owned by the camera, not # the renderer: allocate warp buffers and populate them. - self._data.create_buffers(self._view.count, device_str) + self._data.create_buffers(self._view.count, state_device) self._update_intrinsic_matrices() self._update_poses() self._renderer.set_outputs(self._render_data, self._data.output) diff --git a/source/isaaclab/test/controllers/test_pink_ik.py b/source/isaaclab/test/controllers/test_pink_ik.py index a7935875fa7b..f51b8239bb82 100644 --- a/source/isaaclab/test/controllers/test_pink_ik.py +++ b/source/isaaclab/test/controllers/test_pink_ik.py @@ -7,8 +7,8 @@ from isaaclab.app import AppLauncher -# launch omniverse app -simulation_app = AppLauncher(headless=True).app +# Camera-enabled tasks under test require the rendering experience. +simulation_app = AppLauncher(headless=True, enable_cameras=True).app """Rest everything follows.""" diff --git a/source/isaaclab/test/renderers/test_simulation_render_context.py b/source/isaaclab/test/renderers/test_simulation_render_context.py index 0eb7fbc343b5..f3f8beca26b9 100644 --- a/source/isaaclab/test/renderers/test_simulation_render_context.py +++ b/source/isaaclab/test/renderers/test_simulation_render_context.py @@ -111,6 +111,21 @@ def test_get_renderer_returns_equal_cfg_singleton(): assert r1 is r2 +def test_get_renderer_isolates_per_camera_dlss_settings(): + """Camera-local RTX settings create a distinct backend while equal settings reuse one.""" + ctx = RenderContext() + + default_renderer = ctx.get_renderer(IsaacRtxRendererCfg()) + tuned_cfg = IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=False, dlss_exec_mode="quality") + tuned_renderer = ctx.get_renderer(tuned_cfg) + tuned_renderer_again = ctx.get_renderer( + IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=False, dlss_exec_mode="quality") + ) + + assert tuned_renderer is not default_renderer + assert tuned_renderer_again is tuned_renderer + + def test_get_renderer_two_different_concrete_types_coexist(): """Different renderer_cfg concrete classes register distinct backends (no error).""" diff --git a/source/isaaclab_physx/changelog.d/xr-camera-dlss-ghosting.rst b/source/isaaclab_physx/changelog.d/xr-camera-dlss-ghosting.rst new file mode 100644 index 000000000000..ba2618e52ff7 --- /dev/null +++ b/source/isaaclab_physx/changelog.d/xr-camera-dlss-ghosting.rst @@ -0,0 +1,13 @@ +Added +^^^^^ + +* Added per-camera DLSS Ray Reconstruction enable and execution-mode overrides to + :class:`~isaaclab_physx.renderers.IsaacRtxRendererCfg`, plus a process-global responsive-denoising setting. +* Added an optional RTX camera output-device override so CUDA pixels can be produced with CPU + physics and CPU camera pose state. + +Fixed +^^^^^ + +* Fixed the pre-6.1 Ray Reconstruction compatibility fallback so it applies to every Isaac RTX + camera consumer rather than only cameras selected for XR PiP. diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py index 553d16289381..84c24abd22c5 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer.py @@ -18,7 +18,7 @@ import warp as wp from packaging import version -from pxr import Sdf, Usd, UsdGeom +from pxr import Sdf, Usd, UsdGeom, UsdRender from isaaclab.app.settings_manager import get_settings_manager from isaaclab.renderers import BaseRenderer, RenderBufferKind, RenderBufferSpec @@ -81,6 +81,11 @@ def _raise_missing_ppisp_error(exc: ModuleNotFoundError) -> NoReturn: "simple_shading_full_mdl": 3, } SIMPLE_SHADING_MODE_SETTING = "/rtx/minimal/mode" +DLSS_RAY_RECONSTRUCTION_API_SCHEMA = "OmniRtxDebugSettingsAPI_1" +DLSS_RAY_RECONSTRUCTION_ATTR = "omni:rtx:newDenoiser:enabled" +DLSS_EXEC_MODE_API_SCHEMA = "OmniRtxSettingsRtAPI_1" +DLSS_EXEC_MODE_ATTR = "omni:rtx:post:dlss:execMode" +_DLSS_RAY_RECONSTRUCTION_MIN_ISAAC_SIM_VERSION = (6, 1) def _camera_semantic_filter_predicate(semantic_filter: str | list[str]) -> str: @@ -130,6 +135,10 @@ def __init__(self, cfg: IsaacRtxRendererCfg): ensure_rtx_hydra_engine_attached() # ``/isaaclab/render/rtx_sensors`` is owned by ``Camera.__init__`` (must be set pre-``sim.reset()``). + def resolve_camera_output_device(self, simulation_device: str) -> str: + """Use the optional RTX pixel-output override without moving camera state.""" + return self.cfg.camera_output_device or simulation_device + def prepare_cameras(self, stage: Any, spec: CameraRenderSpec) -> None: """Resolve the camera's PPISP cfg and apply RTX-specific USD overrides. @@ -418,6 +427,10 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: for annotator in annotators.values(): annotator.attach([rp.path]) + # HydraTexture attachment synchronizes legacy global settings onto the render product. + # Author camera-local overrides only after every annotator has finished attaching. + self._apply_render_product_settings(stage, [rp.path]) + ppisp_pipeline = None if spec.cfg.isp_cfg is not None: try: @@ -434,6 +447,62 @@ def create_render_data(self, spec: CameraRenderSpec) -> IsaacRtxRenderData: ppisp_pipeline=ppisp_pipeline, ) + def _apply_render_product_settings(self, stage: Usd.Stage, render_product_paths: list[str]) -> None: + """Author optional renderer settings that must remain local to one camera.""" + ray_reconstruction = self.cfg.enable_dlss_ray_reconstruction + dlss_exec_mode = self.cfg.dlss_exec_mode + if ray_reconstruction is None and dlss_exec_mode is None: + return + if ray_reconstruction is True: + isaac_sim_version = get_isaac_sim_version() + isaac_sim_major_minor = (isaac_sim_version.major, isaac_sim_version.minor) + if isaac_sim_major_minor < _DLSS_RAY_RECONSTRUCTION_MIN_ISAAC_SIM_VERSION: + ray_reconstruction = False + logger.warning( + "DLSS Ray Reconstruction was requested, but Isaac Sim %s predates responsive denoising. " + "Falling back to classic DLSS for this render product.", + isaac_sim_version, + ) + for render_product_path in render_product_paths: + render_product = stage.GetPrimAtPath(render_product_path) + if not render_product.IsValid(): + raise RuntimeError(f"Render product '{render_product_path}' was not materialized on the USD stage.") + if not render_product.IsA(UsdRender.Product): + raise RuntimeError(f"Prim '{render_product_path}' is not a RenderProduct.") + if ray_reconstruction is not None: + self._set_render_product_schema_attribute( + render_product, + DLSS_RAY_RECONSTRUCTION_API_SCHEMA, + DLSS_RAY_RECONSTRUCTION_ATTR, + ray_reconstruction, + ) + if dlss_exec_mode is not None: + self._set_render_product_schema_attribute( + render_product, + DLSS_EXEC_MODE_API_SCHEMA, + DLSS_EXEC_MODE_ATTR, + dlss_exec_mode, + ) + + @staticmethod + def _set_render_product_schema_attribute( + render_product: Usd.Prim, + api_schema: str, + attribute_name: str, + value: bool | str, + ) -> None: + """Apply one RTX API schema and author its validated render-product attribute.""" + if not render_product.ApplyAPI(api_schema): + raise RuntimeError(f"Failed to apply RTX API schema '{api_schema}' to '{render_product.GetPath()}'.") + attribute = render_product.GetAttribute(attribute_name) + if not attribute.IsValid(): + raise RuntimeError( + f"RTX API schema '{api_schema}' does not provide attribute '{attribute_name}' " + f"on '{render_product.GetPath()}'." + ) + if not attribute.Set(value): + raise RuntimeError(f"Failed to set RTX attribute '{attribute_name}' on '{render_product.GetPath()}'.") + def _resolve_simple_shading_mode(self, spec: CameraRenderSpec) -> int | None: """Resolve the requested simple shading mode from data types.""" requested = [dt for dt in spec.cfg.data_types if dt in SIMPLE_SHADING_MODES] diff --git a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py index 9bd1a7e1ad88..43a5f8ed37b1 100644 --- a/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py +++ b/source/isaaclab_physx/isaaclab_physx/renderers/isaac_rtx_renderer_cfg.py @@ -10,6 +10,8 @@ from isaaclab.renderers.renderer_cfg import RendererCfg from isaaclab.utils.configclass import configclass +_DLSS_EXEC_MODES = {"performance", "balanced", "quality", "auto", "rtxaa", "manual"} + @configclass class IsaacRtxRendererGlobalSettingsCfg: @@ -105,6 +107,32 @@ class IsaacRtxRendererCfg(RendererCfg): global_settings: IsaacRtxRendererGlobalSettingsCfg = IsaacRtxRendererGlobalSettingsCfg() """Global Kit/RTX quality settings applied before RTX Hydra attach.""" + camera_output_device: str | None = None + """Optional device for Replicator annotators and persistent camera pixel outputs. + + This leaves physics and camera pose state on the simulation device. For + example, set ``"cuda:0"`` with CPU physics to avoid an RTX GPU-to-CPU + readback when a GPU consumer uses the camera image. ``None`` preserves the + simulation-device behavior. + """ + + enable_dlss_ray_reconstruction: bool | None = None + """Enable DLSS Ray Reconstruction for this renderer's render products. + + Set to ``False`` to use classic DLSS while leaving other render products, + such as XR headset views, on their process-global setting. ``None`` + preserves Kit's render-product default. On Isaac Sim versions before 6.1, + ``True`` falls back to classic DLSS because responsive denoising is unavailable. + """ + + dlss_exec_mode: Literal["performance", "balanced", "quality", "auto", "rtxaa", "manual"] | None = None + """Optional render-product-local DLSS execution mode. + + ``None`` preserves Kit's render-product default. Unlike + :attr:`global_settings`, this setting affects only cameras using this + renderer configuration. + """ + semantic_filter: str | list[str] = "*:*" """A string or a list specifying a semantic filter predicate. Defaults to ``"*:*"``. @@ -172,3 +200,28 @@ class IsaacRtxRendererCfg(RendererCfg): - ``"zero"``: Values are clipped to zero. - ``"none"``: No clipping is applied. Values will be returned as ``inf``. """ + + def __post_init__(self) -> None: + """Validate render-product settings before Kit silently ignores them.""" + if self.camera_output_device is not None and ( + not isinstance(self.camera_output_device, str) + or not ( + self.camera_output_device == "cpu" + or self.camera_output_device == "cuda" + or ( + self.camera_output_device.startswith("cuda:") + and self.camera_output_device.removeprefix("cuda:").isdigit() + ) + ) + ): + raise ValueError( + "camera_output_device must be 'cpu', 'cuda', 'cuda:', or None, " + f"got {self.camera_output_device!r}." + ) + ray_reconstruction = self.enable_dlss_ray_reconstruction + if ray_reconstruction is not None and type(ray_reconstruction) is not bool: + raise TypeError("enable_dlss_ray_reconstruction must be a bool or None.") + if self.dlss_exec_mode is not None and self.dlss_exec_mode not in _DLSS_EXEC_MODES: + raise ValueError( + f"dlss_exec_mode must be one of {sorted(_DLSS_EXEC_MODES)} or None, got {self.dlss_exec_mode!r}." + ) diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_render_product_settings.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_render_product_settings.py new file mode 100644 index 000000000000..8700a86cab79 --- /dev/null +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_render_product_settings.py @@ -0,0 +1,113 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Isaac RTX RenderProduct settings integration tests.""" + +from isaaclab.app import AppLauncher + +# RenderProduct materialization and HydraTexture attachment require a camera-enabled Kit app. +simulation_app = AppLauncher(headless=True, enable_cameras=True, device="cpu").app + +"""Rest everything follows.""" + +import pytest +from isaaclab_physx.renderers import IsaacRtxRendererCfg, IsaacRtxRendererGlobalSettingsCfg + +import omni.replicator.core as rep +import usdrt.Usd as UsdRtUsd +from pxr import UsdUtils + +import isaaclab.sim as sim_utils +from isaaclab.app.settings_manager import get_settings_manager +from isaaclab.sensors.camera import Camera, CameraCfg + +pytestmark = [pytest.mark.integration, pytest.mark.rendering, pytest.mark.isaacsim_ci] + + +def _read_dlss_settings(prim) -> tuple[str, bool]: + """Read the schema-backed settings from a USD or Fabric RenderProduct prim.""" + exec_mode = prim.GetAttribute("omni:rtx:post:dlss:execMode").Get() + ray_reconstruction = prim.GetAttribute("omni:rtx:newDenoiser:enabled").Get() + return str(exec_mode), bool(ray_reconstruction) + + +def test_camera_local_dlss_settings_survive_annotator_attachment(): + """Per-camera settings override conflicting globals in both USD and Fabric after attachment.""" + sim_utils.create_new_stage() + sim = sim_utils.SimulationContext(sim_utils.SimulationCfg(device="cpu", dt=1.0 / 60.0)) + camera = Camera( + CameraCfg( + prim_path="/World/Camera", + height=96, + width=96, + data_types=["rgb"], + spawn=sim_utils.PinholeCameraCfg(), + renderer_cfg=IsaacRtxRendererCfg( + global_settings=IsaacRtxRendererGlobalSettingsCfg( + antialiasing_mode="DLSS", + dlss_mode=0, + enable_dl_denoiser=True, + carb_settings={"/rtx/dldenoiser/responsiveDenoising": True}, + ), + enable_dlss_ray_reconstruction=False, + dlss_exec_mode="quality", + ), + ) + ) + + try: + sim.reset() + for _ in range(2): + sim.step() + camera.update(sim.cfg.dt) + + stage = sim_utils.get_current_stage() + render_product_path = camera._render_data.render_product.path + usd_prim = stage.GetPrimAtPath(render_product_path) + + stage_id = UsdUtils.StageCache.Get().GetId(stage).ToLongInt() + fabric_stage = UsdRtUsd.Stage.Attach(stage_id) + fabric_prim = fabric_stage.GetPrimAtPath(render_product_path) + + expected = ("quality", False) + assert _read_dlss_settings(usd_prim) == expected + assert _read_dlss_settings(fabric_prim) == expected + assert get_settings_manager().get("/rtx/dldenoiser/responsiveDenoising") is True + finally: + camera._invalidate_initialize_callback(None) + rep.vp_manager.destroy_hydra_textures("Replicator") + sim.stop() + sim.clear_instance() + + +def test_cpu_simulation_camera_can_publish_cuda_pixels(): + """Keep camera pose state on CPU while Isaac RTX publishes its image on CUDA.""" + sim_utils.create_new_stage() + sim = sim_utils.SimulationContext(sim_utils.SimulationCfg(device="cpu", dt=1.0 / 60.0)) + camera = Camera( + CameraCfg( + prim_path="/World/Camera", + height=64, + width=64, + data_types=["rgb"], + spawn=sim_utils.PinholeCameraCfg(), + renderer_cfg=IsaacRtxRendererCfg(camera_output_device="cuda:0"), + ) + ) + + try: + sim.reset() + for _ in range(2): + sim.step() + camera.update(sim.cfg.dt) + + assert camera.data.pos_w.torch.device.type == "cpu" + assert camera.data.output["rgb"].torch.device.type == "cuda" + assert camera.data.output["rgb"].torch.device.index == 0 + finally: + camera._invalidate_initialize_callback(None) + rep.vp_manager.destroy_hydra_textures("Replicator") + sim.stop() + sim.clear_instance() diff --git a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py index 2f22e9ff3524..b9fd6def7d15 100644 --- a/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py +++ b/source/isaaclab_physx/test/renderers/test_isaac_rtx_renderer_contract.py @@ -155,6 +155,208 @@ def test_render_product_uuid_name_format_is_sdf_safe(): assert Sdf.Path.IsValidPathString(f"/Render/{name}") +def test_camera_output_device_override_is_rtx_local(monkeypatch): + """RTX may put pixels on CUDA while the base renderer keeps simulation placement.""" + _install_omni_stubs(monkeypatch) + from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + from isaaclab.renderers.base_renderer import BaseRenderer + + renderer = IsaacRtxRenderer.__new__(IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg(camera_output_device="cuda:0") + + assert renderer.resolve_camera_output_device("cpu") == "cuda:0" + assert BaseRenderer.resolve_camera_output_device(renderer, "cpu") == "cpu" + + +@pytest.mark.parametrize("value", ["gpu", "cuda:x", "", 0]) +def test_renderer_cfg_rejects_invalid_camera_output_device(value): + """Invalid output devices fail during configuration rather than Replicator startup.""" + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + with pytest.raises(ValueError, match="camera_output_device"): + IsaacRtxRendererCfg(camera_output_device=value) + + +def test_dlss_settings_are_authored_per_render_product(monkeypatch): + """Camera-local DLSS settings use their schema-backed RenderProduct attributes.""" + _install_omni_stubs(monkeypatch) + from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + renderer = IsaacRtxRenderer.__new__(IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=False, dlss_exec_mode="quality") + stage = MagicMock() + prims = {path: MagicMock() for path in ("/Render/RP_0", "/Render/RP_1")} + stage.GetPrimAtPath.side_effect = prims.__getitem__ + for prim in prims.values(): + prim.IsValid.return_value = True + prim.IsA.return_value = True + prim.ApplyAPI.return_value = True + attributes = { + "omni:rtx:newDenoiser:enabled": MagicMock(), + "omni:rtx:post:dlss:execMode": MagicMock(), + } + for attribute in attributes.values(): + attribute.IsValid.return_value = True + attribute.Set.return_value = True + prim.GetAttribute.side_effect = attributes.__getitem__ + prim._test_attributes = attributes + + renderer._apply_render_product_settings(stage, list(prims)) + + for prim in prims.values(): + assert prim.ApplyAPI.call_args_list == [ + call("OmniRtxDebugSettingsAPI_1"), + call("OmniRtxSettingsRtAPI_1"), + ] + assert prim.GetAttribute.call_args_list == [ + call("omni:rtx:newDenoiser:enabled"), + call("omni:rtx:post:dlss:execMode"), + ] + prim._test_attributes["omni:rtx:newDenoiser:enabled"].Set.assert_called_once_with(False) + prim._test_attributes["omni:rtx:post:dlss:execMode"].Set.assert_called_once_with("quality") + + +@pytest.mark.parametrize( + ("isaac_sim_version", "expected_ray_reconstruction"), + [ + pytest.param("6.0.0", False, id="pre-responsive-denoising"), + pytest.param("6.1.0", True, id="responsive-denoising"), + ], +) +def test_ray_reconstruction_uses_renderer_owned_version_fallback( + monkeypatch, caplog, isaac_sim_version, expected_ray_reconstruction +): + """All Isaac RTX camera consumers share the pre-6.1 compatibility fallback.""" + _install_omni_stubs(monkeypatch) + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=True) + stage = MagicMock() + render_product = stage.GetPrimAtPath.return_value + render_product.IsValid.return_value = True + render_product.IsA.return_value = True + render_product.ApplyAPI.return_value = True + attribute = render_product.GetAttribute.return_value + attribute.IsValid.return_value = True + attribute.Set.return_value = True + + with patch.object(rtx_renderer, "get_isaac_sim_version", return_value=version.parse(isaac_sim_version)): + renderer._apply_render_product_settings(stage, ["/Render/RP"]) + + attribute.Set.assert_called_once_with(expected_ray_reconstruction) + assert ("Falling back to classic DLSS" in caplog.text) == (not expected_ray_reconstruction) + + +def test_default_renderer_cfg_does_not_author_render_product_settings(monkeypatch): + """The default config preserves Kit's render-product settings.""" + _install_omni_stubs(monkeypatch) + from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + renderer = IsaacRtxRenderer.__new__(IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg() + stage = MagicMock() + + renderer._apply_render_product_settings(stage, ["/Render/RP"]) + + stage.GetPrimAtPath.assert_not_called() + + +def test_dlss_settings_are_authored_after_all_annotators_attach(monkeypatch): + """HydraTexture attachment must finish before camera-local DLSS settings are authored.""" + replicator_module, syntheticdata_module = _install_omni_stubs(monkeypatch) + monkeypatch.setattr(syntheticdata_module, "SyntheticData", MagicMock(), raising=False) + + import isaaclab_physx.renderers.isaac_rtx_renderer as rtx_renderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + import isaaclab.sim.utils.stage as stage_utils + + events = [] + color_annotator = MagicMock() + depth_annotator = MagicMock() + color_annotator.attach.side_effect = lambda paths: events.append(("attach", "rgba", paths)) + depth_annotator.attach.side_effect = lambda paths: events.append(("attach", "depth", paths)) + + replicator_module.create = SimpleNamespace( + render_product_tiled=MagicMock(return_value=SimpleNamespace(path="/Render/RP")) + ) + replicator_module.AnnotatorRegistry = MagicMock() + replicator_module.AnnotatorRegistry.get_annotator.side_effect = [color_annotator, depth_annotator] + + stage = MagicMock() + stage.GetPrimAtPath.return_value.IsA.return_value = True + spec = SimpleNamespace( + camera_prim_paths=("/World/Camera",), + cfg=SimpleNamespace( + data_types=["rgba", "distance_to_image_plane"], + width=64, + height=48, + isp_cfg=None, + ), + device="cpu", + ) + renderer = rtx_renderer.IsaacRtxRenderer.__new__(rtx_renderer.IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=False, dlss_exec_mode="quality") + renderer._apply_render_product_settings = MagicMock( + side_effect=lambda authored_stage, paths: events.append(("author", authored_stage, paths)) + ) + + with ( + patch.object(rtx_renderer, "get_settings_manager", return_value=MagicMock()), + patch.object(rtx_renderer, "get_isaac_sim_version", return_value=version.parse("5.1")), + patch.object(stage_utils, "get_current_stage", return_value=stage), + ): + renderer.create_render_data(spec) + + assert events == [ + ("attach", "rgba", ["/Render/RP"]), + ("attach", "depth", ["/Render/RP"]), + ("author", stage, ["/Render/RP"]), + ] + + +@pytest.mark.parametrize("value", ["", "Quality", "ultra", 2]) +def test_renderer_cfg_rejects_invalid_dlss_exec_mode(value): + """Invalid schema tokens fail before Kit can silently ignore them.""" + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + with pytest.raises(ValueError, match="dlss_exec_mode"): + IsaacRtxRendererCfg(dlss_exec_mode=value) + + +@pytest.mark.parametrize("value", [0, 1, "false"]) +def test_renderer_cfg_rejects_non_bool_dlss_ray_reconstruction(value): + """The RenderProduct boolean must not accept truthy values of another type.""" + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + with pytest.raises(TypeError, match="enable_dlss_ray_reconstruction"): + IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=value) + + +def test_render_product_schema_failure_is_reported(monkeypatch): + """A missing RTX schema fails at camera setup instead of creating an inert custom attribute.""" + _install_omni_stubs(monkeypatch) + from isaaclab_physx.renderers.isaac_rtx_renderer import IsaacRtxRenderer + from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + + renderer = IsaacRtxRenderer.__new__(IsaacRtxRenderer) + renderer.cfg = IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=False) + stage = MagicMock() + render_product = stage.GetPrimAtPath.return_value + render_product.IsValid.return_value = True + render_product.IsA.return_value = True + render_product.ApplyAPI.return_value = False + + with pytest.raises(RuntimeError, match="OmniRtxDebugSettingsAPI_1"): + renderer._apply_render_product_settings(stage, ["/Render/RP"]) + + @pytest.mark.parametrize( ("has_gui", "expected_disable_color_render"), [ diff --git a/source/isaaclab_tasks/changelog.d/xr-camera-feedback.minor.rst b/source/isaaclab_tasks/changelog.d/xr-camera-feedback.minor.rst new file mode 100644 index 000000000000..078b652f723f --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/xr-camera-feedback.minor.rst @@ -0,0 +1,12 @@ +Added +^^^^^ + +* Added recorded ``robot_pov_cam`` policy observations to the GR1T2 pick-place and G1 + locomanipulation tasks for XR camera feedback. +* Configured the recorded cameras to produce their RTX pixel output on ``cuda:0`` while preserving + CPU physics. + +Fixed +^^^^^ + +* Fixed the ExhaustPipe ``robot_pov_cam`` rotation after the WXYZ-to-XYZW quaternion migration. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py index 1be577481714..d627c5d7b0d6 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py @@ -3,7 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab_teleop import ControllerHapticFeedbackCfg, IsaacTeleopCfg, XrAnchorRotationMode, XrCfg +from isaaclab_teleop import ( + ControllerHapticFeedbackCfg, + IsaacTeleopCfg, + XrAnchorRotationMode, + XrCameraFeedCfg, + XrCfg, +) import isaaclab.envs.mdp as base_mdp import isaaclab.sim as sim_utils @@ -14,7 +20,7 @@ from isaaclab.managers import SceneEntityCfg from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg -from isaaclab.sensors import ContactSensorCfg +from isaaclab.sensors import CameraCfg, ContactSensorCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR from isaaclab.utils.configclass import configclass @@ -31,6 +37,18 @@ from isaaclab_tasks.contrib.locomanip_pick_place.configs.pink_controller_cfg import ( # isort: skip G1_UPPER_BODY_IK_ACTION_CFG, ) +from isaaclab_physx.renderers import IsaacRtxRendererCfg # isort: skip +from isaaclab_tasks.utils.presets import MultiBackendRendererCfg # isort: skip + + +@configclass +class _RobotPovCameraRendererCfg(MultiBackendRendererCfg): + default: IsaacRtxRendererCfg = IsaacRtxRendererCfg( + camera_output_device="cuda:0", + enable_dlss_ray_reconstruction=True, + dlss_exec_mode="quality", + ) + isaacsim_rtx = default def _build_g1_locomanipulation_pipeline(): @@ -290,6 +308,22 @@ class LocomanipulationG1SceneCfg(InteractiveSceneCfg): # Humanoid robot w/ arms higher robot: ArticulationCfg = G1_29DOF_CFG.replace(prim_path="{ENV_REGEX_NS}/Robot") + # Fixed task camera matching the GR1T2 training-camera placement. + robot_pov_cam = CameraCfg( + prim_path="{ENV_REGEX_NS}/RobotPOVCam", + update_period=0.0, + height=450, + width=720, + data_types=["rgb"], + renderer_cfg=_RobotPovCameraRendererCfg(), + spawn=sim_utils.PinholeCameraCfg(focal_length=18.15, clipping_range=(0.1, 2.0)), + offset=CameraCfg.OffsetCfg( + pos=(0.0, 0.12, 1.67675), + rot=(0.9801, 0.0, 0.0, -0.19848), + convention="ros", + ), + ) + # Per-hand contact sensors over all finger links, used to drive controller # haptics (see HapticFeedbackCfg below). Requires activate_contact_sensors # on the robot spawn, enabled in the env __post_init__. @@ -369,6 +403,16 @@ class PolicyCfg(ObsGroup): params={"left_eef_link_name": "left_wrist_yaw_link", "right_eef_link_name": "right_wrist_yaw_link"}, ) + robot_pov_cam = ObsTerm( + func=base_mdp.image, + params={ + "sensor_cfg": SceneEntityCfg("robot_pov_cam"), + "data_type": "rgb", + "normalize": False, + "clone": False, + }, + ) + def __post_init__(self): self.enable_corruption = False self.concatenate_terms = False @@ -456,7 +500,16 @@ def __post_init__(self): pipeline_builder=_build_g1_locomanipulation_pipeline, sim_device=self.sim.device, xr_cfg=self.xr, + xr_camera_feeds=[ + XrCameraFeedCfg( + camera_name="robot_pov_cam", + # Keep the 0.30 m-tall panel below the eye-level controls. + offset_m=(0.0, -0.15), + max_update_hz=0.0, + ) + ], ) + self.image_obs_list = ["robot_pov_cam"] # Enable contact reporting on the robot so the per-hand ContactSensors # report finger forces, and drive controller haptics from them. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_base_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_base_env_cfg.py index 7ab5fa883440..738230f45b8b 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_base_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_base_env_cfg.py @@ -150,7 +150,7 @@ class ObjectTableSceneCfg(InteractiveSceneCfg): width=256, data_types=["rgb"], spawn=sim_utils.PinholeCameraCfg(focal_length=18.15, clipping_range=(0.1, 2)), - offset=CameraCfg.OffsetCfg(pos=(0.0, 0.12, 1.85418), rot=(0.0, 0.98502, 0.0, -0.17246), convention="ros"), + offset=CameraCfg.OffsetCfg(pos=(0.0, 0.12, 1.85418), rot=(0.98502, 0.0, 0.0, -0.17246), convention="ros"), ) # Ground plane diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_pink_ik_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_pink_ik_env_cfg.py index 8cab1b33fce8..3fe6bdbbe150 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_pink_ik_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/exhaustpipe_gr1t2_pink_ik_env_cfg.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg +from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg, XrCameraFeedCfg from isaaclab.controllers.pink_ik import DampingTaskCfg, FrameTaskCfg, NullSpacePostureTaskCfg, PinkIKControllerCfg from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg @@ -132,4 +132,12 @@ def __post_init__(self): pipeline_builder=lambda: _build_gr1t2_pickplace_pipeline()[0], sim_device=self.sim.device, xr_cfg=self.xr, + xr_camera_feeds=[ + XrCameraFeedCfg( + camera_name="robot_pov_cam", + # Keep the 0.30 m-tall panel below the eye-level controls. + offset_m=(0.0, -0.15), + max_update_hz=0.0, + ) + ], ) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/nutpour_gr1t2_pink_ik_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/nutpour_gr1t2_pink_ik_env_cfg.py index b5c2e3a7d6c7..0ee984b7d4d2 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/nutpour_gr1t2_pink_ik_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/nutpour_gr1t2_pink_ik_env_cfg.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause -from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg +from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg, XrCameraFeedCfg from isaaclab.controllers.pink_ik import DampingTaskCfg, FrameTaskCfg, NullSpacePostureTaskCfg, PinkIKControllerCfg from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg @@ -130,4 +130,12 @@ def __post_init__(self): pipeline_builder=lambda: _build_gr1t2_pickplace_pipeline()[0], sim_device=self.sim.device, xr_cfg=self.xr, + xr_camera_feeds=[ + XrCameraFeedCfg( + camera_name="robot_pov_cam", + # Keep the 0.30 m-tall panel below the eye-level controls. + offset_m=(0.0, -0.15), + max_update_hz=0.0, + ) + ], ) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py index a0220b2464c5..260b0ccdfba2 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py @@ -18,7 +18,7 @@ from isaaclab.managers import SceneEntityCfg from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg -from isaaclab.sensors import ContactSensorCfg +from isaaclab.sensors import CameraCfg, ContactSensorCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR, retrieve_file_path from isaaclab.utils.configclass import configclass @@ -27,8 +27,20 @@ from isaaclab_assets.robots.fourier import GR1T2_HIGH_PD_CFG # isort: skip from isaaclab_teleop.haptic_feedback import GloveHapticFeedbackCfg # isort: skip -from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg # isort: skip +from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg, XrCameraFeedCfg # isort: skip from isaaclab_teleop.xr_cfg import XrCfg # isort: skip +from isaaclab_physx.renderers import IsaacRtxRendererCfg # isort: skip +from isaaclab_tasks.utils.presets import MultiBackendRendererCfg # isort: skip + + +@configclass +class _RobotPovCameraRendererCfg(MultiBackendRendererCfg): + default: IsaacRtxRendererCfg = IsaacRtxRendererCfg( + camera_output_device="cuda:0", + enable_dlss_ray_reconstruction=True, + dlss_exec_mode="quality", + ) + isaacsim_rtx = default def _build_gr1t2_pickplace_pipeline(): @@ -363,6 +375,26 @@ class ObjectTableSceneCfg(InteractiveSceneCfg): ) +@configclass +class PickPlaceGR1T2SceneCfg(ObjectTableSceneCfg): + """GR1T2 pick-place scene with the camera observation shown in XR PiP.""" + + robot_pov_cam = CameraCfg( + prim_path="{ENV_REGEX_NS}/RobotPOVCam", + update_period=0.0, + height=450, + width=720, + data_types=["rgb"], + renderer_cfg=_RobotPovCameraRendererCfg(), + spawn=sim_utils.PinholeCameraCfg(focal_length=18.15, clipping_range=(0.1, 2.0)), + offset=CameraCfg.OffsetCfg( + pos=(0.0, 0.12, 1.67675), + rot=(0.9801, 0.0, 0.0, -0.19848), + convention="ros", + ), + ) + + ## # MDP settings ## @@ -515,6 +547,25 @@ def __post_init__(self): policy: PolicyCfg = PolicyCfg() +@configclass +class PickPlaceGR1T2ObservationsCfg(ObservationsCfg): + """GR1T2 pick-place observations including the camera shown in XR PiP.""" + + @configclass + class PolicyCfg(ObservationsCfg.PolicyCfg): + robot_pov_cam = ObsTerm( + func=base_mdp.image, + params={ + "sensor_cfg": SceneEntityCfg("robot_pov_cam"), + "data_type": "rgb", + "normalize": False, + "clone": False, + }, + ) + + policy: PolicyCfg = PolicyCfg() + + @configclass class TerminationsCfg: """Termination terms for the MDP.""" @@ -553,9 +604,9 @@ class PickPlaceGR1T2EnvCfg(ManagerBasedRLEnvCfg): """Configuration for the GR1T2 environment.""" # Scene settings - scene: ObjectTableSceneCfg = ObjectTableSceneCfg(num_envs=1, env_spacing=2.5, replicate_physics=True) + scene: PickPlaceGR1T2SceneCfg = PickPlaceGR1T2SceneCfg(num_envs=1, env_spacing=2.5, replicate_physics=True) # Basic settings - observations: ObservationsCfg = ObservationsCfg() + observations: PickPlaceGR1T2ObservationsCfg = PickPlaceGR1T2ObservationsCfg() actions: ActionsCfg = ActionsCfg() # MDP settings terminations: TerminationsCfg = TerminationsCfg() @@ -633,7 +684,17 @@ def __post_init__(self): pipeline_builder=lambda: _build_gr1t2_pickplace_pipeline()[0], sim_device=self.sim.device, xr_cfg=self.xr, + xr_camera_feeds=[ + XrCameraFeedCfg( + camera_name="robot_pov_cam", + # The 0.48 m-wide 720x450 image is 0.30 m tall; lower its + # center so its top edge starts at viewer eye height. + offset_m=(0.0, -0.15), + max_update_hz=0.0, + ) + ], ) + self.image_obs_list = ["robot_pov_cam"] # Per-finger haptic glove feedback: vibrate each finger of the operator's # glove in proportion to how tightly it grips the object. The session diff --git a/source/isaaclab_teleop/changelog.d/xr-camera-feedback.minor.rst b/source/isaaclab_teleop/changelog.d/xr-camera-feedback.minor.rst new file mode 100644 index 000000000000..cadf30ff54bd --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/xr-camera-feedback.minor.rst @@ -0,0 +1,19 @@ +Added +^^^^^ + +* Added XR camera feedback to ``teleop_se3_agent.py`` and ``record_demos.py``, with task-configured + existing-camera selection, declarative layouts, and viewer-start, head-locked, or explicit-world + placement. +* Added direct and staged CUDA image presentation paths so camera feedback works with CPU physics. + +Changed +^^^^^^^ + +* Changed Kit Scene UI loading to be lazy so kitless teleoperation warns and continues without PiP. +* Changed ``--disable_external_cameras`` into the master camera-rendering and PiP gate for + ``teleop_se3_agent.py`` and ``record_demos.py``. + +Fixed +^^^^^ + +* Fixed repeated IWER reset messages and refreshed camera feedback immediately after resets. diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index dff9aa5d41a2..8d6a25c5dda0 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -19,6 +19,8 @@ XR anchor synchronization, retargeting pipelines, and action-tensor generation. modes for controlling how the anchor orientation tracks the reference prim. - **Retargeting tuning UI** -- optional ImGui window for real-time adjustment of retargeter parameters when `retargeters_to_tune` is provided. +- **XR camera feedback** -- selected scene cameras can be shown through Kit Scene UI using their + existing RGBA allocation. ## Architecture @@ -37,7 +39,11 @@ XR anchor synchronization, retargeting pipelines, and action-tensor generation. Add an `isaac_teleop` attribute to your environment config: ```python -from isaaclab_teleop import IsaacTeleopCfg, XrCfg +from isaaclab_teleop import ( + IsaacTeleopCfg, + XrCameraFeedCfg, + XrCfg, +) @configclass class MyEnvCfg(ManagerBasedRLEnvCfg): @@ -52,6 +58,9 @@ class MyEnvCfg(ManagerBasedRLEnvCfg): ), pipeline_builder=lambda: pipeline, retargeters_to_tune=lambda: retargeters, + xr_camera_feeds=[ + XrCameraFeedCfg(camera_name="robot_pov_cam"), + ], ) ``` @@ -86,9 +95,21 @@ The existing teleop scripts automatically detect `isaac_teleop` in the environme ```bash uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ - --task My-IsaacTeleop-Env-v0 + --task My-IsaacTeleop-Env-v0 --xr ``` +`IsaacContrib-PickPlace-GR1T2-Abs` and +`IsaacContrib-PickPlace-Locomanipulation-G1-Abs` are camera-feedback reference tasks. Both show +their recorded `robot_pov_cam` observation in PiP: + +```bash +uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ + --task IsaacContrib-PickPlace-GR1T2-Abs --xr --device cpu +``` + +See [XR Camera Feedback](../../../docs/source/features/isaac_teleop.rst#xr-camera-feedback) for +camera selection, layout, placement, renderer, disable, and kitless behavior. + ### 4. Programmatic Usage `IsaacTeleopDevice` supports Python's context-manager protocol: @@ -116,6 +137,8 @@ rendering without blocking. | Field | Type | Default | Description | |---|---|---|---| | `xr_cfg` | `XrCfg` | `XrCfg()` | XR anchor position, rotation, and dynamic-anchoring settings | +| `xr_camera_feeds` | `list[XrCameraFeedCfg]` | `[]` | Existing task scene cameras shown as image panels | +| `xr_camera_feed_layout` | `XrCameraFeedLayoutCfg` | viewer-start manual layout | Placement reference and ordered multi-feed packing | | `pipeline_builder` | `Callable[[], OutputCombiner]` | *required* | Builds the retargeting pipeline | | `retargeters_to_tune` | `Callable[[], list[BaseRetargeter]] \| None` | `None` | Retargeters to expose in the tuning UI | | `plugins` | `list[PluginConfig]` | `[]` | IsaacTeleop plugin configurations | @@ -143,9 +166,8 @@ contend for the GIL at the start of the step. ## Utilities -- **`remove_camera_configs(env_cfg)`** -- strips camera sensors and their associated observation - terms from an environment config. XR does not support additional cameras as they cause rendering - conflicts. +- **`remove_camera_configs(env_cfg)`** -- strips camera sensors and their associated policy + observation terms. ## Run with Docker diff --git a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi index f6838f1dde45..1e699694b4a6 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi +++ b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi @@ -19,6 +19,8 @@ __all__ = [ "SystemCheckItem", "SystemCheckResult", "TELEOP_CONTROL_CHANNEL_UUID", + "XrCameraFeedCfg", + "XrCameraFeedLayoutCfg", "XrAnchorRotationMode", "XrAnchorSynchronizer", "XrCfg", @@ -38,7 +40,14 @@ from .haptic_feedback import ( HapticFeedbackReceiver, create_haptic_feedback_driver, ) -from .isaac_teleop_cfg import CLOUDXR_AVP_ENV, CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV, IsaacTeleopCfg +from .isaac_teleop_cfg import ( + CLOUDXR_AVP_ENV, + CLOUDXR_JS_ENV, + CLOUDXR_STANDALONE_ENV, + IsaacTeleopCfg, + XrCameraFeedCfg, + XrCameraFeedLayoutCfg, +) from .isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device from .system_check import SystemCheckItem, SystemCheckResult, check_system_requirements from .xr_anchor_utils import XrAnchorSynchronizer diff --git a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py new file mode 100644 index 000000000000..706da8d9bb5f --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py @@ -0,0 +1,441 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Lifecycle management for camera image panels in XR.""" + +from __future__ import annotations + +import importlib +import logging +import math +import time +from collections import Counter +from copy import deepcopy +from dataclasses import dataclass +from functools import lru_cache +from typing import TYPE_CHECKING, Any + +import torch + +from .isaac_teleop_cfg import XrCameraFeedCfg, XrCameraFeedLayoutCfg + +if TYPE_CHECKING: + from isaaclab.sensors import Camera + +logger = logging.getLogger(__name__) + + +@lru_cache(maxsize=1) +def _camera_type() -> type[Camera]: + """Load the concrete camera type only when feeds are presented.""" + from isaaclab.sensors import Camera + + return Camera + + +def _load_kit_scene_ui_presenter() -> Any | None: + """Load the optional Kit SceneUI presenter without making Kit a package dependency.""" + try: + module = importlib.import_module(".camera_feed_kit_scene_ui", __package__) + except (ImportError, ModuleNotFoundError) as exc: + logger.warning( + "XR camera PiP is unavailable because Kit SceneUI could not be loaded (%s: %s). Continuing without PiP.", + type(exc).__name__, + exc, + ) + return None + return module._KitSceneUiCameraFeedPresenter() + + +def _prepare_camera_feed_cfgs(env_cfg: Any, cfgs: list[XrCameraFeedCfg]) -> list[XrCameraFeedCfg]: + """Validate selected scene cameras.""" + from isaaclab.sensors import CameraCfg + + prepared: list[XrCameraFeedCfg] = [] + enabled_cfgs = deepcopy(cfgs) + duplicates = sorted(name for name, count in Counter(cfg.camera_name for cfg in enabled_cfgs).items() if count > 1) + if duplicates: + raise ValueError(f"XR camera feeds must have unique camera names. Duplicates: {duplicates}.") + + scene = getattr(env_cfg, "scene", None) + if scene is None: + raise ValueError("XR camera feeds require an environment configuration with a scene.") + for cfg in enabled_cfgs: + camera_cfg = getattr(scene, cfg.camera_name, None) + if camera_cfg is None: + raise ValueError(f"XR camera feed {cfg.camera_name!r} is not present in the scene.") + if not isinstance(camera_cfg, CameraCfg): + raise TypeError(f"XR camera feed {cfg.camera_name!r} does not reference a CameraCfg.") + if not any(data_type in {"rgb", "rgba"} for data_type in camera_cfg.data_types): + raise ValueError(f"XR camera feed {cfg.camera_name!r} camera must provide RGB or RGBA output.") + prepared.append(cfg) + return prepared + + +def _feeds_require_responsive_denoising(env_cfg: Any, cfgs: list[XrCameraFeedCfg]) -> bool: + scene = env_cfg.scene + return any( + getattr(getattr(getattr(scene, cfg.camera_name), "renderer_cfg", None), "enable_dlss_ray_reconstruction", None) + is True + for cfg in cfgs + ) + + +class _XrCameraFeedSession: + """Internal two-phase lifecycle shared by the teleoperation entry points.""" + + def __init__( + self, + cfgs: list[XrCameraFeedCfg], + layout_cfg: XrCameraFeedLayoutCfg | None, + presenter: Any | None, + *, + requires_responsive_denoising: bool, + ): + self._cfgs = cfgs + self._layout_cfg = layout_cfg + self._presenter = presenter + self._requires_responsive_denoising = requires_responsive_denoising + self._env = None + self._manager = None + self._bound = False + + @classmethod + def prepare( + cls, + env_cfg: Any, + *, + enabled: bool, + camera_rendering_enabled: bool, + ) -> _XrCameraFeedSession: + """Prepare task-configured camera feeds before constructing the environment.""" + if type(enabled) is not bool or type(camera_rendering_enabled) is not bool: + raise TypeError("enabled and camera_rendering_enabled must be bool values.") + teleop_cfg = getattr(env_cfg, "isaac_teleop", None) + if not enabled or teleop_cfg is None: + return cls([], None, None, requires_responsive_denoising=False) + + requested = [cfg for cfg in teleop_cfg.xr_camera_feeds if cfg.enabled] + if not camera_rendering_enabled: + if requested: + logger.warning("XR camera PiP is disabled because external camera rendering is disabled.") + return cls([], None, None, requires_responsive_denoising=False) + if not requested: + return cls([], teleop_cfg.xr_camera_feed_layout, None, requires_responsive_denoising=False) + + _validate_layout_cfg(teleop_cfg.xr_camera_feed_layout) + presenter = _load_kit_scene_ui_presenter() + if presenter is None: + return cls([], teleop_cfg.xr_camera_feed_layout, None, requires_responsive_denoising=False) + if int(env_cfg.scene.num_envs) != 1: + raise ValueError("XR camera PiP supports exactly one environment; set --num_envs 1 or disable PiP feeds.") + cfgs = _prepare_camera_feed_cfgs(env_cfg, requested) + return cls( + cfgs, + teleop_cfg.xr_camera_feed_layout, + presenter, + requires_responsive_denoising=_feeds_require_responsive_denoising(env_cfg, cfgs), + ) + + @property + def enabled(self) -> bool: + return bool(self._cfgs) + + @property + def requires_responsive_denoising(self) -> bool: + return self._requires_responsive_denoising + + def bind(self, env: Any) -> _XrCameraFeedSession: + if self._bound: + raise RuntimeError("XR camera feed session is already bound.") + self._env = env + self._bound = True + try: + if self.enabled: + self._manager = _XrCameraFeedManager(env, self._cfgs, self._layout_cfg, self._presenter) + except Exception: + self.close() + raise + return self + + def refresh(self) -> None: + if not self._bound: + raise RuntimeError("XR camera feed session must be bound before refresh.") + if self._manager is not None: + # Request post-reset camera output before each render. Temporal + # RTX cameras need several frames to replace their pre-reset + # annotator contents, matching Isaac Lab's reset rerender path. + for _ in range(3): + self._manager.refresh(publish=False) + self._env.sim.render() + self._manager.refresh() + + def close(self) -> None: + if self._manager is not None: + self._manager.close() + self._manager = None + self._env = None + self._bound = False + + def __enter__(self) -> _XrCameraFeedSession: + if not self._bound: + raise RuntimeError("Call bind(env) before entering an XR camera feed session.") + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.close() + + +def _panel_size_m(cfg: XrCameraFeedCfg, image_size: tuple[int, int]) -> tuple[float, float]: + width, height = image_size + if width <= 0 or height <= 0: + raise ValueError(f"XR camera feed image dimensions must be positive, got {image_size}.") + return cfg.panel_width_m, cfg.panel_width_m * height / width + (0.04 if cfg.label else 0.0) + + +def _centered_positions(sizes: list[float], gap: float) -> list[float]: + cursor = -0.5 * (sum(sizes) + gap * max(0, len(sizes) - 1)) + positions = [] + for size in sizes: + positions.append(cursor + 0.5 * size) + cursor += size + gap + return positions + + +def _validate_layout_cfg(layout_cfg: XrCameraFeedLayoutCfg) -> None: + if layout_cfg.mode not in {"manual", "horizontal", "vertical", "grid"}: + raise ValueError(f"Unknown XR camera feed layout mode {layout_cfg.mode!r}.") + if layout_cfg.placement not in {"viewer_start", "head_locked", "world"}: + raise ValueError(f"Unknown XR camera feed placement {layout_cfg.placement!r}.") + if layout_cfg.placement != "world" and (not math.isfinite(layout_cfg.distance_m) or layout_cfg.distance_m <= 0.0): + raise ValueError("XR camera feed layout distance_m must be finite and positive.") + if not math.isfinite(layout_cfg.panel_gap_m) or layout_cfg.panel_gap_m < 0.0: + raise ValueError("XR camera feed layout panel_gap_m must be finite and non-negative.") + if len(layout_cfg.center_offset_m) != 2 or not all(math.isfinite(value) for value in layout_cfg.center_offset_m): + raise ValueError("XR camera feed layout center_offset_m must contain two finite values.") + if type(layout_cfg.max_columns) is not int or layout_cfg.max_columns <= 0: + raise ValueError("XR camera feed layout max_columns must be positive.") + if layout_cfg.placement == "world": + position = layout_cfg.world_position_m + if position is None or len(position) != 3 or not all(math.isfinite(value) for value in position): + raise ValueError("XR camera feed layout world_position_m must contain three finite values.") + orientation = layout_cfg.world_orientation_xyzw + if len(orientation) != 4 or not all(math.isfinite(value) for value in orientation): + raise ValueError("XR camera feed layout world_orientation_xyzw must contain four finite values.") + if math.sqrt(sum(value * value for value in orientation)) <= 1.0e-8: + raise ValueError("XR camera feed layout world_orientation_xyzw must be non-zero.") + + +def _layout_feed_cfgs( + cfgs: list[XrCameraFeedCfg], + image_sizes: list[tuple[int, int]], + layout_cfg: XrCameraFeedLayoutCfg, +) -> list[XrCameraFeedCfg]: + if len(cfgs) != len(image_sizes): + raise ValueError("XR camera feed configs and image sizes must have the same length.") + _validate_layout_cfg(layout_cfg) + resolved = deepcopy(cfgs) + if layout_cfg.mode == "manual" or not resolved: + return resolved + + panel_sizes = [_panel_size_m(cfg, size) for cfg, size in zip(resolved, image_sizes, strict=True)] + center_x, center_y = layout_cfg.center_offset_m + if layout_cfg.mode == "horizontal": + xs = _centered_positions([width for width, _ in panel_sizes], layout_cfg.panel_gap_m) + offsets = [(center_x + x, center_y) for x in xs] + elif layout_cfg.mode == "vertical": + ys = _centered_positions([height for _, height in panel_sizes], layout_cfg.panel_gap_m) + offsets = [(center_x, center_y - y) for y in ys] + else: + columns = min(layout_cfg.max_columns, len(resolved)) + rows = [list(range(start, min(start + columns, len(resolved)))) for start in range(0, len(resolved), columns)] + row_heights = [max(panel_sizes[index][1] for index in row) for row in rows] + row_ys = _centered_positions(row_heights, layout_cfg.panel_gap_m) + offsets = [(0.0, 0.0)] * len(resolved) + for row, row_y in zip(rows, row_ys, strict=True): + row_xs = _centered_positions([panel_sizes[index][0] for index in row], layout_cfg.panel_gap_m) + for index, row_x in zip(row, row_xs, strict=True): + offsets[index] = (center_x + row_x, center_y - row_y) + + for cfg, offset in zip(resolved, offsets, strict=True): + cfg.offset_m = offset + cfg.distance_m = layout_cfg.distance_m + return resolved + + +@dataclass(frozen=True) +class _PanelDescriptor: + label: str | None + width_m: float + offset_m: tuple[float, float] + distance_m: float + placement: str + world_position_m: tuple[float, float, float] | None + world_orientation_xyzw: tuple[float, float, float, float] + + +def _panel_descriptor(cfg: XrCameraFeedCfg, layout_cfg: XrCameraFeedLayoutCfg) -> _PanelDescriptor: + return _PanelDescriptor( + label=cfg.label, + width_m=cfg.panel_width_m, + offset_m=tuple(cfg.offset_m), + distance_m=cfg.distance_m, + placement=layout_cfg.placement, + world_position_m=None if layout_cfg.world_position_m is None else tuple(layout_cfg.world_position_m), + world_orientation_xyzw=tuple(layout_cfg.world_orientation_xyzw), + ) + + +@dataclass +class _ActiveFeed: + cfg: XrCameraFeedCfg + camera: Camera + image: torch.Tensor + upload_image: torch.Tensor + panel: Any + next_update_time: float = 0.0 + + +class _XrCameraFeedManager: + """Bind camera RGBA buffers to persistent Kit SceneUI panels.""" + + def __init__( + self, + env: Any, + cfgs: list[XrCameraFeedCfg], + layout_cfg: XrCameraFeedLayoutCfg | None, + presenter: Any, + ): + self._env = env + self._feeds: list[_ActiveFeed] = [] + self._presenter = presenter + self._layout_cfg = deepcopy(layout_cfg or XrCameraFeedLayoutCfg()) + self._frame_subscription = None + _validate_layout_cfg(self._layout_cfg) + + try: + bound_feeds = [] + for cfg in cfgs: + self._validate_cfg(cfg, self._layout_cfg.placement) + camera, image = self._bind_image(cfg) + upload_image = self._presenter.prepare_upload_image(cfg.camera_name, image) + bound_feeds.append((camera, image, upload_image)) + image_sizes = [(int(image.shape[1]), int(image.shape[0])) for _, image, _ in bound_feeds] + resolved_cfgs = _layout_feed_cfgs(cfgs, image_sizes, self._layout_cfg) + for cfg, (camera, image, upload_image) in zip(resolved_cfgs, bound_feeds, strict=True): + panel = self._presenter.create_panel( + _panel_descriptor(cfg, self._layout_cfg), + width=int(image.shape[1]), + height=int(image.shape[0]), + ) + self._feeds.append(_ActiveFeed(cfg, camera, image, upload_image, panel)) + self._frame_subscription = self._presenter.subscribe_to_frame_updates(self._on_frame) + except Exception: + self.close() + raise + + def _on_frame(self, _event: Any) -> None: + self.update() + + @staticmethod + def _validate_cfg(cfg: XrCameraFeedCfg, placement: str) -> None: + if not math.isfinite(cfg.panel_width_m) or cfg.panel_width_m <= 0.0: + raise ValueError(f"panel_width_m for XR camera feed {cfg.camera_name!r} must be positive.") + if len(cfg.offset_m) != 2 or not all(math.isfinite(value) for value in cfg.offset_m): + raise ValueError(f"offset_m for XR camera feed {cfg.camera_name!r} must contain two finite values.") + if placement != "world" and (not math.isfinite(cfg.distance_m) or cfg.distance_m <= 0.0): + raise ValueError(f"distance_m for XR camera feed {cfg.camera_name!r} must be positive.") + if not math.isfinite(cfg.max_update_hz) or cfg.max_update_hz < 0.0: + raise ValueError(f"max_update_hz for XR camera feed {cfg.camera_name!r} must be finite and non-negative.") + + def _bind_image(self, cfg: XrCameraFeedCfg) -> tuple[Camera, torch.Tensor]: + sensors = getattr(getattr(self._env, "scene", None), "sensors", {}) + if cfg.camera_name not in sensors: + raise ValueError( + f"XR camera feed {cfg.camera_name!r} is not present in the interactive scene. " + f"Available sensors: {sorted(sensors.keys())}." + ) + camera = sensors[cfg.camera_name] + if not isinstance(camera, _camera_type()): + raise TypeError(f"XR camera feed {cfg.camera_name!r} did not resolve to an Isaac Lab Camera.") + return camera, self._image_from_output(cfg, camera.data.output) + + def _image_from_output(self, cfg: XrCameraFeedCfg, output: Any) -> torch.Tensor: + if output is None or "rgba" not in output: + available = [] if output is None else sorted(output.keys()) + raise ValueError(f"Camera {cfg.camera_name!r} has no RGBA buffer. Available outputs: {available}.") + batch = output["rgba"].torch + if batch.ndim != 4 or int(batch.shape[-1]) != 4: + raise ValueError(f"Camera {cfg.camera_name!r} RGBA output must have shape (N, H, W, 4).") + image = batch[0] + if image.dtype != torch.uint8: + raise TypeError(f"Camera {cfg.camera_name!r} RGBA buffer must be uint8, got {image.dtype}.") + return image + + @staticmethod + def _same_allocation(first: torch.Tensor, second: torch.Tensor) -> bool: + return ( + tuple(first.shape) == tuple(second.shape) + and first.device == second.device + and first.data_ptr() == second.data_ptr() + ) + + def _rebind_feed(self, feed: _ActiveFeed, camera: Camera, image: torch.Tensor) -> None: + if self._same_allocation(feed.image, image): + feed.camera = camera + return + old_shape = tuple(feed.image.shape) + feed.upload_image = self._presenter.prepare_upload_image( + feed.cfg.camera_name, + image, + previous_source=feed.image, + previous_upload=feed.upload_image, + ) + if tuple(image.shape) != old_shape: + replacement = self._presenter.create_panel( + _panel_descriptor(feed.cfg, self._layout_cfg), + width=int(image.shape[1]), + height=int(image.shape[0]), + ) + old_panel = feed.panel + feed.panel = replacement + old_panel.close() + feed.camera = camera + feed.image = image + + def update(self) -> None: + now = time.monotonic() + for feed in self._feeds: + if now < feed.next_update_time: + continue + image = self._image_from_output(feed.cfg, feed.camera.data.output) + self._rebind_feed(feed, feed.camera, image) + self._publish_feed(feed) + period = 0.0 if feed.cfg.max_update_hz == 0.0 else 1.0 / feed.cfg.max_update_hz + feed.next_update_time = now + period + + def _publish_feed(self, feed: _ActiveFeed) -> None: + self._presenter.stage_upload_image(feed.image, feed.upload_image) + feed.panel.upload(feed.upload_image) + + def refresh(self, *, publish: bool = True) -> None: + for feed in self._feeds: + feed.camera.update(0.0, force_recompute=True) + camera, image = self._bind_image(feed.cfg) + self._rebind_feed(feed, camera, image) + if publish: + self._publish_feed(feed) + feed.next_update_time = 0.0 + + def close(self) -> None: + if self._frame_subscription is not None: + self._frame_subscription.close() + self._frame_subscription = None + for feed in reversed(self._feeds): + try: + feed.panel.close() + except Exception: + logger.exception("Failed to close XR camera feed %r.", feed.cfg.camera_name) + self._feeds.clear() diff --git a/source/isaaclab_teleop/isaaclab_teleop/camera_feed_kit_scene_ui.py b/source/isaaclab_teleop/isaaclab_teleop/camera_feed_kit_scene_ui.py new file mode 100644 index 000000000000..c581b12ec384 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/camera_feed_kit_scene_ui.py @@ -0,0 +1,426 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Kit Scene UI presentation for one XR camera feed.""" + +from __future__ import annotations + +import logging +import math +from collections.abc import Callable +from contextlib import suppress +from typing import Any + +import torch + +import omni.gpu_foundation_factory as gf +import omni.ui as ui +from omni.kit.scene_view.xr import XRSceneView +from omni.kit.scene_view.xr_utils import SpatialSource, UiContainer, UpdatePolicy, WidgetComponent +from omni.kit.xr.core import XRCore, XRCoreEventType, XRPoseValidityFlags +from pxr import Gf + +logger = logging.getLogger(__name__) + + +def _meters_per_unit(coordinate_system: Any, name: str) -> float: + """Return a validated coordinate-system scale.""" + value = float(coordinate_system.meters_per_unit) + if not math.isfinite(value) or value <= 0.0: + raise RuntimeError(f"{name} meters_per_unit must be finite and positive, got {value}.") + return value + + +class KitSceneUiViewerStartAnchor: + """Share one frozen viewer-start pose across Kit Scene UI panels.""" + + def __init__(self, xr_core: Any | None = None): + self._xr_core = xr_core or XRCore.get_singleton() + self._registrations: dict[int, tuple[Any, tuple[float, float], float, Callable[[bool], None]]] = {} + self._upright_pose = None + self._post_sync_subscription = None + self._display_disabled_subscription = None + + @property + def captured(self) -> bool: + """Whether a valid viewer-start pose has been frozen.""" + return self._upright_pose is not None + + @property + def xr_core(self) -> Any: + """XR core used to resolve this anchor.""" + return self._xr_core + + def register( + self, + source: Any, + offset_m: tuple[float, float], + distance_m: float, + readiness_callback: Callable[[bool], None], + ) -> int: + """Register a panel source and return its lifecycle token.""" + token = id(source) + self._registrations[token] = (source, offset_m, distance_m, readiness_callback) + try: + self._ensure_subscriptions() + if self._upright_pose is not None: + self._apply_registration(self._registrations[token]) + readiness_callback(True) + else: + readiness_callback(False) + except Exception: + self.unregister(token) + raise + return token + + def unregister(self, token: int) -> None: + """Release one panel and subscriptions after the final panel closes.""" + self._registrations.pop(token, None) + if self._registrations: + return + self._post_sync_subscription = None + self._display_disabled_subscription = None + self._upright_pose = None + + def _ensure_subscriptions(self) -> None: + if self._post_sync_subscription is not None: + return + message_bus = self._xr_core.get_message_bus() + self._post_sync_subscription = message_bus.create_subscription_to_pop_by_type( + XRCoreEventType.post_sync_update, + self._on_post_sync_update, + name="Isaac Lab XR camera PiP viewer-start capture", + ) + self._display_disabled_subscription = message_bus.create_subscription_to_pop_by_type( + XRCoreEventType.xr_display_disabled, + self._on_display_disabled, + name="Isaac Lab XR camera PiP viewer-start reset", + ) + + def _on_post_sync_update(self, _event: Any) -> None: + self._try_capture() + + def _on_display_disabled(self, _event: Any) -> None: + """Hide panels and capture a new starting pose after reconnect.""" + self._upright_pose = None + for _, _, _, callback in tuple(self._registrations.values()): + callback(False) + + def _try_capture(self) -> bool: + if self._upright_pose is not None or not self._registrations: + return self._upright_pose is not None + if not self._xr_core.is_xr_display_enabled(): + return False + input_device = self._xr_core.get_input_device("displayDevice") + if input_device is None: + input_device = self._xr_core.get_input_device("/user/head") + if input_device is None: + return False + pose_desc = input_device.get_virtual_world_pose_desc("") + if pose_desc is None: + return False + required_flags = XRPoseValidityFlags.POSITION_VALID | XRPoseValidityFlags.ORIENTATION_VALID + if pose_desc.validity_flags & required_flags != required_flags: + return False + + coordinate_system = self._xr_core.get_coordinate_system() + self._upright_pose = self._xr_core.reorient_transform_matrix_up_right( + pose_desc.pose_matrix, + coordinate_system.up_axis == "y", + ) + for registration in tuple(self._registrations.values()): + self._apply_registration(registration) + registration[3](True) + logger.info( + "XR camera PiP viewer-start anchor captured at %s (validity_flags=%s).", + tuple(float(value) for value in self._upright_pose.ExtractTranslation()), + pose_desc.validity_flags, + ) + return True + + def _apply_registration( + self, + registration: tuple[Any, tuple[float, float], float, Callable[[bool], None]], + ) -> None: + source, offset_m, distance_m, _ = registration + coordinate_system = self._xr_core.get_coordinate_system() + meters_per_unit = _meters_per_unit(coordinate_system, "XR coordinate-system") + # XR poses use local +X right, +Y up, and -Z forward. Apply the + # panel offset in that local frame before the frozen pose maps it into + # the stage's Y-up or Z-up world. + offset = Gf.Vec3d(offset_m[0], offset_m[1], -distance_m) / meters_per_unit + panel_world = Gf.Matrix4d().SetTranslate(offset) * self._upright_pose + source.source = SpatialSource.new_transform_matrix_source(panel_world).source + + +def _world_panel_matrix( + descriptor: Any, + meters_per_unit: float, +) -> Gf.Matrix4d: + """Build one fixed panel transform from the explicit world layout pose.""" + if not math.isfinite(meters_per_unit) or meters_per_unit <= 0.0: + raise RuntimeError(f"Stage meters_per_unit must be finite and positive, got {meters_per_unit}.") + if descriptor.world_position_m is None: + raise ValueError("world_position_m is required for world XR camera-feed placement.") + x, y, z, w = descriptor.world_orientation_xyzw + norm = math.sqrt(x * x + y * y + z * z + w * w) + if not math.isfinite(norm) or norm <= 1.0e-8: + raise ValueError("world_orientation_xyzw must be a finite, non-zero quaternion.") + rotation = Gf.Quatd(w / norm, Gf.Vec3d(x / norm, y / norm, z / norm)) + anchor = Gf.Matrix4d(1.0).SetRotate(rotation) + anchor.SetTranslateOnly(Gf.Vec3d(*(value / meters_per_unit for value in descriptor.world_position_m))) + local_offset = Gf.Matrix4d().SetTranslate( + Gf.Vec3d( + descriptor.offset_m[0] / meters_per_unit, + descriptor.offset_m[1] / meters_per_unit, + 0.0, + ) + ) + return local_offset * anchor + + +class _CameraImageWidget(ui.Widget): + """Omni UI image surface backed by a Kit byte-image provider.""" + + def __init__(self, provider: ui.ByteImageProvider, label: str | None = None, **kwargs): + super().__init__(**kwargs) + with ui.ZStack(): + ui.Rectangle( + style={ + "Rectangle": { + "background_color": 0xFF101214, + "border_color": 0xFF73787E, + "border_width": 1, + "border_radius": 2, + } + } + ) + with ui.VStack(spacing=0): + if label: + ui.Label( + label, + height=22, + alignment=ui.Alignment.CENTER, + style={"Label": {"color": 0xFFE8EAED, "font_size": 14}}, + ) + ui.ImageWithProvider( + provider, + fill_policy=ui.IwpFillPolicy.IWP_PRESERVE_ASPECT_FIT, + ) + + +class KitSceneUiCameraFeedPanel: + """Persistent Kit Scene UI panel for one RGBA camera.""" + + def __init__( + self, + descriptor: Any, + image_width: int, + image_height: int, + viewer_start_anchor: KitSceneUiViewerStartAnchor | None = None, + ): + """Create the image provider and attach its panel to the XR scene.""" + self._closed = False + self._provider = None + self._component = None + self._container = None + self._viewer_start_anchor = viewer_start_anchor + self._viewer_start_registration = None + + if descriptor.placement == "viewer_start": + if viewer_start_anchor is None: + raise ValueError("viewer_start placement requires a shared viewer-start anchor.") + xr_core = viewer_start_anchor.xr_core + coordinate_system = xr_core.get_coordinate_system() + coordinate_system_name = "XR coordinate-system" + else: + xr_core = XRCore.get_singleton() + coordinate_system = ( + xr_core.get_stage_coordinate_system() + if descriptor.placement == "world" + else xr_core.get_coordinate_system() + ) + coordinate_system_name = "Stage" if descriptor.placement == "world" else "XR coordinate-system" + meters_per_unit = _meters_per_unit(coordinate_system, coordinate_system_name) + image_height_m = descriptor.width_m * image_height / image_width + label_height_m = 0.04 if descriptor.label else 0.0 + panel_width_units = descriptor.width_m / meters_per_unit + panel_height_units = (image_height_m + label_height_m) / meters_per_unit + resolution_scale = max(image_width / descriptor.width_m, image_height / image_height_m) + if descriptor.placement == "viewer_start": + anchor_source = SpatialSource.new_transform_matrix_source(Gf.Matrix4d(1.0)) + space_stack = [anchor_source] + elif descriptor.placement == "head_locked": + anchor_source = None + space_stack = [ + SpatialSource.new_prim_path_source("/_xr/stage/xrCamera"), + SpatialSource.new_translation_source( + Gf.Vec3f( + float(descriptor.offset_m[0] / meters_per_unit), + float(descriptor.offset_m[1] / meters_per_unit), + -float(descriptor.distance_m / meters_per_unit), + ) + ), + SpatialSource.new_look_at_camera_source(), + ] + elif descriptor.placement == "world": + anchor_source = None + space_stack = [ + SpatialSource.new_transform_matrix_source( + _world_panel_matrix(descriptor, meters_per_unit), + ) + ] + else: + raise ValueError(f"Unknown XR camera-feed placement {descriptor.placement!r}.") + try: + self._provider = ui.ByteImageProvider() + self._component = WidgetComponent( + _CameraImageWidget, + width=panel_width_units, + height=panel_height_units, + resolution_scale=resolution_scale, + unit_to_pixel_scale=meters_per_unit, + update_policy=UpdatePolicy.ALWAYS, + widget_args=[self._provider, descriptor.label], + ) + self._container = UiContainer( + XRSceneView, + self._component, + space_stack=space_stack, + ) + if descriptor.placement == "viewer_start": + self._container.hide() + self._viewer_start_registration = viewer_start_anchor.register( + anchor_source, + descriptor.offset_m, + descriptor.distance_m, + self._on_viewer_start_readiness_changed, + ) + except Exception: + with suppress(Exception): + self.close() + raise + + def _on_viewer_start_readiness_changed(self, ready: bool) -> None: + if self._container is None: + return + if ready: + self._container.show() + else: + self._container.hide() + + def upload(self, image: torch.Tensor) -> None: + """Upload a contiguous cuda:0 RGBA tensor through Kit's GPU copy path.""" + if self._closed: + return + self._provider.set_bytes_data_from_gpu( + int(image.data_ptr()), + [int(image.shape[1]), int(image.shape[0])], + gf.TextureFormat.RGBA8_UNORM, + ) + + def close(self) -> None: + """Release the XR scene hierarchy and provider references.""" + if self._closed: + return + self._closed = True + if self._viewer_start_anchor is not None and self._viewer_start_registration is not None: + self._viewer_start_anchor.unregister(self._viewer_start_registration) + self._viewer_start_registration = None + self._viewer_start_anchor = None + container = self._container + self._component = None + self._container = None + self._provider = None + if container is not None: + try: + container.hide() + finally: + container.root.clear() + + +class _KitSceneUiCameraFeedPresenter: + """Private adapter from camera buffers to Kit SceneUI panels.""" + + def __init__(self): + self._viewer_start_anchor = None + + @staticmethod + def _validate_image(camera_name: str, image: torch.Tensor) -> None: + if image.device.type not in {"cpu", "cuda"}: + raise ValueError(f"Camera {camera_name!r} RGBA image must reside on CPU or cuda:0, got {image.device}.") + if image.device.type == "cuda" and image.device.index != 0: + raise ValueError(f"Camera {camera_name!r} RGBA image must reside on cuda:0, got {image.device}.") + if not image.is_contiguous(): + raise ValueError(f"Camera {camera_name!r} RGBA image must be contiguous.") + + def prepare_upload_image( + self, + camera_name: str, + image: torch.Tensor, + previous_source: torch.Tensor | None = None, + previous_upload: torch.Tensor | None = None, + ) -> torch.Tensor: + """Return direct CUDA storage or a persistent cuda:0 staging buffer.""" + self._validate_image(camera_name, image) + if image.device.type == "cuda": + return image + if not torch.cuda.is_available(): + raise RuntimeError(f"Camera {camera_name!r} uses CPU pixels, but cuda:0 is unavailable for XR upload.") + if ( + previous_source is not None + and previous_upload is not None + and previous_upload is not previous_source + and tuple(previous_upload.shape) == tuple(image.shape) + and previous_upload.dtype == image.dtype + and previous_upload.device == torch.device("cuda:0") + ): + return previous_upload + logger.warning( + "XR camera feed %r is using a CPU buffer and requires a CPU-to-GPU staging copy each frame.", + camera_name, + ) + return torch.empty_like(image, device="cuda:0", memory_format=torch.contiguous_format) + + def create_panel(self, descriptor: Any, width: int, height: int) -> KitSceneUiCameraFeedPanel: + viewer_start_anchor = None + if descriptor.placement == "viewer_start": + if self._viewer_start_anchor is None: + self._viewer_start_anchor = KitSceneUiViewerStartAnchor() + viewer_start_anchor = self._viewer_start_anchor + return KitSceneUiCameraFeedPanel( + descriptor=descriptor, + image_width=width, + image_height=height, + viewer_start_anchor=viewer_start_anchor, + ) + + @staticmethod + def stage_upload_image(image: torch.Tensor, upload_image: torch.Tensor) -> None: + if upload_image is not image: + upload_image.copy_(image, non_blocking=False) + + @staticmethod + def subscribe_to_frame_updates(callback: Callable[[Any], None]) -> _KitFrameSubscription: + import carb.eventdispatcher + import omni.kit.app + + observer = carb.eventdispatcher.get_eventdispatcher().observe_event( + order=omni.kit.app.POST_UPDATE_ORDER_PYTHON_EXEC, + event_name=omni.kit.app.GLOBAL_EVENT_POST_UPDATE, + on_event=callback, + observer_name="Isaac Lab XR camera PiP frame update", + ) + return _KitFrameSubscription(observer) + + +class _KitFrameSubscription: + def __init__(self, observer: Any): + self._observer = observer + + def close(self) -> None: + if self._observer is not None: + self._observer.reset() + self._observer = None diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py index 92618f312515..5bd963075925 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py @@ -10,7 +10,7 @@ from collections.abc import Callable from dataclasses import MISSING, field from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal from isaaclab.utils.configclass import configclass @@ -39,6 +39,84 @@ from isaacteleop.teleop_session_manager import PluginConfig, RetargetingExecutionConfig +@configclass +class XrCameraFeedCfg: + """Configuration for one camera image panel shown in XR.""" + + camera_name: str = MISSING + """Name of the :class:`~isaaclab.sensors.Camera` in the interactive scene.""" + + enabled: bool = True + """Whether to create and update this feed.""" + + panel_width_m: float = 0.48 + """Physical panel width [m].""" + + distance_m: float = 0.8 + """Distance in front of the viewer anchor [m]. + + This value is unused when :attr:`XrCameraFeedLayoutCfg.placement` is + ``"world"``. + """ + + offset_m: tuple[float, float] = (0.0, 0.0) + """Horizontal and vertical panel offset in the selected placement frame [m].""" + + max_update_hz: float = 30.0 + """Maximum provider upload rate [Hz]. Set to zero to update after every rendered frame.""" + + label: str | None = None + """Optional short label shown above the image.""" + + +@configclass +class XrCameraFeedLayoutCfg: + """Declarative placement and packing for enabled XR camera feeds.""" + + mode: Literal["manual", "horizontal", "vertical", "grid"] = "manual" + """Layout mode. Manual preserves each feed's offset and distance.""" + + placement: Literal["viewer_start", "head_locked", "world"] = "viewer_start" + """Reference frame used to place the panels. + + ``"viewer_start"`` captures the first valid viewer eye position and yaw, + then leaves the panels fixed in the world. ``"head_locked"`` follows the + viewer with full pose. ``"world"`` uses :attr:`world_position_m` and + :attr:`world_orientation_xyzw` as a fixed pose in the Isaac Lab USD stage + world. + """ + + center_offset_m: tuple[float, float] = (0.0, 0.0) + """Horizontal and vertical center of an automatic layout [m].""" + + distance_m: float = 0.8 + """Distance of every automatically placed panel from the viewer anchor [m]. + + This value is unused when :attr:`placement` is ``"world"``. + """ + + panel_gap_m: float = 0.04 + """Edge-to-edge gap between automatically placed panels [m].""" + + max_columns: int = 2 + """Maximum number of columns in grid mode.""" + + world_position_m: tuple[float, float, float] | None = None + """Layout-plane center in the Isaac Lab USD stage world [m]. + + Isaac Lab stages are Z-up. This value is required when :attr:`placement` + is ``"world"``. + """ + + world_orientation_xyzw: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 1.0) + """Panel-local-to-world orientation as a quaternion in ``xyzw`` order. + + Panel local +X is image right, +Y is image up, and +Z points from the + panel's readable side toward the viewer. Feed and layout offsets are + applied in the resulting local XY plane. + """ + + @configclass class IsaacTeleopCfg: """Configuration for IsaacTeleop-based teleoperation. @@ -82,6 +160,15 @@ def build_pipeline(): to follow a prim (e.g., robot base) during locomotion tasks. """ + xr_camera_feeds: list[XrCameraFeedCfg] = field(default_factory=list) + """Existing task camera outputs to show as XR image panels. + + The default empty list disables PiP. + """ + + xr_camera_feed_layout: XrCameraFeedLayoutCfg = field(default_factory=XrCameraFeedLayoutCfg) + """Placement and packing applied to the ordered enabled camera feeds.""" + pipeline_builder: Callable[[], OutputCombiner] = MISSING """Callable that builds the IsaacTeleop retargeting pipeline. diff --git a/source/isaaclab_teleop/isaaclab_teleop/teleop_message_processor.py b/source/isaaclab_teleop/isaaclab_teleop/teleop_message_processor.py index 15bd7936a08e..18bbc6893597 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/teleop_message_processor.py +++ b/source/isaaclab_teleop/isaaclab_teleop/teleop_message_processor.py @@ -106,6 +106,7 @@ def __init__(self, name: str) -> None: self._run_toggle_queue: list[bool] = [] self._prev_toggle_output = False self._pending_commands: list[str] = [] + self._channel_reset_latched = False super().__init__(name=name) def inject_reset(self, pause: bool = False) -> None: @@ -208,9 +209,12 @@ def _compute_fn( self._inject_reset_pending = False self._inject_reset_pause = False - # Parse incoming messages and enqueue toggle sequences. + # Parse incoming messages and enqueue toggle sequences. Some clients + # repeat a reset command while its control is held, so treat a + # continuous run of reset-bearing frames as one operator action. messages_tracked = inputs[self.INPUT_MESSAGES][0] data = getattr(messages_tracked, "data", None) + channel_reset_seen = False if data: for message in data: payload = getattr(message, "payload", None) @@ -225,9 +229,17 @@ def _compute_fn( if command is None: continue - if self._apply_command_kind(_classify_command(command)): + command_kind = _classify_command(command) + if command_kind == "reset": + channel_reset_seen = True + if self._channel_reset_latched: + continue + self._channel_reset_latched = True + if self._apply_command_kind(command_kind): reset = True pause_on_reset = True + if not channel_reset_seen: + self._channel_reset_latched = False # Apply locally injected commands (e.g. keyboard bindings) identically to # channel messages, so a host can start/stop/reset without an XR client. diff --git a/source/isaaclab_teleop/test/test_control_events.py b/source/isaaclab_teleop/test/test_control_events.py index dc0ddfac699c..3002beea3dd9 100644 --- a/source/isaaclab_teleop/test/test_control_events.py +++ b/source/isaaclab_teleop/test/test_control_events.py @@ -220,6 +220,15 @@ def test_reset_clears_on_next_step(self): result = _step(proc, _empty_tracked()) assert result["reset"] is False + def test_repeated_reset_frames_produce_one_pulse_until_released(self): + proc = TeleopMessageProcessor(name="test") + + assert _step(proc, _tracked(b"reset"))["reset"] is True + assert _step(proc, _tracked(b"reset"))["reset"] is False + assert _step(proc, _tracked(b"reset"))["reset"] is False + assert _step(proc, _empty_tracked())["reset"] is False + assert _step(proc, _tracked(b"reset"))["reset"] is True + class TestKillAlwaysFalse: def test_kill_is_always_false(self): diff --git a/source/isaaclab_teleop/test/test_xr_camera_feed.py b/source/isaaclab_teleop/test/test_xr_camera_feed.py new file mode 100644 index 000000000000..16eb3672a876 --- /dev/null +++ b/source/isaaclab_teleop/test/test_xr_camera_feed.py @@ -0,0 +1,426 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import Mock + +import isaaclab_teleop.camera_feed as camera_feed +import pytest +import torch +from isaaclab_physx.renderers import IsaacRtxRendererCfg +from isaaclab_teleop import IsaacTeleopCfg, XrCameraFeedCfg, XrCameraFeedLayoutCfg + +from isaaclab.sensors import CameraCfg + +from isaaclab_tasks.utils.hydra import resolve_presets +from isaaclab_tasks.utils.presets import MultiBackendRendererCfg + + +class _FakeImage: + def __init__(self, height=8, width=12, *, device="cuda:0", data_ptr=100): + self.dtype = torch.uint8 + self.ndim = 3 + self.shape = (height, width, 4) + self.device = torch.device(device) + self._data_ptr = data_ptr + + def data_ptr(self): + return self._data_ptr + + +class _FakeBatch: + def __init__(self, image): + self.image = image + self.ndim = 4 + self.shape = (1, *image.shape) + + def __getitem__(self, index): + assert index == 0 + return self.image + + +class _FakeCamera: + def __init__(self, image): + self.output = {"rgba": SimpleNamespace(torch=_FakeBatch(image))} + self.update_calls = [] + self.image_on_update = None + + @property + def data(self): + return SimpleNamespace(output=self.output) + + def update(self, dt, force_recompute=False): + self.update_calls.append((dt, force_recompute)) + if self.image_on_update is not None: + self.output["rgba"].torch = _FakeBatch(self.image_on_update) + + +class _FakePanel: + def __init__(self, descriptor, width, height): + self.descriptor = descriptor + self.width = width + self.height = height + self.uploads = [] + self.closed = False + + def upload(self, image): + self.uploads.append(image) + + def close(self): + self.closed = True + + +class _FakeSubscription: + def __init__(self, callback): + self.callback = callback + self.closed = False + + def publish(self): + if not self.closed: + self.callback(None) + + def close(self): + self.closed = True + + +class _FakePresenter: + def __init__(self): + self.panels = [] + self.subscription = None + self.staged = [] + + def prepare_upload_image(self, _name, image, previous_source=None, previous_upload=None): + del previous_source, previous_upload + return image + + def create_panel(self, descriptor, width, height): + panel = _FakePanel(descriptor, width, height) + self.panels.append(panel) + return panel + + def stage_upload_image(self, image, upload_image): + self.staged.append((image, upload_image)) + + def subscribe_to_frame_updates(self, callback): + self.subscription = _FakeSubscription(callback) + return self.subscription + + +def _camera_cfg(renderer_cfg=None): + return CameraCfg( + prim_path="{ENV_REGEX_NS}/Camera", + height=8, + width=12, + data_types=["rgb"], + renderer_cfg=renderer_cfg, + ) + + +def _teleop_env_cfg( + feeds, + *, + num_envs=1, + camera=None, + layout=None, +): + scene = SimpleNamespace(num_envs=num_envs) + if camera is not None: + scene.robot_pov_cam = camera + return SimpleNamespace( + scene=scene, + isaac_teleop=SimpleNamespace( + xr_camera_feeds=feeds, + xr_camera_feed_layout=layout or XrCameraFeedLayoutCfg(), + ), + ) + + +def _manager(monkeypatch, cfgs, images, layout=None): + cameras = {name: _FakeCamera(image) for name, image in images.items()} + monkeypatch.setattr(camera_feed, "_camera_type", lambda: _FakeCamera) + presenter = _FakePresenter() + env = SimpleNamespace(scene=SimpleNamespace(sensors=cameras)) + manager = camera_feed._XrCameraFeedManager(env, cfgs, layout or XrCameraFeedLayoutCfg(), presenter) + return manager, presenter, cameras + + +def test_pip_rejects_multiple_environments_before_camera_creation(monkeypatch): + env_cfg = _teleop_env_cfg([XrCameraFeedCfg(camera_name="robot_pov_cam")], num_envs=2, camera=_camera_cfg()) + load_presenter = Mock(return_value=_FakePresenter()) + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", load_presenter) + + with pytest.raises(ValueError, match="exactly one environment"): + camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=True) + + load_presenter.assert_called_once_with() + + +def test_xr_without_pip_preserves_multiple_environments(monkeypatch): + env_cfg = _teleop_env_cfg([], num_envs=2) + load_presenter = Mock() + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", load_presenter) + + session = camera_feed._XrCameraFeedSession.prepare( + env_cfg, + enabled=True, + camera_rendering_enabled=True, + ) + + assert not session.enabled + load_presenter.assert_not_called() + assert env_cfg.scene.num_envs == 2 + + +def test_kitless_xr_with_configured_pip_preserves_multiple_environments(monkeypatch): + env_cfg = _teleop_env_cfg( + [XrCameraFeedCfg(camera_name="robot_pov_cam")], + num_envs=2, + camera=_camera_cfg(), + ) + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", lambda: None) + + session = camera_feed._XrCameraFeedSession.prepare( + env_cfg, + enabled=True, + camera_rendering_enabled=True, + ) + + assert not session.enabled + assert env_cfg.scene.num_envs == 2 + + +def test_empty_camera_feed_selection_skips_pip(monkeypatch): + env_cfg = _teleop_env_cfg([]) + load_presenter = Mock() + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", load_presenter) + + session = camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=True) + + assert not session.enabled + load_presenter.assert_not_called() + assert vars(env_cfg.scene) == {"num_envs": 1} + + +def test_existing_camera_is_selected_without_replacement(monkeypatch): + selected = _camera_cfg(IsaacRtxRendererCfg(enable_dlss_ray_reconstruction=True)) + env_cfg = _teleop_env_cfg([XrCameraFeedCfg(camera_name="robot_pov_cam")], camera=selected) + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", _FakePresenter) + + session = camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=True) + + assert session.enabled + assert env_cfg.scene.robot_pov_cam is selected + assert session.requires_responsive_denoising + + +def test_session_refresh_renders_reset_state_before_publishing(): + events = [] + session = camera_feed._XrCameraFeedSession([], None, None, requires_responsive_denoising=False) + session._env = SimpleNamespace(sim=SimpleNamespace(render=lambda: events.append("render"))) + + class _Manager: + def refresh(self, *, publish=True): + events.append("publish" if publish else "request") + + session._manager = _Manager() + session._bound = True + + session.refresh() + + assert events == ["request", "render"] * 3 + ["publish"] + + +def test_camera_rendering_switch_disables_pip_before_presenter_load(monkeypatch): + env_cfg = _teleop_env_cfg( + [XrCameraFeedCfg(camera_name="robot_pov_cam")], + num_envs=2, + camera=_camera_cfg(), + ) + load_presenter = Mock() + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", load_presenter) + + session = camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=False) + + assert not session.enabled + load_presenter.assert_not_called() + + +def test_feed_selection_requires_existing_rgb_camera(monkeypatch): + monkeypatch.setattr(camera_feed, "_load_kit_scene_ui_presenter", _FakePresenter) + missing = _teleop_env_cfg([XrCameraFeedCfg(camera_name="missing")]) + with pytest.raises(ValueError, match="not present"): + camera_feed._XrCameraFeedSession.prepare(missing, enabled=True, camera_rendering_enabled=True) + + depth = _camera_cfg() + depth.data_types = ["distance_to_image_plane"] + env_cfg = _teleop_env_cfg([XrCameraFeedCfg(camera_name="robot_pov_cam")], camera=depth) + with pytest.raises(ValueError, match="RGB or RGBA"): + camera_feed._XrCameraFeedSession.prepare(env_cfg, enabled=True, camera_rendering_enabled=True) + + +def test_manual_layout_preserves_per_feed_transforms(): + cfgs = [ + XrCameraFeedCfg(camera_name="left", offset_m=(-0.4, 0.1), distance_m=0.7), + XrCameraFeedCfg(camera_name="right", offset_m=(0.5, -0.2), distance_m=1.1), + ] + + resolved = camera_feed._layout_feed_cfgs(cfgs, [(100, 50), (100, 50)], XrCameraFeedLayoutCfg()) + + assert [(cfg.offset_m, cfg.distance_m) for cfg in resolved] == [ + ((-0.4, 0.1), 0.7), + ((0.5, -0.2), 1.1), + ] + assert resolved is not cfgs + + +@pytest.mark.parametrize( + ("layout", "expected"), + [ + ( + XrCameraFeedLayoutCfg(mode="horizontal", panel_gap_m=0.1), + [(-0.25, 0.0), (0.25, 0.0)], + ), + ( + XrCameraFeedLayoutCfg(mode="vertical", panel_gap_m=0.1), + [(0.0, 0.15), (0.0, -0.15)], + ), + ( + XrCameraFeedLayoutCfg(mode="grid", panel_gap_m=0.1, max_columns=2), + [(-0.25, 0.15), (0.25, 0.15), (0.0, -0.15)], + ), + ], +) +def test_automatic_layouts_preserve_order(layout, expected): + count = len(expected) + cfgs = [XrCameraFeedCfg(camera_name=str(index), panel_width_m=0.4) for index in range(count)] + + resolved = camera_feed._layout_feed_cfgs(cfgs, [(100, 50)] * count, layout) + + for cfg, expected_offset in zip(resolved, expected): + assert cfg.offset_m == pytest.approx(expected_offset) + + +@pytest.mark.parametrize( + "layout", + [ + XrCameraFeedLayoutCfg(mode="diagonal"), + XrCameraFeedLayoutCfg(distance_m=0.0), + XrCameraFeedLayoutCfg(placement="world"), + XrCameraFeedLayoutCfg(placement="world", world_position_m=(0.0, 0.0, 0.0), world_orientation_xyzw=(0, 0, 0, 0)), + ], +) +def test_invalid_layouts_fail_before_panel_creation(layout): + with pytest.raises(ValueError): + camera_feed._validate_layout_cfg(layout) + + +def test_manager_publishes_on_kit_frame_and_closes(monkeypatch): + cfg = XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=0.0) + image = _FakeImage() + manager, presenter, _ = _manager(monkeypatch, [cfg], {"robot_pov_cam": image}) + + presenter.subscription.publish() + manager.close() + + assert presenter.panels[0].uploads == [image] + assert presenter.subscription.closed + assert presenter.panels[0].closed + + +def test_manager_refresh_rebinds_reset_camera_output(monkeypatch): + cfg = XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=0.0) + before = _FakeImage(data_ptr=100) + after = _FakeImage(data_ptr=200) + manager, presenter, cameras = _manager(monkeypatch, [cfg], {"robot_pov_cam": before}) + cameras["robot_pov_cam"].image_on_update = after + + manager.refresh() + + assert cameras["robot_pov_cam"].update_calls == [(0.0, True)] + assert presenter.panels[0].uploads == [after] + + +def test_manager_can_refresh_reset_camera_without_publishing(monkeypatch): + cfg = XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=0.0) + before = _FakeImage(data_ptr=100) + after = _FakeImage(data_ptr=200) + manager, presenter, cameras = _manager(monkeypatch, [cfg], {"robot_pov_cam": before}) + cameras["robot_pov_cam"].image_on_update = after + + manager.refresh(publish=False) + + assert cameras["robot_pov_cam"].update_calls == [(0.0, True)] + assert presenter.panels[0].uploads == [] + + +def test_manager_recreates_panel_when_resolution_changes(monkeypatch): + cfg = XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=0.0) + before = _FakeImage(height=8, width=12) + after = _FakeImage(height=10, width=16, data_ptr=200) + manager, presenter, cameras = _manager(monkeypatch, [cfg], {"robot_pov_cam": before}) + cameras["robot_pov_cam"].output["rgba"].torch = _FakeBatch(after) + + manager.update() + + assert len(presenter.panels) == 2 + assert presenter.panels[0].closed + assert presenter.panels[1].uploads == [after] + + +@pytest.mark.parametrize("value", [-1.0, float("nan"), float("inf")]) +def test_manager_rejects_invalid_update_rate(monkeypatch, value): + cfg = XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=value) + with pytest.raises(ValueError, match="max_update_hz"): + _manager(monkeypatch, [cfg], {"robot_pov_cam": _FakeImage()}) + + +def test_reference_tasks_select_their_recorded_camera(): + from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import LocomanipulationG1EnvCfg + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg + + for cfg in (PickPlaceGR1T2EnvCfg(), LocomanipulationG1EnvCfg()): + assert cfg.isaac_teleop.xr_camera_feeds[0].camera_name == "robot_pov_cam" + assert hasattr(cfg.observations.policy, "robot_pov_cam") + assert isinstance(cfg.scene.robot_pov_cam.renderer_cfg, MultiBackendRendererCfg) + + +def test_reference_camera_renderer_resolves_to_isaac_rtx_and_ovrtx(): + from isaaclab_ov.renderers import OVRTXRendererCfg + + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg + + default = resolve_presets(PickPlaceGR1T2EnvCfg().scene.robot_pov_cam.renderer_cfg) + isaacsim_rtx = resolve_presets(PickPlaceGR1T2EnvCfg().scene.robot_pov_cam.renderer_cfg, selected=("isaacsim_rtx",)) + ovrtx = resolve_presets(PickPlaceGR1T2EnvCfg().scene.robot_pov_cam.renderer_cfg, selected=("ovrtx",)) + + assert isinstance(default, IsaacRtxRendererCfg) + assert default.camera_output_device == "cuda:0" + assert default.enable_dlss_ray_reconstruction + assert isinstance(isaacsim_rtx, IsaacRtxRendererCfg) + assert isaacsim_rtx.camera_output_device == "cuda:0" + assert isaacsim_rtx.enable_dlss_ray_reconstruction + assert isinstance(ovrtx, OVRTXRendererCfg) + + +def test_public_api_only_exports_declarative_camera_feed_types(): + import isaaclab_teleop + + assert isaaclab_teleop.XrCameraFeedCfg is XrCameraFeedCfg + assert isaaclab_teleop.XrCameraFeedLayoutCfg is XrCameraFeedLayoutCfg + for removed_name in ( + "XrCameraFeedManager", + "XrCameraFeedSession", + "XrCameraFeedPresentationBackend", + "XrCameraFeedPresentationCfg", + ): + assert not hasattr(isaaclab_teleop, removed_name) + + +def test_isaac_teleop_default_has_no_camera_feeds(): + cfg = IsaacTeleopCfg(pipeline_builder=lambda: None) + + assert cfg.xr_camera_feeds == [] diff --git a/source/isaaclab_teleop/test/test_xr_camera_feed_kit_scene_ui.py b/source/isaaclab_teleop/test/test_xr_camera_feed_kit_scene_ui.py new file mode 100644 index 000000000000..dbcc8d716e16 --- /dev/null +++ b/source/isaaclab_teleop/test/test_xr_camera_feed_kit_scene_ui.py @@ -0,0 +1,570 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import gc +import importlib.util +import sys +from enum import IntFlag +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any +from unittest.mock import Mock + +import numpy as np +import pytest + +from pxr import Gf + + +class _PoseValidityFlags(IntFlag): + POSITION_VALID = 1 + ORIENTATION_VALID = 2 + POSITION_TRACKED = 4 + ORIENTATION_TRACKED = 8 + + +class _EventType: + post_sync_update = "post_sync_update" + xr_display_disabled = "xr_display_disabled" + + +class _TransformSource: + def __init__(self, matrix: Gf.Matrix4d): + self.source = matrix + + +class _SpatialSource: + @staticmethod + def new_transform_matrix_source(matrix: Gf.Matrix4d) -> _TransformSource: + return _TransformSource(matrix) + + @staticmethod + def new_prim_path_source(path: str): + return SimpleNamespace(kind="prim_path", value=path) + + @staticmethod + def new_translation_source(value): + return SimpleNamespace(kind="translation", value=value) + + @staticmethod + def new_look_at_camera_source(): + return SimpleNamespace(kind="look_at_camera") + + +class _Subscription: + def __init__(self, bus: _MessageBus, token: int): + self._bus = bus + self._token = token + + def __del__(self): + self._bus.release(self._token) + + +class _MessageBus: + def __init__(self): + self._next_token = 0 + self._listeners: dict[int, tuple[Any, Any]] = {} + self.released_event_types: list[Any] = [] + + @property + def listener_count(self) -> int: + return len(self._listeners) + + def create_subscription_to_pop_by_type(self, event_type, callback, *, name): + del name + token = self._next_token + self._next_token += 1 + self._listeners[token] = (event_type, callback) + return _Subscription(self, token) + + def release(self, token: int) -> None: + listener = self._listeners.pop(token, None) + if listener is not None: + self.released_event_types.append(listener[0]) + + def emit(self, event_type) -> None: + for subscribed_type, callback in tuple(self._listeners.values()): + if subscribed_type == event_type: + callback(SimpleNamespace(type=event_type)) + + +class _InputDevice: + def __init__(self, pose_desc): + self.pose_desc = pose_desc + + def get_virtual_world_pose_desc(self, path: str): + assert path == "" + return self.pose_desc + + +class _StageCoordinateSystem: + def __init__( + self, + *, + right: tuple[float, float, float] = (1.0, 0.0, 0.0), + up: tuple[float, float, float] = (0.0, 1.0, 0.0), + forward: tuple[float, float, float] = (0.0, 0.0, -1.0), + meters_per_unit: float = 1.0, + up_axis: str = "y", + ): + self._right = right + self._up = up + self._forward = forward + self.meters_per_unit = meters_per_unit + self.up_axis = up_axis + + def get_right_vector(self): + return self._right + + def get_up_vector(self): + return self._up + + def get_forward_vector(self): + return self._forward + + +class _XrCore: + def __init__( + self, + *, + display_enabled: bool = True, + input_device: _InputDevice | None = None, + head_device: _InputDevice | None = None, + meters_per_unit: float = 1.0, + up_axis: str = "y", + stage_coordinate_system: _StageCoordinateSystem | None = None, + ): + self.display_enabled = display_enabled + self.input_device = input_device + self.head_device = head_device + self.coordinate_system = SimpleNamespace(meters_per_unit=meters_per_unit, up_axis=up_axis) + self.stage_coordinate_system = stage_coordinate_system or _StageCoordinateSystem() + self.message_bus = _MessageBus() + self.reorient_calls: list[tuple[Gf.Matrix4d, bool]] = [] + + def get_message_bus(self): + return self.message_bus + + def is_xr_display_enabled(self): + return self.display_enabled + + def get_input_device(self, path: str): + if path == "displayDevice": + return self.input_device + if path == "/user/head": + return self.head_device + return None + + def get_coordinate_system(self): + return self.coordinate_system + + def get_stage_coordinate_system(self): + return self.stage_coordinate_system + + def reorient_transform_matrix_up_right(self, matrix: Gf.Matrix4d, y_up: bool): + self.reorient_calls.append((matrix, y_up)) + return matrix + + +def _pose(flags: _PoseValidityFlags, translation=(0.0, 0.0, 0.0)): + matrix = Gf.Matrix4d(1.0).SetTranslate(Gf.Vec3d(*translation)) + return SimpleNamespace(validity_flags=flags, pose_matrix=matrix) + + +def _matrix_values(matrix: Gf.Matrix4d) -> np.ndarray: + return np.asarray([[float(matrix[row][column]) for column in range(4)] for row in range(4)]) + + +def _translation(matrix: Gf.Matrix4d) -> tuple[float, float, float]: + return tuple(float(value) for value in matrix.ExtractTranslation()) + + +def _module(name: str) -> ModuleType: + module = ModuleType(name) + module.__path__ = [] + return module + + +@pytest.fixture +def scene_ui_module(monkeypatch): + """Load the Scene UI implementation against fake Kit and OpenXR modules.""" + omni = _module("omni") + gpu_foundation = _module("omni.gpu_foundation_factory") + ui = _module("omni.ui") + kit = _module("omni.kit") + scene_view = _module("omni.kit.scene_view") + scene_view_xr = _module("omni.kit.scene_view.xr") + scene_view_xr_utils = _module("omni.kit.scene_view.xr_utils") + kit_xr = _module("omni.kit.xr") + kit_xr_core = _module("omni.kit.xr.core") + + class _Widget: + pass + + class _XrCoreType: + @staticmethod + def get_singleton(): + raise AssertionError("Tests must inject their XRCore instance.") + + gpu_foundation.TextureFormat = SimpleNamespace(RGBA8_UNORM="rgba8") + ui.Widget = _Widget + scene_view_xr.XRSceneView = object() + scene_view_xr_utils.SpatialSource = _SpatialSource + scene_view_xr_utils.UiContainer = object + scene_view_xr_utils.UpdatePolicy = SimpleNamespace(ALWAYS="always") + scene_view_xr_utils.WidgetComponent = object + kit_xr_core.XRCore = _XrCoreType + kit_xr_core.XRCoreEventType = _EventType + kit_xr_core.XRPoseValidityFlags = _PoseValidityFlags + + omni.gpu_foundation_factory = gpu_foundation + omni.ui = ui + omni.kit = kit + kit.scene_view = scene_view + kit.xr = kit_xr + scene_view.xr = scene_view_xr + scene_view.xr_utils = scene_view_xr_utils + kit_xr.core = kit_xr_core + + fake_modules = { + "omni": omni, + "omni.gpu_foundation_factory": gpu_foundation, + "omni.ui": ui, + "omni.kit": kit, + "omni.kit.scene_view": scene_view, + "omni.kit.scene_view.xr": scene_view_xr, + "omni.kit.scene_view.xr_utils": scene_view_xr_utils, + "omni.kit.xr": kit_xr, + "omni.kit.xr.core": kit_xr_core, + } + for name, module in fake_modules.items(): + monkeypatch.setitem(sys.modules, name, module) + + module_path = Path(__file__).parents[1] / "isaaclab_teleop" / "camera_feed_kit_scene_ui.py" + module_name = "_isaaclab_teleop_camera_feed_kit_scene_ui_under_test" + spec = importlib.util.spec_from_file_location(module_name, module_path) + assert spec is not None and spec.loader is not None + loaded_module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, module_name, loaded_module) + spec.loader.exec_module(loaded_module) + return loaded_module + + +@pytest.mark.parametrize( + "input_device", + [ + None, + _InputDevice(_pose(_PoseValidityFlags.POSITION_VALID)), + ], +) +def test_viewer_start_invalid_or_missing_pose_stays_hidden(scene_ui_module, input_device): + core = _XrCore(input_device=input_device) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + source = _TransformSource(Gf.Matrix4d(1.0)) + visibility = [] + + token = anchor.register(source, (0.0, 0.0), 0.8, visibility.append) + + assert visibility == [False] + assert not anchor.captured + assert _translation(source.source) == pytest.approx((0.0, 0.0, 0.0)) + anchor.unregister(token) + + +def test_viewer_start_accepts_valid_pose_without_tracked_flags(scene_ui_module): + flags = _PoseValidityFlags.POSITION_VALID | _PoseValidityFlags.ORIENTATION_VALID + pose_desc = _pose(flags, translation=(1.0, 2.0, 3.0)) + core = _XrCore(input_device=_InputDevice(pose_desc)) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + source = _TransformSource(Gf.Matrix4d(1.0)) + visibility = [] + + token = anchor.register(source, (0.0, 0.0), 0.0, visibility.append) + + assert visibility == [False] + assert not anchor.captured + + core.message_bus.emit(_EventType.post_sync_update) + + assert visibility == [False, True] + assert anchor.captured + assert core.reorient_calls == [(pose_desc.pose_matrix, True)] + assert _translation(source.source) == pytest.approx((1.0, 2.0, 3.0)) + anchor.unregister(token) + + +def test_viewer_start_freezes_first_valid_pose_across_later_motion(scene_ui_module): + flags = _PoseValidityFlags.POSITION_VALID | _PoseValidityFlags.ORIENTATION_VALID + device = _InputDevice(_pose(flags, translation=(1.0, 2.0, 3.0))) + core = _XrCore(input_device=device) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + source = _TransformSource(Gf.Matrix4d(1.0)) + + token = anchor.register(source, (0.1, -0.2), 0.5, lambda _ready: None) + core.message_bus.emit(_EventType.post_sync_update) + frozen_matrix = _matrix_values(source.source).copy() + device.pose_desc = _pose(flags, translation=(9.0, 8.0, 7.0)) + + core.message_bus.emit(_EventType.post_sync_update) + + np.testing.assert_allclose(_matrix_values(source.source), frozen_matrix) + assert len(core.reorient_calls) == 1 + anchor.unregister(token) + + +def test_viewer_start_panels_share_anchor_and_convert_offsets_to_stage_units(scene_ui_module): + flags = _PoseValidityFlags.POSITION_VALID | _PoseValidityFlags.ORIENTATION_VALID + pose_desc = _pose(flags) + core = _XrCore( + display_enabled=False, + input_device=_InputDevice(pose_desc), + meters_per_unit=0.01, + ) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + first_source = _TransformSource(Gf.Matrix4d(1.0)) + second_source = _TransformSource(Gf.Matrix4d(1.0)) + first_visibility = [] + second_visibility = [] + + first_token = anchor.register(first_source, (0.1, 0.2), 0.5, first_visibility.append) + second_token = anchor.register(second_source, (-0.3, 0.4), 0.8, second_visibility.append) + core.display_enabled = True + core.message_bus.emit(_EventType.post_sync_update) + + assert anchor._upright_pose is pose_desc.pose_matrix + assert len(core.reorient_calls) == 1 + assert first_visibility == [False, True] + assert second_visibility == [False, True] + assert _translation(first_source.source) == pytest.approx((10.0, 20.0, -50.0)) + assert _translation(second_source.source) == pytest.approx((-30.0, 40.0, -80.0)) + np.testing.assert_allclose(_matrix_values(first_source.source)[:3, :3], np.eye(3)) + np.testing.assert_allclose(_matrix_values(second_source.source)[:3, :3], np.eye(3)) + + anchor.unregister(first_token) + anchor.unregister(second_token) + + +def test_viewer_start_uses_pose_local_axes_in_z_up_stage(scene_ui_module): + flags = _PoseValidityFlags.POSITION_VALID | _PoseValidityFlags.ORIENTATION_VALID + # Local +X/+Y/+Z map to stage +X/+Z/-Y. The viewer therefore looks along + # stage +Y from an eye height of 1.6 m. + pose_matrix = Gf.Matrix4d( + (1.0, 0.0, 0.0, 0.0), + (0.0, 0.0, 1.0, 0.0), + (0.0, -1.0, 0.0, 0.0), + (0.0, 0.0, 1.6, 1.0), + ) + core = _XrCore( + input_device=_InputDevice(SimpleNamespace(validity_flags=flags, pose_matrix=pose_matrix)), + up_axis="z", + stage_coordinate_system=_StageCoordinateSystem( + up=(0.0, 0.0, 1.0), + forward=(0.0, -1.0, 0.0), + up_axis="z", + ), + ) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + source = _TransformSource(Gf.Matrix4d(1.0)) + + token = anchor.register(source, (0.1, 0.2), 0.8, lambda _ready: None) + core.message_bus.emit(_EventType.post_sync_update) + + assert _translation(source.source) == pytest.approx((0.1, 0.8, 1.8)) + np.testing.assert_allclose(_matrix_values(source.source)[:3, :3], _matrix_values(pose_matrix)[:3, :3]) + anchor.unregister(token) + + +def test_viewer_start_disconnect_hides_resets_and_recaptures(scene_ui_module): + flags = _PoseValidityFlags.POSITION_VALID | _PoseValidityFlags.ORIENTATION_VALID + device = _InputDevice(_pose(flags, translation=(1.0, 2.0, 3.0))) + core = _XrCore(input_device=device) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + source = _TransformSource(Gf.Matrix4d(1.0)) + visibility = [] + + token = anchor.register(source, (0.0, 0.0), 0.0, visibility.append) + core.message_bus.emit(_EventType.post_sync_update) + first_matrix = _matrix_values(source.source).copy() + core.display_enabled = False + core.message_bus.emit(_EventType.xr_display_disabled) + + assert not anchor.captured + assert visibility[-1] is False + np.testing.assert_allclose(_matrix_values(source.source), first_matrix) + + core.message_bus.emit(_EventType.post_sync_update) + assert visibility[-1] is False + assert len(core.reorient_calls) == 1 + + device.pose_desc = _pose(flags, translation=(4.0, 5.0, 6.0)) + core.display_enabled = True + core.message_bus.emit(_EventType.post_sync_update) + + assert anchor.captured + assert visibility == [False, True, False, True] + assert len(core.reorient_calls) == 2 + assert _translation(source.source) == pytest.approx((4.0, 5.0, 6.0)) + anchor.unregister(token) + + +def test_viewer_start_unregister_releases_listeners_after_last_panel(scene_ui_module): + core = _XrCore(display_enabled=False) + anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) + first_source = _TransformSource(Gf.Matrix4d(1.0)) + second_source = _TransformSource(Gf.Matrix4d(1.0)) + + first_token = anchor.register(first_source, (0.0, 0.0), 0.8, lambda _ready: None) + second_token = anchor.register(second_source, (0.1, 0.1), 0.8, lambda _ready: None) + assert core.message_bus.listener_count == 2 + + anchor.unregister(first_token) + assert core.message_bus.listener_count == 2 + + anchor.unregister(second_token) + gc.collect() + + assert anchor._post_sync_subscription is None + assert anchor._display_disabled_subscription is None + assert core.message_bus.listener_count == 0 + assert set(core.message_bus.released_event_types) == { + _EventType.post_sync_update, + _EventType.xr_display_disabled, + } + + +def test_world_panel_matrix_is_fixed_and_normalizes_quaternion(scene_ui_module): + scaled_descriptor = SimpleNamespace( + world_position_m=(1.0, 2.0, 3.0), + world_orientation_xyzw=(0.0, 0.0, 2.0, 2.0), + offset_m=(0.5, -0.25), + ) + unit = np.sqrt(0.5) + unit_descriptor = SimpleNamespace( + world_position_m=(1.0, 2.0, 3.0), + world_orientation_xyzw=(0.0, 0.0, unit, unit), + offset_m=(0.5, -0.25), + ) + + scaled_matrix = scene_ui_module._world_panel_matrix(scaled_descriptor, 0.01) + repeated_matrix = scene_ui_module._world_panel_matrix(scaled_descriptor, 0.01) + unit_matrix = scene_ui_module._world_panel_matrix(unit_descriptor, 0.01) + + np.testing.assert_allclose(_matrix_values(scaled_matrix), _matrix_values(repeated_matrix), atol=1.0e-12) + np.testing.assert_allclose(_matrix_values(scaled_matrix), _matrix_values(unit_matrix), atol=1.0e-12) + np.testing.assert_allclose( + _matrix_values(scaled_matrix)[:3, :3], + np.asarray( + [ + [0.0, 1.0, 0.0], + [-1.0, 0.0, 0.0], + [0.0, 0.0, 1.0], + ] + ), + atol=1.0e-12, + ) + assert _translation(scaled_matrix) == pytest.approx((125.0, 250.0, 300.0)) + + +@pytest.mark.parametrize( + ("position", "orientation"), + [ + (None, (0.0, 0.0, 0.0, 1.0)), + ((0.0, 0.0, 0.0), (0.0, 0.0, 0.0, 0.0)), + ((0.0, 0.0, 0.0), (0.0, 0.0, float("nan"), 1.0)), + ], +) +def test_world_panel_matrix_rejects_missing_position_or_invalid_quaternion( + scene_ui_module, + position, + orientation, +): + descriptor = SimpleNamespace( + world_position_m=position, + world_orientation_xyzw=orientation, + offset_m=(0.0, 0.0), + ) + + with pytest.raises(ValueError): + scene_ui_module._world_panel_matrix(descriptor, 1.0) + + +@pytest.mark.parametrize("meters_per_unit", [0.0, -1.0, float("nan")]) +def test_world_panel_matrix_rejects_invalid_stage_scale(scene_ui_module, meters_per_unit): + descriptor = SimpleNamespace( + world_position_m=(0.0, 0.0, 0.0), + world_orientation_xyzw=(0.0, 0.0, 0.0, 1.0), + offset_m=(0.0, 0.0), + ) + + with pytest.raises(RuntimeError, match="meters_per_unit"): + scene_ui_module._world_panel_matrix(descriptor, meters_per_unit) + + +@pytest.mark.parametrize( + ("placement", "expected_meters_per_unit", "expected_width", "expected_height"), + [ + ("viewer_start", 0.02, 24.0, 16.0), + ("head_locked", 0.02, 24.0, 16.0), + ("world", 0.5, 0.96, 0.64), + ], +) +def test_panel_converts_metric_geometry_in_selected_coordinate_system( + scene_ui_module, + monkeypatch, + placement, + expected_meters_per_unit, + expected_width, + expected_height, +): + core = _XrCore( + display_enabled=False, + meters_per_unit=0.02, + stage_coordinate_system=_StageCoordinateSystem(meters_per_unit=0.5, up_axis="z"), + ) + monkeypatch.setattr(scene_ui_module, "XRCore", SimpleNamespace(get_singleton=lambda: core)) + component = SimpleNamespace() + component_type = Mock(return_value=component) + monkeypatch.setattr(scene_ui_module, "WidgetComponent", component_type) + monkeypatch.setattr(scene_ui_module.ui, "ByteImageProvider", Mock(return_value=object()), raising=False) + + class _Container: + def __init__(self, _view, _component, *, space_stack): + self.space_stack = space_stack + self.root = SimpleNamespace(clear=Mock()) + self.show = Mock() + self.hide = Mock() + + monkeypatch.setattr(scene_ui_module, "UiContainer", _Container) + descriptor = SimpleNamespace( + label=None, + width_m=0.48, + offset_m=(0.1, -0.2), + distance_m=0.8, + placement=placement, + world_position_m=(1.0, 2.0, 3.0) if placement == "world" else None, + world_orientation_xyzw=(0.0, 0.0, 0.0, 1.0), + ) + viewer_start_anchor = scene_ui_module.KitSceneUiViewerStartAnchor(core) if placement == "viewer_start" else None + + panel = scene_ui_module.KitSceneUiCameraFeedPanel( + descriptor, + image_width=720, + image_height=480, + viewer_start_anchor=viewer_start_anchor, + ) + + kwargs = component_type.call_args.kwargs + assert kwargs["width"] == pytest.approx(expected_width) + assert kwargs["height"] == pytest.approx(expected_height) + assert kwargs["resolution_scale"] == pytest.approx(1500.0) + assert kwargs["unit_to_pixel_scale"] == pytest.approx(expected_meters_per_unit) + if placement == "head_locked": + translation = panel._container.space_stack[1].value + assert tuple(translation) == pytest.approx((5.0, -10.0, -40.0)) + elif placement == "world": + assert _translation(panel._container.space_stack[0].source) == pytest.approx((2.2, 3.6, 6.0)) + panel.close()