[Task Clean-up][Manager][Camera] Dexterous Part 10/11: Add the Shadow camera manager counterpart

source/isaaclab_tasks/changelog.d/task-cleanup-dex-part10.minor.rst

@@ -0,0 +1,11 @@
+Added
+^^^^^
+
+* Added a manager-based counterpart for the Shadow Hand camera reorientation
+  environment.
+
+Deprecated
+^^^^^^^^^^
+
+* Deprecated ``shadow_hand_camera_env.compute_keypoints`` in favor of
+  ``feature_extractor.compute_cube_keypoints``.

source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/__init__.py

@@ -17,6 +17,18 @@
 
 reorient_direct_entry = "isaaclab_tasks.core.reorient.reorient_direct_env:ReorientDirectEnv"
 
+gym.register(
+    id="Isaac-Reorient-Cube-Shadow",
+    entry_point="isaaclab.envs:ManagerBasedRLEnv",
+    disable_env_checker=True,
+    kwargs={
+        "env_cfg_entry_point": f"{__name__}.shadow_hand_manager_env_cfg:ShadowHandManagerEnvCfg",
+        "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_cfg.yaml",
+        "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandPPORunnerCfg",
+        "skrl_cfg_entry_point": f"{agents.__name__}:skrl_ppo_cfg.yaml",
+    },
+)
+
 gym.register(
     id="Isaac-Reorient-Cube-Shadow-Direct",
     entry_point=reorient_direct_entry,
@@ -29,6 +41,29 @@
     },
 )
 
+gym.register(
+    id="Isaac-Reorient-Cube-Shadow-OpenAI-FF",
+    entry_point="isaaclab.envs:ManagerBasedRLEnv",
+    disable_env_checker=True,
+    kwargs={
+        "env_cfg_entry_point": f"{__name__}.shadow_hand_manager_env_cfg:ShadowHandOpenAIManagerEnvCfg",
+        "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_ff_cfg.yaml",
+        "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandAsymFFPPORunnerCfg",
+        "skrl_cfg_entry_point": f"{agents.__name__}:skrl_ff_ppo_cfg.yaml",
+    },
+)
+
+gym.register(
+    id="Isaac-Reorient-Cube-Shadow-OpenAI-LSTM",
+    entry_point="isaaclab.envs:ManagerBasedRLEnv",
+    disable_env_checker=True,
+    kwargs={
+        "env_cfg_entry_point": f"{__name__}.shadow_hand_manager_env_cfg:ShadowHandOpenAIManagerEnvCfg",
+        "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_lstm_cfg.yaml",
+        "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandAsymLSTMPPORunnerCfg",
+    },
+)
+
 gym.register(
     id="Isaac-Reorient-Cube-Shadow-OpenAI-FF-Direct",
     entry_point=reorient_direct_entry,
@@ -48,13 +83,36 @@
     kwargs={
         "env_cfg_entry_point": f"{__name__}.shadow_hand_env_cfg:ShadowHandOpenAIEnvCfg",
         "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_lstm_cfg.yaml",
+        "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandAsymLSTMPPORunnerCfg",
     },
 )
 
 # -------
 # Vision
 # -------
 
+gym.register(
+    id="Isaac-Reorient-Cube-Shadow-Camera",
+    entry_point="isaaclab.envs:ManagerBasedRLEnv",
+    disable_env_checker=True,
+    kwargs={
+        "env_cfg_entry_point": f"{__name__}.shadow_hand_camera_manager_env_cfg:ShadowHandCameraManagerEnvCfg",
+        "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandCameraFFPPORunnerCfg",
+        "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_camera_cfg.yaml",
+    },
+)
+
+gym.register(
+    id="Isaac-Reorient-Cube-Shadow-Camera-Play",
+    entry_point="isaaclab.envs:ManagerBasedRLEnv",
+    disable_env_checker=True,
+    kwargs={
+        "env_cfg_entry_point": f"{__name__}.shadow_hand_camera_manager_env_cfg:ShadowHandCameraManagerPlayEnvCfg",
+        "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandCameraFFPPORunnerCfg",
+        "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_camera_cfg.yaml",
+    },
+)
+
 gym.register(
     id="Isaac-Reorient-Cube-Shadow-Camera-Direct",
     entry_point=f"{__name__}.shadow_hand_camera_env:ShadowHandCameraEnv",

source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/feature_extractor.py

@@ -3,8 +3,11 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+from __future__ import annotations
+
 import glob
 import os
+from typing import TYPE_CHECKING
 
 import torch
 import torch.nn as nn
@@ -13,6 +16,12 @@
 from isaaclab.sensors import save_images_to_file
 from isaaclab.utils.configclass import configclass
 
+# re-exported for backward compatibility; the shared implementation lives in the family math root
+from isaaclab_tasks.core.reorient.reorient_kernels import compute_cube_keypoints  # noqa: F401
+
+if TYPE_CHECKING:
+    pass
+
 # Number of output channels for each supported camera data type.
 _DATA_TYPE_CHANNELS: dict[str, int] = {
     "rgb": 3,

source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_env.py

@@ -6,6 +6,7 @@
 
 from __future__ import annotations
 
+import warnings
 from typing import TYPE_CHECKING
 
 import torch
@@ -14,10 +15,12 @@
 from isaaclab import cloner
 from isaaclab.assets import Articulation, RigidObject
 from isaaclab.sensors import Camera
-from isaaclab.utils.math import quat_apply, scale_transform
+from isaaclab.utils.math import scale_transform
 
 from isaaclab_tasks.core.reorient.config.shadow_hand.feature_extractor import FeatureExtractor
 from isaaclab_tasks.core.reorient.reorient_direct_env import ReorientDirectEnv
+from isaaclab_tasks.core.reorient.reorient_kernels import compute_cube_keypoints
+from isaaclab_tasks.core.reorient.reorient_task_base import CAMERA_GOAL_MARKER_POSITION
 
 if TYPE_CHECKING:
     from isaaclab_tasks.core.reorient.config.shadow_hand.shadow_hand_camera_env_cfg import ShadowHandCameraEnvCfg
@@ -40,7 +43,7 @@ def __init__(self, cfg: ShadowHandCameraEnvCfg, render_mode: str | None = None,
             width=self.cfg.tiled_camera.width,
         )
         # hide goal cubes
-        self.goal_pos[:, :] = torch.tensor([-0.2, 0.1, 0.6], device=self.device)
+        self.goal_pos[:, :] = torch.tensor(CAMERA_GOAL_MARKER_POSITION, device=self.device)
         # keypoints buffer
         self.gt_keypoints = torch.ones(self.num_envs, 8, 3, dtype=torch.float32, device=self.device)
         self.goal_keypoints = torch.ones(self.num_envs, 8, 3, dtype=torch.float32, device=self.device)
@@ -66,7 +69,7 @@ def _setup_scene(self):
 
     def _compute_image_observations(self):
         # generate ground truth keypoints for in-hand cube
-        compute_keypoints(pose=torch.cat((self.object_pos, self.object_rot), dim=1), out=self.gt_keypoints)
+        compute_cube_keypoints(pose=torch.cat((self.object_pos, self.object_rot), dim=1), out=self.gt_keypoints)
 
         object_pose = torch.cat([self.object_pos, self.gt_keypoints.view(-1, 24)], dim=-1)
 
@@ -78,7 +81,7 @@ def _compute_image_observations(self):
 
         self.embeddings = embeddings.clone().detach()
         # compute keypoints for goal cube
-        compute_keypoints(
+        compute_cube_keypoints(
             pose=torch.cat((torch.zeros_like(self.goal_pos), self.goal_rot), dim=-1), out=self.goal_keypoints
         )
 
@@ -126,44 +129,43 @@ def _compute_states(self):
         return state
 
     def _get_observations(self) -> dict:
+        # refresh the torch-side state snapshots this observation path reads; the base
+        # environment computes its observations in Warp kernels and no longer updates them
+        self._compute_intermediate_values()
         # proprioception observations
         state_obs = self._compute_proprio_observations()
         # vision observations from CMM
         image_obs = self._compute_image_observations()
         obs = torch.cat((state_obs, image_obs), dim=-1)
-        self._update_fingertip_force_sensors()
         state = self._compute_states()
 
         observations = {"policy": obs, "critic": state}
         return observations
 
 
-@torch.jit.script
 def compute_keypoints(
     pose: torch.Tensor,
     num_keypoints: int = 8,
     size: tuple[float, float, float] = (2 * 0.03, 2 * 0.03, 2 * 0.03),
     out: torch.Tensor | None = None,
-):
-    """Computes positions of 8 corner keypoints of a cube.
+) -> torch.Tensor:
+    """Compute cube keypoints using the shared implementation.
+
+    .. deprecated:: 9.0.0
+        Use :func:`compute_cube_keypoints` instead.
 
     Args:
-        pose: Position and orientation of the center of the cube. Shape is (N, 7)
-        num_keypoints: Number of keypoints to compute. Default = 8
-        size: Length of X, Y, Z dimensions of cube. Default = [0.06, 0.06, 0.06]
-        out: Buffer to store keypoints. If None, a new buffer will be created.
+        pose: Cube center poses ``(x, y, z, qx, qy, qz, qw)`` [m, unit quaternion].
+        num_keypoints: Number of binary-sign corners to compute.
+        size: Cube side lengths along each axis [m].
+        out: Optional output buffer [m], shape ``(num_envs, num_keypoints, 3)``.
+
+    Returns:
+        Cube-corner positions [m], shape ``(num_envs, num_keypoints, 3)``.
     """
-    num_envs = pose.shape[0]
-    if out is None:
-        out = torch.ones(num_envs, num_keypoints, 3, dtype=torch.float32, device=pose.device)
-    else:
-        out[:] = 1.0
-    for i in range(num_keypoints):
-        # which dimensions to negate
-        n = [((i >> k) & 1) == 0 for k in range(3)]
-        corner_loc = ([(1 if n[k] else -1) * s / 2 for k, s in enumerate(size)],)
-        corner = torch.tensor(corner_loc, dtype=torch.float32, device=pose.device) * out[:, i, :]
-        # express corner position in the world frame
-        out[:, i, :] = pose[:, :3] + quat_apply(pose[:, 3:7], corner)
-
-    return out
+    warnings.warn(
+        "compute_keypoints() is deprecated; use compute_cube_keypoints() instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return compute_cube_keypoints(pose, num_keypoints=num_keypoints, size=size, out=out)

source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_env_cfg.py

@@ -12,10 +12,48 @@
 
 from isaaclab_tasks.core.reorient.config.shadow_hand.feature_extractor import FeatureExtractorCfg
 from isaaclab_tasks.core.reorient.config.shadow_hand.shadow_hand_env_cfg import ShadowHandEnvCfg
+from isaaclab_tasks.core.reorient.reorient_task_base import (
+    CAMERA_PLAY_NUM_ENVS,
+)
 from isaaclab_tasks.utils import PresetCfg
 from isaaclab_tasks.utils.presets import MultiBackendRendererCfg
 
 
+def validate_shadow_hand_camera_settings(
+    tiled_camera: CameraCfg | ShadowHandTiledCameraCfg,
+    feature_extractor: FeatureExtractorCfg,
+) -> None:
+    """Validate one resolved or defaulted Shadow Hand camera pipeline."""
+    while isinstance(tiled_camera, PresetCfg):
+        tiled_camera = tiled_camera.default
+    renderer_cfg = tiled_camera.renderer_cfg
+    while isinstance(renderer_cfg, PresetCfg):
+        renderer_cfg = renderer_cfg.default
+
+    renderer_type = getattr(renderer_cfg, "renderer_type", None)
+    warp_supported = {"rgb", "depth", "normals"}
+    if renderer_type == "newton_warp":
+        unsupported = set(tiled_camera.data_types) - warp_supported
+        if unsupported:
+            raise ValueError(
+                f"Warp renderer only supports data types {sorted(warp_supported)}, "
+                f"but the camera is configured with unsupported types: {sorted(unsupported)}. "
+                "Choose a compatible preset, e.g. presets=newton_renderer,rgb."
+            )
+
+    non_depth_data_types = set(tiled_camera.data_types).difference(
+        {"depth", "distance_to_image_plane", "distance_to_camera"}
+    )
+    if tiled_camera.data_types and not non_depth_data_types and feature_extractor.enabled:
+        raise ValueError(
+            "Depth-only camera data type is intended for benchmarking only. "
+            "The keypoint-regression CNN cannot be meaningfully trained from depth alone. "
+            "Disable the feature extractor with 'feature_extractor.enabled=False' "
+            "(e.g. use Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct), "
+            "or choose a data type that includes colour, e.g. presets=rgb."
+        )
+
+
 @configclass
 class _ShadowHandBaseTiledCameraCfg(CameraCfg):
     """Base camera configuration for the shadow hand vision environment.
@@ -49,6 +87,7 @@ class ShadowHandTiledCameraCfg(PresetCfg):
     Select a data-type preset via the ``presets`` CLI argument, e.g.::
 
         presets = rgb  # RGB only (3 channels)
+        presets = rgb_depth  # RGB + depth (4 channels)
         presets = albedo  # albedo (3 channels)
         presets = simple_shading_constant_diffuse  # simple shading, constant diffuse (3 channels)
 
@@ -70,6 +109,9 @@ class ShadowHandTiledCameraCfg(PresetCfg):
     rgb: _ShadowHandBaseTiledCameraCfg = _ShadowHandBaseTiledCameraCfg(data_types=["rgb"])
     """RGB only (3 CNN input channels)."""
 
+    rgb_depth: _ShadowHandBaseTiledCameraCfg = _ShadowHandBaseTiledCameraCfg(data_types=["rgb", "depth"])
+    """RGB and depth (4 CNN input channels)."""
+
     albedo: _ShadowHandBaseTiledCameraCfg = _ShadowHandBaseTiledCameraCfg(data_types=["albedo"])
     """Albedo (3 CNN input channels)."""
 
@@ -123,34 +165,15 @@ class ShadowHandCameraEnvCfg(ShadowHandEnvCfg):
 
     def validate_config(self):
         """Check renderer/data-type and feature-extractor compatibility."""
-        renderer_type = getattr(self.tiled_camera.renderer_cfg, "renderer_type", None)
-        warp_supported = {"rgb", "depth", "normals"}
-        if renderer_type == "newton_warp":
-            unsupported = set(self.tiled_camera.data_types) - warp_supported
-            if unsupported:
-                raise ValueError(
-                    f"Warp renderer only supports data types {sorted(warp_supported)}, "
-                    f"but the camera is configured with unsupported types: {sorted(unsupported)}. "
-                    "Choose a compatible preset, e.g. presets=newton_renderer,rgb."
-                )
-
-        non_depth_data_types = set(self.tiled_camera.data_types).difference(
-            {"depth", "distance_to_image_plane", "distance_to_camera"}
-        )
-        if self.tiled_camera.data_types and not non_depth_data_types and self.feature_extractor.enabled:
-            raise ValueError(
-                "Depth-only camera data type is intended for benchmarking only. "
-                "The keypoint-regression CNN cannot be meaningfully trained from depth alone. "
-                "Disable the feature extractor with 'feature_extractor.enabled=False' "
-                "(e.g. use Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct), "
-                "or choose a data type that includes colour, e.g. presets=rgb."
-            )
+        validate_shadow_hand_camera_settings(self.tiled_camera, self.feature_extractor)
 
 
 @configclass
 class ShadowHandCameraEnvPlayCfg(ShadowHandCameraEnvCfg):
     # scene
-    scene: InteractiveSceneCfg = InteractiveSceneCfg(num_envs=64, env_spacing=2.0, replicate_physics=True)
+    scene: InteractiveSceneCfg = InteractiveSceneCfg(
+        num_envs=CAMERA_PLAY_NUM_ENVS, env_spacing=2.0, replicate_physics=True
+    )
     # inference for CNN
     feature_extractor: FeatureExtractorCfg = FeatureExtractorCfg(train=False, load_checkpoint=True)
 
@@ -159,6 +182,11 @@ class ShadowHandCameraEnvPlayCfg(ShadowHandCameraEnvCfg):
 class ShadowHandCameraBenchmarkEnvCfg(ShadowHandCameraEnvCfg):
     """Benchmark configuration with the feature extractor CNN disabled.
 
+    .. deprecated:: 9.0.0
+        Use the regular camera task with the ``env.feature_extractor.enabled=False``
+        override instead. The ``Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct``
+        registration will be removed in a future release.
+
     The tiled camera renders frames each step as normal, but the CNN forward pass is
     bypassed — zero embeddings are returned instead. This isolates rendering throughput
     from CNN inference overhead when profiling.