diff --git a/source/isaaclab_tasks/changelog.d/task-cleanup-dex-part10.minor.rst b/source/isaaclab_tasks/changelog.d/task-cleanup-dex-part10.minor.rst new file mode 100644 index 000000000000..7192fa59a5b7 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/task-cleanup-dex-part10.minor.rst @@ -0,0 +1,11 @@ +Added +^^^^^ + +* Added a manager-based counterpart for the Shadow Hand camera reorientation + environment. + +Deprecated +^^^^^^^^^^ + +* Deprecated ``shadow_hand_camera_env.compute_keypoints`` in favor of + ``feature_extractor.compute_cube_keypoints``. diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/__init__.py b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/__init__.py index 5135565424d8..d7c465c8d2c9 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/__init__.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/__init__.py @@ -17,6 +17,18 @@ reorient_direct_entry = "isaaclab_tasks.core.reorient.reorient_direct_env:ReorientDirectEnv" +gym.register( + id="Isaac-Reorient-Cube-Shadow", + entry_point="isaaclab.envs:ManagerBasedRLEnv", + disable_env_checker=True, + kwargs={ + "env_cfg_entry_point": f"{__name__}.shadow_hand_manager_env_cfg:ShadowHandManagerEnvCfg", + "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_cfg.yaml", + "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandPPORunnerCfg", + "skrl_cfg_entry_point": f"{agents.__name__}:skrl_ppo_cfg.yaml", + }, +) + gym.register( id="Isaac-Reorient-Cube-Shadow-Direct", entry_point=reorient_direct_entry, @@ -29,6 +41,29 @@ }, ) +gym.register( + id="Isaac-Reorient-Cube-Shadow-OpenAI-FF", + entry_point="isaaclab.envs:ManagerBasedRLEnv", + disable_env_checker=True, + kwargs={ + "env_cfg_entry_point": f"{__name__}.shadow_hand_manager_env_cfg:ShadowHandOpenAIManagerEnvCfg", + "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_ff_cfg.yaml", + "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandAsymFFPPORunnerCfg", + "skrl_cfg_entry_point": f"{agents.__name__}:skrl_ff_ppo_cfg.yaml", + }, +) + +gym.register( + id="Isaac-Reorient-Cube-Shadow-OpenAI-LSTM", + entry_point="isaaclab.envs:ManagerBasedRLEnv", + disable_env_checker=True, + kwargs={ + "env_cfg_entry_point": f"{__name__}.shadow_hand_manager_env_cfg:ShadowHandOpenAIManagerEnvCfg", + "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_lstm_cfg.yaml", + "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandAsymLSTMPPORunnerCfg", + }, +) + gym.register( id="Isaac-Reorient-Cube-Shadow-OpenAI-FF-Direct", entry_point=reorient_direct_entry, @@ -48,6 +83,7 @@ kwargs={ "env_cfg_entry_point": f"{__name__}.shadow_hand_env_cfg:ShadowHandOpenAIEnvCfg", "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_lstm_cfg.yaml", + "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandAsymLSTMPPORunnerCfg", }, ) @@ -55,6 +91,28 @@ # Vision # ------- +gym.register( + id="Isaac-Reorient-Cube-Shadow-Camera", + entry_point="isaaclab.envs:ManagerBasedRLEnv", + disable_env_checker=True, + kwargs={ + "env_cfg_entry_point": f"{__name__}.shadow_hand_camera_manager_env_cfg:ShadowHandCameraManagerEnvCfg", + "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandCameraFFPPORunnerCfg", + "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_camera_cfg.yaml", + }, +) + +gym.register( + id="Isaac-Reorient-Cube-Shadow-Camera-Play", + entry_point="isaaclab.envs:ManagerBasedRLEnv", + disable_env_checker=True, + kwargs={ + "env_cfg_entry_point": f"{__name__}.shadow_hand_camera_manager_env_cfg:ShadowHandCameraManagerPlayEnvCfg", + "rsl_rl_cfg_entry_point": f"{agents.__name__}.rsl_rl_ppo_cfg:ShadowHandCameraFFPPORunnerCfg", + "rl_games_cfg_entry_point": f"{agents.__name__}:rl_games_ppo_camera_cfg.yaml", + }, +) + gym.register( id="Isaac-Reorient-Cube-Shadow-Camera-Direct", entry_point=f"{__name__}.shadow_hand_camera_env:ShadowHandCameraEnv", diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/feature_extractor.py b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/feature_extractor.py index 56c159a1446e..e950f5eece70 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/feature_extractor.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/feature_extractor.py @@ -3,8 +3,11 @@ # # SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations + import glob import os +from typing import TYPE_CHECKING import torch import torch.nn as nn @@ -13,6 +16,12 @@ from isaaclab.sensors import save_images_to_file from isaaclab.utils.configclass import configclass +# re-exported for backward compatibility; the shared implementation lives in the family math root +from isaaclab_tasks.core.reorient.reorient_kernels import compute_cube_keypoints # noqa: F401 + +if TYPE_CHECKING: + pass + # Number of output channels for each supported camera data type. _DATA_TYPE_CHANNELS: dict[str, int] = { "rgb": 3, diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_env.py b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_env.py index 90de75ba3ee7..143a557ac6a3 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_env.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_env.py @@ -6,6 +6,7 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING import torch @@ -14,10 +15,12 @@ from isaaclab import cloner from isaaclab.assets import Articulation, RigidObject from isaaclab.sensors import Camera -from isaaclab.utils.math import quat_apply, scale_transform +from isaaclab.utils.math import scale_transform from isaaclab_tasks.core.reorient.config.shadow_hand.feature_extractor import FeatureExtractor from isaaclab_tasks.core.reorient.reorient_direct_env import ReorientDirectEnv +from isaaclab_tasks.core.reorient.reorient_kernels import compute_cube_keypoints +from isaaclab_tasks.core.reorient.reorient_task_base import CAMERA_GOAL_MARKER_POSITION if TYPE_CHECKING: from isaaclab_tasks.core.reorient.config.shadow_hand.shadow_hand_camera_env_cfg import ShadowHandCameraEnvCfg @@ -40,7 +43,7 @@ def __init__(self, cfg: ShadowHandCameraEnvCfg, render_mode: str | None = None, width=self.cfg.tiled_camera.width, ) # hide goal cubes - self.goal_pos[:, :] = torch.tensor([-0.2, 0.1, 0.6], device=self.device) + self.goal_pos[:, :] = torch.tensor(CAMERA_GOAL_MARKER_POSITION, device=self.device) # keypoints buffer self.gt_keypoints = torch.ones(self.num_envs, 8, 3, dtype=torch.float32, device=self.device) self.goal_keypoints = torch.ones(self.num_envs, 8, 3, dtype=torch.float32, device=self.device) @@ -66,7 +69,7 @@ def _setup_scene(self): def _compute_image_observations(self): # generate ground truth keypoints for in-hand cube - compute_keypoints(pose=torch.cat((self.object_pos, self.object_rot), dim=1), out=self.gt_keypoints) + compute_cube_keypoints(pose=torch.cat((self.object_pos, self.object_rot), dim=1), out=self.gt_keypoints) object_pose = torch.cat([self.object_pos, self.gt_keypoints.view(-1, 24)], dim=-1) @@ -78,7 +81,7 @@ def _compute_image_observations(self): self.embeddings = embeddings.clone().detach() # compute keypoints for goal cube - compute_keypoints( + compute_cube_keypoints( pose=torch.cat((torch.zeros_like(self.goal_pos), self.goal_rot), dim=-1), out=self.goal_keypoints ) @@ -126,44 +129,43 @@ def _compute_states(self): return state def _get_observations(self) -> dict: + # refresh the torch-side state snapshots this observation path reads; the base + # environment computes its observations in Warp kernels and no longer updates them + self._compute_intermediate_values() # proprioception observations state_obs = self._compute_proprio_observations() # vision observations from CMM image_obs = self._compute_image_observations() obs = torch.cat((state_obs, image_obs), dim=-1) - self._update_fingertip_force_sensors() state = self._compute_states() observations = {"policy": obs, "critic": state} return observations -@torch.jit.script def compute_keypoints( pose: torch.Tensor, num_keypoints: int = 8, size: tuple[float, float, float] = (2 * 0.03, 2 * 0.03, 2 * 0.03), out: torch.Tensor | None = None, -): - """Computes positions of 8 corner keypoints of a cube. +) -> torch.Tensor: + """Compute cube keypoints using the shared implementation. + + .. deprecated:: 9.0.0 + Use :func:`compute_cube_keypoints` instead. Args: - pose: Position and orientation of the center of the cube. Shape is (N, 7) - num_keypoints: Number of keypoints to compute. Default = 8 - size: Length of X, Y, Z dimensions of cube. Default = [0.06, 0.06, 0.06] - out: Buffer to store keypoints. If None, a new buffer will be created. + pose: Cube center poses ``(x, y, z, qx, qy, qz, qw)`` [m, unit quaternion]. + num_keypoints: Number of binary-sign corners to compute. + size: Cube side lengths along each axis [m]. + out: Optional output buffer [m], shape ``(num_envs, num_keypoints, 3)``. + + Returns: + Cube-corner positions [m], shape ``(num_envs, num_keypoints, 3)``. """ - num_envs = pose.shape[0] - if out is None: - out = torch.ones(num_envs, num_keypoints, 3, dtype=torch.float32, device=pose.device) - else: - out[:] = 1.0 - for i in range(num_keypoints): - # which dimensions to negate - n = [((i >> k) & 1) == 0 for k in range(3)] - corner_loc = ([(1 if n[k] else -1) * s / 2 for k, s in enumerate(size)],) - corner = torch.tensor(corner_loc, dtype=torch.float32, device=pose.device) * out[:, i, :] - # express corner position in the world frame - out[:, i, :] = pose[:, :3] + quat_apply(pose[:, 3:7], corner) - - return out + warnings.warn( + "compute_keypoints() is deprecated; use compute_cube_keypoints() instead.", + DeprecationWarning, + stacklevel=2, + ) + return compute_cube_keypoints(pose, num_keypoints=num_keypoints, size=size, out=out) diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_env_cfg.py index 3ecb37b503d7..7217ffc93221 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_env_cfg.py @@ -12,10 +12,48 @@ from isaaclab_tasks.core.reorient.config.shadow_hand.feature_extractor import FeatureExtractorCfg from isaaclab_tasks.core.reorient.config.shadow_hand.shadow_hand_env_cfg import ShadowHandEnvCfg +from isaaclab_tasks.core.reorient.reorient_task_base import ( + CAMERA_PLAY_NUM_ENVS, +) from isaaclab_tasks.utils import PresetCfg from isaaclab_tasks.utils.presets import MultiBackendRendererCfg +def validate_shadow_hand_camera_settings( + tiled_camera: CameraCfg | ShadowHandTiledCameraCfg, + feature_extractor: FeatureExtractorCfg, +) -> None: + """Validate one resolved or defaulted Shadow Hand camera pipeline.""" + while isinstance(tiled_camera, PresetCfg): + tiled_camera = tiled_camera.default + renderer_cfg = tiled_camera.renderer_cfg + while isinstance(renderer_cfg, PresetCfg): + renderer_cfg = renderer_cfg.default + + renderer_type = getattr(renderer_cfg, "renderer_type", None) + warp_supported = {"rgb", "depth", "normals"} + if renderer_type == "newton_warp": + unsupported = set(tiled_camera.data_types) - warp_supported + if unsupported: + raise ValueError( + f"Warp renderer only supports data types {sorted(warp_supported)}, " + f"but the camera is configured with unsupported types: {sorted(unsupported)}. " + "Choose a compatible preset, e.g. presets=newton_renderer,rgb." + ) + + non_depth_data_types = set(tiled_camera.data_types).difference( + {"depth", "distance_to_image_plane", "distance_to_camera"} + ) + if tiled_camera.data_types and not non_depth_data_types and feature_extractor.enabled: + raise ValueError( + "Depth-only camera data type is intended for benchmarking only. " + "The keypoint-regression CNN cannot be meaningfully trained from depth alone. " + "Disable the feature extractor with 'feature_extractor.enabled=False' " + "(e.g. use Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct), " + "or choose a data type that includes colour, e.g. presets=rgb." + ) + + @configclass class _ShadowHandBaseTiledCameraCfg(CameraCfg): """Base camera configuration for the shadow hand vision environment. @@ -49,6 +87,7 @@ class ShadowHandTiledCameraCfg(PresetCfg): Select a data-type preset via the ``presets`` CLI argument, e.g.:: presets = rgb # RGB only (3 channels) + presets = rgb_depth # RGB + depth (4 channels) presets = albedo # albedo (3 channels) presets = simple_shading_constant_diffuse # simple shading, constant diffuse (3 channels) @@ -70,6 +109,9 @@ class ShadowHandTiledCameraCfg(PresetCfg): rgb: _ShadowHandBaseTiledCameraCfg = _ShadowHandBaseTiledCameraCfg(data_types=["rgb"]) """RGB only (3 CNN input channels).""" + rgb_depth: _ShadowHandBaseTiledCameraCfg = _ShadowHandBaseTiledCameraCfg(data_types=["rgb", "depth"]) + """RGB and depth (4 CNN input channels).""" + albedo: _ShadowHandBaseTiledCameraCfg = _ShadowHandBaseTiledCameraCfg(data_types=["albedo"]) """Albedo (3 CNN input channels).""" @@ -123,34 +165,15 @@ class ShadowHandCameraEnvCfg(ShadowHandEnvCfg): def validate_config(self): """Check renderer/data-type and feature-extractor compatibility.""" - renderer_type = getattr(self.tiled_camera.renderer_cfg, "renderer_type", None) - warp_supported = {"rgb", "depth", "normals"} - if renderer_type == "newton_warp": - unsupported = set(self.tiled_camera.data_types) - warp_supported - if unsupported: - raise ValueError( - f"Warp renderer only supports data types {sorted(warp_supported)}, " - f"but the camera is configured with unsupported types: {sorted(unsupported)}. " - "Choose a compatible preset, e.g. presets=newton_renderer,rgb." - ) - - non_depth_data_types = set(self.tiled_camera.data_types).difference( - {"depth", "distance_to_image_plane", "distance_to_camera"} - ) - if self.tiled_camera.data_types and not non_depth_data_types and self.feature_extractor.enabled: - raise ValueError( - "Depth-only camera data type is intended for benchmarking only. " - "The keypoint-regression CNN cannot be meaningfully trained from depth alone. " - "Disable the feature extractor with 'feature_extractor.enabled=False' " - "(e.g. use Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct), " - "or choose a data type that includes colour, e.g. presets=rgb." - ) + validate_shadow_hand_camera_settings(self.tiled_camera, self.feature_extractor) @configclass class ShadowHandCameraEnvPlayCfg(ShadowHandCameraEnvCfg): # scene - scene: InteractiveSceneCfg = InteractiveSceneCfg(num_envs=64, env_spacing=2.0, replicate_physics=True) + scene: InteractiveSceneCfg = InteractiveSceneCfg( + num_envs=CAMERA_PLAY_NUM_ENVS, env_spacing=2.0, replicate_physics=True + ) # inference for CNN feature_extractor: FeatureExtractorCfg = FeatureExtractorCfg(train=False, load_checkpoint=True) @@ -159,6 +182,11 @@ class ShadowHandCameraEnvPlayCfg(ShadowHandCameraEnvCfg): class ShadowHandCameraBenchmarkEnvCfg(ShadowHandCameraEnvCfg): """Benchmark configuration with the feature extractor CNN disabled. + .. deprecated:: 9.0.0 + Use the regular camera task with the ``env.feature_extractor.enabled=False`` + override instead. The ``Isaac-Reorient-Cube-Shadow-Camera-Benchmark-Direct`` + registration will be removed in a future release. + The tiled camera renders frames each step as normal, but the CNN forward pass is bypassed — zero embeddings are returned instead. This isolates rendering throughput from CNN inference overhead when profiling. diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_manager_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_manager_env_cfg.py new file mode 100644 index 000000000000..de7d91e7f6d6 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/config/shadow_hand/shadow_hand_camera_manager_env_cfg.py @@ -0,0 +1,174 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Manager-based counterpart of the Shadow Hand camera reorientation task.""" + +from isaaclab.managers import ObservationTermCfg as ObsTerm +from isaaclab.managers import SceneEntityCfg +from isaaclab.sensors import JointWrenchSensorCfg +from isaaclab.utils.configclass import configclass + +import isaaclab_tasks.core.reorient.mdp as mdp +from isaaclab_tasks.core.reorient.config.shadow_hand.feature_extractor import FeatureExtractorCfg +from isaaclab_tasks.core.reorient.config.shadow_hand.shadow_hand_camera_env_cfg import ( + ShadowHandTiledCameraCfg, + validate_shadow_hand_camera_settings, +) +from isaaclab_tasks.core.reorient.config.shadow_hand.shadow_hand_manager_env_cfg import ( + ActionsCfg, + CommandsCfg, + EventCfg, + FullStateWithoutActionCfg, + RewardsCfg, + ShadowHandManagerEnvCfg, + TerminationsCfg, + _ShadowHandManagerSceneCfg, +) +from isaaclab_tasks.core.reorient.reorient_task_base import ( + CAMERA_GOAL_MARKER_POSITION, + CAMERA_PLAY_NUM_ENVS, + SHADOW_FINGERTIP_BODY_NAMES, +) +from isaaclab_tasks.utils import PresetCfg + + +@configclass +class _ShadowHandCameraManagerSceneCfg(_ShadowHandManagerSceneCfg): + """State Manager scene augmented with camera and fingertip-wrench sensors.""" + + ground = None + tiled_camera: ShadowHandTiledCameraCfg = ShadowHandTiledCameraCfg() + joint_wrench = JointWrenchSensorCfg(prim_path="{ENV_REGEX_NS}/Robot") + + +@configclass +class ShadowHandCameraManagerSceneCfg(PresetCfg): + """Backend-specific camera scene alternatives for training and benchmarking.""" + + physx = _ShadowHandCameraManagerSceneCfg( + num_envs=1225, + env_spacing=2.0, + replicate_physics=True, + clone_in_fabric=True, + ) + newton_mjwarp = _ShadowHandCameraManagerSceneCfg( + num_envs=1225, + env_spacing=2.0, + replicate_physics=True, + clone_in_fabric=False, + ) + ovphysx = physx + default = physx + + +@configclass +class ShadowHandCameraManagerPlaySceneCfg(PresetCfg): + """Reduced backend-specific camera scenes for checkpoint playback.""" + + physx = _ShadowHandCameraManagerSceneCfg( + num_envs=CAMERA_PLAY_NUM_ENVS, + env_spacing=2.0, + replicate_physics=True, + clone_in_fabric=True, + ) + newton_mjwarp = _ShadowHandCameraManagerSceneCfg( + num_envs=CAMERA_PLAY_NUM_ENVS, + env_spacing=2.0, + replicate_physics=True, + clone_in_fabric=False, + ) + ovphysx = _ShadowHandCameraManagerSceneCfg( + num_envs=CAMERA_PLAY_NUM_ENVS, + env_spacing=2.0, + replicate_physics=True, + clone_in_fabric=True, + ) + default = physx + + +@configclass +class CameraPolicyCfg(FullStateWithoutActionCfg): + """Direct-compatible 191-dimensional camera actor observation.""" + + last_action = ObsTerm(func=mdp.reorient_last_action, params={"action_name": "joint_pos"}) + camera_features = ObsTerm( + func=mdp.ShadowHandCameraFeatures, + params={ + "feature_extractor_cfg": FeatureExtractorCfg(), + "sensor_cfg": SceneEntityCfg("tiled_camera"), + "object_cfg": SceneEntityCfg("object"), + }, + ) + goal_keypoints = ObsTerm(func=mdp.shadow_hand_goal_keypoints, params={"command_name": "object_pose"}) + + def __post_init__(self): + super().__post_init__() + # Camera actor observations infer object state from pixels. These five + # privileged state terms are present only in the critic. + self.object_pos = None + self.object_quat = None + self.object_lin_vel = None + self.object_ang_vel = None + self.goal_quat_diff = None + + +@configclass +class CameraCriticCfg(FullStateWithoutActionCfg): + """Direct-compatible 214-dimensional asymmetric camera critic state.""" + + fingertip_wrench = ObsTerm( + func=mdp.fingertip_wrench, + scale=10.0, + params={ + "sensor_cfg": SceneEntityCfg("joint_wrench", body_names=SHADOW_FINGERTIP_BODY_NAMES, preserve_order=False) + }, + ) + last_action = ObsTerm(func=mdp.reorient_last_action, params={"action_name": "joint_pos"}) + camera_features = ObsTerm(func=mdp.shadow_hand_camera_cached_features) + + +@configclass +class CameraObservationsCfg: + """Camera actor and asymmetric critic observation groups.""" + + policy: CameraPolicyCfg = CameraPolicyCfg() + critic: CameraCriticCfg = CameraCriticCfg() + + +@configclass +class ShadowHandCameraManagerEnvCfg(ShadowHandManagerEnvCfg): + """Manager-based camera task with exact Direct dynamics and observations.""" + + scene: ShadowHandCameraManagerSceneCfg = ShadowHandCameraManagerSceneCfg() + observations: CameraObservationsCfg = CameraObservationsCfg() + actions: ActionsCfg = ActionsCfg() + commands: CommandsCfg = CommandsCfg() + rewards: RewardsCfg = RewardsCfg() + terminations: TerminationsCfg = TerminationsCfg() + events: EventCfg = EventCfg() + feature_extractor: FeatureExtractorCfg = FeatureExtractorCfg() + + def __post_init__(self): + super().__post_init__() + # camera tasks display the goal inside the tiled camera's frustum + self.commands.object_pose.fixed_marker_pos = CAMERA_GOAL_MARKER_POSITION + self.observations.policy.camera_features.params["feature_extractor_cfg"] = self.feature_extractor + + def validate_config(self): + """Check every unresolved scene alternative or the selected camera pipeline.""" + if isinstance(self.scene, PresetCfg): + scenes = (self.scene.physx, self.scene.newton_mjwarp, self.scene.ovphysx) + else: + scenes = (self.scene,) + for scene in scenes: + validate_shadow_hand_camera_settings(scene.tiled_camera, self.feature_extractor) + + +@configclass +class ShadowHandCameraManagerPlayEnvCfg(ShadowHandCameraManagerEnvCfg): + """Manager camera task configured for checkpoint playback.""" + + scene: ShadowHandCameraManagerPlaySceneCfg = ShadowHandCameraManagerPlaySceneCfg() + feature_extractor: FeatureExtractorCfg = FeatureExtractorCfg(train=False, load_checkpoint=True) diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/mdp/__init__.pyi b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/mdp/__init__.pyi index e835f887dd8a..5097dadb6668 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/mdp/__init__.pyi +++ b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/mdp/__init__.pyi @@ -4,19 +4,69 @@ # SPDX-License-Identifier: BSD-3-Clause __all__ = [ + "NoisyEMAJointPositionToLimitsAction", + "NoisyEMAJointPositionToLimitsActionCfg", + "ShadowHandCameraFeatures", + "shadow_hand_camera_cached_features", + "shadow_hand_goal_keypoints", "ReorientCommand", "ReorientCommandCfg", + "ReorientEpisodeCommand", + "ReorientEpisodeCommandCfg", + "reset_reorient_state", + "fingertip_pos", + "fingertip_quat", + "fingertip_vel", + "fingertip_wrench", + "reorient_last_action", + "OpenAIPolicyObservation", "goal_quat_diff", "success_bonus", "track_orientation_inv_l2", "track_pos_l2", + "rotation_distance_kernel", + "reorient_success_kernel", + "direct_reorient_reward", + "DirectReorientReward", "max_consecutive_success", "object_away_from_goal", "object_away_from_robot", + "object_reorientation_out_of_reach", + "DirectReorientTimeout", ] -from .commands import ReorientCommand, ReorientCommandCfg -from .observations import goal_quat_diff -from .rewards import success_bonus, track_orientation_inv_l2, track_pos_l2 -from .terminations import max_consecutive_success, object_away_from_goal, object_away_from_robot +from .commands import ReorientCommand, ReorientCommandCfg, ReorientEpisodeCommand, ReorientEpisodeCommandCfg +from .events import reset_reorient_state +from .actions import ( + NoisyEMAJointPositionToLimitsAction, + NoisyEMAJointPositionToLimitsActionCfg, +) +from .observations import ( + ShadowHandCameraFeatures, + shadow_hand_camera_cached_features, + shadow_hand_goal_keypoints, + OpenAIPolicyObservation, + fingertip_pos, + fingertip_quat, + fingertip_vel, + fingertip_wrench, + goal_quat_diff, + reorient_last_action, +) +from .rewards import ( + DirectReorientReward, + direct_reorient_reward, + reorient_success_kernel, + rotation_distance_kernel, + success_bonus, + track_orientation_inv_l2, + track_pos_l2, +) +from .terminations import ( + DirectReorientTimeout, + max_consecutive_success, + object_away_from_goal, + object_away_from_robot, + object_reorientation_out_of_reach, +) from isaaclab.envs.mdp import * diff --git a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/mdp/observations.py b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/mdp/observations.py index b54b2beeab9f..2cb93527ee24 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/core/reorient/mdp/observations.py +++ b/source/isaaclab_tasks/isaaclab_tasks/core/reorient/mdp/observations.py @@ -7,36 +7,421 @@ from __future__ import annotations +from collections.abc import Sequence from typing import TYPE_CHECKING import torch +import warp as wp -import isaaclab.utils.math as math_utils -from isaaclab.managers import SceneEntityCfg +from isaaclab.managers import ManagerTermBase, ObservationTermCfg, SceneEntityCfg +from isaaclab.utils.noise import NoiseModelCfg + +# CUBE_HALF_SIZE, cube_keypoints_from_quat_kernel, and compute_cube_keypoints are +# re-exported for API stability: the cube-keypoint math moved to the family math +# root in :mod:`isaaclab_tasks.core.reorient.reorient_kernels` +from isaaclab_tasks.core.reorient.reorient_kernels import ( # noqa: F401 + CUBE_HALF_SIZE, + compute_cube_keypoints, + cube_keypoints_from_quat_kernel, + fingertip_pos_kernel, + fingertip_quat_kernel, + fingertip_vel_kernel, +) if TYPE_CHECKING: from isaaclab.assets import RigidObject from isaaclab.envs import ManagerBasedRLEnv + from isaaclab.sensors import Camera, JointWrenchSensor + + from isaaclab_tasks.core.reorient.config.shadow_hand.feature_extractor import FeatureExtractorCfg from .commands import ReorientCommand -def goal_quat_diff( - env: ManagerBasedRLEnv, asset_cfg: SceneEntityCfg, command_name: str, make_quat_unique: bool -) -> torch.Tensor: +@wp.kernel +def _fingertip_wrench_kernel( + force: wp.array2d(dtype=wp.vec3f), + torque: wp.array2d(dtype=wp.vec3f), + body_ids: wp.array(dtype=wp.int32), + out: wp.array2d(dtype=wp.float32), +): + i, j = wp.tid() + f = force[i, body_ids[j]] + t = torque[i, body_ids[j]] + out[i, 6 * j + 0] = f[0] + out[i, 6 * j + 1] = f[1] + out[i, 6 * j + 2] = f[2] + out[i, 6 * j + 3] = t[0] + out[i, 6 * j + 4] = t[1] + out[i, 6 * j + 5] = t[2] + + +@wp.kernel +def _goal_quat_error_kernel( + asset_quat: wp.array(dtype=wp.quatf), + goal_quat: wp.array(dtype=wp.quatf), + make_unique: int, + out: wp.array(dtype=wp.quatf), +): + """Per-environment quaternion error ``asset * conjugate(goal)`` in (x, y, z, w) order.""" + i = wp.tid() + # quat_inverse == conjugate for these unit quaternions, matching + # isaaclab.utils.math.quat_mul/quat_conjugate semantics + qe = asset_quat[i] * wp.quat_inverse(goal_quat[i]) + sign = 1.0 + # make_unique keeps the real part non-negative (isaaclab.utils.math.quat_unique) + if make_unique != 0 and qe[3] < 0.0: + sign = -1.0 + out[i] = wp.quatf(sign * qe[0], sign * qe[1], sign * qe[2], sign * qe[3]) + + +class goal_quat_diff(ManagerTermBase): """Goal orientation relative to the asset's root frame. - The quaternion is represented as (w, x, y, z). The real part is always positive. + The real part is always positive when ``make_quat_unique`` is set. + """ + + def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedRLEnv): + super().__init__(cfg, env) + self._out = torch.empty(env.num_envs, 4, dtype=torch.float32, device=env.device) + # cached Warp views; the hot loop launches the kernel without conversions + self._out_wp = wp.from_torch(self._out, dtype=wp.quatf) + # resolved on first call: the command term does not exist yet during manager construction + self._goal_quat_wp: wp.array | None = None + + def __call__( + self, env: ManagerBasedRLEnv, asset_cfg: SceneEntityCfg, command_name: str, make_quat_unique: bool + ) -> torch.Tensor: + """Return the per-environment quaternion error, shape ``(num_envs, 4)``.""" + asset: RigidObject = env.scene[asset_cfg.name] + if self._goal_quat_wp is None: + command_term: ReorientCommand = env.command_manager.get_term(command_name) + self._goal_quat_wp = wp.from_torch(command_term.quat_command_w, dtype=wp.quatf) + wp.launch( + _goal_quat_error_kernel, + dim=self.num_envs, + inputs=[asset.data.root_quat_w.warp, self._goal_quat_wp, make_quat_unique], + outputs=[self._out_wp], + device=self._out_wp.device, + ) + return self._out + + +class fingertip_pos(ManagerTermBase): + """Flattened fingertip positions in the environment frame [m].""" + + def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedRLEnv): + super().__init__(cfg, env) + body_ids = cfg.params["asset_cfg"].body_ids + self._body_ids_wp = wp.array(body_ids, dtype=wp.int32, device=str(env.device)) + self._out = torch.empty(env.num_envs, len(body_ids) * 3, dtype=torch.float32, device=env.device) + self._out_wp = wp.from_torch(self._out) + self._env_origins_wp = wp.from_torch(env.scene.env_origins, dtype=wp.vec3f) + + def __call__(self, env: ManagerBasedRLEnv, asset_cfg: SceneEntityCfg) -> torch.Tensor: + """Return the flattened per-fingertip block, shape ``(num_envs, num_fingertips * 3)``.""" + asset = env.scene[asset_cfg.name] + wp.launch( + fingertip_pos_kernel, + dim=(self.num_envs, self._body_ids_wp.shape[0]), + inputs=[asset.data.body_pos_w.warp, self._env_origins_wp, self._body_ids_wp], + outputs=[self._out_wp], + device=self._out_wp.device, + ) + return self._out + + +class fingertip_quat(ManagerTermBase): + """Flattened fingertip ``(x, y, z, w)`` orientations.""" + + def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedRLEnv): + super().__init__(cfg, env) + body_ids = cfg.params["asset_cfg"].body_ids + self._body_ids_wp = wp.array(body_ids, dtype=wp.int32, device=str(env.device)) + self._out = torch.empty(env.num_envs, len(body_ids) * 4, dtype=torch.float32, device=env.device) + self._out_wp = wp.from_torch(self._out) + + def __call__(self, env: ManagerBasedRLEnv, asset_cfg: SceneEntityCfg) -> torch.Tensor: + """Return the flattened per-fingertip block, shape ``(num_envs, num_fingertips * 4)``.""" + asset = env.scene[asset_cfg.name] + wp.launch( + fingertip_quat_kernel, + dim=(self.num_envs, self._body_ids_wp.shape[0]), + inputs=[asset.data.body_quat_w.warp, self._body_ids_wp], + outputs=[self._out_wp], + device=self._out_wp.device, + ) + return self._out + + +class fingertip_vel(ManagerTermBase): + """Flattened fingertip spatial velocities [m/s, rad/s].""" + + def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedRLEnv): + super().__init__(cfg, env) + body_ids = cfg.params["asset_cfg"].body_ids + self._body_ids_wp = wp.array(body_ids, dtype=wp.int32, device=str(env.device)) + self._out = torch.empty(env.num_envs, len(body_ids) * 6, dtype=torch.float32, device=env.device) + self._out_wp = wp.from_torch(self._out) + + def __call__(self, env: ManagerBasedRLEnv, asset_cfg: SceneEntityCfg) -> torch.Tensor: + """Return the flattened per-fingertip block, shape ``(num_envs, num_fingertips * 6)``.""" + asset = env.scene[asset_cfg.name] + wp.launch( + fingertip_vel_kernel, + dim=(self.num_envs, self._body_ids_wp.shape[0]), + inputs=[asset.data.body_vel_w.warp, self._body_ids_wp], + outputs=[self._out_wp], + device=self._out_wp.device, + ) + return self._out + + +class fingertip_wrench(ManagerTermBase): + """Fingertip reaction wrenches [N, N·m] with Direct-compatible zero fallback.""" + + def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedRLEnv): + super().__init__(cfg, env) + body_ids = cfg.params["sensor_cfg"].body_ids + self._body_ids_wp = wp.array(body_ids, dtype=wp.int32, device=str(env.device)) + self._out = torch.zeros(env.num_envs, len(body_ids) * 6, dtype=torch.float32, device=env.device) + self._out_wp = wp.from_torch(self._out) + + def __call__(self, env: ManagerBasedRLEnv, sensor_cfg: SceneEntityCfg) -> torch.Tensor: + """Return the flattened wrench block, shape ``(num_envs, num_fingertips * 6)``.""" + sensor: JointWrenchSensor = env.scene.sensors[sensor_cfg.name] + force_data = sensor.data.force + torque_data = sensor.data.torque + if force_data is None or torque_data is None: + # Direct-compatible fallback: report zero wrenches until the sensor produces data + return self._out + wp.launch( + _fingertip_wrench_kernel, + dim=(self.num_envs, self._body_ids_wp.shape[0]), + inputs=[force_data.warp, torque_data.warp, self._body_ids_wp], + outputs=[self._out_wp], + device=self._out_wp.device, + ) + return self._out + + +def reorient_last_action(env: ManagerBasedRLEnv, action_name: str) -> torch.Tensor: + """Return the Direct-compatible last action across same-step autoreset. + + Args: + env: Environment containing the action term and reset buffers. + action_name: Action term whose raw action is observed. + + Returns: + Raw actions, retaining each terminal action in its same-step reset observation. """ - # extract useful elements - asset: RigidObject = env.scene[asset_cfg.name] - command_term: ReorientCommand = env.command_manager.get_term(command_name) - - # obtain the orientations - goal_quat_w = command_term.command[:, 3:7] - asset_quat_w = asset.data.root_quat_w.torch - - # compute quaternion difference - quat = math_utils.quat_mul(asset_quat_w, math_utils.quat_conjugate(goal_quat_w)) - # make sure the quaternion real-part is always positive - return math_utils.quat_unique(quat) if make_quat_unique else quat + raw_action = env.action_manager.get_term(action_name).raw_actions + reset_action = getattr(env, "_reorient_reset_action", None) + reset_step = getattr(env, "_reorient_reset_step", None) + common_step_counter = getattr(env, "common_step_counter", None) + if reset_action is None or reset_step is None or common_step_counter is None: + return raw_action + return torch.where((reset_step == common_step_counter).unsqueeze(-1), reset_action, raw_action) + + +class OpenAIPolicyObservation(ManagerTermBase): + """Apply one stateful noise model to the concatenated OpenAI actor observation.""" + + def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedRLEnv): + super().__init__(cfg, env) + noise_model: NoiseModelCfg = cfg.params["noise_model"] + self._noise_model = noise_model.class_type(noise_model, num_envs=self.num_envs, device=self.device) + self._quat_error = torch.empty(env.num_envs, 4, dtype=torch.float32, device=env.device) + robot_body_ids = cfg.params["robot_cfg"].body_ids + self._robot_body_ids_wp = wp.array(robot_body_ids, dtype=wp.int32, device=str(env.device)) + self._fingertip_buf = torch.empty(env.num_envs, len(robot_body_ids) * 3, dtype=torch.float32, device=env.device) + self._fingertip_buf_wp = wp.from_torch(self._fingertip_buf) + self._env_origins_wp = wp.from_torch(env.scene.env_origins, dtype=wp.vec3f) + # cached Warp views; the hot loop launches the kernel without conversions + self._quat_error_wp = wp.from_torch(self._quat_error, dtype=wp.quatf) + # resolved on first call: the command term does not exist yet during manager construction + self._goal_quat_wp: wp.array | None = None + # ObservationManager probes callable terms once for their shape and then + # calls reset. Keep that probe side-effect free so initialization matches + # DirectRLEnv's first noise-model reset and application. + self._shape_probe_pending = True + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + """Reset the actor observation bias for selected environments. + + Args: + env_ids: Environment indices to reset, or ``None`` for every environment. + """ + if self._shape_probe_pending: + self._shape_probe_pending = False + return + self._noise_model.reset(env_ids) + + def __call__( + self, + env: ManagerBasedRLEnv, + command_name: str, + action_name: str, + noise_model: NoiseModelCfg, + robot_cfg: SceneEntityCfg, + object_cfg: SceneEntityCfg, + ) -> torch.Tensor: + """Return the corrupted 42-dimensional actor observation.""" + del noise_model + object_asset: RigidObject = env.scene[object_cfg.name] + object_pos = object_asset.data.root_pos_w.torch - env.scene.env_origins + if self._goal_quat_wp is None: + command_term: ReorientCommand = env.command_manager.get_term(command_name) + self._goal_quat_wp = wp.from_torch(command_term.quat_command_w, dtype=wp.quatf) + wp.launch( + _goal_quat_error_kernel, + dim=self.num_envs, + inputs=[object_asset.data.root_quat_w.warp, self._goal_quat_wp, 0], + outputs=[self._quat_error_wp], + device=self._quat_error_wp.device, + ) + robot = env.scene[robot_cfg.name] + wp.launch( + fingertip_pos_kernel, + dim=(self.num_envs, self._robot_body_ids_wp.shape[0]), + inputs=[robot.data.body_pos_w.warp, self._env_origins_wp, self._robot_body_ids_wp], + outputs=[self._fingertip_buf_wp], + device=self._fingertip_buf_wp.device, + ) + # Direct actor-observation order: fingertips, object position, goal quat error, last action + observation = torch.cat( + (self._fingertip_buf, object_pos, self._quat_error, reorient_last_action(env, action_name)), + dim=-1, + ) + if self._shape_probe_pending: + return observation + return self._noise_model(observation) + + +# --------------------------------------------------------------------------- +# Shadow Hand camera observation terms. +# +# These terms wrap the CNN feature pipeline defined in the shadow-hand config +# package. The config layer imports the mdp layer, so the FeatureExtractor +# machinery is imported lazily at term construction time. +# --------------------------------------------------------------------------- + + +class ShadowHandCameraFeatures(ManagerTermBase): + """Run the Direct camera feature pipeline as one Manager observation term.""" + + def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedRLEnv): + super().__init__(cfg, env) + sensor_cfg: SceneEntityCfg = cfg.params["sensor_cfg"] + camera: Camera = env.scene.sensors[sensor_cfg.name] + # Runtime-only import: the mdp layer must not import the task-config layer + # at module load (config modules import mdp; see the layering note above). + from isaaclab_tasks.core.reorient.config.shadow_hand.feature_extractor import FeatureExtractor + + feature_extractor_cfg: FeatureExtractorCfg = env.cfg.feature_extractor + self._feature_extractor = FeatureExtractor( + feature_extractor_cfg, + env.device, + camera.cfg.data_types, + env.cfg.log_dir, + height=camera.cfg.height, + width=camera.cfg.width, + ) + # ObservationManager calls terms once to infer their shape. Do not train + # or save a CNN checkpoint during that initialization probe. + self._shape_probe_pending = True + self._keypoints_buf = torch.empty(env.num_envs, 8, 3, dtype=torch.float32, device=env.device) + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + """Finish the shape-probe phase on the first Manager reset. + + Args: + env_ids: Environment indices being reset. The feature extractor + has no per-environment state, so the indices are unused. + """ + del env_ids + if self._shape_probe_pending: + self._shape_probe_pending = False + + def __call__( + self, + env: ManagerBasedRLEnv, + feature_extractor_cfg: FeatureExtractorCfg, + sensor_cfg: SceneEntityCfg, + object_cfg: SceneEntityCfg, + ) -> torch.Tensor: + """Return the detached 27-dimensional cube-pose embedding. + + Args: + env: Environment containing the object and tiled camera. + feature_extractor_cfg: Feature-extractor configuration captured by + the observation term. The initialized extractor owns its copy. + sensor_cfg: Tiled-camera scene entity. + object_cfg: Reoriented-object scene entity. + + Returns: + Predicted object position and cube keypoints [m], shape + ``(num_envs, 27)``. + """ + del feature_extractor_cfg + if self._shape_probe_pending: + embeddings = torch.zeros(env.num_envs, 27, dtype=torch.float32, device=env.device) + env._shadow_hand_camera_embeddings = embeddings + return embeddings + + camera: Camera = env.scene.sensors[sensor_cfg.name] + object_asset: RigidObject = env.scene[object_cfg.name] + object_pos = object_asset.data.root_pos_w.torch - env.scene.env_origins + object_pose = torch.cat((object_pos, object_asset.data.root_quat_w.torch), dim=-1) + keypoints = compute_cube_keypoints(object_pose, out=self._keypoints_buf) + target = torch.cat((object_pos, keypoints.flatten(start_dim=1)), dim=-1) + camera_output = { + data_type: value if isinstance(value, torch.Tensor) else value.torch + for data_type, value in camera.data.output.items() + } + pose_loss, embeddings = self._feature_extractor.step(camera_output, target) + embeddings = embeddings.clone().detach() + env._shadow_hand_camera_embeddings = embeddings + if pose_loss is not None: + env.extras.setdefault("log", {})["pose_loss"] = pose_loss + return embeddings + + +def shadow_hand_camera_cached_features(env: ManagerBasedRLEnv) -> torch.Tensor: + """Return camera features computed by the preceding policy observation group. + + Args: + env: Environment whose policy group cached the current camera embedding. + + Returns: + Detached camera embeddings, shape ``(num_envs, 27)``. + """ + embeddings = getattr(env, "_shadow_hand_camera_embeddings", None) + if embeddings is None: + raise RuntimeError("Shadow Hand camera policy features must be computed before critic observations.") + return embeddings + + +class shadow_hand_goal_keypoints(ManagerTermBase): + """Flattened zero-origin cube keypoints [m] for the current goal orientation.""" + + def __init__(self, cfg: ObservationTermCfg, env: ManagerBasedRLEnv): + super().__init__(cfg, env) + self._out = torch.empty(env.num_envs, 24, dtype=torch.float32, device=env.device) + self._out_wp = wp.from_torch(self._out) + # resolved on first call: the command term does not exist yet during manager construction + self._goal_quat_wp: wp.array | None = None + + def __call__(self, env: ManagerBasedRLEnv, command_name: str) -> torch.Tensor: + """Return flattened zero-origin cube keypoints [m], shape ``(num_envs, 24)``.""" + if self._goal_quat_wp is None: + command_term = env.command_manager.get_term(command_name) + self._goal_quat_wp = wp.from_torch(command_term.quat_command_w, dtype=wp.quatf) + wp.launch( + cube_keypoints_from_quat_kernel, + dim=(self.num_envs, 8), + inputs=[self._goal_quat_wp, CUBE_HALF_SIZE], + outputs=[self._out_wp], + device=self._out_wp.device, + ) + return self._out diff --git a/source/isaaclab_tasks/test/benchmarking/configs.yaml b/source/isaaclab_tasks/test/benchmarking/configs.yaml index 62a652e995e3..ceee490150ac 100644 --- a/source/isaaclab_tasks/test/benchmarking/configs.yaml +++ b/source/isaaclab_tasks/test/benchmarking/configs.yaml @@ -268,6 +268,17 @@ full: episode_length: 400 upper_thresholds: duration: 40000 + Isaac-Reorient-Cube-Shadow-Camera: + max_iterations: 3000 + lower_thresholds: + # interim reward gate: catches pipeline breakage well below the truncated + # manager plateau (~174-290); tighten toward the Direct row's 1000 once a + # full-budget manager run calibrates it (success gates stay off until then + # — success is near-binary below convergence) + reward: 150 + episode_length: 400 + upper_thresholds: + duration: 40000 Isaac-Shadow-Handover-Direct: max_iterations: 3000 lower_thresholds: diff --git a/source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py b/source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py index fd2f7535a7da..8c6484dd67a5 100644 --- a/source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py +++ b/source/isaaclab_tasks/test/core/test_rendering_registered_tasks.py @@ -82,6 +82,12 @@ def _collect_camera_outputs(env: object) -> dict[str, dict[str, torch.Tensor]]: # require at least one pass while we tighten the validation tolerances for this scene. marks=pytest.mark.flaky(max_runs=3, min_passes=1), ), + pytest.param( + "Isaac-Reorient-Cube-Shadow-Camera", + None, + "shadow_hand", + marks=pytest.mark.flaky(max_runs=3, min_passes=1), + ), ] diff --git a/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-depth.png b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-depth.png new file mode 100644 index 000000000000..c229b583dfb7 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-depth.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a37f6bca30bb2d093eb68186c601551d52aafe8ed19c6c090de149b3210d81a5 +size 3665 diff --git a/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-rgb.png b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-rgb.png new file mode 100644 index 000000000000..472bbe6e9db0 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-rgb.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc17ca40a050eb357a607326ccb4fc553cb525abb3f3fa96d7b496f5dc51c93e +size 19878 diff --git a/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-rgba.png b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-rgba.png new file mode 100644 index 000000000000..885109b2f728 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-rgba.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5aa01e4ef3a1dbd299fbbc31d47c0db785c0db99252236376470d547fac4f509 +size 22056 diff --git a/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-semantic_segmentation.png b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-semantic_segmentation.png new file mode 100644 index 000000000000..4bad29d72ce6 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera-Direct/default_physics-default_renderer-semantic_segmentation.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75576f31118081f96b0cec2151ada7016794a933a035ea35665a752e8552b503 +size 1474 diff --git a/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-depth.png b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-depth.png new file mode 100644 index 000000000000..c229b583dfb7 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-depth.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a37f6bca30bb2d093eb68186c601551d52aafe8ed19c6c090de149b3210d81a5 +size 3665 diff --git a/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-rgb.png b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-rgb.png new file mode 100644 index 000000000000..eace991f49eb --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-rgb.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b60263834743f0508a437281d36bd2f46296789b28ad930627fb72e406ab8700 +size 19962 diff --git a/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-rgba.png b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-rgba.png new file mode 100644 index 000000000000..7d6fa735693d --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-rgba.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8da49db87ab2afe4f2a32c9354a72fd1b5a32aaf76484a21f787b01c0a47197a +size 22127 diff --git a/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-semantic_segmentation.png b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-semantic_segmentation.png new file mode 100644 index 000000000000..4bad29d72ce6 --- /dev/null +++ b/source/isaaclab_tasks/test/golden_images/registered_tasks/Isaac-Reorient-Cube-Shadow-Camera/default_physics-default_renderer-semantic_segmentation.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75576f31118081f96b0cec2151ada7016794a933a035ea35665a752e8552b503 +size 1474