Adds support for manus and vive (#3357)

# Description
Support getting hand tracking data from manus gloves (joint poses
relative to wrists) and vive trackers (wrist poses, calibrated with AVP
wrist poses).

## Type of change
- New feature (non-breaking change which adds functionality)

## Checklist

- [ x] I have run the [`pre-commit` checks](https://pre-commit.com/)
with `./isaaclab.sh --format`
- [ x] I have made corresponding changes to the documentation
- [ x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [ x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

---------
Signed-off-by: Kelly Guo <kellyg@nvidia.com>
Co-authored-by: Kelly Guo <kellyg@nvidia.com>

CONTRIBUTORS.md

@@ -53,6 +53,7 @@ Guidelines for modifications:
 * Brian Bingham
 * Cameron Upright
 * Calvin Yu
+* Cathy Y. Li
 * Cheng-Rong Lai
 * Chenyu Yang
 * Connor Smith

docs/source/api/lab/isaaclab.devices.rst

@@ -16,6 +16,7 @@
     Se2SpaceMouse
     Se3SpaceMouse
     OpenXRDevice
+    ManusVive
     isaaclab.devices.openxr.retargeters.GripperRetargeter
     isaaclab.devices.openxr.retargeters.Se3AbsRetargeter
     isaaclab.devices.openxr.retargeters.Se3RelRetargeter
@@ -86,6 +87,14 @@ OpenXR
     :inherited-members:
     :show-inheritance:
 
+Manus + Vive
+------------
+
+.. autoclass:: ManusVive
+    :members:
+    :inherited-members:
+    :show-inheritance:
+
 Retargeters
 -----------
 

docs/source/how-to/cloudxr_teleoperation.rst

@@ -390,6 +390,38 @@ Back on your Apple Vision Pro:
    and build teleoperation and imitation learning workflows with Isaac Lab.
 
 
+.. _manus-vive-handtracking:
+
+Manus + Vive Hand Tracking
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Manus gloves and HTC Vive trackers can provide hand tracking when optical hand tracking from a headset is occluded.
+This setup expects Manus gloves with a Manus SDK license and Vive trackers attached to the gloves.
+Requires Isaac Sim >=5.1.
+
+Run the teleoperation example with Manus + Vive tracking:
+
+.. code-block:: bash
+
+   ./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py \
+       --task Isaac-PickPlace-GR1T2-Abs-v0 \
+       --teleop_device manusvive \
+       --xr \
+       --enable_pinocchio
+
+Begin the session with your palms facing up.
+This is necessary for calibrating Vive tracker poses using Apple Vision Pro wrist poses from a few initial frames,
+as the Vive trackers attached to the back of the hands occlude the optical hand tracking.
+
+.. note::
+
+   To avoid resource contention and crashes, ensure Manus and Vive devices are connected to different USB controllers/buses.
+   Use ``lsusb -t`` to identify different buses and connect devices accordingly.
+
+   Vive trackers are automatically calculated to map to the left and right wrist joints.
+   This auto-mapping calculation supports up to 2 Vive trackers;
+   if more than 2 Vive trackers are detected, it uses the first two trackers detected for calibration, which may not be correct.
+
 .. _develop-xr-isaac-lab:
 
 Develop for XR in Isaac Lab

scripts/environments/teleoperation/teleop_se3_agent.py

@@ -19,8 +19,7 @@ parser.add_argument(
     "--teleop_device",
     type=str,
     default="keyboard",
-    choices=["keyboard", "spacemouse", "gamepad", "handtracking"],
-    help="Device for interacting with environment",
+    help="Device for interacting with environment. Examples: keyboard, spacemouse, gamepad, handtracking, manusvive",
 )
 parser.add_argument("--task", type=str, default=None, help="Name of the task.")
 parser.add_argument("--sensitivity", type=float, default=1.0, help="Sensitivity factor.")

source/isaaclab/config/extension.toml

 [package]
 
 # Note: Semantic Versioning is used: https://semver.org/
-version = "0.45.12"
+version = "0.45.13"
 
 # Description
 title = "Isaac Lab framework for Robot Learning"

source/isaaclab/docs/CHANGELOG.rst

 Changelog
 ---------
 
+0.45.13 (2025-09-08)
+~~~~~~~~~~~~~~~~~~~~
+
+Added
+^^^^^
+
+* Added :class:`~isaaclab.devices.openxr.manus_vive.ManusVive` to support teleoperation with Manus gloves and Vive trackers.
+
+
 0.45.12 (2025-09-05)
 ~~~~~~~~~~~~~~~~~~~~
 

source/isaaclab/isaaclab/devices/__init__.py

@@ -22,7 +22,7 @@ the peripheral device.
 from .device_base import DeviceBase, DeviceCfg, DevicesCfg
 from .gamepad import Se2Gamepad, Se2GamepadCfg, Se3Gamepad, Se3GamepadCfg
 from .keyboard import Se2Keyboard, Se2KeyboardCfg, Se3Keyboard, Se3KeyboardCfg
-from .openxr import OpenXRDevice, OpenXRDeviceCfg
+from .openxr import ManusVive, ManusViveCfg, OpenXRDevice, OpenXRDeviceCfg
 from .retargeter_base import RetargeterBase, RetargeterCfg
 from .spacemouse import Se2SpaceMouse, Se2SpaceMouseCfg, Se3SpaceMouse, Se3SpaceMouseCfg
 from .teleop_device_factory import create_teleop_device

source/isaaclab/isaaclab/devices/openxr/__init__.py

@@ -5,5 +5,6 @@
 
 """Keyboard device for SE(2) and SE(3) control."""
 
+from .manus_vive import ManusVive, ManusViveCfg
 from .openxr_device import OpenXRDevice, OpenXRDeviceCfg
 from .xr_cfg import XrCfg, remove_camera_configs

source/isaaclab/isaaclab/devices/openxr/openxr_device.py

@@ -40,10 +40,12 @@ class OpenXRDevice(DeviceBase):
     """An OpenXR-powered device for teleoperation and interaction.
 
     This device tracks hand joints using OpenXR and makes them available as:
+
     1. A dictionary of tracking data (when used without retargeters)
     2. Retargeted commands for robot control (when retargeters are provided)
 
     Raw data format (_get_raw_data output):
+
     * A dictionary with keys matching TrackingTarget enum values (HAND_LEFT, HAND_RIGHT, HEAD)
     * Each hand tracking entry contains a dictionary of joint poses
     * Each joint pose is a 7D vector (x, y, z, qw, qx, qy, qz) in meters and quaternion units
@@ -52,6 +54,7 @@ class OpenXRDevice(DeviceBase):
 
     Teleop commands:
     The device responds to several teleop commands that can be subscribed to via add_callback():
+
     * "START": Resume hand tracking data flow
     * "STOP": Pause hand tracking data flow
     * "RESET": Reset the tracking and signal simulation reset

source/isaaclab/isaaclab/devices/teleop_device_factory.py

@@ -29,7 +29,7 @@ from isaaclab.devices.spacemouse import Se2SpaceMouse, Se2SpaceMouseCfg, Se3Spac
 
 with contextlib.suppress(ModuleNotFoundError):
     # May fail if xr is not in use
-    from isaaclab.devices.openxr import OpenXRDevice, OpenXRDeviceCfg
+    from isaaclab.devices.openxr import ManusVive, ManusViveCfg, OpenXRDevice, OpenXRDeviceCfg
 
 # Map device types to their constructor and expected config type
 DEVICE_MAP: dict[type[DeviceCfg], type[DeviceBase]] = {
@@ -40,6 +40,7 @@ DEVICE_MAP: dict[type[DeviceCfg], type[DeviceBase]] = {
     Se2GamepadCfg: Se2Gamepad,
     Se2SpaceMouseCfg: Se2SpaceMouse,
     OpenXRDeviceCfg: OpenXRDevice,
+    ManusViveCfg: ManusVive,
 }
 
 

source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/pick_place/pickplace_gr1t2_env_cfg.py

@@ -14,7 +14,7 @@ import isaaclab.sim as sim_utils
 from isaaclab.assets import ArticulationCfg, AssetBaseCfg, RigidObjectCfg
 from isaaclab.controllers.pink_ik import NullSpacePostureTask, PinkIKControllerCfg
 from isaaclab.devices.device_base import DevicesCfg
-from isaaclab.devices.openxr import OpenXRDeviceCfg, XrCfg
+from isaaclab.devices.openxr import ManusViveCfg, OpenXRDeviceCfg, XrCfg
 from isaaclab.devices.openxr.retargeters.humanoid.fourier.gr1t2_retargeter import GR1T2RetargeterCfg
 from isaaclab.envs import ManagerBasedRLEnvCfg
 from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg
@@ -437,5 +437,17 @@ class PickPlaceGR1T2EnvCfg(ManagerBasedRLEnvCfg):
                     sim_device=self.sim.device,
                     xr_cfg=self.xr,
                 ),
+                "manusvive": ManusViveCfg(
+                    retargeters=[
+                        GR1T2RetargeterCfg(
+                            enable_visualization=True,
+                            num_open_xr_hand_joints=2 * 26,
+                            sim_device=self.sim.device,
+                            hand_joint_names=self.actions.pink_ik_cfg.hand_joint_names,
+                        ),
+                    ],
+                    sim_device=self.sim.device,
+                    xr_cfg=self.xr,
+                ),
             }
         )