Updates teleop_se3_agent.py and CloudXR docs with GR1 task (#327)

# Description

<!--
Thank you for your interest in sending a pull request. Please make sure
to check the contribution guidelines.

Link:
https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html
-->

This MR updated `teleop_se3_agent.py` for supporting
`Isaac-PickPlace-GR1T2-Abs-v0` task. In addition, we updated the CloudXR
docs.


## Type of change

<!-- As you go through the list, delete the ones that are not
applicable. -->

- New feature (non-breaking change which adds functionality)
- This change requires a documentation update


<!--
Example:

| Before | After |
| ------ | ----- |
| _gif/png before_ | _gif/png after_ |

To upload images to a PR -- simply drag and drop an image while in edit
mode and it should upload the image directly. You can then paste that
source into the above before/after sections.
-->

## Checklist

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

<!--
As you go through the checklist above, you can mark something as done by
putting an x character in it

For example,
- [x] I have done this task
- [ ] I have not done this task
-->

---------
Co-authored-by: Connor Smith <cosmith@nvidia.com>

docs/source/how-to/cloudxr_teleoperation.rst

@@ -147,11 +147,9 @@ There are two options to run the CloudXR Runtime Docker container:
       .. code:: bash
 
          ./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py \
-             --xr \
-             --task Isaac-Lift-Cube-Franka-IK-Abs-v0 \
-             --num_envs 1 \
-             --device cpu \
-             --teleop_device handtracking_abs
+             --task Isaac-PickPlace-GR1T2-Abs-v0 \
+             --teleop_device dualhandtracking_abs \
+             --enable_pinocchio
 
    #. You'll want to leave the container running for the next steps. But once you are finished, you can
       stop the containers with:
@@ -210,11 +208,9 @@ There are two options to run the CloudXR Runtime Docker container:
       .. code:: bash
 
          ./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py \
-             --xr \
-             --task Isaac-Lift-Cube-Franka-IK-Abs-v0 \
-             --num_envs 1 \
-             --device cpu \
-             --teleop_device handtracking_abs
+             --task Isaac-PickPlace-GR1T2-Abs-v0 \
+             --teleop_device dualhandtracking_abs \
+             --enable_pinocchio
 
 With Isaac Lab and the CloudXR Runtime running:
 
@@ -286,11 +282,9 @@ On your Isaac Lab workstation:
    .. code-block:: bash
 
       ./isaaclab.sh -p scripts/environments/teleoperation/teleop_se3_agent.py \
-          --xr \
-          --task Isaac-Lift-Cube-Franka-IK-Abs-v0 \
-          --num_envs 1 \
-          --device cpu \
-          --teleop_device handtracking_abs
+          --task Isaac-PickPlace-GR1T2-Abs-v0 \
+          --teleop_device dualhandtracking_abs \
+          --enable_pinocchio
 
    .. note::
       Recall that the script above should either be run within the Isaac Lab Docker container
@@ -361,6 +355,7 @@ Back on your Apple Vision Pro:
 #. When you are finished with the example, click **Disconnect** to disconnect from Isaac Lab.
 
 
+
 .. _develop-xr-isaac-lab:
 
 Develop for XR in Isaac Lab

scripts/environments/teleoperation/teleop_se3_agent.py

@@ -17,14 +17,27 @@ parser.add_argument("--num_envs", type=int, default=1, help="Number of environme
 parser.add_argument("--teleop_device", type=str, default="keyboard", help="Device for interacting with environment")
 parser.add_argument("--task", type=str, default=None, help="Name of the task.")
 parser.add_argument("--sensitivity", type=float, default=1.0, help="Sensitivity factor.")
+parser.add_argument(
+    "--enable_pinocchio",
+    action="store_true",
+    default=False,
+    help="Enable Pinocchio.",
+)
 # append AppLauncher cli args
 AppLauncher.add_app_launcher_args(parser)
 # parse the arguments
 args_cli = parser.parse_args()
 
 app_launcher_args = vars(args_cli)
+
+if args_cli.enable_pinocchio:
+    # Import pinocchio before AppLauncher to force the use of the version installed by IsaacLab and
+    # not the one installed by Isaac Sim pinocchio is required by the Pink IK controllers and the
+    # GR1T2 retargeter
+    import pinocchio  # noqa: F401
 if "handtracking" in args_cli.teleop_device.lower():
     app_launcher_args["xr"] = True
+
 # launch omniverse app
 app_launcher = AppLauncher(app_launcher_args)
 simulation_app = app_launcher.app
@@ -33,11 +46,19 @@ simulation_app = app_launcher.app
 
 
 import gymnasium as gym
+import numpy as np
 import torch
 
 import omni.log
 
+if "handtracking" in args_cli.teleop_device.lower():
+    from isaacsim.xr.openxr import OpenXRSpec
+
 from isaaclab.devices import OpenXRDevice, Se3Gamepad, Se3Keyboard, Se3SpaceMouse
+
+if args_cli.enable_pinocchio:
+    from isaaclab.devices.openxr.retargeters.humanoid.fourier.gr1t2_retargeter import GR1T2Retargeter
+    import isaaclab_tasks.manager_based.manipulation.pick_place  # noqa: F401
 from isaaclab.devices.openxr.retargeters.manipulator import GripperRetargeter, Se3AbsRetargeter, Se3RelRetargeter
 from isaaclab.managers import TerminationTermCfg as DoneTerm
 
@@ -46,17 +67,48 @@ from isaaclab_tasks.manager_based.manipulation.lift import mdp
 from isaaclab_tasks.utils import parse_env_cfg
 
 
-def pre_process_actions(delta_pose: torch.Tensor, gripper_command: bool) -> torch.Tensor:
-    """Pre-process actions for the environment."""
+def pre_process_actions(
+    teleop_data: tuple[np.ndarray, bool] | list[tuple[np.ndarray, np.ndarray, np.ndarray]], num_envs: int, device: str
+) -> torch.Tensor:
+    """Convert teleop data to the format expected by the environment action space.
+
+    Args:
+        teleop_data: Data from the teleoperation device.
+        num_envs: Number of environments.
+        device: Device to create tensors on.
+
+    Returns:
+        Processed actions as a tensor.
+    """
     # compute actions based on environment
     if "Reach" in args_cli.task:
+        delta_pose, gripper_command = teleop_data
+        # convert to torch
+        delta_pose = torch.tensor(delta_pose, dtype=torch.float, device=device).repeat(num_envs, 1)
         # note: reach is the only one that uses a different action space
         # compute actions
         return delta_pose
+    elif "PickPlace-GR1T2" in args_cli.task:
+        (left_wrist_pose, right_wrist_pose, hand_joints) = teleop_data[0]
+        # Reconstruct actions_arms tensor with converted positions and rotations
+        actions = torch.tensor(
+            np.concatenate([
+                left_wrist_pose,  # left ee pose
+                right_wrist_pose,  # right ee pose
+                hand_joints,  # hand joint angles
+            ]),
+            device=device,
+            dtype=torch.float32,
+        ).unsqueeze(0)
+        # Concatenate arm poses and hand joint angles
+        return actions
     else:
         # resolve gripper command
-        gripper_vel = torch.zeros(delta_pose.shape[0], 1, device=delta_pose.device)
-        gripper_vel[:] = -1.0 if gripper_command else 1.0
+        delta_pose, gripper_command = teleop_data
+        # convert to torch
+        delta_pose = torch.tensor(delta_pose, dtype=torch.float, device=device).repeat(num_envs, 1)
+        gripper_vel = torch.zeros((delta_pose.shape[0], 1), dtype=torch.float, device=device)
+        gripper_vel[:] = -1 if gripper_command else 1
         # compute actions
         return torch.concat([delta_pose, gripper_vel], dim=1)
 
@@ -65,6 +117,7 @@ def main():
     """Running keyboard teleoperation with Isaac Lab manipulation environment."""
     # parse configuration
     env_cfg = parse_env_cfg(args_cli.task, device=args_cli.device, num_envs=args_cli.num_envs)
+    env_cfg.env_name = args_cli.task
     # modify configuration
     env_cfg.terminations.time_out = None
     if "Lift" in args_cli.task:
@@ -140,6 +193,28 @@ def main():
         teleop_interface = Se3Gamepad(
             pos_sensitivity=0.1 * args_cli.sensitivity, rot_sensitivity=0.1 * args_cli.sensitivity
         )
+    elif "dualhandtracking_abs" in args_cli.teleop_device.lower() and "GR1T2" in args_cli.task:
+        # Create GR1T2 retargeter with desired configuration
+        gr1t2_retargeter = GR1T2Retargeter(
+            enable_visualization=True,
+            num_open_xr_hand_joints=2 * (int(OpenXRSpec.HandJointEXT.XR_HAND_JOINT_LITTLE_TIP_EXT) + 1),
+            device=env.unwrapped.device,
+            hand_joint_names=env.scene["robot"].data.joint_names[-22:],
+        )
+
+        # Create hand tracking device with retargeter
+        teleop_interface = OpenXRDevice(
+            env_cfg.xr,
+            hand=OpenXRDevice.Hand.BOTH,
+            retargeters=[gr1t2_retargeter],
+        )
+        teleop_interface.add_callback("RESET", reset_recording_instance)
+        teleop_interface.add_callback("START", start_teleoperation)
+        teleop_interface.add_callback("STOP", stop_teleoperation)
+
+        # Hand tracking needs explicit start gesture to activate
+        teleoperation_active = False
+
     elif "handtracking" in args_cli.teleop_device.lower():
         # Create EE retargeter with desired configuration
         if "_abs" in args_cli.teleop_device.lower():
@@ -180,15 +255,12 @@ def main():
         # run everything in inference mode
         with torch.inference_mode():
             # get device command
-            delta_pose, gripper_command = teleop_interface.advance()
+            teleop_data = teleop_interface.advance()
 
             # Only apply teleop commands when active
             if teleoperation_active:
-                delta_pose = delta_pose.astype("float32")
-                # convert to torch
-                delta_pose = torch.tensor(delta_pose, device=env.device).repeat(env.num_envs, 1)
-                # pre-process actions
-                actions = pre_process_actions(delta_pose, gripper_command)
+                # compute actions based on environment
+                actions = pre_process_actions(teleop_data, env.num_envs, env.device)
                 # apply actions
                 env.step(actions)
             else: