Configures XR teleop camera placement per-task with an XrCfg env configuration (#282)

Currently, the camera placement for the stacking task is set to a
task-specific
view in each main script (record_demos.py, teleop_se3_agent.py). This is
not
scalable for when we need to add more tasks with different camera
placements,
or more main scripts.

Instead, this change adds an XrCfg with "xr_anchor_pos" and
"xr_anchor_rot" fields
which can specify the XR anchor prim per-task (e.g. in the stacking base
task).

Specifically: the pose of this prim will be placed at the origin of the
XR
device's local coordinate frame. This is achieved by setting the XR
anchor mode
to "custom anchor", which determines the coordinate frame transformation
between
the simulation and the XR device's local frame.

The default camera placement is set to the scene origin pose (in case a
custom
anchor is not set for a task).

- Bug fix (non-breaking change which fixes an issue)

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

CONTRIBUTORS.md

@@ -47,6 +47,7 @@ Guidelines for modifications:
 * Cheng-Rong Lai
 * Chenyu Yang
 * Clemens Schwarke
+* Connor Smith
 * CY (Chien-Ying) Chen
 * David Yang
 * Dorsa Rohani

apps/isaaclab.python.xr.openxr.kit

@@ -39,9 +39,9 @@ app.xr.enabled = true
 
 # xr settings
 xr.ui.enabled = false
-xrstage.profile.ar.anchorMode = "active camera"
 xr.depth.aov = "GBufferDepth"
 defaults.xr.profile.ar.renderQuality = "off"
+defaults.xr.profile.ar.anchorMode = "custom anchor"
 rtx.rendermode = "RaytracedLighting"
 
 # Register extension folder from this repo in kit

scripts/environments/teleoperation/teleop_se3_agent.py

@@ -38,8 +38,6 @@ import torch
 import omni.log
 
 from isaaclab.devices import Se3Gamepad, Se3HandTracking, Se3Keyboard, Se3SpaceMouse
-from isaaclab.envs import ViewerCfg
-from isaaclab.envs.ui import ViewportCameraController
 from isaaclab.managers import TerminationTermCfg as DoneTerm
 
 import isaaclab_tasks  # noqa: F401
@@ -97,10 +95,8 @@ def main():
     elif args_cli.teleop_device.lower() == "handtracking":
         from isaacsim.xr.openxr import OpenXRSpec
 
-        teleop_interface = Se3HandTracking(OpenXRSpec.XrHandEXT.XR_HAND_RIGHT_EXT, False, True)
+        teleop_interface = Se3HandTracking(env_cfg.xr, OpenXRSpec.XrHandEXT.XR_HAND_RIGHT_EXT, False, True)
         teleop_interface.add_callback("RESET", env.reset)
-        viewer = ViewerCfg(eye=(-0.25, -0.3, 0.5), lookat=(0.6, 0, 0), asset_name="viewer")
-        ViewportCameraController(env, viewer)
     else:
         raise ValueError(
             f"Invalid device interface '{args_cli.teleop_device}'. Supported: 'keyboard', 'spacemouse', 'gamepad',"

scripts/tools/record_demos.py

@@ -69,9 +69,7 @@ import torch
 import omni.log
 
 from isaaclab.devices import Se3HandTracking, Se3Keyboard, Se3SpaceMouse
-from isaaclab.envs import ViewerCfg
 from isaaclab.envs.mdp.recorders.recorders_cfg import ActionStateRecorderManagerCfg
-from isaaclab.envs.ui import ViewportCameraController
 
 import isaaclab_tasks  # noqa: F401
 from isaaclab_tasks.utils.parse_cfg import parse_env_cfg
@@ -180,10 +178,8 @@ def main():
     elif args_cli.teleop_device.lower() == "handtracking":
         from isaacsim.xr.openxr import OpenXRSpec
 
-        teleop_interface = Se3HandTracking(OpenXRSpec.XrHandEXT.XR_HAND_RIGHT_EXT, False, True)
+        teleop_interface = Se3HandTracking(env_cfg.xr, OpenXRSpec.XrHandEXT.XR_HAND_RIGHT_EXT, False, True)
         teleop_interface.add_callback("RESET", reset_recording_instance)
-        viewer = ViewerCfg(eye=(-0.25, -0.3, 0.5), lookat=(0.6, 0, 0), asset_name="viewer")
-        ViewportCameraController(env, viewer)
     else:
         raise ValueError(
             f"Invalid device interface '{args_cli.teleop_device}'. Supported: 'keyboard', 'spacemouse', 'handtracking'."

source/isaaclab/docs/CHANGELOG.rst

@@ -118,8 +118,8 @@ Changed
   * ``set_fixed_tendon_limit`` → ``set_fixed_tendon_position_limit``
 
 
-0.34.9 (2025-03-04)
-~~~~~~~~~~~~~~~~~~~
+0.34.10 (2025-03-04)
+~~~~~~~~~~~~~~~~~~~~
 
 Fixed
 ^^^^^
@@ -129,7 +129,7 @@ Fixed
   outputs are requested.
 
 
-0.34.8 (2025-03-04)
+0.34.9 (2025-03-04)
 ~~~~~~~~~~~~~~~~~~~
 
 Fixed
@@ -139,7 +139,7 @@ Fixed
   with other modalities such as RGBA and depth.
 
 
-0.34.7 (2025-03-04)
+0.34.8 (2025-03-04)
 ~~~~~~~~~~~~~~~~~~~
 
 Added
@@ -151,7 +151,7 @@ Added
   which didn't allow setting the joint position and velocity separately.
 
 
-0.34.6 (2025-03-02)
+0.34.7 (2025-03-02)
 ~~~~~~~~~~~~~~~~~~~
 
 Fixed
@@ -162,7 +162,7 @@ Fixed
   was always set to False, which led to incorrect contact sensor settings for the spawned assets.
 
 
-0.34.5 (2025-03-02)
+0.34.6 (2025-03-02)
 ~~~~~~~~~~~~~~~~~~~
 
 Changed
@@ -180,7 +180,7 @@ Removed
 * Removed the attribute ``disable_contact_processing`` from :class:`~isaaclab.sim.SimulationContact`.
 
 
-0.34.4 (2025-03-01)
+0.34.5 (2025-03-01)
 ~~~~~~~~~~~~~~~~~~~
 
 Added
@@ -209,7 +209,7 @@ Changed
   they should use the :attr:`isaaclab.actuators.ImplicitActuatorCfg.velocity_limit_sim` attribute instead.
 
 
-0.34.3 (2025-02-28)
+0.34.4 (2025-02-28)
 ~~~~~~~~~~~~~~~~~~~
 
 Added
@@ -220,6 +220,16 @@ Added
   Support for other Isaac Sim builds will become available in Isaac Sim 5.0.
 
 
+0.34.3 (2025-02-26)
+~~~~~~~~~~~~~~~~~~~
+
+Added
+^^^^^
+
+* Enablec specifying the placement of the simulation when viewed in an XR device. This is achieved by
+  adding an ``XrCfg`` environment configuration with ``anchor_pos`` and ``anchor_rot`` parameters.
+
+
 0.34.2 (2025-02-21)
 ~~~~~~~~~~~~~~~~~~~
 

source/isaaclab/isaaclab/devices/gamepad/se2_gamepad.py

@@ -10,6 +10,7 @@ import weakref
 from collections.abc import Callable
 
 import carb
+import carb.input
 import omni
 
 from ..device_base import DeviceBase

source/isaaclab/isaaclab/devices/openxr/__init__.py

@@ -6,3 +6,4 @@
 """Keyboard device for SE(2) and SE(3) control."""
 
 from .se3_handtracking import Se3HandTracking
+from .xr_cfg import XrCfg

source/isaaclab/isaaclab/devices/openxr/se3_handtracking.py

@@ -13,6 +13,7 @@ from typing import Final
 import carb
 
 from ..device_base import DeviceBase
+from .xr_cfg import XrCfg
 
 with contextlib.suppress(ModuleNotFoundError):
     from isaacsim.xr.openxr import OpenXR, OpenXRSpec
@@ -20,6 +21,8 @@ with contextlib.suppress(ModuleNotFoundError):
 
     from . import teleop_command
 
+import isaacsim.core.utils.prims as prim_utils
+
 from isaaclab.markers import VisualizationMarkers
 from isaaclab.markers.config import FRAME_MARKER_CFG
 
@@ -49,6 +52,7 @@ class Se3HandTracking(DeviceBase):
 
     def __init__(
         self,
+        xr_cfg: XrCfg | None,
         hand,
         abs=True,
         zero_out_xy_rotation=False,
@@ -62,6 +66,7 @@ class Se3HandTracking(DeviceBase):
         self._previous_rot = Rotation.identity()
         self._previous_grip_distance = 0.0
         self._previous_gripper_command = False
+        self._xr_cfg = xr_cfg or XrCfg()
         self._hand = hand
         self._abs = abs
         self._zero_out_xy_rotation = zero_out_xy_rotation
@@ -88,6 +93,13 @@ class Se3HandTracking(DeviceBase):
         self._frame_marker_cfg.markers["frame"].scale = (0.1, 0.1, 0.1)
         self._goal_marker = VisualizationMarkers(self._frame_marker_cfg.replace(prim_path="/Visuals/ee_goal"))
 
+        # Specify the placement of the simulation when viewed in an XR device using a prim.
+        prim_utils.create_prim(
+            "/XRAnchor", "Xform", position=self._xr_cfg.anchor_pos, orientation=self._xr_cfg.anchor_rot
+        )
+        carb.settings.get_settings().set_string("/persistent/xr/profile/ar/anchorMode", "custom anchor")
+        carb.settings.get_settings().set_string("/xrstage/profile/ar/customAnchor", "/XRAnchor")
+
     def __del__(self):
         return
 

source/isaaclab/isaaclab/devices/openxr/test_se3_handtracking.py

+# Copyright (c) 2025, The Isaac Lab Project Developers.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# Ignore private usage of variables warning.
+# pyright: reportPrivateUsage=none
+
+from __future__ import annotations
+
+from isaaclab.app import AppLauncher, run_tests
+
+# Can set this to False to see the GUI for debugging.
+HEADLESS = True
+
+# Launch omniverse app.
+app_launcher = AppLauncher(headless=HEADLESS, kit_args="--enable isaacsim.xr.openxr")
+simulation_app = app_launcher.app
+
+import numpy as np
+import unittest
+
+import carb
+import omni.usd
+from isaacsim.core.prims import XFormPrim
+from isaacsim.xr.openxr import OpenXRSpec
+
+from isaaclab.devices import Se3HandTracking
+from isaaclab.devices.openxr import XrCfg
+from isaaclab.envs import ManagerBasedEnv, ManagerBasedEnvCfg
+from isaaclab.scene import InteractiveSceneCfg
+from isaaclab.utils import configclass
+
+
+@configclass
+class EmptyManagerCfg:
+    """Empty manager."""
+
+    pass
+
+
+@configclass
+class EmptySceneCfg(InteractiveSceneCfg):
+    """Configuration for an empty scene."""
+
+    pass
+
+
+@configclass
+class EmptyEnvCfg(ManagerBasedEnvCfg):
+    """Configuration for the empty test environment."""
+
+    scene: EmptySceneCfg = EmptySceneCfg(num_envs=1, env_spacing=1.0)
+    actions: EmptyManagerCfg = EmptyManagerCfg()
+    observations: EmptyManagerCfg = EmptyManagerCfg()
+
+    def __post_init__(self):
+        """Post initialization."""
+        self.decimation = 5
+        self.episode_length_s = 30.0
+        self.sim.dt = 0.01  # 100Hz
+        self.sim.render_interval = 2
+
+
+class TestSe3HandTracking(unittest.TestCase):
+    """Test for Se3HandTracking"""
+
+    def test_xr_anchor(self):
+        env_cfg = EmptyEnvCfg()
+        env_cfg.xr = XrCfg(anchor_pos=(1, 2, 3), anchor_rot=(0, 1, 0, 0))
+
+        # Create a new stage.
+        omni.usd.get_context().new_stage()
+        # Create environment.
+        env = ManagerBasedEnv(cfg=env_cfg)
+
+        device = Se3HandTracking(env_cfg.xr, OpenXRSpec.XrHandEXT.XR_HAND_RIGHT_EXT)
+
+        # Check that the xr anchor prim is created with the correct pose.
+        xr_anchor_prim = XFormPrim("/XRAnchor")
+        self.assertTrue(xr_anchor_prim.is_valid())
+        position, orientation = xr_anchor_prim.get_world_poses()
+        np.testing.assert_almost_equal(position.tolist(), [[1, 2, 3]])
+        np.testing.assert_almost_equal(orientation.tolist(), [[0, 1, 0, 0]])
+
+        # Check that xr anchor mode and custom anchor are set correctly.
+        self.assertEqual(carb.settings.get_settings().get("/persistent/xr/profile/ar/anchorMode"), "custom anchor")
+        self.assertEqual(carb.settings.get_settings().get("/xrstage/profile/ar/customAnchor"), "/XRAnchor")
+
+        device.reset()
+        env.close()
+
+    def test_xr_anchor_default(self):
+        env_cfg = EmptyEnvCfg()
+
+        # Create a new stage.
+        omni.usd.get_context().new_stage()
+        # Create environment.
+        env = ManagerBasedEnv(cfg=env_cfg)
+
+        device = Se3HandTracking(None, OpenXRSpec.XrHandEXT.XR_HAND_RIGHT_EXT)
+
+        # Check that the xr anchor prim is created with the correct default pose.
+        xr_anchor_prim = XFormPrim("/XRAnchor")
+        self.assertTrue(xr_anchor_prim.is_valid())
+        position, orientation = xr_anchor_prim.get_world_poses()
+        np.testing.assert_almost_equal(position.tolist(), [[0, 0, 0]])
+        np.testing.assert_almost_equal(orientation.tolist(), [[1, 0, 0, 0]])
+
+        # Check that xr anchor mode and custom anchor are set correctly.
+        self.assertEqual(carb.settings.get_settings().get("/persistent/xr/profile/ar/anchorMode"), "custom anchor")
+        self.assertEqual(carb.settings.get_settings().get("/xrstage/profile/ar/customAnchor"), "/XRAnchor")
+
+        device.reset()
+        env.close()
+
+
+if __name__ == "__main__":
+    run_tests()

source/isaaclab/isaaclab/devices/openxr/xr_cfg.py

+# Copyright (c) 2025, The Isaac Lab Project Developers.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+# ignore private usage of variables warning
+# pyright: reportPrivateUsage=none
+
+from __future__ import annotations
+
+from isaaclab.utils import configclass
+
+
+@configclass
+class XrCfg:
+    """Configuration for viewing and interacting with the environment through an XR device."""
+
+    anchor_pos: tuple[float, float, float] = (0.0, 0.0, 0.0)
+    """Specifies the position (in m) of the simulation when viewed in an XR device.
+
+    Specifically: this position will appear at the origin of the XR device's local coordinate frame.
+    """
+
+    anchor_rot: tuple[float, float, float] = (1.0, 0.0, 0.0, 0.0)
+    """Specifies the rotation (as a quaternion) of the simulation when viewed in an XR device.
+
+    Specifically: this rotation will determine how the simulation is rotated with respect to the
+    origin of the XR device's local coordinate frame.
+
+    This quantity is only effective if :attr:`xr_anchor_pos` is set.
+    """

source/isaaclab/isaaclab/envs/direct_marl_env_cfg.py

@@ -5,6 +5,7 @@
 
 from dataclasses import MISSING
 
+from isaaclab.devices.openxr import XrCfg
 from isaaclab.scene import InteractiveSceneCfg
 from isaaclab.sim import SimulationCfg
 from isaaclab.utils import configclass
@@ -221,3 +222,6 @@ class DirectMARLEnvCfg:
 
     The contents of the list cannot be modified during the entire training process.
     """
+
+    xr: XrCfg | None = None
+    """Configuration for viewing and interacting with the environment through an XR device."""

source/isaaclab/isaaclab/envs/direct_rl_env_cfg.py

@@ -5,6 +5,7 @@
 
 from dataclasses import MISSING
 
+from isaaclab.devices.openxr import XrCfg
 from isaaclab.scene import InteractiveSceneCfg
 from isaaclab.sim import SimulationCfg
 from isaaclab.utils import configclass
@@ -225,3 +226,6 @@ class DirectRLEnvCfg:
 
     wait_for_textures: bool = True
     """True to wait for assets to be loaded completely, False otherwise. Defaults to True."""
+
+    xr: XrCfg | None = None
+    """Configuration for viewing and interacting with the environment through an XR device."""

source/isaaclab/isaaclab/envs/manager_based_env_cfg.py

@@ -12,6 +12,7 @@ configuring the environment instances, viewer settings, and simulation parameter
 from dataclasses import MISSING
 
 import isaaclab.envs.mdp as mdp
+from isaaclab.devices.openxr import XrCfg
 from isaaclab.managers import EventTermCfg as EventTerm
 from isaaclab.managers import RecorderManagerBaseCfg as DefaultEmptyRecorderManagerCfg
 from isaaclab.scene import InteractiveSceneCfg
@@ -117,3 +118,6 @@ class ManagerBasedEnvCfg:
 
     wait_for_textures: bool = True
     """True to wait for assets to be loaded completely, False otherwise. Defaults to True."""
+
+    xr: XrCfg | None = None
+    """Configuration for viewing and interacting with the environment through an XR device."""

source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/stack/stack_env_cfg.py

@@ -7,6 +7,7 @@ from dataclasses import MISSING
 
 import isaaclab.sim as sim_utils
 from isaaclab.assets import ArticulationCfg, AssetBaseCfg
+from isaaclab.devices.openxr import XrCfg
 from isaaclab.envs import ManagerBasedRLEnvCfg
 from isaaclab.managers import ObservationGroupCfg as ObsGroup
 from isaaclab.managers import ObservationTermCfg as ObsTerm
@@ -177,6 +178,11 @@ class StackEnvCfg(ManagerBasedRLEnvCfg):
     events = None
     curriculum = None
 
+    xr: XrCfg = XrCfg(
+        anchor_pos=(-0.1, -0.5, -1.05),
+        anchor_rot=(0.866, 0, 0, -0.5),
+    )
+
     def __post_init__(self):
         """Post initialization."""
         # general settings