Adds Headless XR support and Centralize XR-specific Settings (#278)

Adds headless XR support to Isaac Lab and refactors the AppLauncher:
* Fixed headless XR rendering with Kit XR's OpenXR extension
* Automatically enables AR mode when running headless
* Centralized XR-specific settings from code to Kit app configuration
files
* Added new `isaaclab.python.xr.openxr.headless.kit` configuration file
* Introduced `--xr` flag in AppLauncher for explicit XR mode control
(also supports `XR=1` environment variable)
* Device resolution for XR should default to CPU unless overridden
* In a separate commit, the AppLauncher configuration resolution is
broken down to resolve the flake8 C901 complexity violation

<!-- As you go through the list, delete the ones that are not
applicable. -->

- New feature (non-breaking change which adds functionality)

Please attach before and after screenshots of the change if applicable.

<!--
Example:

| Before | After |
| ------ | ----- |
| _gif/png before_ | _gif/png after_ |

To upload images to a PR -- simply drag and drop an image while in edit
mode and it should upload the image directly. You can then paste that
source into the above before/after sections.
-->

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

<!--
As you go through the checklist above, you can mark something as done by
putting an x character in it

For example,
- [x] I have done this task
- [ ] I have not done this task
-->

apps/isaaclab.python.xr.openxr.headless.kit

+##
+# Adapted from: apps/isaaclab.python.xr.openxr.kit
+##
+
+[package]
+title = "Isaac Lab Python OpenXR Headless"
+description = "An app for running Isaac Lab with OpenXR in headless mode"
+version = "2.0.0"
+
+# That makes it browsable in UI with "experience" filter
+keywords = ["experience", "app", "usd", "headless"]
+
+[settings]
+# Note: This path was adapted to be respective to the kit-exe file location
+app.versionFile = "${exe-path}/VERSION"
+app.folder = "${exe-path}/"
+app.name = "Isaac-Sim"
+app.version = "4.5.0"
+
+[dependencies]
+"isaaclab.python.xr.openxr" = {}
+
+[settings]
+xr.profile.ar.enabled = true
+
+# Register extension folder from this repo in kit
+[settings.app.exts]
+folders = [
+    "${exe-path}/exts",  # kit extensions
+    "${exe-path}/extscore",  # kit core extensions
+    "${exe-path}/../exts",  # isaac extensions
+    "${exe-path}/../extsDeprecated",  # deprecated isaac extensions
+    "${exe-path}/../extscache",  # isaac cache extensions
+    "${exe-path}/../extsPhysics",  # isaac physics extensions
+    "${exe-path}/../isaacsim/exts",  # isaac extensions for pip
+    "${exe-path}/../isaacsim/extsDeprecated",  # deprecated isaac extensions
+    "${exe-path}/../isaacsim/extscache",  # isaac cache extensions for pip
+    "${exe-path}/../isaacsim/extsPhysics",  # isaac physics extensions for pip
+    "${app}", # needed to find other app files
+    "${app}/../source", # needed to find extensions in Isaac Lab
+]

apps/isaaclab.python.xr.openxr.kit

@@ -24,6 +24,7 @@ app.asyncRenderingLowLatency = true
 
 # For XR, set this back to default "#define OMNI_MAX_DEVICE_GROUP_DEVICE_COUNT 16"
 renderer.multiGpu.maxGpuCount = 16
+renderer.gpuEnumeration.glInterop.enabled = true # Allow Kit XR OpenXR to render headless
 
 [dependencies]
 "isaaclab.python.rendering" = {}
@@ -34,11 +35,14 @@ renderer.multiGpu.maxGpuCount = 16
 "omni.kit.xr.profile.ar" = {}
 
 [settings]
+app.xr.enabled = true
+
 # xr settings
 xr.ui.enabled = false
 xrstage.profile.ar.anchorMode = "active camera"
 xr.depth.aov = "GBufferDepth"
 defaults.xr.profile.ar.renderQuality = "off"
+rtx.rendermode = "RaytracedLighting"
 
 # Register extension folder from this repo in kit
 [settings.app.exts]

scripts/environments/teleoperation/teleop_se3_agent.py

@@ -8,15 +8,11 @@
 """Launch Isaac Sim Simulator first."""
 
 import argparse
-import os
 
 from isaaclab.app import AppLauncher
 
 # add argparse arguments
 parser = argparse.ArgumentParser(description="Keyboard teleoperation for Isaac Lab environments.")
-parser.add_argument(
-    "--disable_fabric", action="store_true", default=False, help="Disable fabric and use USD I/O operations."
-)
 parser.add_argument("--num_envs", type=int, default=1, help="Number of environments to simulate.")
 parser.add_argument("--teleop_device", type=str, default="keyboard", help="Device for interacting with environment")
 parser.add_argument("--task", type=str, default=None, help="Name of the task.")
@@ -28,7 +24,7 @@ args_cli = parser.parse_args()
 
 app_launcher_args = vars(args_cli)
 if args_cli.teleop_device.lower() == "handtracking":
-    app_launcher_args["experience"] = f'{os.environ["ISAACLAB_PATH"]}/apps/isaaclab.python.xr.openxr.kit'
+    app_launcher_args["xr"] = True
 # launch omniverse app
 app_launcher = AppLauncher(app_launcher_args)
 simulation_app = app_launcher.app
@@ -69,9 +65,7 @@ def pre_process_actions(delta_pose: torch.Tensor, gripper_command: bool) -> torc
 def main():
     """Running keyboard teleoperation with Isaac Lab manipulation environment."""
     # parse configuration
-    env_cfg = parse_env_cfg(
-        args_cli.task, device=args_cli.device, num_envs=args_cli.num_envs, use_fabric=not args_cli.disable_fabric
-    )
+    env_cfg = parse_env_cfg(args_cli.task, device=args_cli.device, num_envs=args_cli.num_envs)
     # modify configuration
     env_cfg.terminations.time_out = None
     if "Lift" in args_cli.task:

scripts/tools/record_demos.py

@@ -51,11 +51,12 @@ AppLauncher.add_app_launcher_args(parser)
 # parse the arguments
 args_cli = parser.parse_args()
 
+app_launcher_args = vars(args_cli)
 if args_cli.teleop_device.lower() == "handtracking":
-    vars(args_cli)["experience"] = f'{os.environ["ISAACLAB_PATH"]}/apps/isaaclab.python.xr.openxr.kit'
+    app_launcher_args["xr"] = True
 
 # launch the simulator
-app_launcher = AppLauncher(args_cli)
+app_launcher = AppLauncher(app_launcher_args)
 simulation_app = app_launcher.app
 
 """Rest everything follows."""

source/isaaclab/isaaclab/app/app_launcher.py

@@ -248,10 +248,15 @@ class AppLauncher:
             default=AppLauncher._APPLAUNCHER_CFG_INFO["enable_cameras"][1],
             help="Enable camera sensors and relevant extension dependencies.",
         )
+        arg_group.add_argument(
+            "--xr",
+            action="store_true",
+            default=AppLauncher._APPLAUNCHER_CFG_INFO["xr"][1],
+            help="Enable XR mode for VR/AR applications.",
+        )
         arg_group.add_argument(
             "--device",
             type=str,
-            default=AppLauncher._APPLAUNCHER_CFG_INFO["device"][1],
             help='The device to run the simulation on. Can be "cpu", "cuda", "cuda:N", where N is the device ID',
         )
         # Add the deprecated cpu flag to raise an error if it is used
@@ -300,6 +305,7 @@ class AppLauncher:
         "headless": ([bool], False),
         "livestream": ([int], -1),
         "enable_cameras": ([bool], False),
+        "xr": ([bool], False),
         "device": ([str], "cuda:0"),
         "experience": ([str], ""),
     }
@@ -396,10 +402,31 @@ class AppLauncher:
         Args:
             launcher_args: A dictionary of all input arguments passed to the class object.
         """
-        # Handle all control logic resolution
+        # Handle core settings
+        livestream_arg, livestream_env = self._resolve_livestream_settings(launcher_args)
+        self._resolve_headless_settings(launcher_args, livestream_arg, livestream_env)
+        self._resolve_camera_settings(launcher_args)
+        self._resolve_xr_settings(launcher_args)
+        self._resolve_viewport_settings(launcher_args)
 
-        # --LIVESTREAM logic--
-        #
+        # Handle device and distributed settings
+        self._resolve_device_settings(launcher_args)
+
+        # Handle experience file settings
+        self._resolve_experience_file(launcher_args)
+
+        # Handle additional arguments
+        self._resolve_kit_args(launcher_args)
+
+        # Prepare final simulation app config
+        # Remove all values from input keyword args which are not meant for SimulationApp
+        # Assign all the passed settings to a dictionary for the simulation app
+        self._sim_app_config = {
+            key: launcher_args[key] for key in set(AppLauncher._SIM_APP_CFG_TYPES.keys()) & set(launcher_args.keys())
+        }
+
+    def _resolve_livestream_settings(self, launcher_args: dict) -> tuple[int, int]:
+        """Resolve livestream related settings."""
         livestream_env = int(os.environ.get("LIVESTREAM", 0))
         livestream_arg = launcher_args.pop("livestream", AppLauncher._APPLAUNCHER_CFG_INFO["livestream"][1])
         livestream_valid_vals = {0, 1, 2}
@@ -426,8 +453,38 @@ class AppLauncher:
         else:
             self._livestream = livestream_env
 
-        # --HEADLESS logic--
-        #
+        # Process livestream here before launching kit because some of the extensions only work when launched with the kit file
+        self._livestream_args = []
+        if self._livestream >= 1:
+            # Note: Only one livestream extension can be enabled at a time
+            if self._livestream == 1:
+                warnings.warn(
+                    "Native Livestream is deprecated. Please use WebRTC Livestream instead with --livestream 2."
+                )
+                self._livestream_args += [
+                    '--/app/livestream/proto="ws"',
+                    "--/app/livestream/allowResize=true",
+                    "--enable",
+                    "omni.kit.livestream.core-4.1.2",
+                    "--enable",
+                    "omni.kit.livestream.native-5.0.1",
+                    "--enable",
+                    "omni.kit.streamsdk.plugins-4.1.1",
+                ]
+            elif self._livestream == 2:
+                self._livestream_args += [
+                    "--/app/livestream/allowResize=false",
+                    "--enable",
+                    "omni.kit.livestream.webrtc",
+                ]
+            else:
+                raise ValueError(f"Invalid value for livestream: {self._livestream}. Expected: 1, 2 .")
+            sys.argv += self._livestream_args
+
+        return livestream_arg, livestream_env
+
+    def _resolve_headless_settings(self, launcher_args: dict, livestream_arg: int, livestream_env: int):
+        """Resolve headless related settings."""
         # Resolve headless execution of simulation app
         # HEADLESS is initially passed as an int instead of
         # the bool of headless_arg to avoid messy string processing,
@@ -463,8 +520,8 @@ class AppLauncher:
         # Headless needs to be passed to the SimulationApp so we keep it here
         launcher_args["headless"] = self._headless
 
-        # --enable_cameras logic--
-        #
+    def _resolve_camera_settings(self, launcher_args: dict):
+        """Resolve camera related settings."""
         enable_cameras_env = int(os.environ.get("ENABLE_CAMERAS", 0))
         enable_cameras_arg = launcher_args.pop("enable_cameras", AppLauncher._APPLAUNCHER_CFG_INFO["enable_cameras"][1])
         enable_cameras_valid_vals = {0, 1}
@@ -482,6 +539,21 @@ class AppLauncher:
         if self._enable_cameras and self._headless:
             self._offscreen_render = True
 
+    def _resolve_xr_settings(self, launcher_args: dict):
+        """Resolve XR related settings."""
+        xr_env = int(os.environ.get("XR", 0))
+        xr_arg = launcher_args.pop("xr", AppLauncher._APPLAUNCHER_CFG_INFO["xr"][1])
+        xr_valid_vals = {0, 1}
+        if xr_env not in xr_valid_vals:
+            raise ValueError(f"Invalid value for environment variable `XR`: {xr_env} .Expected: {xr_valid_vals} .")
+        # We allow xr kwarg to supersede XR envvar
+        if xr_arg is True:
+            self._xr = xr_arg
+        else:
+            self._xr = bool(xr_env)
+
+    def _resolve_viewport_settings(self, launcher_args: dict):
+        """Resolve viewport related settings."""
         # Check if we can disable the viewport to improve performance
         #   This should only happen if we are running headless and do not require livestreaming or video recording
         #   This is different from offscreen_render because this only affects the default viewport and not other renderproducts in the scene
@@ -497,14 +569,20 @@ class AppLauncher:
         # avoid creating new stage at startup by default for performance reasons
         launcher_args["create_new_stage"] = False
 
-        # --simulation GPU device logic --
+    def _resolve_device_settings(self, launcher_args: dict):
+        """Resolve simulation GPU device related settings."""
         self.device_id = 0
-        device = launcher_args.get("device", AppLauncher._APPLAUNCHER_CFG_INFO["device"][1])
+        device = launcher_args.get("device")
+        if device is None:
+            # If no device is specified, default to the GPU device if we are not running in XR
+            device = "cpu" if self._xr else AppLauncher._APPLAUNCHER_CFG_INFO["device"][1]
+
         if "cuda" not in device and "cpu" not in device:
             raise ValueError(
                 f"Invalid value for input keyword argument `device`: {device}."
                 " Expected: a string with the format 'cuda', 'cuda:<device_id>', or 'cpu'."
             )
+
         if "cuda:" in device:
             self.device_id = int(device.split(":")[-1])
 
@@ -534,6 +612,10 @@ class AppLauncher:
         launcher_args["physics_gpu"] = self.device_id
         launcher_args["active_gpu"] = self.device_id
 
+        print(f"[INFO][AppLauncher]: Using device: {device}")
+
+    def _resolve_experience_file(self, launcher_args: dict):
+        """Resolve experience file related settings."""
         # Check if input keywords contain an 'experience' file setting
         # Note: since experience is taken as a separate argument by Simulation App, we store it separately
         self._sim_experience_file = launcher_args.pop("experience", "")
@@ -550,6 +632,13 @@ class AppLauncher:
                     )
                 else:
                     self._sim_experience_file = os.path.join(isaaclab_app_exp_path, "isaaclab.python.rendering.kit")
+            elif self._xr:
+                if self._headless:
+                    self._sim_experience_file = os.path.join(
+                        isaaclab_app_exp_path, "isaaclab.python.xr.openxr.headless.kit"
+                    )
+                else:
+                    self._sim_experience_file = os.path.join(isaaclab_app_exp_path, "isaaclab.python.xr.openxr.kit")
             elif self._headless and not self._livestream:
                 self._sim_experience_file = os.path.join(isaaclab_app_exp_path, "isaaclab.python.headless.kit")
             else:
@@ -605,22 +694,18 @@ class AppLauncher:
             else:
                 raise ValueError(f"Invalid value for livestream: {self._livestream}. Expected: 1, 2 .")
             sys.argv += self._livestream_args
+        # Resolve the absolute path of the experience file
+        self._sim_experience_file = os.path.abspath(self._sim_experience_file)
+        print(f"[INFO][AppLauncher]: Loading experience file: {self._sim_experience_file}")
 
+    def _resolve_kit_args(self, launcher_args: dict):
+        """Resolve additional arguments passed to Kit."""
         # Resolve additional arguments passed to Kit
         self._kit_args = []
         if "kit_args" in launcher_args:
             self._kit_args = [arg for arg in launcher_args["kit_args"].split()]
             sys.argv += self._kit_args
 
-        # Resolve the absolute path of the experience file
-        self._sim_experience_file = os.path.abspath(self._sim_experience_file)
-        print(f"[INFO][AppLauncher]: Loading experience file: {self._sim_experience_file}")
-        # Remove all values from input keyword args which are not meant for SimulationApp
-        # Assign all the passed settings to a dictionary for the simulation app
-        self._sim_app_config = {
-            key: launcher_args[key] for key in set(AppLauncher._SIM_APP_CFG_TYPES.keys()) & set(launcher_args.keys())
-        }
-
     def _create_app(self):
         """Launch and create the SimulationApp based on the parsed simulation config."""
         # Initialize SimulationApp
@@ -659,7 +744,7 @@ class AppLauncher:
         """Check if rendering is required by the app."""
         # Indicates whether rendering is required by the app.
         # Extensions required for rendering bring startup and simulation costs, so we do not enable them if not required.
-        return not self._headless or self._livestream >= 1 or self._enable_cameras
+        return not self._headless or self._livestream >= 1 or self._enable_cameras or self._xr
 
     def _load_extensions(self):
         """Load correct extensions based on AppLauncher's resolved config member variables."""

source/isaaclab/isaaclab/devices/openxr/se3_handtracking.py

@@ -11,7 +11,6 @@ from scipy.spatial.transform import Rotation, Slerp
 from typing import Final
 
 import carb
-from omni.kit.viewport.utility import get_active_viewport
 
 from ..device_base import DeviceBase
 
@@ -83,11 +82,6 @@ class Se3HandTracking(DeviceBase):
         self._alpha_rot = 0.5
         self._smoothed_delta_pos = np.zeros(3)
         self._smoothed_delta_rot = np.zeros(3)
-        # Set the XR anchormode to active camera
-        carb.settings.get_settings().set_string("/xrstage/profile/ar/anchorMode", "active camera")
-        # Select RTX - RealTime for Renderer
-        viewport_api = get_active_viewport()
-        viewport_api.set_hd_engine("rtx", "RaytracedLighting")
         self._delta_pos_scale_factor = delta_pos_scale_factor
         self._delta_rot_scale_factor = delta_rot_scale_factor
         self._frame_marker_cfg = FRAME_MARKER_CFG.copy()

source/isaaclab/isaaclab/sim/simulation_context.py

@@ -154,6 +154,8 @@ class SimulationContext(_SimulationContext):
         self._local_gui = carb_settings_iface.get("/app/window/enabled")
         # read flag for whether livestreaming GUI is enabled
         self._livestream_gui = carb_settings_iface.get("/app/livestream/enabled")
+        # read flag for whether XR GUI is enabled
+        self._xr_gui = carb_settings_iface.get("/app/xr/enabled")
 
         # read flag for whether the Isaac Lab viewport capture pipeline will be used,
         # casting None to False if the flag doesn't exist
@@ -162,7 +164,7 @@ class SimulationContext(_SimulationContext):
         # read flag for whether the default viewport should be enabled
         self._render_viewport = bool(carb_settings_iface.get("/isaaclab/render/active_viewport"))
         # flag for whether any GUI will be rendered (local, livestreamed or viewport)
-        self._has_gui = self._local_gui or self._livestream_gui
+        self._has_gui = self._local_gui or self._livestream_gui or self._xr_gui
 
         # apply render settings from render config
         if self.cfg.render.enable_translucency is not None:
@@ -193,6 +195,8 @@ class SimulationContext(_SimulationContext):
                 import omni.replicator.core as rep
 
                 rep.settings.set_render_rtx_realtime(antialiasing=self.cfg.render.antialiasing_mode)
+                # WAR: The omni.replicator.core extension sets /rtx/renderMode=RayTracedLighting with incorrect casing.
+                carb_settings_iface.set("/rtx/rendermode", "RaytracedLighting")
             except Exception:
                 pass