Adds notification widgets at IK error status and Teleop task completion (#3356)

# Description

1. Add a notification widget when ik error happens
2. At the end of Teleop data collection, display a notification before
the application termination


<!--
Thank you for your interest in sending a pull request. Please make sure
to check the contribution guidelines.

Link:
https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html
-->

Please include a summary of the change and which issue is fixed. Please
also include relevant motivation and context.
List any dependencies that are required for this change.

Fixes # (issue)

<!-- As a practice, it is recommended to open an issue to have
discussions on the proposed pull request.
This makes it easier for the community to keep track of what is being
developed or added, and if a given feature
is demanded by more than one party. -->

## Type of change

<!-- As you go through the list, delete the ones that are not
applicable. -->

- Bug fix (non-breaking change which fixes an issue)
- New feature (non-breaking change which adds functionality)
- Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- This change requires a documentation update

## Screenshots

Please attach before and after screenshots of the change if applicable.

<!--
Example:

| Before | After |
| ------ | ----- |
| _gif/png before_ | _gif/png after_ |

To upload images to a PR -- simply drag and drop an image while in edit
mode and it should upload the image directly. You can then paste that
source into the above before/after sections.
-->

## Checklist

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [ ] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

<!--
As you go through the checklist above, you can mark something as done by
putting an x character in it

For example,
- [x] I have done this task
- [ ] I have not done this task
-->

---------
Signed-off-by: Kelly Guo <kellyg@nvidia.com>
Co-authored-by: Kelly Guo <kellyg@nvidia.com>

CONTRIBUTORS.md

@@ -88,6 +88,7 @@ Guidelines for modifications:
 * Kourosh Darvish
 * Kousheek Chakraborty
 * Lionel Gulich
+* Lotus Li
 * Louis Le Lay
 * Lorenz Wellhausen
 * Lukas Fröhlich

docs/source/how-to/cloudxr_teleoperation.rst

@@ -382,6 +382,18 @@ Back on your Apple Vision Pro:
       motion of the dots and the robot may be caused by the limits of the robot joints and/or robot
       controller.
 
+   .. note::
+      When the inverse kinematics solver fails to find a valid solution, an error message will appear
+      in the XR device display. To recover from this state, click the **Reset** button to return
+      the robot to its original pose and continue teleoperation.
+
+      .. figure:: ../_static/setup/cloudxr_avp_ik_error.jpg
+         :align: center
+         :figwidth: 80%
+         :alt: IK Error Message Display in XR Device
+
+
+
 #. When you are finished with the example, click **Disconnect** to disconnect from Isaac Lab.
 
 .. admonition:: Learn More about Teleoperation and Imitation Learning in Isaac Lab

scripts/tools/record_demos.py

@@ -469,16 +469,24 @@ def run_simulation_loop(
                 label_text = f"Recorded {current_recorded_demo_count} successful demonstrations."
                 print(label_text)
 
+            # Check if we've reached the desired number of demos
+            if args_cli.num_demos > 0 and env.recorder_manager.exported_successful_episode_count >= args_cli.num_demos:
+                label_text = f"All {current_recorded_demo_count} demonstrations recorded.\nExiting the app."
+                instruction_display.show_demo(label_text)
+                print(label_text)
+                target_time = time.time() + 0.8
+                while time.time() < target_time:
+                    if rate_limiter:
+                        rate_limiter.sleep(env)
+                    else:
+                        env.sim.render()
+                break
+
             # Handle reset if requested
             if should_reset_recording_instance:
                 success_step_count = handle_reset(env, success_step_count, instruction_display, label_text)
                 should_reset_recording_instance = False
 
-            # Check if we've reached the desired number of demos
-            if args_cli.num_demos > 0 and env.recorder_manager.exported_successful_episode_count >= args_cli.num_demos:
-                print(f"All {args_cli.num_demos} demonstrations recorded. Exiting the app.")
-                break
-
             # Check if simulation is stopped
             if env.sim.is_stopped():
                 break
@@ -506,6 +514,10 @@ def main() -> None:
     # if handtracking is selected, rate limiting is achieved via OpenXR
     if args_cli.xr:
         rate_limiter = None
+        from isaaclab.ui.xr_widgets import TeleopVisualizationManager, XRVisualization
+
+        # Assign the teleop visualization manager to the visualization system
+        XRVisualization.assign_manager(TeleopVisualizationManager)
     else:
         rate_limiter = RateLimiter(args_cli.step_hz)
 

source/isaaclab/config/extension.toml

 [package]
 
 # Note: Semantic Versioning is used: https://semver.org/
-version = "0.45.13"
+version = "0.45.14"
 
 # Description
 title = "Isaac Lab framework for Robot Learning"

source/isaaclab/docs/CHANGELOG.rst

 Changelog
 ---------
 
+0.45.14 (2025-09-08)
+~~~~~~~~~~~~~~~~~~~~
+
+Added
+^^^^^
+
+* * Added :class:`~isaaclab.ui.xr_widgets.TeleopVisualizationManager` and :class:`~isaaclab.ui.xr_widgets.XRVisualization`
+  classes to provide real-time visualization of teleoperation and inverse kinematics status in XR environments.
+
 0.45.13 (2025-09-08)
 ~~~~~~~~~~~~~~~~~~~~
 

source/isaaclab/isaaclab/controllers/pink_ik/pink_ik.py

@@ -173,6 +173,11 @@ class PinkIKController:
                     "Warning: IK quadratic solver could not find a solution! Did not update the target joint"
                     f" positions.\nError: {e}"
                 )
+
+            if self.cfg.xr_enabled:
+                from isaaclab.ui.xr_widgets import XRVisualization
+
+                XRVisualization.push_event("ik_error", {"error": e})
             return torch.tensor(curr_joint_pos, device=self.device, dtype=torch.float32)
 
         # Discard the first 6 values (for root and universal joints)

source/isaaclab/isaaclab/controllers/pink_ik/pink_ik_cfg.py

@@ -62,3 +62,6 @@ class PinkIKControllerCfg:
     """If True, the Pink IK solver will fail and raise an error if any joint limit is violated during optimization. PinkIKController
     will handle the error by setting the last joint positions. If False, the solver will ignore joint limit violations and return the
     closest solution found."""
+
+    xr_enabled: bool = False
+    """If True, the Pink IK controller will send information to the XRVisualization."""

source/isaaclab/isaaclab/ui/xr_widgets/__init__.py

@@ -2,4 +2,6 @@
 # All rights reserved.
 #
 # SPDX-License-Identifier: BSD-3-Clause
-from .instruction_widget import SimpleTextWidget, show_instruction
+from .instruction_widget import hide_instruction, show_instruction, update_instruction
+from .scene_visualization import DataCollector, TriggerType, VisualizationManager, XRVisualization
+from .teleop_visualization_manager import TeleopVisualizationManager

source/isaaclab/isaaclab/ui/xr_widgets/teleop_visualization_manager.py

+# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from typing import Any
+
+from isaaclab.ui.xr_widgets import DataCollector, TriggerType, VisualizationManager
+from isaaclab.ui.xr_widgets.instruction_widget import hide_instruction
+
+
+class TeleopVisualizationManager(VisualizationManager):
+    """Specialized visualization manager for teleoperation scenarios.
+    For sample and debug use.
+
+    Provides teleoperation-specific visualization features including:
+    - IK error handling and display
+    """
+
+    def __init__(self, data_collector: DataCollector):
+        """Initialize the teleop visualization manager and register callbacks.
+
+        Args:
+            data_collector: DataCollector instance to read data for visualization use.
+        """
+        super().__init__(data_collector)
+
+        # Handle error event
+        self._error_text_color = 0xFF0000FF
+        self.ik_error_widget_id = "/ik_solver_failed"
+
+        self.register_callback(TriggerType.TRIGGER_ON_EVENT, {"event_name": "ik_error"}, self._handle_ik_error)
+
+    def _handle_ik_error(self, mgr: VisualizationManager, data_collector: DataCollector, params: Any = None) -> None:
+        """Handle IK error events by displaying an error message widget.
+
+        Args:
+            data_collector: DataCollector instance (unused in this handler)
+        """
+        # Todo: move display_widget to instruction_widget.py
+        if not hasattr(mgr, "_ik_error_widget_timer"):
+            self.display_widget(
+                "IK Error Detected",
+                mgr.ik_error_widget_id,
+                VisualizationManager.message_widget_preset()
+                | {"text_color": self._error_text_color, "display_duration": None},
+            )
+            mgr._ik_error_widget_timer = mgr.register_callback(
+                TriggerType.TRIGGER_ON_PERIOD, {"period": 3.0, "initial_countdown": 3.0}, self._hide_ik_error_widget
+            )
+            if mgr._ik_error_widget_timer is None:
+                mgr.cancel_rule(TriggerType.TRIGGER_ON_PERIOD, mgr._ik_error_widget_timer)
+                mgr.cancel_rule(TriggerType.TRIGGER_ON_EVENT, "ik_solver_failed")
+                raise RuntimeWarning("Failed to register IK error widget timer")
+        else:
+            mgr._ik_error_widget_timer.countdown = 3.0
+
+    def _hide_ik_error_widget(self, mgr: VisualizationManager, data_collector: DataCollector) -> None:
+        """Hide the IK error widget.
+
+        Args:
+            data_collector: DataCollector instance (unused in this handler)
+        """
+
+        hide_instruction(mgr.ik_error_widget_id)
+        mgr.cancel_rule(TriggerType.TRIGGER_ON_PERIOD, mgr._ik_error_widget_timer)
+        delattr(mgr, "_ik_error_widget_timer")

source/isaaclab/test/visualization/check_scene_xr_visualization.py

+# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""
+This script checks if the XR visualization widgets are visible from the camera.
+
+.. code-block:: bash
+
+    # Usage
+    ./isaaclab.sh -p source/isaaclab/test/visualization/check_scene_visualization.py
+
+"""
+
+"""Launch Isaac Sim Simulator first."""
+
+import argparse
+
+from isaaclab.app import AppLauncher
+
+# add argparse arguments
+parser = argparse.ArgumentParser(description="Check XR visualization widgets in Isaac Lab.")
+parser.add_argument("--num_envs", type=int, default=2, help="Number of environments to spawn.")
+# append AppLauncher cli args
+AppLauncher.add_app_launcher_args(parser)
+# parse the arguments
+args_cli = parser.parse_args()
+
+# launch omniverse app with XR support
+args_cli.xr = True
+app_launcher = AppLauncher(args_cli)
+simulation_app = app_launcher.app
+
+"""Rest everything follows."""
+
+import time
+from typing import Any
+
+from pxr import Gf
+
+import isaaclab.sim as sim_utils
+from isaaclab.assets import AssetBaseCfg
+from isaaclab.scene import InteractiveScene, InteractiveSceneCfg
+from isaaclab.ui.xr_widgets import DataCollector, TriggerType, VisualizationManager, XRVisualization, update_instruction
+from isaaclab.utils import configclass
+
+##
+# Pre-defined configs
+##
+
+
+@configclass
+class SimpleSceneCfg(InteractiveSceneCfg):
+    """Design the scene with sensors on the robot."""
+
+    # ground plane
+    ground = AssetBaseCfg(prim_path="/World/defaultGroundPlane", spawn=sim_utils.GroundPlaneCfg())
+
+    # lights
+    dome_light = AssetBaseCfg(
+        prim_path="/World/Light", spawn=sim_utils.DomeLightCfg(intensity=3000.0, color=(0.75, 0.75, 0.75))
+    )
+
+
+def get_camera_position():
+    """Get the current camera position from the USD stage.
+
+    Returns:
+        tuple: (x, y, z) camera position or None if not available
+    """
+    try:
+        import isaacsim.core.utils.stage as stage_utils
+        from pxr import UsdGeom
+
+        stage = stage_utils.get_current_stage()
+        if stage is not None:
+            # Get the viewport camera prim
+            camera_prim_path = "/OmniverseKit_Persp"
+            camera_prim = stage.GetPrimAtPath(camera_prim_path)
+
+            if camera_prim and camera_prim.IsValid():
+                # Get the camera's world transform
+                camera_xform = UsdGeom.Xformable(camera_prim)
+                world_transform = camera_xform.ComputeLocalToWorldTransform(0)  # 0 = current time
+
+                # Extract position from the transform matrix
+                camera_pos = world_transform.ExtractTranslation()
+                return (camera_pos[0], camera_pos[1], camera_pos[2])
+        return None
+    except Exception as e:
+        print(f"[ERROR]: Failed to get camera position: {e}")
+        return None
+
+
+def _sample_handle_ik_error(mgr: VisualizationManager, data_collector: DataCollector, params: Any = None) -> None:
+    error_text_color = getattr(mgr, "_error_text_color", 0xFF0000FF)
+    mgr.display_widget(
+        "IK Error Detected",
+        "/ik_error",
+        VisualizationManager.message_widget_preset()
+        | {
+            "text_color": error_text_color,
+            "prim_path_source": "/World/defaultGroundPlane/GroundPlane",
+            "translation": Gf.Vec3f(0, 0, 1),
+        },
+    )
+
+
+def _sample_update_error_text_color(mgr: VisualizationManager, data_collector: DataCollector) -> None:
+    current_color = getattr(mgr, "_error_text_color", 0xFF0000FF)
+    new_color = current_color + 0x100
+    if new_color >= 0xFFFFFFFF:
+        new_color = 0xFF0000FF
+    mgr.set_attr("_error_text_color", new_color)
+
+
+def _sample_update_left_panel(mgr: VisualizationManager, data_collector: DataCollector) -> None:
+    left_panel_id = getattr(mgr, "left_panel_id", None)
+
+    if left_panel_id is None:
+        return
+
+    left_panel_created = getattr(mgr, "_left_panel_created", False)
+    if left_panel_created is False:
+        # create a new left panel
+        mgr.display_widget(
+            "Left Panel",
+            left_panel_id,
+            VisualizationManager.panel_widget_preset()
+            | {
+                "text_color": 0xFFFFFFFF,
+                "prim_path_source": "/World/defaultGroundPlane/GroundPlane",
+                "translation": Gf.Vec3f(0, -3, 1),
+            },
+        )
+        mgr.set_attr("_left_panel_created", True)
+
+    updated_times = getattr(mgr, "_left_panel_updated_times", 0)
+    # Create a simple panel content since make_panel_content doesn't exist
+    content = f"Left Panel\nUpdated #{updated_times} times"
+    update_instruction(left_panel_id, content)
+    mgr.set_attr("_left_panel_updated_times", updated_times + 1)
+
+
+def _sample_update_right_panel(mgr: VisualizationManager, data_collector: DataCollector) -> None:
+    right_panel_id = getattr(mgr, "right_panel_id", None)
+
+    if right_panel_id is None:
+        return
+
+    updated_times = getattr(mgr, "_right_panel_updated_times", 0)
+    # Create a simple panel content since make_panel_content doesn't exist
+    right_panel_data = data_collector.get_data("right_panel_data")
+    if right_panel_data is not None:
+        assert isinstance(right_panel_data, (tuple, list)), "Right panel data must be a tuple or list"
+        # Format each element to 3 decimal places
+        formatted_data = tuple(f"{x:.3f}" for x in right_panel_data)
+        content = f"Right Panel\nUpdated #{updated_times} times\nData: {formatted_data}"
+    else:
+        content = f"Right Panel\nUpdated #{updated_times} times\nData: None"
+
+    right_panel_created = getattr(mgr, "_right_panel_created", False)
+    if right_panel_created is False:
+        # create a new left panel
+        mgr.display_widget(
+            content,
+            right_panel_id,
+            VisualizationManager.panel_widget_preset()
+            | {
+                "text_color": 0xFFFFFFFF,
+                "prim_path_source": "/World/defaultGroundPlane/GroundPlane",
+                "translation": Gf.Vec3f(0, 3, 1),
+            },
+        )
+        mgr.set_attr("_right_panel_created", True)
+
+    update_instruction(right_panel_id, content)
+    mgr.set_attr("_right_panel_updated_times", updated_times + 1)
+
+
+def apply_sample_visualization():
+    # Error Message
+    XRVisualization.register_callback(TriggerType.TRIGGER_ON_EVENT, {"event_name": "ik_error"}, _sample_handle_ik_error)
+
+    # Display a panel on the left to display DataCollector data
+    # Refresh periodically
+    XRVisualization.set_attrs({
+        "left_panel_id": "/left_panel",
+        "left_panel_translation": Gf.Vec3f(-2, 2.6, 2),
+        "left_panel_updated_times": 0,
+        "right_panel_updated_times": 0,
+    })
+    XRVisualization.register_callback(TriggerType.TRIGGER_ON_PERIOD, {"period": 1.0}, _sample_update_left_panel)
+
+    # Display a panel on the right to display DataCollector data
+    # Refresh when camera position changes
+    XRVisualization.set_attrs({
+        "right_panel_id": "/right_panel",
+        "right_panel_translation": Gf.Vec3f(1.5, 2, 2),
+    })
+    XRVisualization.register_callback(
+        TriggerType.TRIGGER_ON_CHANGE, {"variable_name": "right_panel_data"}, _sample_update_right_panel
+    )
+
+    # Change error text color every second
+    XRVisualization.set_attrs({
+        "error_text_color": 0xFF0000FF,
+    })
+    XRVisualization.register_callback(TriggerType.TRIGGER_ON_UPDATE, {}, _sample_update_error_text_color)
+
+
+def run_simulator(
+    sim: sim_utils.SimulationContext,
+    scene: InteractiveScene,
+):
+    """Run the simulator."""
+
+    # Define simulation stepping
+    sim_dt = sim.get_physics_dt()
+
+    apply_sample_visualization()
+
+    # Simulate
+    while simulation_app.is_running():
+        if int(time.time()) % 10 < 1:
+            XRVisualization.push_event("ik_error")
+
+        XRVisualization.push_data({"right_panel_data": get_camera_position()})
+
+        sim.step()
+        scene.update(sim_dt)
+
+
+def main():
+    """Main function."""
+
+    # Initialize the simulation context
+    sim_cfg = sim_utils.SimulationCfg(dt=0.005)
+    sim = sim_utils.SimulationContext(sim_cfg)
+    # Set main camera
+    sim.set_camera_view(eye=(8, 0, 4), target=(0.0, 0.0, 0.0))
+    # design scene
+    scene = InteractiveScene(SimpleSceneCfg(num_envs=args_cli.num_envs, env_spacing=2.0))
+    # Play the simulator
+    sim.reset()
+    # Now we are ready!
+    print("[INFO]: Setup complete...")
+    # Run the simulator
+    run_simulator(sim, scene)
+
+
+if __name__ == "__main__":
+    # run the main function
+    main()
+    # close sim app
+    simulation_app.close()

source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/pick_place/exhaustpipe_gr1t2_pink_ik_env_cfg.py

@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+import carb
 from pink.tasks import DampingTask, FrameTask
 
 import isaaclab.controllers.utils as ControllerUtils
@@ -171,6 +172,7 @@ class ExhaustPipeGR1T2PinkIKEnvCfg(ExhaustPipeGR1T2BaseEnvCfg):
                     #     orientation_cost=0.05,  # [cost] / [rad]
                     # ),
                 ],
+                xr_enabled=bool(carb.settings.get_settings().get("/app/xr/enabled")),
             ),
         )
         # Convert USD to URDF and change revolute joints to fixed

source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/pick_place/nutpour_gr1t2_pink_ik_env_cfg.py

@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+import carb
 from pink.tasks import DampingTask, FrameTask
 
 import isaaclab.controllers.utils as ControllerUtils
@@ -169,6 +170,7 @@ class NutPourGR1T2PinkIKEnvCfg(NutPourGR1T2BaseEnvCfg):
                     #     orientation_cost=0.05,  # [cost] / [rad]
                     # ),
                 ],
+                xr_enabled=bool(carb.settings.get_settings().get("/app/xr/enabled")),
             ),
         )
         # Convert USD to URDF and change revolute joints to fixed

source/isaaclab_tasks/isaaclab_tasks/manager_based/manipulation/pick_place/pickplace_gr1t2_env_cfg.py

@@ -6,6 +6,7 @@
 import tempfile
 import torch
 
+import carb
 from pink.tasks import DampingTask, FrameTask
 
 import isaaclab.controllers.utils as ControllerUtils
@@ -255,6 +256,7 @@ class ActionsCfg:
                 ),
             ],
             fixed_input_tasks=[],
+            xr_enabled=bool(carb.settings.get_settings().get("/app/xr/enabled")),
         ),
     )