Fixes the camera frame delay (#124)

# Description

- Fixes the camera frame delay
- Adds `update_articulations_kinematic` in various places to make sure
link poses are always up to date

## Type of change

- Bug fix (non-breaking change which fixes an issue)

## Checklist

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [x] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

VERSION

-1.1.0
+1.2.0

source/apps/isaaclab.python.headless.kit

@@ -5,7 +5,7 @@
 [package]
 title = "Isaac Lab Python Headless"
 description = "An app for running Isaac Lab headlessly"
-version = "1.1.0"
+version = "1.2.0"
 
 # That makes it browsable in UI with "experience" filter
 keywords = ["experience", "app", "isaaclab", "python", "headless"]

source/apps/isaaclab.python.headless.rendering.kit

@@ -9,7 +9,7 @@
 [package]
 title = "Isaac Lab Python Headless Camera"
 description = "An app for running Isaac Lab headlessly with rendering enabled"
-version = "1.1.0"
+version = "1.2.0"
 
 # That makes it browsable in UI with "experience" filter
 keywords = ["experience", "app", "isaaclab", "python", "camera", "minimal"]
@@ -24,6 +24,10 @@ keywords = ["experience", "app", "isaaclab", "python", "camera", "minimal"]
 "omni.kit.material.library" = {}
 "omni.kit.viewport.rtx" = {}
 
+[settings.isaaclab]
+# This is used to check that this experience file is loaded when using cameras
+cameras_enabled = true
+
 [settings]
 # Note: This path was adapted to be respective to the kit-exe file location
 app.versionFile = "${exe-path}/VERSION"

source/apps/isaaclab.python.kit

@@ -5,7 +5,7 @@
 [package]
 title = "Isaac Lab Python"
 description = "An app for running Isaac Lab"
-version = "1.1.0"
+version = "1.2.0"
 
 # That makes it browsable in UI with "experience" filter
 keywords = ["experience", "app", "usd"]

source/apps/isaaclab.python.rendering.kit

@@ -9,7 +9,7 @@
 [package]
 title = "Isaac Lab Python Camera"
 description = "An app for running Isaac Lab with rendering enabled"
-version = "1.1.0"
+version = "1.2.0"
 
 # That makes it browsable in UI with "experience" filter
 keywords = ["experience", "app", "isaaclab", "python", "camera", "minimal"]
@@ -24,6 +24,10 @@ keywords = ["experience", "app", "isaaclab", "python", "camera", "minimal"]
 # Rendering
 "omni.kit.material.library" = {}
 
+[settings.isaaclab]
+# This is used to check that this experience file is loaded when using cameras
+cameras_enabled = true
+
 [settings]
 # Note: This path was adapted to be respective to the kit-exe file location
 app.versionFile = "${exe-path}/VERSION"
@@ -86,7 +90,6 @@ fabricUpdateVelocities = false
 fabricUpdateForceSensors = false
 fabricUpdateJointStates = false
 
-
 [settings.exts."omni.kit.registry.nucleus"]
 registries = [
     { name = "kit/default", url = "https://ovextensionsprod.blob.core.windows.net/exts/kit/prod/shared" },

source/extensions/omni.isaac.lab/config/extension.toml

 [package]
 
 # Note: Semantic Versioning is used: https://semver.org/
-version = "0.24.11"
+version = "0.24.12"
 
 # Description
 title = "Isaac Lab framework for Robot Learning"

source/extensions/omni.isaac.lab/docs/CHANGELOG.rst

@@ -2,6 +2,19 @@ Changelog
 ---------
 
 
+0.24.12 (2024-09-18)
+~~~~~~~~~~~~~~~~~~~~
+
+Fixed
+^^^^^
+
+* Fixed outdated fetching of articulation data by using the method ``update_articulations_kinematic`` in
+  :class:`omni.isaac.lab.assets.ArticulationData`. Before if an articulation was moved during a reset, the pose of the
+  links were outdated if fetched before the next physics step. Adding this method ensures that the pose of the links
+  is always up-to-date. Similarly ``update_articulations_kinematic`` was added before any render step to ensure that the
+  articulation displays correctly after a reset.
+
+
 0.24.11 (2024-09-11)
 ~~~~~~~~~~~~~~~~~~~~
 

source/extensions/omni.isaac.lab/omni/isaac/lab/assets/articulation/articulation.py

@@ -304,6 +304,8 @@ class Articulation(AssetBase):
         # convert root quaternion from wxyz to xyzw
         root_poses_xyzw = self._data.root_state_w[:, :7].clone()
         root_poses_xyzw[:, 3:] = math_utils.convert_quat(root_poses_xyzw[:, 3:], to="xyzw")
+        # Need to invalidate the buffer to trigger the update with the new root pose.
+        self._data._body_state_w.timestamp = -1.0
         # set into simulation
         self.root_physx_view.set_root_transforms(root_poses_xyzw, indices=physx_env_ids)
 
@@ -356,6 +358,8 @@ class Articulation(AssetBase):
         self._data.joint_vel[env_ids, joint_ids] = velocity
         self._data._previous_joint_vel[env_ids, joint_ids] = velocity
         self._data.joint_acc[env_ids, joint_ids] = 0.0
+        # Need to invalidate the buffer to trigger the update with the new root pose.
+        self._data._body_state_w.timestamp = -1.0
         # set into simulation
         self.root_physx_view.set_dof_positions(self._data.joint_pos, indices=physx_env_ids)
         self.root_physx_view.set_dof_velocities(self._data.joint_vel, indices=physx_env_ids)

source/extensions/omni.isaac.lab/omni/isaac/lab/assets/articulation/articulation_data.py

@@ -48,9 +48,9 @@ class ArticulationData:
         self._sim_timestamp = 0.0
 
         # Obtain global physics sim view
-        physics_sim_view = physx.create_simulation_view("torch")
-        physics_sim_view.set_subspace_roots("/")
-        gravity = physics_sim_view.get_gravity()
+        self._physics_sim_view = physx.create_simulation_view("torch")
+        self._physics_sim_view.set_subspace_roots("/")
+        gravity = self._physics_sim_view.get_gravity()
         # Convert to direction vector
         gravity_dir = torch.tensor((gravity[0], gravity[1], gravity[2]), device=self.device)
         gravity_dir = math_utils.normalize(gravity_dir.unsqueeze(0)).squeeze(0)
@@ -285,6 +285,7 @@ class ArticulationData:
         velocities are of the articulation links's center of mass frame.
         """
         if self._body_state_w.timestamp < self._sim_timestamp:
+            self._physics_sim_view.update_articulations_kinematic()
             # read data from simulation
             poses = self._root_physx_view.get_link_transforms().clone()
             poses[..., 3:7] = math_utils.convert_quat(poses[..., 3:7], to="wxyz")

source/extensions/omni.isaac.lab/omni/isaac/lab/envs/direct_rl_env.py

@@ -270,6 +270,10 @@ class DirectRLEnv(gym.Env):
         indices = torch.arange(self.num_envs, dtype=torch.int64, device=self.device)
         self._reset_idx(indices)
 
+        # if sensors are added to the scene, make sure we render to reflect changes in reset
+        if self.sim.has_rtx_sensors() and self.cfg.rerender_on_reset:
+            self.sim.render()
+
         # return observations
         return self._get_observations(), self.extras
 
@@ -339,6 +343,9 @@ class DirectRLEnv(gym.Env):
         reset_env_ids = self.reset_buf.nonzero(as_tuple=False).squeeze(-1)
         if len(reset_env_ids) > 0:
             self._reset_idx(reset_env_ids)
+            # if sensors are added to the scene, make sure we render to reflect changes in reset
+            if self.sim.has_rtx_sensors() and self.cfg.rerender_on_reset:
+                self.sim.render()
 
         # post-step: step interval event
         if self.cfg.events:
@@ -529,10 +536,6 @@ class DirectRLEnv(gym.Env):
         """
         self.scene.reset(env_ids)
 
-        # if sensors are added to the scene, make sure we render to reflect changes in reset
-        if len(env_ids) > 0 and self.sim.has_rtx_sensors() and self.cfg.rerender_on_reset:
-            self.sim.render()
-
         # apply events such as randomization for environments that need a reset
         if self.cfg.events:
             if "reset" in self.event_manager.available_modes:

source/extensions/omni.isaac.lab/omni/isaac/lab/envs/manager_based_env.py

@@ -251,6 +251,10 @@ class ManagerBasedEnv:
         indices = torch.arange(self.num_envs, dtype=torch.int64, device=self.device)
         self._reset_idx(indices)
 
+        # if sensors are added to the scene, make sure we render to reflect changes in reset
+        if self.sim.has_rtx_sensors() and self.cfg.rerender_on_reset:
+            self.sim.render()
+
         # return observations
         return self.observation_manager.compute(), self.extras
 
@@ -350,9 +354,7 @@ class ManagerBasedEnv:
         """
         # reset the internal buffers of the scene elements
         self.scene.reset(env_ids)
-        # if sensors are added to the scene, make sure we render to reflect changes in reset
-        if len(env_ids) > 0 and self.sim.has_rtx_sensors() and self.cfg.rerender_on_reset:
-            self.sim.render()
+
         # apply events such as randomization for environments that need a reset
         if "reset" in self.event_manager.available_modes:
             env_step_count = self._sim_step_counter // self.cfg.decimation

source/extensions/omni.isaac.lab/omni/isaac/lab/envs/manager_based_rl_env.py

@@ -194,6 +194,10 @@ class ManagerBasedRLEnv(ManagerBasedEnv, gym.Env):
         reset_env_ids = self.reset_buf.nonzero(as_tuple=False).squeeze(-1)
         if len(reset_env_ids) > 0:
             self._reset_idx(reset_env_ids)
+            # if sensors are added to the scene, make sure we render to reflect changes in reset
+            if self.sim.has_rtx_sensors() and self.cfg.rerender_on_reset:
+                self.sim.render()
+
         # -- update command
         self.command_manager.compute(dt=self.step_dt)
         # -- step interval events
@@ -319,9 +323,6 @@ class ManagerBasedRLEnv(ManagerBasedEnv, gym.Env):
         self.curriculum_manager.compute(env_ids=env_ids)
         # reset the internal buffers of the scene elements
         self.scene.reset(env_ids)
-        # if sensors are added to the scene, make sure we render to reflect changes in reset
-        if len(env_ids) > 0 and self.sim.has_rtx_sensors() and self.cfg.rerender_on_reset:
-            self.sim.render()
         # apply events such as randomizations for environments that need a reset
         if "reset" in self.event_manager.available_modes:
             env_step_count = self._sim_step_counter // self.cfg.decimation

source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/camera.py

@@ -378,12 +378,14 @@ class Camera(SensorBase):
             RuntimeError: If the number of camera prims in the view does not match the number of environments.
             RuntimeError: If replicator was not found.
         """
-        try:
-            import omni.replicator.core as rep
-        except ModuleNotFoundError:
+        carb_settings_iface = carb.settings.get_settings()
+        if not carb_settings_iface.get("/isaaclab/cameras_enabled"):
             raise RuntimeError(
-                "Replicator was not found for rendering. Please use --enable_cameras to enable rendering."
+                "A camera was spawned without the --enable_cameras flag. Please use --enable_cameras to enable"
+                " rendering."
             )
+
+        import omni.replicator.core as rep
         from omni.syntheticdata.scripts.SyntheticData import SyntheticData
 
         # Initialize parent class

source/extensions/omni.isaac.lab/omni/isaac/lab/sensors/camera/tiled_camera.py

@@ -12,6 +12,7 @@ from collections.abc import Sequence
 from tensordict import TensorDict
 from typing import TYPE_CHECKING, Any
 
+import carb
 import omni.usd
 import warp as wp
 from omni.isaac.core.prims import XFormPrimView
@@ -146,13 +147,15 @@ class TiledCamera(Camera):
             RuntimeError: If the number of camera prims in the view does not match the number of environments.
             RuntimeError: If replicator was not found.
         """
-        try:
-            import omni.replicator.core as rep
-        except ModuleNotFoundError:
+        carb_settings_iface = carb.settings.get_settings()
+        if not carb_settings_iface.get("/isaaclab/cameras_enabled"):
             raise RuntimeError(
-                "Replicator was not found for rendering. Please use --enable_cameras to enable rendering."
+                "A camera was spawned without the --enable_cameras flag. Please use --enable_cameras to enable"
+                " rendering."
             )
 
+        import omni.replicator.core as rep
+
         # Initialize parent class
         SensorBase._initialize_impl(self)
         # Create a view for the sensor

source/extensions/omni.isaac.lab/omni/isaac/lab/sim/simulation_context.py

@@ -7,6 +7,7 @@ import builtins
 import enum
 import numpy as np
 import sys
+import torch
 import traceback
 import weakref
 from collections.abc import Iterator
@@ -205,6 +206,12 @@ class SimulationContext(_SimulationContext):
         # note: we do it once here because it reads the VERSION file from disk and is not expected to change.
         self._isaacsim_version = get_version()
 
+        # create a tensor for gravity
+        # note: this line is needed to create a "tensor" in the device to avoid issues with torch 2.1 onwards.
+        #   the issue is with some heap memory corruption when torch tensor is created inside the asset class.
+        #   you can reproduce the issue by commenting out this line and running the test `test_articulation.py`.
+        self._gravity_tensor = torch.tensor(self.cfg.gravity, dtype=torch.float32, device=self.cfg.device)
+
         # add callback to deal the simulation app when simulation is stopped.
         # this is needed because physics views go invalid once we stop the simulation
         if not builtins.ISAAC_LAUNCHED_FROM_TERMINAL:
@@ -455,7 +462,10 @@ class SimulationContext(_SimulationContext):
         else:
             # manually flush the fabric data to update Hydra textures
             if self._fabric_iface is not None:
-                self._fabric_iface.update(0.0, 0.0)
+                if self.physics_sim_view is not None and self.is_playing():
+                    # Update the articulations' link's poses before rendering
+                    self.physics_sim_view.update_articulations_kinematic()
+                self._update_fabric(0.0, 0.0)
             # render the simulation
             # note: we don't call super().render() anymore because they do above operation inside
             #  and we don't want to do it twice. We may remove it once we drop support for Isaac Sim 2022.2.
@@ -569,6 +579,15 @@ class SimulationContext(_SimulationContext):
 
             # acquire fabric interface
             self._fabric_iface = get_physx_fabric_interface()
+            if hasattr(self._fabric_iface, "force_update"):
+                # The update method in the fabric interface only performs an update if a physics step has occurred.
+                # However, for rendering, we need to force an update since any element of the scene might have been
+                # modified in a reset (which occurs after the physics step) and we want the renderer to be aware of
+                # these changes.
+                self._update_fabric = self._fabric_iface.force_update
+            else:
+                # Needed for backward compatibility with older Isaac Sim versions
+                self._update_fabric = self._fabric_iface.update
 
     """
     Callbacks.

source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/direct/shadow_hand/shadow_hand_vision_env.py

@@ -43,7 +43,7 @@ class ShadowHandVisionEnvCfg(ShadowHandEnvCfg):
     feature_extractor = FeatureExtractorCfg()
 
     # env
-    num_observations = 188 + 27  # state observation + vision CNN embedding
+    num_observations = 164 + 27  # state observation + vision CNN embedding
     num_states = 187 + 27  # asymettric states + vision CNN embedding