Fixes rendering of RTX-related sensors in `RLTaskEnv` (#501)

# Description

Previously, rendering of RTX-related sensors was not happening within
the step call of the environment. Instead, it was expected that users
call `env.render()` instead. However, this was unintuitive when sensors
are involved.

Fixes https://github.com/NVIDIA-Omniverse/orbit/discussions/291

## Type of change

- Bug fix (non-breaking change which fixes an issue)

## Checklist

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./orbit.sh --format`
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [ ] I have run all the tests with `./orbit.sh --test` and they pass
- [x] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

source/extensions/omni.isaac.orbit/config/extension.toml

 [package]
 
 # Note: Semantic Versioning is used: https://semver.org/
-version = "0.15.10"
+version = "0.15.11"
 
 # Description
 title = "ORBIT framework for Robot Learning"

source/extensions/omni.isaac.orbit/docs/CHANGELOG.rst

 Changelog
 ---------
 
+0.15.11 (2024-04-15)
+~~~~~~~~~~~~~~~~~~~~
+
+Added
+^^^^^
+
+* Added the :meth:`omni.isaac.orbit.sim.SimulationContext.has_rtx_sensors` method to check if any
+  RTX-related sensors such as cameras have been created in the simulation. This is useful to determine
+  if simulation requires RTX rendering during step or not.
+
+Fixed
+^^^^^
+
+* Fixed the rendering of RTX-related sensors such as cameras inside the :class:`omni.isaac.orbit.envs.RLTaskEnv` class.
+  Earlier the rendering did not happen inside the step function, which caused the sensor data to be empty.
+
+
 0.15.10 (2024-04-11)
 ~~~~~~~~~~~~~~~~~~~~
 

source/extensions/omni.isaac.orbit/omni/isaac/orbit/app/app_launcher.py

@@ -534,7 +534,12 @@ class AppLauncher:
         # set carb setting to indicate orbit's offscreen_render pipeline should be enabled
         # this flag is used by the SimulationContext class to enable the offscreen_render pipeline
         # when the render() method is called.
-        carb_settings_iface.set_bool("/orbit/offscreen_render/enabled", self._offscreen_render)
+        carb_settings_iface.set_bool("/orbit/render/offscreen", self._offscreen_render)
+
+        # set carb setting to indicate no RTX sensors are used
+        # this flag is set to True when an RTX-rendering related sensor is created
+        # for example: the `Camera` sensor class
+        carb_settings_iface.set_bool("/orbit/render/rtx_sensors", False)
 
         # enable extensions for off-screen rendering
         # Depending on the app file, some extensions might not be available in it.

source/extensions/omni.isaac.orbit/omni/isaac/orbit/envs/base_env.py

@@ -101,7 +101,9 @@ class BaseEnv:
 
         # create a simulation context to control the simulator
         if SimulationContext.instance() is None:
-            self.sim = SimulationContext(self.cfg.sim)
+            # the type-annotation is required to avoid a type-checking error
+            # since it gets confused with Isaac Sim's SimulationContext class
+            self.sim: SimulationContext = SimulationContext(self.cfg.sim)
         else:
             raise RuntimeError("Simulation context already exists. Cannot create a new one.")
 
@@ -252,7 +254,7 @@ class BaseEnv:
         # return observations
         return self.observation_manager.compute(), self.extras
 
-    def step(self, action: torch.Tensor) -> VecEnvObs:
+    def step(self, action: torch.Tensor) -> tuple[VecEnvObs, dict]:
         """Execute one time-step of the environment's dynamics.
 
         The environment steps forward at a fixed time-step, while the physics simulation is
@@ -280,7 +282,7 @@ class BaseEnv:
             # update buffers at sim dt
             self.scene.update(dt=self.physics_dt)
         # perform rendering if gui is enabled
-        if self.sim.has_gui():
+        if self.sim.has_gui() or self.sim.has_rtx_sensors():
             self.sim.render()
 
         # post-step: step interval event

source/extensions/omni.isaac.orbit/omni/isaac/orbit/envs/rl_task_env.py

@@ -173,7 +173,7 @@ class RLTaskEnv(BaseEnv, gym.Env):
             # update buffers at sim dt
             self.scene.update(dt=self.physics_dt)
         # perform rendering if gui is enabled
-        if self.sim.has_gui():
+        if self.sim.has_gui() or self.sim.has_rtx_sensors():
             self.sim.render()
 
         # post-step:
@@ -203,7 +203,7 @@ class RLTaskEnv(BaseEnv, gym.Env):
         # return observations, rewards, resets and extras
         return self.obs_buf, self.reward_buf, self.reset_terminated, self.reset_time_outs, self.extras
 
-    def render(self) -> np.ndarray | None:
+    def render(self, recompute: bool = False) -> np.ndarray | None:
         """Run rendering without stepping through the physics.
 
         By convention, if mode is:
@@ -212,6 +212,10 @@ class RLTaskEnv(BaseEnv, gym.Env):
         - **rgb_array**: Return an numpy.ndarray with shape (x, y, 3), representing RGB values for an
           x-by-y pixel image, suitable for turning into a video.
 
+        Args:
+            recompute: Whether to force a render even if the simulator has already rendered the scene.
+                Defaults to False.
+
         Returns:
             The rendered image as a numpy array if mode is "rgb_array". Otherwise, returns None.
 
@@ -222,7 +226,9 @@ class RLTaskEnv(BaseEnv, gym.Env):
             NotImplementedError: If an unsupported rendering mode is specified.
         """
         # run a rendering step of the simulator
-        self.sim.render()
+        # if we have rtx sensors, we do not need to render again sin
+        if not self.sim.has_rtx_sensors() and not recompute:
+            self.sim.render()
         # decide the rendering mode
         if self.render_mode == "human" or self.render_mode is None:
             return None

source/extensions/omni.isaac.orbit/omni/isaac/orbit/sensors/camera/camera.py

@@ -13,10 +13,10 @@ from collections.abc import Sequence
 from tensordict import TensorDict
 from typing import TYPE_CHECKING, Any, Literal
 
+import carb
 import omni.kit.commands
 import omni.usd
 from omni.isaac.core.prims import XFormPrimView
-from omni.syntheticdata.scripts.SyntheticData import SyntheticData
 from pxr import UsdGeom
 
 import omni.isaac.orbit.sim as sim_utils
@@ -106,6 +106,11 @@ class Camera(SensorBase):
         # initialize base class
         super().__init__(cfg)
 
+        # toggle rendering of rtx sensors as True
+        # this flag is read by SimulationContext to determine if rtx sensors should be rendered
+        carb_settings_iface = carb.settings.get_settings()
+        carb_settings_iface.set_bool("/orbit/render/rtx_sensors", True)
+
         # spawn the asset
         if self.cfg.spawn is not None:
             # compute the rotation offset
@@ -355,6 +360,7 @@ class Camera(SensorBase):
             RuntimeError: If the number of camera prims in the view does not match the number of environments.
         """
         import omni.replicator.core as rep
+        from omni.syntheticdata.scripts.SyntheticData import SyntheticData
 
         # Initialize parent class
         super()._initialize_impl()

source/extensions/omni.isaac.orbit/omni/isaac/orbit/sim/simulation_context.py

@@ -147,7 +147,7 @@ class SimulationContext(_SimulationContext):
         # read flag for whether the orbit viewport capture pipeline will be used,
         # casting None to False if the flag doesn't exist
         # this flag is set from the AppLauncher class
-        self._offscreen_render = bool(carb_settings_iface.get("/orbit/offscreen_render/enabled"))
+        self._offscreen_render = bool(carb_settings_iface.get("/orbit/render/offscreen"))
         # flag for whether any GUI will be rendered (local, livestreamed or viewport)
         self._has_gui = self._local_gui or self._livestream_gui
 
@@ -237,6 +237,21 @@ class SimulationContext(_SimulationContext):
         """
         return self._has_gui
 
+    def has_rtx_sensors(self) -> bool:
+        """Returns whether the simulation has any RTX-rendering related sensors.
+
+        This function returns the value of the simulation parameter ``"/orbit/render/rtx_sensors"``.
+        The parameter is set to True when instances of RTX-related sensors (cameras or LiDARs) are
+        created using Orbit's sensor classes.
+
+        True if the simulation has RTX sensors (such as USD Cameras or LiDARs).
+
+        For more information, please check `NVIDIA RTX documentation`_.
+
+        .. _NVIDIA RTX documentation: https://www.nvidia.com/design-visualization/solutions/rendering/
+        """
+        return self._settings.get_as_bool("/orbit/render/rtx_sensors")
+
     def is_fabric_enabled(self) -> bool:
         """Returns whether the fabric interface is enabled.
 

source/extensions/omni.isaac.orbit/test/sensors/test_camera.py

@@ -92,6 +92,8 @@ class TestCamera(unittest.TestCase):
         """Test camera initialization."""
         # Create camera
         camera = Camera(self.camera_cfg)
+        # Check simulation parameter is set correctly
+        self.assertTrue(self.sim.has_rtx_sensors())
         # Play sim
         self.sim.reset()
         # Check if camera is initialized

source/extensions/omni.isaac.orbit/test/sim/test_simulation_context.py

@@ -63,6 +63,7 @@ class TestSimulationContext(unittest.TestCase):
         # check valid settings
         self.assertEqual(sim.get_physics_dt(), cfg.dt)
         self.assertEqual(sim.get_rendering_dt(), cfg.dt * cfg.substeps)
+        self.assertFalse(sim.has_rtx_sensors())
         # check valid paths
         self.assertTrue(prim_utils.is_prim_path_valid("/Physics/PhysX"))
         self.assertTrue(prim_utils.is_prim_path_valid("/Physics/PhysX/defaultMaterial"))