Fixes the image feature extractor in observations (#1340)

# Description

- Fixes the image feature extractor in observations
- Adds missing dependencies in setup.py

## Type of change

- Bug fix (non-breaking change which fixes an issue)

## Checklist

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

CONTRIBUTORS.md

@@ -50,6 +50,7 @@ Guidelines for modifications:
 * Jan Kerner
 * Jean Tampon
 * Jia Lin Yuan
+* Jinghuan Shang
 * Jingzhou Liu
 * Johnson Sun
 * Kaixi Bao

docs/licenses/dependencies/einops-license.txt

+MIT License
+
+Copyright (c) 2018 Alex Rogozhnikov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docs/source/_static/refs.bib

@@ -139,3 +139,18 @@
   pages={3740-3747},
   doi={10.1109/LRA.2023.3270034}
 }
+
+@article{shang2024theia,
+  title={Theia: Distilling diverse vision foundation models for robot learning},
+  author={Shang, Jinghuan and Schmeckpeper, Karl and May, Brandon B and Minniti, Maria Vittoria and Kelestemur, Tarik and Watkins, David and Herlant, Laura},
+  journal={arXiv preprint arXiv:2407.20179},
+  year={2024}
+}
+
+@inproceedings{he2016deep,
+  title={Deep residual learning for image recognition},
+  author={He, Kaiming and Zhang, Xiangyu and Ren, Shaoqing and Sun, Jian},
+  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
+  pages={770--778},
+  year={2016}
+}

source/apps/isaaclab.python.headless.rendering.kit

@@ -35,6 +35,10 @@ app.folder = "${exe-path}/"
 app.name = "Isaac-Sim"
 app.version = "4.2.0"
 
+# Disable print outs on extension startup information
+# this only disables the app print_and_log function
+app.enableStdoutOutput = false
+
 # set the default ros bridge to disable on startup
 isaac.startup.ros_bridge_extension = ""
 

source/apps/isaaclab.python.rendering.kit

@@ -35,6 +35,10 @@ app.folder = "${exe-path}/"
 app.name = "Isaac-Sim"
 app.version = "4.2.0"
 
+# Disable print outs on extension startup information
+# this only disables the app print_and_log function
+app.enableStdoutOutput = false
+
 # set the default ros bridge to disable on startup
 isaac.startup.ros_bridge_extension = ""
 

source/extensions/omni.isaac.lab/setup.py

@@ -31,6 +31,9 @@ INSTALL_REQUIRES = [
     # procedural-generation
     "trimesh",
     "pyglet<2",
+    # image processing
+    "transformers",
+    "einops",  # needed for transformers, doesn't always auto-install
 ]
 
 PYTORCH_INDEX_URL = ["https://download.pytorch.org/whl/cu118"]

source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/agents/rl_games_feature_ppo_cfg.yaml

@@ -63,7 +63,7 @@ params:
     lr_schedule: adaptive
     kl_threshold: 0.008
     score_to_win: 20000
-    max_epochs: 5000
+    max_epochs: 200
     save_best_after: 50
     save_frequency: 25
     grad_norm: 1.0

source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/cartpole_camera_env_cfg.py

@@ -134,30 +134,43 @@ class TheiaTinyObservationCfg:
 class CartpoleRGBCameraEnvCfg(CartpoleEnvCfg):
     """Configuration for the cartpole environment with RGB camera."""
 
-    scene: CartpoleSceneCfg = CartpoleRGBCameraSceneCfg(num_envs=1024, env_spacing=20)
+    scene: CartpoleRGBCameraSceneCfg = CartpoleRGBCameraSceneCfg(num_envs=1024, env_spacing=20)
     observations: RGBObservationsCfg = RGBObservationsCfg()
 
+    def __post_init__(self):
+        super().__post_init__()
+        # remove ground as it obstructs the camera
+        self.scene.ground = None
+        # viewer settings
+        self.viewer.eye = (7.0, 0.0, 2.5)
+        self.viewer.lookat = (0.0, 0.0, 2.5)
+
 
 @configclass
 class CartpoleDepthCameraEnvCfg(CartpoleEnvCfg):
     """Configuration for the cartpole environment with depth camera."""
 
-    scene: CartpoleSceneCfg = CartpoleDepthCameraSceneCfg(num_envs=1024, env_spacing=20)
+    scene: CartpoleDepthCameraSceneCfg = CartpoleDepthCameraSceneCfg(num_envs=1024, env_spacing=20)
     observations: DepthObservationsCfg = DepthObservationsCfg()
 
+    def __post_init__(self):
+        super().__post_init__()
+        # remove ground as it obstructs the camera
+        self.scene.ground = None
+        # viewer settings
+        self.viewer.eye = (7.0, 0.0, 2.5)
+        self.viewer.lookat = (0.0, 0.0, 2.5)
+
 
 @configclass
 class CartpoleResNet18CameraEnvCfg(CartpoleRGBCameraEnvCfg):
+    """Configuration for the cartpole environment with ResNet18 features as observations."""
+
     observations: ResNet18ObservationCfg = ResNet18ObservationCfg()
 
 
 @configclass
 class CartpoleTheiaTinyCameraEnvCfg(CartpoleRGBCameraEnvCfg):
-    """
-    Due to TheiaTiny's size in GPU memory, we reduce the number of environments by default.
-    This helps reduce the possibility of crashing on more modest hardware.
-    The following configuration uses ~12gb VRAM at peak.
-    """
+    """Configuration for the cartpole environment with Theia-Tiny features as observations."""
 
-    scene: CartpoleSceneCfg = CartpoleRGBCameraSceneCfg(num_envs=128, env_spacing=20)
     observations: TheiaTinyObservationCfg = TheiaTinyObservationCfg()

source/extensions/omni.isaac.lab_tasks/omni/isaac/lab_tasks/manager_based/classic/cartpole/cartpole_env_cfg.py

@@ -48,11 +48,6 @@ class CartpoleSceneCfg(InteractiveSceneCfg):
         prim_path="/World/DomeLight",
         spawn=sim_utils.DomeLightCfg(color=(0.9, 0.9, 0.9), intensity=500.0),
     )
-    distant_light = AssetBaseCfg(
-        prim_path="/World/DistantLight",
-        spawn=sim_utils.DistantLightCfg(color=(0.9, 0.9, 0.9), intensity=2500.0),
-        init_state=AssetBaseCfg.InitialStateCfg(rot=(0.738, 0.477, 0.477, 0.0)),
-    )
 
 
 ##