Fixes test scripts to report their status to the runner (#475)

# Description

@Mayankm96 recently found that tests failures weren't properly reporting
in runs of `orbit -t`. This is because we were previously relying on
individual test's main to exit with 1 if the test failed, but this
recently changed when we stopped them from exiting with `exit=False`
argument to `unittest.main()` to ensure the simulation app could be
closed properly.
 
We no longer call `simulation_app.close()` as this will cause the exit to not reach `run_all_tests.py`. The downside here is a bit more console spam.

Fixes #474

## Type of change

- Bug fix (non-breaking change which fixes an issue)

## Checklist

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./orbit.sh --format`
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [x] I have run all the tests with `./orbit.sh --test` and they pass
- [ ] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

---------
Signed-off-by: James Smith <142246516+jsmith-bdai@users.noreply.github.com>
Co-authored-by: Mayank Mittal <mittalma@leggedrobotics.com>
Co-authored-by: Mayank Mittal <12863862+Mayankm96@users.noreply.github.com>

source/extensions/omni.isaac.orbit/omni/isaac/orbit/app/__init__.py

+# Copyright (c) 2022-2024, The ORBIT Project Developers.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Sub-package containing app-specific functionalities.
+
+These include:
+
+* Ability to launch the simulation app with different configurations
+* Run tests with the simulation app
+
+"""
+
+from .app_launcher import AppLauncher  # noqa: F401, F403
+from .runners import run_tests  # noqa: F401, F403

source/extensions/omni.isaac.orbit/omni/isaac/orbit/app/runners.py

+# Copyright (c) 2022-2024, The ORBIT Project Developers.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Sub-module with runners to simplify running main via unittest."""
+
+from __future__ import annotations
+
+import unittest
+
+
+def run_tests(verbosity: int = 2, **kwargs):
+    """Wrapper for running tests via ``unittest``.
+
+    Args:
+        verbosity: Verbosity level for the test runner.
+        **kwargs: Additional arguments to pass to the `unittest.main` function.
+    """
+    # run main
+    unittest.main(verbosity=verbosity, exit=True, **kwargs)

source/extensions/omni.isaac.orbit/test/app/test_argparser_launch.py

@@ -9,7 +9,7 @@ import argparse
 import unittest
 from unittest import mock
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 
 class TestAppLauncher(unittest.TestCase):
@@ -46,4 +46,4 @@ class TestAppLauncher(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    run_tests()

source/extensions/omni.isaac.orbit/test/app/test_env_var_launch.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 import os
 import unittest
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 
 class TestAppLauncher(unittest.TestCase):
@@ -37,4 +37,4 @@ class TestAppLauncher(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    run_tests()

source/extensions/omni.isaac.orbit/test/app/test_kwarg_launch.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 import unittest
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 
 class TestAppLauncher(unittest.TestCase):
@@ -34,4 +34,4 @@ class TestAppLauncher(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    run_tests()

source/extensions/omni.isaac.orbit/test/assets/test_articulation.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 app_launcher = AppLauncher(headless=True)
@@ -503,7 +503,4 @@ class TestArticulation(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/assets/test_rigid_object.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # Can set this to False to see the GUI for debugging
 # This will also add lights to the scene
@@ -562,7 +562,4 @@ class TestRigidObject(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/controllers/test_differential_ik.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 simulation_app = AppLauncher(headless=True).app
@@ -205,7 +205,4 @@ class TestDifferentialIKController(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/deps/test_scipy.py

@@ -15,6 +15,8 @@ import numpy as np
 import scipy.interpolate as interpolate
 import unittest
 
+from omni.isaac.orbit.app import run_tests
+
 
 class TestScipyOperations(unittest.TestCase):
     """Tests for assuring scipy related operations used in Orbit."""
@@ -72,4 +74,4 @@ class TestScipyOperations(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    run_tests()

source/extensions/omni.isaac.orbit/test/deps/test_torch.py

@@ -9,6 +9,8 @@ import torch
 import torch.utils.benchmark as benchmark
 import unittest
 
+from omni.isaac.orbit.app import run_tests
+
 
 class TestTorchOperations(unittest.TestCase):
     """Tests for assuring torch related operations used in Orbit."""
@@ -151,4 +153,4 @@ class TestTorchOperations(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    run_tests()

source/extensions/omni.isaac.orbit/test/envs/test_null_command_term.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 app_launcher = AppLauncher(headless=True)
@@ -50,7 +50,4 @@ class TestNullCommandTerm(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/managers/test_observation_manager.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 config = {"headless": True}
@@ -297,7 +297,4 @@ class TestObservationManager(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/managers/test_reward_manager.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 config = {"headless": True}
@@ -168,7 +168,4 @@ class TestRewardManager(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/markers/test_visualization_markers.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 config = {"headless": True}
@@ -207,7 +207,4 @@ class TestUsdVisualizationMarkers(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/scene/test_interactive_scene.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 simulation_app = AppLauncher(headless=True).app
@@ -111,8 +111,4 @@ class TestInteractiveScene(unittest.TestCase):
 
 
 if __name__ == "__main__":
-
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sensors/test_camera.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 app_launcher = AppLauncher(headless=True)
@@ -302,7 +302,7 @@ class TestCamera(unittest.TestCase):
         camera.update(self.dt)
 
         # expected sizes
-        hw_3c_shape = (1, camera_cfg.height, camera_cfg.width, 3)
+        hw_3c_shape = (1, camera_cfg.height, camera_cfg.width, 4)
         hw_1c_shape = (1, camera_cfg.height, camera_cfg.width)
         # access image data and compare shapes
         output = camera.data.output
@@ -352,7 +352,7 @@ class TestCamera(unittest.TestCase):
         camera.update(self.dt)
 
         # expected sizes
-        hw_3c_shape = (1, camera_cfg.height, camera_cfg.width, 3)
+        hw_3c_shape = (1, camera_cfg.height, camera_cfg.width, 4)
         hw_1c_shape = (1, camera_cfg.height, camera_cfg.width)
         # access image data and compare shapes
         output = camera.data.output
@@ -462,7 +462,4 @@ class TestCamera(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sensors/test_frame_transformer.py

@@ -11,7 +11,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 app_launcher = AppLauncher(headless=True)
@@ -306,7 +306,4 @@ class TestFrameTransformer(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sensors/test_ray_caster_camera.py

@@ -8,7 +8,7 @@
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 app_launcher = AppLauncher(headless=True)
@@ -370,7 +370,6 @@ class TestWarpCamera(unittest.TestCase):
             spawn=PinholeCameraCfg(
                 focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(1e-4, 1.0e5)
             ),
-            colorize=False,
         )
         camera_usd = Camera(camera_cfg_usd)
 
@@ -440,7 +439,6 @@ class TestWarpCamera(unittest.TestCase):
             spawn=PinholeCameraCfg(
                 focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(1e-6, 1.0e5)
             ),
-            colorize=False,
             offset=CameraCfg.OffsetCfg(pos=(2.5, 2.5, 4.0), rot=offset_rot, convention="ros"),
         )
         camera_usd = Camera(camera_cfg_usd)
@@ -518,7 +516,6 @@ class TestWarpCamera(unittest.TestCase):
             spawn=PinholeCameraCfg(
                 focal_length=24.0, focus_distance=400.0, horizontal_aperture=20.955, clipping_range=(1e-6, 1.0e5)
             ),
-            colorize=False,
             offset=CameraCfg.OffsetCfg(pos=(0, 0, 2.0), rot=offset_rot, convention="ros"),
         )
         prim_usd = prim_utils.create_prim("/World/Camera_usd", "Xform")
@@ -565,7 +562,4 @@ class TestWarpCamera(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_build_simulation_context_headless.py

@@ -14,7 +14,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 app_launcher = AppLauncher(headless=True)
@@ -100,7 +100,4 @@ class TestBuildSimulationContextHeadless(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_build_simulation_context_nonheadless.py

@@ -13,7 +13,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 app_launcher = AppLauncher(headless=False)
@@ -100,7 +100,4 @@ class TestBuildSimulationContextNonheadless(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_mesh_converter.py

@@ -6,7 +6,7 @@
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 simulation_app = AppLauncher(headless=True).app
@@ -263,7 +263,4 @@ class TestMeshConverter(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_schemas.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 simulation_app = AppLauncher(headless=True).app
@@ -327,7 +327,4 @@ class TestPhysicsSchema(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_simulation_context.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 simulation_app = AppLauncher(headless=True).app
@@ -135,7 +135,4 @@ class TestSimulationContext(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_spawn_from_files.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 """Launch Isaac Sim Simulator first."""
 
@@ -85,7 +85,4 @@ class TestSpawningFromFiles(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_spawn_lights.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 simulation_app = AppLauncher(headless=True).app
@@ -159,7 +159,4 @@ class TestSpawningLights(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_spawn_materials.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 simulation_app = AppLauncher(headless=True).app
@@ -171,7 +171,4 @@ class TestSpawningMaterials(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_spawn_sensors.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 simulation_app = AppLauncher(headless=True).app
@@ -120,7 +120,4 @@ class TestSpawningSensors(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_spawn_shapes.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 simulation_app = AppLauncher(headless=True).app
@@ -303,7 +303,4 @@ class TestSpawningUsdGeometries(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_urdf_converter.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 config = {"headless": True}
@@ -24,7 +24,7 @@ import omni.isaac.core.utils.prims as prim_utils
 import omni.isaac.core.utils.stage as stage_utils
 from omni.isaac.core.articulations import ArticulationView
 from omni.isaac.core.simulation_context import SimulationContext
-from omni.isaac.core.utils.extensions import get_extension_path_from_name
+from omni.isaac.core.utils.extensions import enable_extension, get_extension_path_from_name
 
 from omni.isaac.orbit.sim.converters import UrdfConverter, UrdfConverterCfg
 
@@ -37,6 +37,7 @@ class TestUrdfConverter(unittest.TestCase):
         # Create a new stage
         stage_utils.create_new_stage()
         # retrieve path to urdf importer extension
+        enable_extension("omni.importer.urdf")
         extension_path = get_extension_path_from_name("omni.importer.urdf")
         # default configuration
         self.config = UrdfConverterCfg(
@@ -158,7 +159,4 @@ class TestUrdfConverter(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/sim/test_utils.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 config = {"headless": True}
@@ -86,7 +86,4 @@ class TestUtilities(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/utils/test_configclass.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 #       because warp is only available in the context of a running simulation
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 app_launcher = AppLauncher(headless=True)
@@ -705,7 +705,4 @@ class TestConfigClass(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/utils/test_dict.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 #       because warp is only available in the context of a running simulation
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 app_launcher = AppLauncher(headless=True)
@@ -83,7 +83,4 @@ class TestDictUtilities(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/utils/test_math.py

@@ -14,7 +14,7 @@ from math import pi as PI
 This is only needed because of warp dependency.
 """
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app in headless mode
 simulation_app = AppLauncher(headless=True).app
@@ -112,7 +112,4 @@ class TestMathUtilities(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/utils/test_string.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 #       because warp is only available in the context of a running simulation
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 app_launcher = AppLauncher(headless=True)
@@ -130,7 +130,4 @@ class TestStringUtilities(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit/test/utils/test_timer.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 #       because warp is only available in the context of a running simulation
 """Launch Isaac Sim Simulator first."""
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch omniverse app
 app_launcher = AppLauncher(headless=True)
@@ -49,7 +49,4 @@ class TestTimer(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit_tasks/test/test_data_collector.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import os
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch the simulator
 app_experience = f"{os.environ['EXP_PATH']}/omni.isaac.sim.python.gym.headless.kit"
@@ -87,4 +87,4 @@ class TestRobomimicDataCollector(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    run_tests()

source/extensions/omni.isaac.orbit_tasks/test/test_environments.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import os
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch the simulator
 app_experience = f"{os.environ['EXP_PATH']}/omni.isaac.sim.python.gym.headless.kit"
@@ -134,7 +134,4 @@ class TestEnvironments(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit_tasks/test/test_record_video.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import os
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch the simulator
 app_experience = f"{os.environ['EXP_PATH']}/omni.isaac.sim.python.gym.headless.kit"
@@ -92,7 +92,4 @@ class TestRecordVideoWrapper(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit_tasks/test/wrappers/test_rl_games_wrapper.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import os
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch the simulator
 app_experience = f"{os.environ['EXP_PATH']}/omni.isaac.sim.python.gym.headless.kit"
@@ -117,7 +117,4 @@ class TestRlGamesVecEnvWrapper(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit_tasks/test/wrappers/test_rsl_rl_wrapper.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import os
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch the simulator
 app_experience = f"{os.environ['EXP_PATH']}/omni.isaac.sim.python.gym.headless.kit"
@@ -153,7 +153,4 @@ class TestRslRlVecEnvWrapper(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

source/extensions/omni.isaac.orbit_tasks/test/wrappers/test_sb3_wrapper.py

@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import os
 
-from omni.isaac.orbit.app import AppLauncher
+from omni.isaac.orbit.app import AppLauncher, run_tests
 
 # launch the simulator
 app_experience = f"{os.environ['EXP_PATH']}/omni.isaac.sim.python.gym.headless.kit"
@@ -123,7 +123,4 @@ class TestStableBaselines3VecEnvWrapper(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    # run main
-    unittest.main(verbosity=2, exit=False)
-    # close sim app
-    simulation_app.close()
+    run_tests()

tools/run_all_tests.py

@@ -26,6 +26,7 @@ import argparse
 import logging
 import os
 import subprocess
+import sys
 import time
 from datetime import datetime
 from pathlib import Path
@@ -137,7 +138,7 @@ def test_all(
 
     # Print tests to be run
     logging.info("\n" + "=" * 60 + "\n")
-    logging.info(f"The following {len(all_test_paths)} tests will be run:")
+    logging.info(f"The following {len(all_test_paths)} tests were found:")
     for i, test_path in enumerate(all_test_paths):
         logging.info(f"{i + 1:02d}: {test_path}")
     logging.info("\n" + "=" * 60 + "\n")
@@ -153,8 +154,6 @@ def test_all(
 
     results = {}
 
-    # Resolve python executable to use
-    orbit_shell_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "orbit.sh")
     # Run each script and store results
     for test_path in test_paths:
         results[test_path] = {}
@@ -163,19 +162,28 @@ def test_all(
         logging.info(f"[INFO] Running '{test_path}'\n")
         try:
             completed_process = subprocess.run(
-                ["bash", orbit_shell_path, "-p", test_path], check=True, capture_output=True, timeout=timeout
+                [sys.executable, test_path], check=True, capture_output=True, timeout=timeout
             )
         except subprocess.TimeoutExpired as e:
             logging.error(f"Timeout occurred: {e}")
             result = "TIMEDOUT"
             stdout = e.stdout
             stderr = e.stderr
+        except subprocess.CalledProcessError as e:
+            # When check=True is passed to subprocess.run() above, CalledProcessError is raised if the process returns a
+            # non-zero exit code. The caveat is returncode is not correctly updated in this case, so we simply
+            # catch the exception and set this test as FAILED
+            result = "FAILED"
+            stdout = e.stdout
+            stderr = e.stderr
         except Exception as e:
-            logging.error(f"Exception {e}!")
+            logging.error(f"Unexpected exception {e}. Please report this issue on the repository.")
             result = "FAILED"
             stdout = e.stdout
             stderr = e.stderr
         else:
+            # Should only get here if the process ran successfully, e.g. no exceptions were raised
+            # but we still check the returncode just in case
             result = "PASSED" if completed_process.returncode == 0 else "FAILED"
             stdout = completed_process.stdout
             stderr = completed_process.stderr