Updates to latest RSL-RL v2.3.0 release (#2154)

# Description

This MR introduces multi-GPU training for RSL-RL library. Also adds
configuration options for symmetry and RND.

Compatible only with RSL-RL v2.3.0 onwards so fixing the version.

Fixes #2180 

## Type of change

- New feature (non-breaking change which adds functionality)

## Checklist

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [x] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

docs/source/_static/refs.bib

@@ -154,3 +154,24 @@
   pages={770--778},
   year={2016}
 }
+
+@InProceedings{schwarke2023curiosity,
+  title =    {Curiosity-Driven Learning of Joint Locomotion and Manipulation Tasks},
+  author =       {Schwarke, Clemens and Klemm, Victor and Boon, Matthijs van der and Bjelonic, Marko and Hutter, Marco},
+  booktitle =    {Proceedings of The 7th Conference on Robot Learning},
+  pages =    {2594--2610},
+  year =     {2023},
+  volume =   {229},
+  series =   {Proceedings of Machine Learning Research},
+  publisher =    {PMLR},
+  url =      {https://proceedings.mlr.press/v229/schwarke23a.html},
+}
+
+@InProceedings{mittal2024symmetry,
+  author={Mittal, Mayank and Rudin, Nikita and Klemm, Victor and Allshire, Arthur and Hutter, Marco},
+  booktitle={2024 IEEE International Conference on Robotics and Automation (ICRA)},
+  title={Symmetry Considerations for Learning Task Symmetric Robot Policies},
+  year={2024},
+  pages={7433-7439},
+  doi={10.1109/ICRA57147.2024.10611493}
+}

docs/source/features/multi_gpu.rst

@@ -4,7 +4,7 @@ Multi-GPU and Multi-Node Training
 .. currentmodule:: isaaclab
 
 Isaac Lab supports multi-GPU and multi-node reinforcement learning. Currently, this feature is only
-available for RL-Games and skrl libraries workflows. We are working on extending this feature to
+available for RL-Games, RSL-RL and skrl libraries workflows. We are working on extending this feature to
 other workflows.
 
 .. attention::
@@ -57,6 +57,13 @@ To train with multiple GPUs, use the following command, where ``--nproc_per_node
 
             python -m torch.distributed.run --nnodes=1 --nproc_per_node=2 scripts/reinforcement_learning/rl_games/train.py --task=Isaac-Cartpole-v0 --headless --distributed
 
+    .. tab-item:: rsl_rl
+        :sync: rsl_rl
+
+        .. code-block:: shell
+
+            python -m torch.distributed.run --nnodes=1 --nproc_per_node=2 scripts/reinforcement_learning/rsl_rl/train.py --task=Isaac-Cartpole-v0 --headless --distributed
+
     .. tab-item:: skrl
         :sync: skrl
 
@@ -95,6 +102,13 @@ For the master node, use the following command, where ``--nproc_per_node`` repre
 
             python -m torch.distributed.run --nproc_per_node=2 --nnodes=2 --node_rank=0 --rdzv_id=123 --rdzv_backend=c10d --rdzv_endpoint=localhost:5555 scripts/reinforcement_learning/rl_games/train.py --task=Isaac-Cartpole-v0 --headless --distributed
 
+    .. tab-item:: rsl_rl
+        :sync: rsl_rl
+
+        .. code-block:: shell
+
+            python -m torch.distributed.run --nproc_per_node=2 --nnodes=2 --node_rank=0 --rdzv_id=123 --rdzv_backend=c10d --rdzv_endpoint=localhost:5555 scripts/reinforcement_learning/rsl_rl/train.py --task=Isaac-Cartpole-v0 --headless --distributed
+
     .. tab-item:: skrl
         :sync: skrl
 
@@ -128,6 +142,13 @@ For non-master nodes, use the following command, replacing ``--node_rank`` with
 
             python -m torch.distributed.run --nproc_per_node=2 --nnodes=2 --node_rank=1 --rdzv_id=123 --rdzv_backend=c10d --rdzv_endpoint=ip_of_master_machine:5555 scripts/reinforcement_learning/rl_games/train.py --task=Isaac-Cartpole-v0 --headless --distributed
 
+    .. tab-item:: rsl_rl
+        :sync: rsl_rl
+
+        .. code-block:: shell
+
+            python -m torch.distributed.run --nproc_per_node=2 --nnodes=2 --node_rank=1 --rdzv_id=123 --rdzv_backend=c10d --rdzv_endpoint=ip_of_master_machine:5555 scripts/reinforcement_learning/rsl_rl/train.py --task=Isaac-Cartpole-v0 --headless --distributed
+
     .. tab-item:: skrl
         :sync: skrl
 

docs/source/overview/reinforcement-learning/rl_frameworks.rst

@@ -27,7 +27,7 @@ Feature Comparison
      - Stable Baselines3
    * - Algorithms Included
      - PPO, SAC, A2C
-     - PPO
+     - PPO, Distillation
      - `Extensive List <https://skrl.readthedocs.io/en/latest/#agents>`__
      - `Extensive List <https://github.com/DLR-RM/stable-baselines3?tab=readme-ov-file#implemented-algorithms>`__
    * - Vectorized Training
@@ -37,7 +37,7 @@ Feature Comparison
      - No
    * - Distributed Training
      - Yes
-     - No
+     - Yes
      - Yes
      - No
    * - ML Frameworks Supported

scripts/benchmarks/benchmark_rsl_rl.py

@@ -31,6 +31,9 @@ parser.add_argument("--num_envs", type=int, default=4096, help="Number of enviro
 parser.add_argument("--task", type=str, default=None, help="Name of the task.")
 parser.add_argument("--seed", type=int, default=42, help="Seed used for the environment")
 parser.add_argument("--max_iterations", type=int, default=10, help="RL Policy training iterations.")
+parser.add_argument(
+    "--distributed", action="store_true", default=False, help="Run training with multiple GPUs or nodes."
+)
 parser.add_argument(
     "--benchmark_backend",
     type=str,
@@ -126,8 +129,27 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agen
     """Train with RSL-RL agent."""
     # parse configuration
     benchmark.set_phase("loading", start_recording_frametime=False, start_recording_runtime=True)
+    # override configurations with non-hydra CLI arguments
     agent_cfg = cli_args.update_rsl_rl_cfg(agent_cfg, args_cli)
     env_cfg.scene.num_envs = args_cli.num_envs if args_cli.num_envs is not None else env_cfg.scene.num_envs
+    agent_cfg.max_iterations = (
+        args_cli.max_iterations if args_cli.max_iterations is not None else agent_cfg.max_iterations
+    )
+
+    # set the environment seed
+    # note: certain randomizations occur in the environment initialization so we set the seed here
+    env_cfg.seed = agent_cfg.seed
+    env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device
+
+    # multi-gpu training configuration
+    if args_cli.distributed:
+        env_cfg.sim.device = f"cuda:{app_launcher.local_rank}"
+        agent_cfg.device = f"cuda:{app_launcher.local_rank}"
+
+        # set seed to have diversity in different threads
+        seed = agent_cfg.seed + app_launcher.local_rank
+        env_cfg.seed = seed
+        agent_cfg.seed = seed
 
     # specify directory for logging experiments
     log_root_path = os.path.join("logs", "rsl_rl", agent_cfg.experiment_name)

scripts/reinforcement_learning/rsl_rl/play.py

@@ -5,6 +5,21 @@
 
 """Script to play a checkpoint if an RL agent from RSL-RL."""
 
+import platform
+from importlib.metadata import version
+
+if version("rsl-rl-lib") != "2.3.0":
+    if platform.system() == "Windows":
+        cmd = [r".\isaaclab.bat", "-p", "-m", "pip", "install", "rsl-rl-lib==2.3.0"]
+    else:
+        cmd = ["./isaaclab.sh", "-p", "-m", "pip", "install", "rsl-rl-lib==2.3.0"]
+    print(
+        f"Please install the correct version of RSL-RL.\nExisting version is: '{version('rsl-rl-lib')}'"
+        " and required version is: '2.3.0'.\nTo install the correct version, run:"
+        f"\n\n\t{' '.join(cmd)}\n"
+    )
+    exit(1)
+
 """Launch Isaac Sim Simulator first."""
 
 import argparse
@@ -120,11 +135,9 @@ def main():
 
     # export policy to onnx/jit
     export_model_dir = os.path.join(os.path.dirname(resume_path), "exported")
-    export_policy_as_jit(
-        ppo_runner.alg.actor_critic, ppo_runner.obs_normalizer, path=export_model_dir, filename="policy.pt"
-    )
+    export_policy_as_jit(ppo_runner.alg.policy, ppo_runner.obs_normalizer, path=export_model_dir, filename="policy.pt")
     export_policy_as_onnx(
-        ppo_runner.alg.actor_critic, normalizer=ppo_runner.obs_normalizer, path=export_model_dir, filename="policy.onnx"
+        ppo_runner.alg.policy, normalizer=ppo_runner.obs_normalizer, path=export_model_dir, filename="policy.onnx"
     )
 
     dt = env.unwrapped.physics_dt

scripts/reinforcement_learning/rsl_rl/train.py

@@ -5,6 +5,21 @@
 
 """Script to train RL agent with RSL-RL."""
 
+import platform
+from importlib.metadata import version
+
+if version("rsl-rl-lib") != "2.3.0":
+    if platform.system() == "Windows":
+        cmd = [r".\isaaclab.bat", "-p", "-m", "pip", "install", "rsl-rl-lib==2.3.0"]
+    else:
+        cmd = ["./isaaclab.sh", "-p", "-m", "pip", "install", "rsl-rl-lib==2.3.0"]
+    print(
+        f"Please install the correct version of RSL-RL.\nExisting version is: '{version('rsl-rl-lib')}'"
+        " and required version is: '2.3.0'.\nTo install the correct version, run:"
+        f"\n\n\t{' '.join(cmd)}\n"
+    )
+    exit(1)
+
 """Launch Isaac Sim Simulator first."""
 
 import argparse
@@ -25,6 +40,9 @@ parser.add_argument("--num_envs", type=int, default=None, help="Number of enviro
 parser.add_argument("--task", type=str, default=None, help="Name of the task.")
 parser.add_argument("--seed", type=int, default=None, help="Seed used for the environment")
 parser.add_argument("--max_iterations", type=int, default=None, help="RL Policy training iterations.")
+parser.add_argument(
+    "--distributed", action="store_true", default=False, help="Run training with multiple GPUs or nodes."
+)
 # append RSL-RL cli arguments
 cli_args.add_rsl_rl_args(parser)
 # append AppLauncher cli args
@@ -90,6 +108,16 @@ def main(env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg | DirectMARLEnvCfg, agen
     env_cfg.seed = agent_cfg.seed
     env_cfg.sim.device = args_cli.device if args_cli.device is not None else env_cfg.sim.device
 
+    # multi-gpu training configuration
+    if args_cli.distributed:
+        env_cfg.sim.device = f"cuda:{app_launcher.local_rank}"
+        agent_cfg.device = f"cuda:{app_launcher.local_rank}"
+
+        # set seed to have diversity in different threads
+        seed = agent_cfg.seed + app_launcher.local_rank
+        env_cfg.seed = seed
+        agent_cfg.seed = seed
+
     # specify directory for logging experiments
     log_root_path = os.path.join("logs", "rsl_rl", agent_cfg.experiment_name)
     log_root_path = os.path.abspath(log_root_path)

source/isaaclab_rl/config/extension.toml

 [package]
 
 # Note: Semantic Versioning is used: https://semver.org/
-version = "0.1.1"
+version = "0.1.2"
 
 # Description
 title = "Isaac Lab RL"

source/isaaclab_rl/docs/CHANGELOG.rst

 Changelog
 ---------
 
+0.1.2 (2025-03-28)
+~~~~~~~~~~~~~~~~~~
+
+Added
+^^^^^
+
+* Added symmetry and curiosity-based exploration configurations for RSL-RL wrapper.
+
+
 0.1.1 (2025-03-10)
 ~~~~~~~~~~~~~~~~~~
 

source/isaaclab_rl/isaaclab_rl/rsl_rl/__init__.py

@@ -17,4 +17,6 @@ The following example shows how to wrap an environment for RSL-RL:
 
 from .exporter import export_policy_as_jit, export_policy_as_onnx
 from .rl_cfg import RslRlOnPolicyRunnerCfg, RslRlPpoActorCriticCfg, RslRlPpoAlgorithmCfg
+from .rnd_cfg import RslRlRndCfg
+from .symmetry_cfg import RslRlSymmetryCfg
 from .vecenv_wrapper import RslRlVecEnvWrapper

source/isaaclab_rl/isaaclab_rl/rsl_rl/rl_cfg.py

@@ -8,6 +8,9 @@ from typing import Literal
 
 from isaaclab.utils import configclass
 
+from .rnd_cfg import RslRlRndCfg
+from .symmetry_cfg import RslRlSymmetryCfg
+
 
 @configclass
 class RslRlPpoActorCriticCfg:
@@ -19,6 +22,9 @@ class RslRlPpoActorCriticCfg:
     init_noise_std: float = MISSING
     """The initial noise standard deviation for the policy."""
 
+    noise_std_type: Literal["scalar", "log"] = "scalar"
+    """The type of noise standard deviation for the policy. Default is scalar."""
+
     actor_hidden_dims: list[int] = MISSING
     """The hidden dimensions of the actor network."""
 
@@ -72,6 +78,21 @@ class RslRlPpoAlgorithmCfg:
     max_grad_norm: float = MISSING
     """The maximum gradient norm."""
 
+    normalize_advantage_per_mini_batch: bool = False
+    """Whether to normalize the advantage per mini-batch. Default is False.
+
+    If True, the advantage is normalized over the entire collected trajectories.
+    Otherwise, the advantage is normalized over the mini-batches only.
+    """
+
+    symmetry_cfg: RslRlSymmetryCfg | None = None
+    """The symmetry configuration. Default is None, in which case symmetry is not used."""
+
+    rnd_cfg: RslRlRndCfg | None = None
+    """The configuration for the Random Network Distillation (RND) module. Default is None,
+    in which case RND is not used.
+    """
+
 
 @configclass
 class RslRlOnPolicyRunnerCfg:
@@ -99,7 +120,11 @@ class RslRlOnPolicyRunnerCfg:
     """The algorithm configuration."""
 
     clip_actions: float | None = None
-    """The clipping value for actions. If ``None``, then no clipping is done."""
+    """The clipping value for actions. If ``None``, then no clipping is done.
+
+    .. note::
+        This clipping is performed inside the :class:`RslRlVecEnvWrapper` wrapper.
+    """
 
     ##
     # Checkpointing parameters

source/isaaclab_rl/isaaclab_rl/rsl_rl/rnd_cfg.py

+# Copyright (c) 2022-2025, The Isaac Lab Project Developers.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from dataclasses import MISSING
+
+from isaaclab.utils import configclass
+
+
+@configclass
+class RslRlRndCfg:
+    """Configuration for the Random Network Distillation (RND) module.
+
+    For more information, please check the work from :cite:`schwarke2023curiosity`.
+    """
+
+    @configclass
+    class WeightScheduleCfg:
+        """Configuration for the weight schedule."""
+
+        mode: str = "constant"
+        """The type of weight schedule. Default is "constant"."""
+
+    @configclass
+    class LinearWeightScheduleCfg(WeightScheduleCfg):
+        """Configuration for the linear weight schedule.
+
+        This schedule decays the weight linearly from the initial value to the final value
+        between :attr:`initial_step` and before :attr:`final_step`.
+        """
+
+        mode: str = "linear"
+
+        final_value: float = MISSING
+        """The final value of the weight parameter."""
+
+        initial_step: int = MISSING
+        """The initial step of the weight schedule.
+
+        For steps before this step, the weight is the initial value specified in :attr:`RslRlRndCfg.weight`.
+        """
+
+        final_step: int = MISSING
+        """The final step of the weight schedule.
+
+        For steps after this step, the weight is the final value specified in :attr:`final_value`.
+        """
+
+    @configclass
+    class StepWeightScheduleCfg(WeightScheduleCfg):
+        """Configuration for the step weight schedule.
+
+        This schedule sets the weight to the value specified in :attr:`final_value` at step :attr:`final_step`.
+        """
+
+        mode: str = "step"
+
+        final_step: int = MISSING
+        """The final step of the weight schedule.
+
+        For steps after this step, the weight is the value specified in :attr:`final_value`.
+        """
+
+        final_value: float = MISSING
+        """The final value of the weight parameter."""
+
+    weight: float = 0.0
+    """The weight for the RND reward (also known as intrinsic reward). Default is 0.0.
+
+    Similar to other reward terms, the RND reward is scaled by this weight.
+    """
+
+    weight_schedule: WeightScheduleCfg | None = None
+    """The weight schedule for the RND reward. Default is None, which means the weight is constant."""
+
+    reward_normalization: bool = False
+    """Whether to normalize the RND reward. Default is False."""
+
+    state_normalization: bool = False
+    """Whether to normalize the RND state. Default is False."""
+
+    learning_rate: float = 1e-3
+    """The learning rate for the RND module. Default is 1e-3."""
+
+    num_outputs: int = 1
+    """The number of outputs for the RND module. Default is 1."""
+
+    predictor_hidden_dims: list[int] = [-1]
+    """The hidden dimensions for the RND predictor network. Default is [-1].
+
+    If the list contains -1, then the hidden dimensions are the same as the input dimensions.
+    """
+
+    target_hidden_dims: list[int] = [-1]
+    """The hidden dimensions for the RND target network. Default is [-1].
+
+    If the list contains -1, then the hidden dimensions are the same as the input dimensions.
+    """

source/isaaclab_rl/isaaclab_rl/rsl_rl/symmetry_cfg.py

+# Copyright (c) 2022-2025, The Isaac Lab Project Developers.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+from dataclasses import MISSING
+
+from isaaclab.utils import configclass
+
+
+@configclass
+class RslRlSymmetryCfg:
+    """Configuration for the symmetry-augmentation in the training.
+
+    When :meth:`use_data_augmentation` is True, the :meth:`data_augmentation_func` is used to generate
+    augmented observations and actions. These are then used to train the model.
+
+    When :meth:`use_mirror_loss` is True, the :meth:`mirror_loss_coeff` is used to weight the
+    symmetry-mirror loss. This loss is directly added to the agent's loss function.
+
+    If both :meth:`use_data_augmentation` and :meth:`use_mirror_loss` are False, then no symmetry-based
+    training is enabled. However, the :meth:`data_augmentation_func` is called to compute and log
+    symmetry metrics. This is useful for performing ablations.
+
+    For more information, please check the work from :cite:`mittal2024symmetry`.
+    """
+
+    use_data_augmentation: bool = False
+    """Whether to use symmetry-based data augmentation. Default is False."""
+
+    use_mirror_loss: bool = False
+    """Whether to use the symmetry-augmentation loss. Default is False."""
+
+    data_augmentation_func: callable = MISSING
+    """The symmetry data augmentation function.
+
+    The function signature should be as follows:
+
+    Args:
+
+        env (VecEnv): The environment object. This is used to access the environment's properties.
+        obs (torch.Tensor | None): The observation tensor. If None, the observation is not used.
+        action (torch.Tensor | None): The action tensor. If None, the action is not used.
+        obs_type (str): The name of the observation type. Defaults to "policy".
+            This is useful when handling augmentation for different observation groups.
+
+    Returns:
+        A tuple containing the augmented observation and action tensors. The tensors can be None,
+        if their respective inputs are None.
+    """
+
+    mirror_loss_coeff: float = 0.0
+    """The weight for the symmetry-mirror loss. Default is 0.0."""

source/isaaclab_rl/setup.py

@@ -44,7 +44,7 @@ EXTRAS_REQUIRE = {
     "sb3": ["stable-baselines3>=2.1"],
     "skrl": ["skrl>=1.4.2"],
     "rl-games": ["rl-games==1.6.1", "gym"],  # rl-games still needs gym :(
-    "rsl-rl": ["rsl-rl-lib>=2.1.1"],
+    "rsl-rl": ["rsl-rl-lib==2.3.0"],
 }
 # Add the names with hyphens as aliases for convenience
 EXTRAS_REQUIRE["rl_games"] = EXTRAS_REQUIRE["rl-games"]