Unverified Commit 755111c1 authored by James Tigue's avatar James Tigue Committed by GitHub

Adds record at close functionality to the RecorderManager (#2826)

# Description

Introducing a close function to the recorder manager which exports the
data to file when the environment is closed and closes the recorder
terms.

Fixes # (issue)

## Type of change

<!-- As you go through the list, delete the ones that are not
applicable. -->

- New feature (non-breaking change which adds functionality)

## Screenshots

## Checklist

- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [x] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [x] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

<!--
As you go through the checklist above, you can mark something as done by
putting an x character in it

For example,
- [x] I have done this task
- [ ] I have not done this task
-->

---------
Signed-off-by: 's avatarJames Tigue <166445701+jtigue-bdai@users.noreply.github.com>
Signed-off-by: 's avatarooctipus <zhengyuz@nvidia.com>
Co-authored-by: 's avatarEva M. <164949346+mmungai-bdai@users.noreply.github.com>
Co-authored-by: 's avatarJames Smith <142246516+jsmith-bdai@users.noreply.github.com>
Co-authored-by: 's avatarooctipus <zhengyuz@nvidia.com>
parent 017a1ee3
[package] [package]
# Note: Semantic Versioning is used: https://semver.org/ # Note: Semantic Versioning is used: https://semver.org/
version = "0.49.0" version = "0.49.1"
# Description # Description
title = "Isaac Lab framework for Robot Learning" title = "Isaac Lab framework for Robot Learning"
......
Changelog Changelog
--------- ---------
0.49.1 (2025-12-08)
~~~~~~~~~~~~~~~~~~~
Added
^^^^^
* Added write to file on close to :class:`~isaaclab.manager.RecorderManager`.
* Added :attr:`~isaaclab.manager.RecorderManagerCfg.export_in_close` configuration parameter.
0.49.0 (2025-11-10) 0.49.0 (2025-11-10)
~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~
......
...@@ -50,6 +50,9 @@ class RecorderManagerBaseCfg: ...@@ -50,6 +50,9 @@ class RecorderManagerBaseCfg:
export_in_record_pre_reset: bool = True export_in_record_pre_reset: bool = True
"""Whether to export episodes in the record_pre_reset call.""" """Whether to export episodes in the record_pre_reset call."""
export_in_close: bool = False
"""Whether to export episodes in the close call."""
class RecorderTerm(ManagerTermBase): class RecorderTerm(ManagerTermBase):
"""Base class for recorder terms. """Base class for recorder terms.
...@@ -132,6 +135,17 @@ class RecorderTerm(ManagerTermBase): ...@@ -132,6 +135,17 @@ class RecorderTerm(ManagerTermBase):
""" """
return None, None return None, None
def close(self, file_path: str):
"""Finalize and "clean up" the recorder term.
This can include tasks such as appending metadata (e.g. labels) to a file
and properly closing any associated file handles or resources.
Args:
file_path: the absolute path to the file
"""
pass
class RecorderManager(ManagerBase): class RecorderManager(ManagerBase):
"""Manager for recording data from recorder terms.""" """Manager for recording data from recorder terms."""
...@@ -202,15 +216,7 @@ class RecorderManager(ManagerBase): ...@@ -202,15 +216,7 @@ class RecorderManager(ManagerBase):
def __del__(self): def __del__(self):
"""Destructor for recorder.""" """Destructor for recorder."""
# Do nothing if no active recorder terms are provided self.close()
if len(self.active_terms) == 0:
return
if self._dataset_file_handler is not None:
self._dataset_file_handler.close()
if self._failed_episode_dataset_file_handler is not None:
self._failed_episode_dataset_file_handler.close()
""" """
Properties. Properties.
...@@ -519,6 +525,20 @@ class RecorderManager(ManagerBase): ...@@ -519,6 +525,20 @@ class RecorderManager(ManagerBase):
if self._failed_episode_dataset_file_handler is not None: if self._failed_episode_dataset_file_handler is not None:
self._failed_episode_dataset_file_handler.flush() self._failed_episode_dataset_file_handler.flush()
def close(self):
"""Closes the recorder manager by exporting any remaining data to file as well as properly closes the recorder terms."""
# Do nothing if no active recorder terms are provided
if len(self.active_terms) == 0:
return
if self._dataset_file_handler is not None:
if self.cfg.export_in_close:
self.export_episodes()
self._dataset_file_handler.close()
if self._failed_episode_dataset_file_handler is not None:
self._failed_episode_dataset_file_handler.close()
for term in self._terms.values():
term.close(os.path.join(self.cfg.dataset_export_dir_path, self.cfg.dataset_filename))
""" """
Helper functions. Helper functions.
""" """
...@@ -538,6 +558,7 @@ class RecorderManager(ManagerBase): ...@@ -538,6 +558,7 @@ class RecorderManager(ManagerBase):
"dataset_export_dir_path", "dataset_export_dir_path",
"dataset_export_mode", "dataset_export_mode",
"export_in_record_pre_reset", "export_in_record_pre_reset",
"export_in_close",
]: ]:
continue continue
# check if term config is None # check if term config is None
......
...@@ -14,6 +14,7 @@ simulation_app = AppLauncher(headless=True).app ...@@ -14,6 +14,7 @@ simulation_app = AppLauncher(headless=True).app
"""Rest everything follows.""" """Rest everything follows."""
import h5py
import os import os
import shutil import shutil
import tempfile import tempfile
...@@ -21,14 +22,20 @@ import torch ...@@ -21,14 +22,20 @@ import torch
import uuid import uuid
from collections import namedtuple from collections import namedtuple
from collections.abc import Sequence from collections.abc import Sequence
from typing import TYPE_CHECKING
import omni.usd
import pytest import pytest
from isaaclab.envs import ManagerBasedEnv from isaaclab.envs import ManagerBasedEnv, ManagerBasedEnvCfg
from isaaclab.managers import DatasetExportMode, RecorderManager, RecorderManagerBaseCfg, RecorderTerm, RecorderTermCfg from isaaclab.managers import DatasetExportMode, RecorderManager, RecorderManagerBaseCfg, RecorderTerm, RecorderTermCfg
from isaaclab.scene import InteractiveSceneCfg
from isaaclab.sim import SimulationContext from isaaclab.sim import SimulationContext
from isaaclab.utils import configclass from isaaclab.utils import configclass
if TYPE_CHECKING:
import numpy as np
class DummyResetRecorderTerm(RecorderTerm): class DummyResetRecorderTerm(RecorderTerm):
"""Dummy recorder term that records dummy data.""" """Dummy recorder term that records dummy data."""
...@@ -78,6 +85,72 @@ class DummyRecorderManagerCfg(RecorderManagerBaseCfg): ...@@ -78,6 +85,72 @@ class DummyRecorderManagerCfg(RecorderManagerBaseCfg):
dataset_export_mode = DatasetExportMode.EXPORT_ALL dataset_export_mode = DatasetExportMode.EXPORT_ALL
@configclass
class EmptyManagerCfg:
"""Empty manager specifications for the environment."""
pass
@configclass
class EmptySceneCfg(InteractiveSceneCfg):
"""Configuration for an empty scene."""
pass
def get_empty_base_env_cfg(device: str = "cuda", num_envs: int = 1, env_spacing: float = 1.0):
"""Generate base environment config based on device"""
@configclass
class EmptyEnvCfg(ManagerBasedEnvCfg):
"""Configuration for the empty test environment."""
# Scene settings
scene: EmptySceneCfg = EmptySceneCfg(num_envs=num_envs, env_spacing=env_spacing)
# Basic settings
actions: EmptyManagerCfg = EmptyManagerCfg()
observations: EmptyManagerCfg = EmptyManagerCfg()
recorders: EmptyManagerCfg = EmptyManagerCfg()
def __post_init__(self):
"""Post initialization."""
# step settings
self.decimation = 4 # env step every 4 sim steps: 200Hz / 4 = 50Hz
# simulation settings
self.sim.dt = 0.005 # sim step every 5ms: 200Hz
self.sim.render_interval = self.decimation # render every 4 sim steps
# pass device down from test
self.sim.device = device
return EmptyEnvCfg()
def get_file_contents(file_name: str, num_steps: int) -> dict[str, np.ndarray]:
"""Retrieves the contents of the hdf5 file
Args:
file_name: absolute path to the hdf5 file
num_steps: number of steps taken in the environment
Returns:
dict[str, np.ndarray]: dictionary where keys are HDF5 paths and values are the corresponding data arrays.
"""
data = {}
with h5py.File(file_name, "r") as f:
def get_data(name, obj):
if isinstance(obj, h5py.Dataset):
if "record_post_step" in name:
assert obj[()].shape == (num_steps, 5)
elif "record_pre_step" in name:
assert obj[()].shape == (num_steps, 4)
else:
raise Exception(f"The hdf5 file contains an unexpected data path, {name}")
data[name] = obj[()]
f.visititems(get_data)
return data
@configclass @configclass
class DummyEnvCfg: class DummyEnvCfg:
"""Dummy environment configuration.""" """Dummy environment configuration."""
...@@ -146,9 +219,9 @@ def test_initialize_dataset_file(dataset_dir): ...@@ -146,9 +219,9 @@ def test_initialize_dataset_file(dataset_dir):
assert os.path.exists(os.path.join(cfg.dataset_export_dir_path, cfg.dataset_filename)) assert os.path.exists(os.path.join(cfg.dataset_export_dir_path, cfg.dataset_filename))
def test_record(dataset_dir): @pytest.mark.parametrize("device", ("cpu", "cuda"))
def test_record(device, dataset_dir):
"""Test the recording of the data.""" """Test the recording of the data."""
for device in ("cuda:0", "cpu"):
env = create_dummy_env(device) env = create_dummy_env(device)
# create recorder manager # create recorder manager
cfg = DummyRecorderManagerCfg() cfg = DummyRecorderManagerCfg()
...@@ -179,3 +252,30 @@ def test_record(dataset_dir): ...@@ -179,3 +252,30 @@ def test_record(dataset_dir):
for env_id in range(env.num_envs): for env_id in range(env.num_envs):
episode = recorder_manager.get_episode(env_id) episode = recorder_manager.get_episode(env_id)
assert torch.stack(episode.data["record_post_reset"]).shape == (1, 3) assert torch.stack(episode.data["record_post_reset"]).shape == (1, 3)
@pytest.mark.parametrize("device", ("cpu", "cuda"))
def test_close(device, dataset_dir):
"""Test whether data is correctly exported in the close function when fully integrated with ManagerBasedEnv and
`export_in_close` is True."""
# create a new stage
omni.usd.get_context().new_stage()
# create environment
env_cfg = get_empty_base_env_cfg(device=device, num_envs=2)
cfg = DummyRecorderManagerCfg()
cfg.export_in_close = True
cfg.dataset_export_dir_path = dataset_dir
cfg.dataset_filename = f"{uuid.uuid4()}.hdf5"
env_cfg.recorders = cfg
env = ManagerBasedEnv(cfg=env_cfg)
num_steps = 3
for _ in range(num_steps):
act = torch.randn_like(env.action_manager.action)
obs, ext = env.step(act)
# check contents of hdf5 file
file_name = f"{env_cfg.recorders.dataset_export_dir_path}/{env_cfg.recorders.dataset_filename}"
data_pre_close = get_file_contents(file_name, num_steps)
assert len(data_pre_close) == 0
env.close()
data_post_close = get_file_contents(file_name, num_steps)
assert len(data_post_close.keys()) == 2 * env_cfg.scene.num_envs
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment