Skip to content

K
KincoActuatorIsaacLab
Unverified Commit ba0ac372 authored by Alexander Poddubny's avatar Alexander Poddubny Committed by GitHub
Browse files
Options

Fixes and improvements for CI pipeline (#546) 

- Fixing a minor bug in the compatibility pipeline
- Adding an HTML link to the comment with test results
- Adding log reporting for skipped tests
- Running the post-merge pipeline in DinD
parent 95e37603
Hide whitespace changes
Inline Side-by-side
Showing with 226 additions and 279 deletions
.github/actions/process-results/action.yml deleted 100644 → 0
View file @ 95e37603
# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
# All rights reserved.
#
# SPDX-License-Identifier: BSD-3-Clause
name: 'Process Test Results'
description: 'Processes XML test results and comments on PR with summary'
inputs:
results-file:
description: 'Path to the XML test results file'
required: true
default: 'reports/combined-results.xml'
github-token:
description: 'GitHub token for API access'
required: true
issue-number:
description: 'PR issue number to comment on'
required: true
repo-owner:
description: 'Repository owner'
required: true
repo-name:
description: 'Repository name'
required: true
runs:
using: composite
steps:
- name: Process Test Results and Comment on PR
uses: actions/github-script@v6
with:
github-token: ${{ inputs.github-token }}
script: |
const fs = require('fs');
let summary;
let shouldFail = false;
try {
// Read the test results XML
const xmlContent = fs.readFileSync('${{ inputs.results-file }}', 'utf8');
// Find all <testsuite ...> tags
const testsuitePattern = /<testsuite[^>]*>/g;
const testsuiteMatches = xmlContent.match(testsuitePattern);
if (testsuiteMatches && testsuiteMatches.length > 0) {
let totalTests = 0;
let totalFailures = 0;
let totalErrors = 0;
let totalSkipped = 0;
testsuiteMatches.forEach((tag, idx) => {
const testsMatch = tag.match(/tests="([^"]*)"/);
const failuresMatch = tag.match(/failures="([^"]*)"/);
const errorsMatch = tag.match(/errors="([^"]*)"/);
const skippedMatch = tag.match(/skipped="([^"]*)"/);
const tests = testsMatch ? parseInt(testsMatch[1]) : 0;
const failures = failuresMatch ? parseInt(failuresMatch[1]) : 0;
const errors = errorsMatch ? parseInt(errorsMatch[1]) : 0;
const skipped = skippedMatch ? parseInt(skippedMatch[1]) : 0;
totalTests += tests;
totalFailures += failures;
totalErrors += errors;
totalSkipped += skipped;
});
const passed = totalTests - totalFailures - totalErrors - totalSkipped;
summary = `## Combined Test Results\n\n- Total Tests: ${totalTests}\n- Passed: ${passed}\n- Failures: ${totalFailures}\n- Errors: ${totalErrors}\n- Skipped: ${totalSkipped}\n- Test Suites: ${testsuiteMatches.length}\n\n${totalFailures + totalErrors === 0 ? '✅ All tests passed!' : '❌ Some tests failed.'}\n\nDetailed test results are available in the workflow artifacts.`;
// Set flag to fail if there are any failures or errors
if (totalFailures > 0 || totalErrors > 0) {
console.error('❌ Tests failed or had errors');
shouldFail = true;
} else {
console.log('✅ All tests passed successfully');
}
} else {
summary = `## Combined Test Results\n❌ Could not parse XML structure. Raw content preview:\n\`\`\`\n${xmlContent.substring(0, 200)}...\n\`\`\`\n\nPlease check the workflow logs for more details.`;
shouldFail = true;
}
} catch (error) {
summary = `## Combined Test Results
❌ Failed to read test results: ${error.message}
Please check the workflow logs for more details.`;
shouldFail = true;
}
// Comment on the PR with the test results
await github.rest.issues.createComment({
issue_number: ${{ inputs.issue-number }},
owner: '${{ inputs.repo-owner }}',
repo: '${{ inputs.repo-name }}',
body: summary
});
// Fail the workflow after commenting if needed
if (shouldFail) {
process.exit(1);
}
.github/actions/run-tests/action.yml
View file @ ba0ac372
......@@ -36,7 +36,7 @@ runs:
using: composite
steps:
- name: Run Tests in Docker Container
shell: sh
shell: bash
run: |
# Function to run tests in Docker container
run_tests() {
......@@ -63,7 +63,15 @@ runs:
docker rm -f $container_name 2>/dev/null || true
# Build Docker environment variables
docker_env_vars="-e CUDA_VISIBLE_DEVICES=0 -e OMNI_KIT_ACCEPT_EULA=yes -e OMNI_KIT_DISABLE_CUP=1 -e ISAAC_SIM_HEADLESS=1 -e ISAAC_SIM_LOW_MEMORY=1 -e PYTHONUNBUFFERED=1 -e PYTHONIOENCODING=utf-8 -e TEST_RESULT_FILE=$result_file"
docker_env_vars="\
-e OMNI_KIT_ACCEPT_EULA=yes \
-e ACCEPT_EULA=Y \
-e OMNI_KIT_DISABLE_CUP=1 \
-e ISAAC_SIM_HEADLESS=1 \
-e ISAAC_SIM_LOW_MEMORY=1 \
-e PYTHONUNBUFFERED=1 \
-e PYTHONIOENCODING=utf-8 \
-e TEST_RESULT_FILE=$result_file"
if [ -n "$filter_pattern" ]; then
if [[ "$filter_pattern" == not* ]]; then
......@@ -87,8 +95,8 @@ runs:
if docker run --name $container_name \
--entrypoint bash --gpus all --network=host \
--security-opt=no-new-privileges:true \
--memory=$(echo "$(free -m | awk '/^Mem:/{print $2}') * 0.8 / 1" | bc)m \
--cpus=$(echo "$(nproc) * 0.8" | bc) \
--memory=$(echo "$(free -m | awk '/^Mem:/{print $2}') * 0.9 / 1" | bc)m \
--cpus=$(echo "$(nproc) * 0.9" | bc) \
--oom-kill-disable=false \
--ulimit nofile=65536:65536 \
--ulimit nproc=4096:4096 \
......@@ -98,12 +106,8 @@ runs:
set -e
cd /workspace/isaaclab
mkdir -p tests
echo 'Environment variables in container:'
echo 'TEST_FILTER_PATTERN: '\"'\"'$TEST_FILTER_PATTERN'\"'\"''
echo 'TEST_EXCLUDE_PATTERN: '\"'\"'$TEST_EXCLUDE_PATTERN'\"'\"''
echo 'TEST_RESULT_FILE: '\"'\"'$TEST_RESULT_FILE'\"'\"''
echo 'Starting pytest with path: $test_path'
/isaac-sim/python.sh -m pytest $test_path $pytest_options -v || echo 'Pytest completed with exit code: $?'
/isaac-sim/python.sh -m pytest --ignore=tools/conftest.py $test_path $pytest_options -v --junitxml=tests/$result_file || echo 'Pytest completed with exit code: $?'
"; then
echo "✅ Docker container completed successfully"
else
......
.github/workflows/build.yml
View file @ ba0ac372
......@@ -20,6 +20,7 @@ permissions:
contents: read
pull-requests: write
checks: write
issues: read
env:
NGC_API_KEY: ${{ secrets.NGC_API_KEY }}
......@@ -32,25 +33,8 @@ jobs:
runs-on: [self-hosted, gpu]
timeout-minutes: 180
continue-on-error: true
env:
CUDA_VISIBLE_DEVICES: all
NVIDIA_VISIBLE_DEVICES: all
NVIDIA_DRIVER_CAPABILITIES: all
CUDA_HOME: /usr/local/cuda
LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
DOCKER_HOST: unix:///var/run/docker.sock
DOCKER_TLS_CERTDIR: ""
container:
image: docker:dind
options: --gpus all --security-opt=no-new-privileges:true --privileged
steps:
- name: Install Git LFS
run: |
apk update
apk add --no-cache git-lfs
git lfs install
- name: Checkout Code
uses: actions/checkout@v4
with:
......@@ -74,11 +58,11 @@ jobs:
pytest-options: ""
filter-pattern: "isaaclab_tasks"
- name: Copy All Test Results from IsaacLab Tasks Container
- name: Copy Test Results from IsaacLab Tasks Container
run: |
CONTAINER_NAME="isaac-lab-tasks-test-$$"
if docker ps -a | grep -q $CONTAINER_NAME; then
echo "Copying all test results from IsaacLab Tasks container..."
echo "Copying test results from IsaacLab Tasks container..."
docker cp $CONTAINER_NAME:/workspace/isaaclab/tests/isaaclab-tasks-report.xml reports/ 2>/dev/null || echo "No test results to copy from IsaacLab Tasks container"
fi
......@@ -94,25 +78,8 @@ jobs:
test-general:
runs-on: [self-hosted, gpu]
timeout-minutes: 180
env:
CUDA_VISIBLE_DEVICES: all
NVIDIA_VISIBLE_DEVICES: all
NVIDIA_DRIVER_CAPABILITIES: all
CUDA_HOME: /usr/local/cuda
LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
DOCKER_HOST: unix:///var/run/docker.sock
DOCKER_TLS_CERTDIR: ""
container:
image: docker:dind
options: --gpus all --security-opt=no-new-privileges:true --privileged
steps:
- name: Install Git LFS
run: |
apk update
apk add --no-cache git-lfs
git lfs install
- name: Checkout Code
uses: actions/checkout@v4
with:
......@@ -136,11 +103,11 @@ jobs:
pytest-options: ""
filter-pattern: "not isaaclab_tasks"
- name: Copy All Test Results from General Tests Container
- name: Copy Test Results from General Tests Container
run: |
CONTAINER_NAME="isaac-lab-general-test-$$"
if docker ps -a | grep -q $CONTAINER_NAME; then
echo "Copying all test results from General Tests container..."
echo "Copying test results from General Tests container..."
docker cp $CONTAINER_NAME:/workspace/isaaclab/tests/general-tests-report.xml reports/ 2>/dev/null || echo "No test results to copy from General Tests container"
fi
......@@ -157,39 +124,17 @@ jobs:
needs: [test-isaaclab-tasks, test-general]
runs-on: [self-hosted, gpu]
if: always()
env:
DOCKER_HOST: unix:///var/run/docker.sock
DOCKER_TLS_CERTDIR: ""
container:
image: docker:dind
options: --security-opt=no-new-privileges:true --privileged
steps:
- name: Install Git LFS
run: |
apk update
apk add --no-cache git-lfs
git lfs install
- name: Checkout Code
uses: actions/checkout@v4
with:
fetch-depth: 0
lfs: false
- name: Configure Git Safe Directory
run: |
git config --global --add safe.directory ${{ github.workspace }}
git config --global --add safe.directory /workspace/isaaclab
git config --global --add safe.directory .
- name: Create Reports Directory
run: |
mkdir -p reports
chmod 777 reports
ls -la reports/
echo "Current user: $(whoami)"
echo "Current directory: $(pwd)"
- name: Download Test Results
uses: actions/download-artifact@v4
......@@ -219,20 +164,26 @@ jobs:
retention-days: 7
compression-level: 9
- name: Comment on Test Results
id: test-reporter
uses: EnricoMi/publish-unit-test-result-action@v2
with:
files: "reports/combined-results.xml"
check_name: "Tests Summary"
comment_mode: changes
comment_title: "Test Results Summary"
report_individual_runs: false
deduplicate_classes_by_file_name: true
compare_to_earlier_commit: true
fail_on: errors
action_fail_on_inconclusive: true
- name: Report Test Results
uses: dorny/test-reporter@v1
if: always()
with:
name: IsaacLab Tests
name: IsaacLab Build and Test Results
path: reports/combined-results.xml
reporter: java-junit
fail-on-error: false
- name: Process Combined Test Results
uses: ./.github/actions/process-results
with:
results-file: "reports/combined-results.xml"
github-token: ${{ secrets.GITHUB_TOKEN }}
issue-number: ${{ github.event.pull_request.number }}
repo-owner: ${{ github.repository_owner }}
repo-name: ${{ github.event.repository.name }}
fail-on-error: true
only-summary: false
report-title: "IsaacLab Test Results - ${{ github.workflow }}"
.github/workflows/daily-compatibility.yml
View file @ ba0ac372
......@@ -43,19 +43,8 @@ jobs:
NVIDIA_DRIVER_CAPABILITIES: all
CUDA_HOME: /usr/local/cuda
LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
DOCKER_HOST: unix:///var/run/docker.sock
DOCKER_TLS_CERTDIR: ""
container:
image: docker:dind
options: --gpus all --security-opt=no-new-privileges:true --privileged
steps:
- name: Install Git LFS
run: |
apk update
apk add --no-cache git-lfs
git lfs install
- name: Checkout Code
uses: actions/checkout@v3
with:
......@@ -105,19 +94,8 @@ jobs:
NVIDIA_DRIVER_CAPABILITIES: all
CUDA_HOME: /usr/local/cuda
LD_LIBRARY_PATH: /usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
DOCKER_HOST: unix:///var/run/docker.sock
DOCKER_TLS_CERTDIR: ""
container:
image: docker:dind
options: --gpus all --security-opt=no-new-privileges:true --privileged
steps:
- name: Install Git LFS
run: |
apk update
apk add --no-cache git-lfs
git lfs install
- name: Checkout Code
uses: actions/checkout@v3
with:
......@@ -162,20 +140,8 @@ jobs:
needs: [test-isaaclab-tasks-compat, test-general-compat]
runs-on: [self-hosted, gpu]
if: always()
env:
DOCKER_HOST: unix:///var/run/docker.sock
DOCKER_TLS_CERTDIR: ""
container:
image: docker:dind
options: --security-opt=no-new-privileges:true --privileged
steps:
- name: Install Git LFS
run: |
apk update
apk add --no-cache git-lfs
git lfs install
- name: Checkout Code
uses: actions/checkout@v3
with:
......@@ -185,10 +151,6 @@ jobs:
- name: Create Reports Directory
run: |
mkdir -p reports
chmod 777 reports
ls -la reports/
echo "Current user: $(whoami)"
echo "Current directory: $(pwd)"
- name: Download Test Results
uses: actions/download-artifact@v4
......@@ -219,30 +181,18 @@ jobs:
retention-days: 30
compression-level: 9
- name: Process Combined Test Results
uses: ./.github/actions/process-results
- name: Publish Test Results
uses: EnricoMi/publish-unit-test-result-action@v2
if: always()
with:
results-file: "reports/combined-compat-results.xml"
github-token: ${{ secrets.GH_TOKEN }}
issue-number: 0 # No PR for scheduled runs
repo-owner: ${{ github.repository_owner }}
repo-name: ${{ github.event.repository.name }}
files: "reports/combined-compat-results.xml"
notify-compatibility-status:
needs: [combine-compat-results]
runs-on: [self-hosted, gpu]
if: always()
container:
image: docker:dind
options: --security-opt=no-new-privileges:true --privileged
steps:
- name: Install Git LFS
run: |
apk update
apk add --no-cache git-lfs
git lfs install
- name: Checkout Code
uses: actions/checkout@v3
with:
......
.github/workflows/postmerge-ci.yml
View file @ ba0ac372
......@@ -27,13 +27,25 @@ env:
jobs:
build-and-push-images:
runs-on: self-hosted
runs-on: [self-hosted, gpu]
timeout-minutes: 180
environment:
name: postmerge-production
url: https://github.com/${{ github.repository }}
env:
DOCKER_HOST: unix:///var/run/docker.sock
DOCKER_TLS_CERTDIR: ""
container:
image: docker:dind
options: --security-opt=no-new-privileges:true --privileged
steps:
- name: Install Git LFS
run: |
apk update
apk add --no-cache git-lfs
git lfs install
- name: Checkout Code
uses: actions/checkout@v4
with:
......
tools/conftest.py
View file @ ba0ac372
......@@ -3,13 +3,20 @@
#
# SPDX-License-Identifier: BSD-3-Clause
import contextlib
import os
# Platform-specific imports for real-time output streaming
import select
import subprocess
import sys
import time
# Third-party imports
from prettytable import PrettyTable
import pytest
from junitparser import JUnitXml
from junitparser import Error, JUnitXml, TestCase, TestSuite
import tools.test_settings as test_settings
......@@ -19,6 +26,112 @@ def pytest_ignore_collect(collection_path, config):
return True
def capture_test_output_with_timeout(cmd, timeout, env):
"""Run a command with timeout and capture all output while streaming in real-time."""
stdout_data = b""
stderr_data = b""
try:
# Use Popen to capture output in real-time
process = subprocess.Popen(
cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=0, universal_newlines=False
)
# Set up file descriptors for non-blocking reads
stdout_fd = process.stdout.fileno()
stderr_fd = process.stderr.fileno()
# Set non-blocking mode (Unix systems only)
try:
import fcntl
for fd in [stdout_fd, stderr_fd]:
flags = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
except ImportError:
# fcntl not available on Windows, use a simpler approach
pass
start_time = time.time()
while process.poll() is None:
# Check for timeout
if time.time() - start_time > timeout:
process.kill()
try:
remaining_stdout, remaining_stderr = process.communicate(timeout=5)
stdout_data += remaining_stdout
stderr_data += remaining_stderr
except subprocess.TimeoutExpired:
process.terminate()
remaining_stdout, remaining_stderr = process.communicate(timeout=1)
stdout_data += remaining_stdout
stderr_data += remaining_stderr
return -1, stdout_data, stderr_data, True # -1 indicates timeout
# Check for available output
try:
ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], 0.1)
for fd in ready_fds:
with contextlib.suppress(OSError):
if fd == stdout_fd:
chunk = process.stdout.read(1024)
if chunk:
stdout_data += chunk
# Print to stdout in real-time
sys.stdout.buffer.write(chunk)
sys.stdout.buffer.flush()
elif fd == stderr_fd:
chunk = process.stderr.read(1024)
if chunk:
stderr_data += chunk
# Print to stderr in real-time
sys.stderr.buffer.write(chunk)
sys.stderr.buffer.flush()
except OSError:
# select failed, fall back to simple polling
time.sleep(0.1)
continue
# Get any remaining output
remaining_stdout, remaining_stderr = process.communicate()
stdout_data += remaining_stdout
stderr_data += remaining_stderr
return process.returncode, stdout_data, stderr_data, False
except Exception as e:
return -1, str(e).encode(), b"", False
def create_timeout_test_case(test_file, timeout, stdout_data, stderr_data):
"""Create a test case entry for a timeout test with captured logs."""
test_suite = TestSuite(name=f"timeout_{os.path.splitext(os.path.basename(test_file))[0]}")
test_case = TestCase(name="test_execution", classname=os.path.splitext(os.path.basename(test_file))[0])
# Create error message with timeout info and captured logs
error_msg = f"Test timed out after {timeout} seconds"
# Add captured output to error details
details = f"Timeout after {timeout} seconds\n\n"
if stdout_data:
details += "=== STDOUT ===\n"
details += stdout_data.decode("utf-8", errors="replace") + "\n"
if stderr_data:
details += "=== STDERR ===\n"
details += stderr_data.decode("utf-8", errors="replace") + "\n"
error = Error(message=error_msg)
error.text = details
test_case.result = error
test_suite.add_testcase(test_case)
return test_suite
def run_individual_tests(test_files, workspace_root):
"""Run each test file separately, ensuring one finishes before starting the next."""
failed_tests = []
......@@ -30,46 +143,54 @@ def run_individual_tests(test_files, workspace_root):
file_name = os.path.basename(test_file)
env = os.environ.copy()
try:
# Run each test file with pytest but skip collection
process = subprocess.run(
[
sys.executable,
"-m",
"pytest",
"--no-header",
f"--junitxml=tests/test-reports-{str(file_name)}.xml",
str(test_file),
"-v",
],
env=env,
timeout=(
test_settings.PER_TEST_TIMEOUTS[file_name]
if file_name in test_settings.PER_TEST_TIMEOUTS
else test_settings.DEFAULT_TIMEOUT
),
)
if process.returncode != 0:
failed_tests.append(test_file)
except subprocess.TimeoutExpired:
print(f"Test {test_file} timed out...")
# Determine timeout for this test
timeout = (
test_settings.PER_TEST_TIMEOUTS[file_name]
if file_name in test_settings.PER_TEST_TIMEOUTS
else test_settings.DEFAULT_TIMEOUT
)
# Prepare command
cmd = [
sys.executable,
"-m",
"pytest",
"--no-header",
f"--junitxml=tests/test-reports-{str(file_name)}.xml",
str(test_file),
"-v",
"--tb=short",
]
# Run test with timeout and capture output
returncode, stdout_data, stderr_data, timed_out = capture_test_output_with_timeout(cmd, timeout, env)
if timed_out:
print(f"Test {test_file} timed out after {timeout} seconds...")
failed_tests.append(test_file)
# Create a special XML report for timeout tests with captured logs
timeout_suite = create_timeout_test_case(test_file, timeout, stdout_data, stderr_data)
timeout_report = JUnitXml()
timeout_report.add_testsuite(timeout_suite)
# Write timeout report
report_file = f"tests/test-reports-{str(file_name)}.xml"
timeout_report.write(report_file)
test_status[test_file] = {
"errors": 1,
"failures": 0,
"skipped": 0,
"tests": 0,
"tests": 1,
"result": "TIMEOUT",
"time_elapsed": (
test_settings.PER_TEST_TIMEOUTS[file_name]
if file_name in test_settings.PER_TEST_TIMEOUTS
else test_settings.DEFAULT_TIMEOUT
),
"time_elapsed": timeout,
}
continue
if returncode != 0:
failed_tests.append(test_file)
# check report for any failures
report_file = f"tests/test-reports-{str(file_name)}.xml"
if not os.path.exists(report_file):
......@@ -87,12 +208,23 @@ def run_individual_tests(test_files, workspace_root):
try:
report = JUnitXml.fromfile(report_file)
# Parse the integer values
errors = int(report.errors)
failures = int(report.failures)
skipped = int(report.skipped)
tests = int(report.tests)
time_elapsed = float(report.time)
# Rename test suites to be more descriptive
for suite in report:
if suite.name == "pytest":
# Remove .py extension and use the filename as the test suite name
suite_name = os.path.splitext(file_name)[0]
suite.name = suite_name
# Write the updated report back
report.write(report_file)
# Parse the integer values with None handling
errors = int(report.errors) if report.errors is not None else 0
failures = int(report.failures) if report.failures is not None else 0
skipped = int(report.skipped) if report.skipped is not None else 0
tests = int(report.tests) if report.tests is not None else 0
time_elapsed = float(report.time) if report.time is not None else 0.0
except Exception as e:
print(f"Error reading test report {report_file}: {e}")
failed_tests.append(test_file)
......@@ -270,3 +402,6 @@ def pytest_sessionstart(session):
# Print summary to console and log file
print(summary_str)
# Exit pytest after custom execution to prevent normal pytest from overwriting our report
pytest.exit("Custom test execution completed", returncode=0)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment