Skip to content

CICD NeMo RL

CICD NeMo RL #13663

Workflow file for this run

# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: "CICD NeMo RL"
on:
push:
branches:
- main
- "pull-request/[0-9]+"
schedule:
- cron: "0 9 * * *"
workflow_dispatch:
inputs:
test_to_run:
required: false
default: L2
type: choice
options:
- docs
- Lfast
- L0
- L1
- L2
description: Test level to run. docs = doc tests only (reuses main container), Lfast = fast subset (reuses main container), L0 = unit/docs/lint, L1 = L0 + functional, L2 = L1 + convergence
image_tag:
description: "Override container image tag (e.g. 'main'). Skips container build."
required: false
default: ""
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name || 'main' }}-${{ github.event_name }}
cancel-in-progress: true
env:
GB200_CONTAINER_REGISTRY: ${{ vars.GB200_CONTAINER_REGISTRY }}
jobs:
pre-flight:
runs-on: ubuntu-latest
outputs:
test_level: ${{ steps.evaluate.outputs.test_level }}
image_tag: ${{ steps.evaluate.outputs.image_tag }}
base_ref: ${{ steps.base-head-ref.outputs.base_ref }}
base_sha: ${{ steps.base-head-ref.outputs.base_sha }}
head_ref: ${{ steps.base-head-ref.outputs.head_ref }}
head_sha: ${{ steps.base-head-ref.outputs.head_sha }}
head_label: ${{ steps.base-head-ref.outputs.head_label }}
has_skip_cicd: ${{ steps.base-head-ref.outputs.has_skip_cicd }}
test_sha: ${{ steps.base-head-ref.outputs.test_sha }}
steps:
- name: Get PR info
id: get-pr-info
if: startsWith(github.ref, 'refs/heads/pull-request/')
uses: nv-gha-runners/get-pr-info@main
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Determine base and head references
id: base-head-ref
env:
IS_PULL_REQUEST_REF: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }}
PR_INFO_JSON: ${{ steps.get-pr-info.outputs.pr-info }}
run: |
if [[ "$IS_PULL_REQUEST_REF" == "true" && -n "$PR_INFO_JSON" ]]; then
base_ref=$(echo "$PR_INFO_JSON" | jq -r '.base.ref')
base_sha=$(echo "$PR_INFO_JSON" | jq -r '.base.sha')
head_ref=$(echo "$PR_INFO_JSON" | jq -r '.head.ref')
head_sha=$(echo "$PR_INFO_JSON" | jq -r '.head.sha')
test_sha=$(echo "$PR_INFO_JSON" | jq -r '.merge_commit_sha')
head_label=$(echo "$PR_INFO_JSON" | jq -r '.head.label // empty')
ci_label=$(echo "$PR_INFO_JSON" | jq -r '[.labels[]? | (if type == "string" then . else .name end) | select(startswith("CI:"))] | first // empty')
has_skip_cicd=$(echo "$PR_INFO_JSON" | jq -r '[.labels[]? | (if type == "string" then . else .name end) | select(. == "Skip CICD")] | length > 0')
else
base_ref="HEAD~1"
base_sha=$(git rev-parse HEAD~1)
head_ref="HEAD"
head_sha="${{ github.sha }}"
test_sha="${{ github.sha }}"
head_label="${{ github.ref_name }}"
ci_label=""
has_skip_cicd="false"
fi
[[ "$has_skip_cicd" != "true" ]] && has_skip_cicd="false"
echo "base_ref=$base_ref" >> "$GITHUB_OUTPUT"
echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT"
echo "head_ref=$head_ref" >> "$GITHUB_OUTPUT"
echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT"
echo "head_label=$head_label" >> "$GITHUB_OUTPUT"
echo "ci_label=$ci_label" >> "$GITHUB_OUTPUT"
echo "has_skip_cicd=$has_skip_cicd" >> "$GITHUB_OUTPUT"
- name: Get changed files
id: changed-files
if: startsWith(github.ref, 'refs/heads/pull-request/')
uses: step-security/changed-files@v45.0.1
with:
base_sha: ${{ steps.base-head-ref.outputs.base_sha }}
files_yaml: |
doc:
- '**.md'
- docs/**
src:
- '!**.md'
- '!docs/**'
- name: Evaluate conditions
id: evaluate
env:
DOCS_ONLY: ${{ steps.changed-files.outputs.doc_any_changed == 'true' && steps.changed-files.outputs.src_any_changed == 'false' }}
CHANGED_DOCS: ${{ steps.changed-files.outputs.doc_all_changed_files }}
CHANGED_SRC: ${{ steps.changed-files.outputs.src_all_changed_files }}
IS_PULLREQUEST: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }}
LABEL: ${{ steps.base-head-ref.outputs.ci_label }}
MERGE_GROUP: ${{ github.event_name == 'merge_group' }}
run: |
# Some output that's helpful for debugging
echo "Docs changed: $CHANGED_DOCS"
echo "Src changed: $CHANGED_SRC"
echo "LABEL: $LABEL"
echo "IS_PULLREQUEST: $IS_PULLREQUEST"
echo "DOCS_ONLY: $DOCS_ONLY"
# Run CI only (on main or if label is attached) and if it's not only docs
# Determine test level based on conditions
if [[ "$DOCS_ONLY" == "true" || "$LABEL" == "CI:docs" ]]; then
# For doc-only changes, run only doc tests
TEST_LEVEL="docs"
elif [[ "$LABEL" == "CI:Lfast" ]]; then
TEST_LEVEL="Lfast"
elif [[ "$LABEL" == "CI:L0" ]]; then
TEST_LEVEL="L0"
elif [[ "$LABEL" == "CI:L1" || "$IS_PULLREQUEST" == "false" || "$MERGE_GROUP" == "true" ]]; then
# For labeled PRs, pushes to main (IS_PULL_REQUEST=false), or merge group events, run L1 by default
TEST_LEVEL="L1"
elif [[ "$LABEL" == "CI:L2" ]]; then
TEST_LEVEL="L2"
else
# Skip tests by default for non-labeled PRs
TEST_LEVEL="none"
fi
if [[ "${{ github.event_name }}" == "schedule" ]]; then
echo "Setting test level to L1 for nightly scheduled run"
TEST_LEVEL="L1"
fi
# Override test level if specified in workflow_dispatch
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
echo "Overriding test level from $TEST_LEVEL to ${{ inputs.test_to_run }}"
TEST_LEVEL="${{ inputs.test_to_run }}"
fi
echo "test_level=$TEST_LEVEL" | tee -a "$GITHUB_OUTPUT"
# Determine image tag: Lfast and docs reuse the main container, workflow_dispatch can override
IMAGE_TAG=""
if [[ "$TEST_LEVEL" == "Lfast" || "$TEST_LEVEL" == "docs" ]]; then
IMAGE_TAG="main"
fi
if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ inputs.image_tag }}" ]]; then
IMAGE_TAG="${{ inputs.image_tag }}"
fi
echo "image_tag=$IMAGE_TAG" | tee -a "$GITHUB_OUTPUT"
org-member-pre-flight:
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.80.1
with:
default_runner_prefix: ${{ vars.DEFAULT_H100_RUNNER }}
non_nvidia_runner_prefix: ${{ vars.NON_NVIDIA_H100_RUNNER }}
default_test_data_path: ${{ vars.DEFAULT_H100_TEST_DATA_PATH }}
non_nvidia_test_data_path: ${{ vars.NON_NVIDIA_H100_TEST_DATA_PATH }}
default_registry: ${{ vars.DEFAULT_H100_CONTAINER_REGISTRY }}
non_nvidia_registry: ${{ vars.NON_NVIDIA_H100_CONTAINER_REGISTRY }}
sso_users_filename: ${{ vars.SSO_USERS_FILENAME }}
secrets:
NVIDIA_MANAGEMENT_ORG_PAT: ${{ secrets.NVIDIA_MANAGEMENT_ORG_PAT }}
gb200-config:
runs-on: ubuntu-latest
outputs:
registry: ${{ steps.config.outputs.registry }}
steps:
- name: Configure GB200 registry
id: config
env:
GB200_REGISTRY: ${{ env.GB200_CONTAINER_REGISTRY }}
run: echo "registry=$GB200_REGISTRY" | tee -a "$GITHUB_OUTPUT"
pr-branch-up-to-date-check:
name: Check if PR branch is up to date
needs: [pre-flight]
if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }}
runs-on: ubuntu-latest
env:
MAX_COMMITS_BEHIND: 10
steps:
- name: Check how many commits behind target branch
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
BASE_SHA: ${{ needs.pre-flight.outputs.base_sha }}
HEAD_SHA: ${{ needs.pre-flight.outputs.head_sha }}
BASE_REF: ${{ needs.pre-flight.outputs.base_ref }}
HEAD_LABEL: ${{ needs.pre-flight.outputs.head_label }}
run: |
echo "Repository: $REPO"
echo "Base branch: $BASE_REF (SHA: $BASE_SHA)"
echo "PR head: $HEAD_LABEL (SHA: $HEAD_SHA)"
echo "Maximum commits behind allowed: $MAX_COMMITS_BEHIND"
API_RESPONSE=$(gh api "repos/$REPO/compare/$HEAD_SHA...$BASE_REF" --jq '{behind_by: .behind_by, ahead_by: .ahead_by, status: .status}')
COMMITS_BEHIND=$(echo "$API_RESPONSE" | jq -r '.ahead_by')
COMMITS_AHEAD=$(echo "$API_RESPONSE" | jq -r '.behind_by')
STATUS=$(echo "$API_RESPONSE" | jq -r '.status')
echo "Comparison status: $STATUS"
echo "PR is $COMMITS_BEHIND commits behind and $COMMITS_AHEAD commits ahead of $BASE_REF"
# Check if we're behind by more than the allowed number
if [ "$COMMITS_BEHIND" -gt "$MAX_COMMITS_BEHIND" ]; then
echo "❌ ERROR: This PR is $COMMITS_BEHIND commits behind $BASE_REF, which exceeds the maximum allowed ($MAX_COMMITS_BEHIND commits)."
echo "Please rebase or merge the latest changes from $BASE_REF into your PR branch."
exit 1
else
echo "✅ PR is acceptably fresh ($COMMITS_BEHIND commits behind, limit is $MAX_COMMITS_BEHIND)"
fi
lint-check:
name: Lint check
needs: [pre-flight]
runs-on: ubuntu-latest
steps:
- name: Free up disk space
run: |
# Remove unnecessary packages and files on Ubuntu
sudo apt-get clean
sudo rm -rf /usr/local/lib/android || true
sudo rm -rf /opt/ghc || true
sudo rm -rf /usr/local/.ghcup || true
sudo rm -rf /usr/share/dotnet || true
sudo rm -rf /opt/az || true
# Clear pip and npm caches
pip cache purge || true
sudo npm cache clean --force || true
- name: Checkout repository
uses: actions/checkout@v6
with:
submodules: "recursive"
- name: Install uv
uses: astral-sh/setup-uv@v5
with:
version: "0.11.18"
enable-cache: true
prune-cache: false
# Faster than uv python install since it caches python alongside runner
- name: "Set up Python"
uses: actions/setup-python@v6
with:
python-version-file: ".python-version"
- name: Check lint
run: |
uv venv
uv run --group dev pre-commit install
uv run --group dev pre-commit run --all-files --show-diff-on-failure --color=always
# TODO: this is a temporary check and should be removed once we have 100% correctness
- name: Check if any files with zero errors not in whitelist
run: |
missing_count=0
for file in $(uv run --group dev pyrefly check $(git ls-files 'nemo_rl/**/*.py' 'examples/**/*.py' 'docs/*.py' 'tools/**/*.py') --output-format json | jq -r --slurpfile all_files <(git ls-files 'nemo_rl/**/*.py' 'examples/**/*.py' 'docs/*.py' 'tools/**/*.py' | jq -R -s 'split("\n")[:-1]') --arg pwd "$(pwd)/" '(.errors | group_by(.path) | map({(.[0].path | sub($pwd; "")): length}) | add // {}) as $error_counts | $all_files[0][] | . as $file | if ($error_counts[$file] // 0) == 0 then $file else empty end'); do
if ! fgrep -q "$file" pyrefly.toml; then
echo "File $file has zero errors but is not in pyrefly.toml in the 'project-includes' list. Please add it to this whitelist."
((missing_count++))
fi
done
exit $missing_count
- name: Minimize uv cache
run: uv cache prune --ci
cicd-wait-in-queue:
name: Wait in test approval queue
needs: [pre-flight, lint-check]
runs-on: ubuntu-latest
environment:
name: test
deployment: false
if: >-
${{
always() &&
startsWith(github.ref, 'refs/heads/pull-request/') &&
contains('Lfast L0 L1 L2', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.result == 'success' &&
needs.lint-check.result == 'success' &&
!cancelled()
}}
steps:
- name: Approved
run: echo "Approved to run CI tests."
sphinx-build:
needs: [pre-flight, cicd-wait-in-queue]
if: >-
${{
always() &&
needs.pre-flight.result == 'success' &&
needs.pre-flight.outputs.test_level != 'none' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
needs.pre-flight.outputs.test_level == 'docs' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
!cancelled()
}}
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_docs.yml@v0.57.0
build-container:
name: Build H100 container
if: >-
${{
always() &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
needs.pre-flight.outputs.test_level != 'none' &&
needs.pre-flight.outputs.image_tag == '' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
!cancelled()
}}
needs: [pre-flight, org-member-pre-flight, cicd-wait-in-queue]
permissions:
contents: read
pull-requests: read
uses: ./.github/workflows/_build_container.yml
with:
build-ref: ${{ needs.pre-flight.outputs.test_sha }}
image-name: ${{ vars.CI_CONTAINER_NAME }}
dockerfile: docker/Dockerfile
platform: linux/amd64
registry: ${{ needs.org-member-pre-flight.outputs.registry }}
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
target: release
build-contexts: |
nemo-rl=.
${{ vars.UV_BUILD_CACHE == 'enabled' && format('uv-cache-seed=docker-image://{0}/{1}:uv-cache', needs.org-member-pre-flight.outputs.registry, vars.CI_CONTAINER_NAME) || '' }}
trtllm-ccache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-ccache' || '' }}
trtllm-wheel-cache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-wheel-cache' || '' }}
build-args: |
MAX_JOBS=4
TRTLLM_BUILD_JOBS=24
NEMO_RL_COMMIT=${{ needs.pre-flight.outputs.test_sha }}
build-container-gb200:
name: Build GB200/GCP container
if: >-
${{
always() &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
needs.gb200-config.result == 'success' &&
needs.pre-flight.outputs.test_level != 'none' &&
needs.pre-flight.outputs.image_tag == '' &&
needs.org-member-pre-flight.outputs.is_member == 'true' &&
vars.DISABLE_GB200_TESTS != 'true' &&
contains('L1 L2', needs.pre-flight.outputs.test_level) &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
!cancelled()
}}
needs: [pre-flight, org-member-pre-flight, gb200-config, cicd-wait-in-queue]
permissions:
contents: read
pull-requests: read
uses: ./.github/workflows/_build_container.yml
with:
build-ref: ${{ needs.pre-flight.outputs.test_sha }}
image-name: ${{ vars.CI_CONTAINER_NAME }}
dockerfile: docker/Dockerfile
platform: linux/arm64
registry: ${{ needs.gb200-config.outputs.registry }}
runner: ${{ vars.GB200_RUNNER }}
target: release
build-contexts: |
nemo-rl=.
${{ vars.UV_BUILD_CACHE == 'enabled' && format('uv-cache-seed=docker-image://{0}/{1}:uv-cache', needs.gb200-config.outputs.registry, vars.CI_CONTAINER_NAME) || '' }}
trtllm-ccache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-ccache' || '' }}
trtllm-wheel-cache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-wheel-cache' || '' }}
build-args: |
MAX_JOBS=4
TRTLLM_BUILD_JOBS=8
NEMO_RL_COMMIT=${{ needs.pre-flight.outputs.test_sha }}
update-uv-cache:
name: Update uv build cache
needs: [build-container, org-member-pre-flight]
if: >-
${{
github.ref == 'refs/heads/main' &&
vars.UV_BUILD_CACHE == 'enabled' &&
needs.build-container.result == 'success'
}}
runs-on: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
env:
REGISTRY: ${{ needs.org-member-pre-flight.outputs.registry }}
IMAGE_NAME: ${{ vars.CI_CONTAINER_NAME }}
steps:
- name: Extract and push uv cache image
run: |
set -euo pipefail
SRC="${REGISTRY}/${IMAGE_NAME}:${{ github.run_id }}"
DST="${REGISTRY}/${IMAGE_NAME}:uv-cache"
docker pull "${SRC}"
CID=$(docker create "${SRC}" true)
mkdir -p /tmp/uv-cache
docker cp "${CID}:/root/.cache/uv/." /tmp/uv-cache/
docker rm "${CID}"
printf 'FROM scratch\nCOPY uv-cache/ /\n' > /tmp/Dockerfile.uv-cache
docker build -t "${DST}" -f /tmp/Dockerfile.uv-cache /tmp
docker push "${DST}"
docker rmi "${SRC}" "${DST}" 2>/dev/null || true
rm -rf /tmp/uv-cache /tmp/Dockerfile.uv-cache
update-uv-cache-gb200:
name: Update GB200 uv build cache
needs: [build-container-gb200, gb200-config]
if: >-
${{
github.ref == 'refs/heads/main' &&
vars.UV_BUILD_CACHE == 'enabled' &&
needs.build-container-gb200.result == 'success'
}}
runs-on: ${{ vars.GB200_RUNNER }}
env:
REGISTRY: ${{ needs.gb200-config.outputs.registry }}
IMAGE_NAME: ${{ vars.CI_CONTAINER_NAME }}
steps:
- name: Extract and push uv cache image
run: |
set -euo pipefail
SRC="${REGISTRY}/${IMAGE_NAME}:${{ github.run_id }}"
DST="${REGISTRY}/${IMAGE_NAME}:uv-cache"
docker pull "${SRC}"
CID=$(docker create "${SRC}" true)
mkdir -p /tmp/uv-cache
docker cp "${CID}:/root/.cache/uv/." /tmp/uv-cache/
docker rm "${CID}"
printf 'FROM scratch\nCOPY uv-cache/ /\n' > /tmp/Dockerfile.uv-cache
docker build -t "${DST}" -f /tmp/Dockerfile.uv-cache /tmp
docker push "${DST}"
docker rmi "${SRC}" "${DST}" 2>/dev/null || true
rm -rf /tmp/uv-cache /tmp/Dockerfile.uv-cache
cicd-doc-tests:
strategy:
fail-fast: false
matrix:
include:
- script: Docs_Tests
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
needs: [pre-flight, build-container, org-member-pre-flight, cicd-wait-in-queue]
if: >-
${{
(
always() &&
contains('docs Lfast L0 L1 L2', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
needs.pre-flight.outputs.test_level == 'docs' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
(needs.build-container.result == 'success' || needs.build-container.result == 'skipped')
) && !cancelled()
}}
runs-on: ${{ matrix.runner }}
name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
steps:
- name: Checkout
uses: actions/checkout@v6
- name: main
uses: ./.github/actions/test-template
with:
runner: ${{ matrix.runner }}
registry: ${{ needs.org-member-pre-flight.outputs.registry }}
image: ${{ vars.CI_CONTAINER_NAME }}
image-tag: ${{ needs.pre-flight.outputs.image_tag }}
test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }}
script: ${{ matrix.script }}
is_doc_test: "true"
test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }}
cicd-unit-tests:
strategy:
fail-fast: false
max-parallel: 16
matrix:
include:
- script: L0_Unit_Tests_Vllm_1
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Vllm_2
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Vllm_3
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Sglang
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Trtllm
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Mcore
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Mcore_Policy_1
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Mcore_Policy_2
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Mcore_Policy_3
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Automodel
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Automodel_Policy_1
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Automodel_Policy_2
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Automodel_Policy_3
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Models_1
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Models_2
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Models_3
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Models_4
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Environments
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Nemo_Gym
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Algorithms
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Data
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Distributed
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L0_Unit_Tests_Other
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
needs: [pre-flight, build-container, cicd-doc-tests, org-member-pre-flight, cicd-wait-in-queue]
if: >-
${{
(
always() &&
contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
(needs.build-container.result == 'success' || needs.build-container.result == 'skipped') &&
(needs.cicd-doc-tests.result == 'success' || needs.cicd-doc-tests.result == 'skipped')
) && !cancelled()
}}
runs-on: ${{ matrix.runner }}
name: ${{ matrix.script }}
steps:
- name: Checkout
uses: actions/checkout@v6
- name: main
# Lfast reuses the main image. Until TRT-LLM lands on main, running this
# shard would make prefetch_venvs compile the cp313 wheel from source.
if: ${{ needs.pre-flight.outputs.test_level != 'Lfast' || matrix.script != 'L0_Unit_Tests_Trtllm' }}
uses: ./.github/actions/test-template
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
with:
runner: ${{ matrix.runner }}
script: ${{ matrix.script }}
registry: ${{ needs.org-member-pre-flight.outputs.registry }}
test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }}
image: ${{ vars.CI_CONTAINER_NAME }}
image-tag: ${{ needs.pre-flight.outputs.image_tag }}
is_unit_test: "true"
cpu-only: ${{ matrix.cpu-only || false }}
test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }}
unit-test-script-check:
name: Check unit test script coverage
needs: [pre-flight, cicd-wait-in-queue]
if: >-
${{
always() &&
contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.result == 'success' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
!cancelled()
}}
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ needs.pre-flight.outputs.test_sha }}
- name: Verify L0 unit scripts are in the workflow
run: |
set -euo pipefail
expected=$(mktemp)
configured=$(mktemp)
find tests/unit -maxdepth 1 -type f -name 'L0_Unit*.sh' \
-exec basename {} .sh \; | sort -u > "$expected"
{
grep -E '^[[:space:]]*-[[:space:]]*script:[[:space:]]*L0_Unit' .github/workflows/cicd-main.yml || true
} | sed -E 's/^[[:space:]]*-[[:space:]]*script:[[:space:]]*//' | sort -u > "$configured"
missing=$(comm -23 "$expected" "$configured")
if [[ -n "$missing" ]]; then
echo "The following tests/unit/L0_Unit*.sh scripts are missing from .github/workflows/cicd-main.yml:"
printf '%s\n' "$missing"
exit 1
fi
echo "All L0 unit scripts are included in .github/workflows/cicd-main.yml."
functional-test-script-check:
name: Check functional test script coverage
needs: [pre-flight, cicd-wait-in-queue]
if: >-
${{
always() &&
contains('L1 L2 Lfast', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.result == 'success' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
!cancelled()
}}
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@v6
with:
ref: ${{ needs.pre-flight.outputs.test_sha }}
- name: Verify L1 functional scripts are in the workflow
run: |
set -euo pipefail
expected=$(mktemp)
configured=$(mktemp)
find tests/functional -maxdepth 1 -type f -name 'L1_Functional*.sh' \
-exec basename {} .sh \; | sort -u > "$expected"
{
grep -E '^[[:space:]]*-[[:space:]]*script:[[:space:]]*L1_Functional' .github/workflows/cicd-main.yml || true
} | sed -E 's/^[[:space:]]*-[[:space:]]*script:[[:space:]]*//' | sort -u > "$configured"
missing=$(comm -23 "$expected" "$configured")
if [[ -n "$missing" ]]; then
echo "The following tests/functional/L1_Functional*.sh scripts are missing from .github/workflows/cicd-main.yml:"
printf '%s\n' "$missing"
exit 1
fi
echo "All L1 functional scripts are included in .github/workflows/cicd-main.yml."
cicd-functional-tests:
strategy:
fail-fast: false
max-parallel: 16
matrix:
include:
- script: L1_Functional_Tests_Megatron_1
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Megatron_2
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Megatron_3
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Megatron_4
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_AutoModel
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_SGLang
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Trtllm
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Gym
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_GRPO_1
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_GRPO_2
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_GRPO_3
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_SFT
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_PPO
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Eval
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Other_1
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Other_2
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
needs: [pre-flight, build-container, cicd-unit-tests, functional-test-script-check, org-member-pre-flight, cicd-wait-in-queue]
runs-on: ${{ matrix.runner }}
if: >-
${{
always() &&
contains('L1 L2', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
(needs.build-container.result == 'success' || needs.build-container.result == 'skipped') &&
needs.cicd-unit-tests.result == 'success' &&
needs.functional-test-script-check.result == 'success' &&
!cancelled()
}}
name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
steps:
- name: Checkout
uses: actions/checkout@v6
- name: main
uses: ./.github/actions/test-template
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
with:
runner: ${{ matrix.runner }}
registry: ${{ needs.org-member-pre-flight.outputs.registry }}
image: ${{ vars.CI_CONTAINER_NAME }}
test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }}
script: ${{ matrix.script }}
test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }}
cicd-functional-tests-gb200:
strategy:
fail-fast: false
max-parallel: 16
matrix:
include:
- script: L1_Functional_Tests_Megatron_1
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_Megatron_2
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_Megatron_3
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_Megatron_4
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_AutoModel
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_SGLang
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_Gym
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_GRPO_1
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_GRPO_2
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_GRPO_3
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_SFT
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_PPO
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_Eval
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_Other_1
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_Other_2
runner: ${{ vars.GB200_RUNNER }}
- script: L1_Functional_Tests_GB200_MXFP8
runner: ${{ vars.GB200_RUNNER }}
needs: [pre-flight, build-container-gb200, cicd-unit-tests, functional-test-script-check, org-member-pre-flight, gb200-config, cicd-wait-in-queue]
runs-on: ${{ matrix.runner }}
if: >-
${{
always() &&
contains('L1 L2', needs.pre-flight.outputs.test_level) &&
needs.org-member-pre-flight.outputs.is_member == 'true' &&
vars.DISABLE_GB200_TESTS != 'true' &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
(needs.build-container-gb200.result == 'success' || needs.build-container-gb200.result == 'skipped') &&
needs.cicd-unit-tests.result == 'success' &&
needs.functional-test-script-check.result == 'success' &&
!cancelled()
}}
name: gb200_${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }}
steps:
- name: Checkout
uses: actions/checkout@v6
- name: main
uses: ./.github/actions/test-template
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
with:
runner: ${{ matrix.runner }}
registry: ${{ needs.gb200-config.outputs.registry }}
image: ${{ vars.CI_CONTAINER_NAME }}
test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }}
image-tag: ${{ needs.pre-flight.outputs.image_tag }}
script: ${{ matrix.script }}
test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }}
cicd-fast-functional-tests:
strategy:
fail-fast: false
matrix:
include:
- script: L1_Functional_Tests_Megatron_1
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Megatron_2
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Megatron_3
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Megatron_4
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_AutoModel
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_SGLang
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Gym
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_GRPO_1
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_GRPO_2
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_GRPO_3
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_SFT
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_PPO
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Eval
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Other_1
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
- script: L1_Functional_Tests_Other_2
runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }}
needs: [pre-flight, functional-test-script-check, org-member-pre-flight, cicd-wait-in-queue]
if: >-
${{
always() &&
contains('Lfast', needs.pre-flight.outputs.test_level) &&
needs.pre-flight.result == 'success' &&
needs.org-member-pre-flight.result == 'success' &&
(
needs.cicd-wait-in-queue.result == 'success' ||
!startsWith(github.ref, 'refs/heads/pull-request/')
) &&
needs.functional-test-script-check.result == 'success' &&
!cancelled()
}}
runs-on: ${{ matrix.runner }}
name: fast_${{ matrix.script }}
steps:
- name: Checkout
uses: actions/checkout@v6
- name: main
uses: ./.github/actions/test-template
env:
HF_TOKEN: ${{ secrets.HF_TOKEN }}
with:
runner: ${{ matrix.runner }}
script: ${{ matrix.script }}
image-tag: ${{ needs.pre-flight.outputs.image_tag }}
registry: ${{ needs.org-member-pre-flight.outputs.registry }}
image: ${{ vars.CI_CONTAINER_NAME }}
test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }}
test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }}
CI_QA_Gate:
name: "CI quality check${{ needs.pre-flight.outputs.test_level == 'none' && ' (No tests run: Label CI:L*)' || '' }}"
if: always()
runs-on: ubuntu-latest
needs:
- pre-flight
- org-member-pre-flight
- pr-branch-up-to-date-check
- lint-check
- cicd-wait-in-queue
- sphinx-build
- build-container
- build-container-gb200
- cicd-doc-tests
- cicd-unit-tests
- unit-test-script-check
- functional-test-script-check
- cicd-functional-tests
- cicd-functional-tests-gb200
- cicd-fast-functional-tests
steps:
- name: main
env:
JOB_RESULTS: ${{ toJSON(needs) }}
# Job is considered successful if nothing was run, or if all jobs were successful (the tests run even if only docs were run b/c doctests are selected)
ALL_SUCCESS: >-
${{
needs.lint-check.result == 'success' &&
(needs.cicd-wait-in-queue.result == 'success' || needs.cicd-wait-in-queue.result == 'skipped') &&
(needs.pr-branch-up-to-date-check.result == 'success' || needs.pr-branch-up-to-date-check.result == 'skipped') &&
(
needs.pre-flight.outputs.test_level != 'none' &&
needs.sphinx-build.result == 'success' &&
(needs.build-container.result == 'success' || needs.build-container.result == 'skipped') &&
(needs.build-container-gb200.result == 'success' || needs.build-container-gb200.result == 'skipped') &&
(
(
(needs.cicd-doc-tests.result == 'success' || needs.cicd-doc-tests.result == 'skipped') &&
(
!contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) ||
needs.cicd-unit-tests.result == 'success'
) &&
(
!contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) ||
needs.unit-test-script-check.result == 'success'
) &&
(
!contains('L1 L2 Lfast', needs.pre-flight.outputs.test_level) ||
needs.functional-test-script-check.result == 'success'
) &&
(
!contains('L1 L2', needs.pre-flight.outputs.test_level) ||
needs.cicd-functional-tests.result == 'success'
) &&
(
needs.org-member-pre-flight.outputs.is_member != 'true' ||
vars.DISABLE_GB200_TESTS == 'true' ||
!contains('L1 L2', needs.pre-flight.outputs.test_level) ||
needs.cicd-functional-tests-gb200.result == 'success'
) &&
(
!contains('Lfast', needs.pre-flight.outputs.test_level) ||
needs.cicd-fast-functional-tests.result == 'success'
)
)
)
)
}}
CI_SKIP: ${{ needs.pre-flight.outputs.has_skip_cicd }}
TEST_LEVEL: ${{ needs.pre-flight.outputs.test_level }}
run: |
SUMMARY=$(echo $JOB_RESULTS | jq 'to_entries[] | .key + ": " + .value.result' | tr -d '"')
echo '🤖: CICD Result for test level: ${{ needs.pre-flight.outputs.test_level }}' >> $GITHUB_STEP_SUMMARY
echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY
if [[ "$TEST_LEVEL" == "none" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "---" >> $GITHUB_STEP_SUMMARY
echo "⚠️ **No tests were run.** This PR does not have a CI label." >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "To trigger tests, add one of the following labels to your PR:" >> $GITHUB_STEP_SUMMARY
echo "| Label | What it runs |" >> $GITHUB_STEP_SUMMARY
echo "|-------|-------------|" >> $GITHUB_STEP_SUMMARY
echo "| \`CI:docs\` | Doc tests only |" >> $GITHUB_STEP_SUMMARY
echo "| \`CI:Lfast\` | Fast subset (reuses main container) |" >> $GITHUB_STEP_SUMMARY
echo "| \`CI:L0\` | Unit tests + docs + lint |" >> $GITHUB_STEP_SUMMARY
echo "| \`CI:L1\` | L0 + functional tests |" >> $GITHUB_STEP_SUMMARY
echo "| \`CI:L2\` | L1 + convergence tests |" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "This check will remain failed until a CI label is added." >> $GITHUB_STEP_SUMMARY
fi
test "$ALL_SUCCESS" = "true" || test "$CI_SKIP" = "true"
notify-nightly-failure:
name: Notify nightly test failure
runs-on: ubuntu-latest
needs: [CI_QA_Gate]
environment:
name: main
deployment: false
if: ${{ always() && github.event_name == 'schedule' && needs.CI_QA_Gate.result == 'failure' }}
steps:
- name: Send Slack notification
env:
SLACK_WEBHOOK: ${{ secrets.SLACK_TEAM_CHANNEL_WEBHOOK }}
run: |
MESSAGE='{
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "🚨 Nightly GitHub CI test failed on main branch\n\n• Repository: ${{ github.repository }}\n• Commit: `${{ github.sha }}`\n• Workflow: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
}
}
]
}'
curl -X POST -H "Content-type: application/json" --data "$MESSAGE" "$SLACK_WEBHOOK"
Coverage:
runs-on: ubuntu-latest
needs:
- CI_QA_Gate
- cicd-doc-tests
- cicd-unit-tests
- cicd-functional-tests
if: always()
strategy:
matrix:
flag: [doc-test, unit-test, e2e]
steps:
- name: Get PR info
id: get-pr-info
if: startsWith(github.ref, 'refs/heads/pull-request/')
uses: nv-gha-runners/get-pr-info@main
- name: Checkout
uses: actions/checkout@v6
- name: Download coverage reports of current branch
uses: actions/download-artifact@v7
with:
pattern: coverage-${{ matrix.flag }}-*
path: coverage-downloads
- name: Check if artifacts were downloaded
id: check-artifacts
run: |
# Check if any .coverage files were downloaded
if find coverage-downloads -name ".coverage" -type f 2>/dev/null | grep -q .; then
echo "artifacts-found=true" >> $GITHUB_OUTPUT
echo "Found coverage artifacts for ${{ matrix.flag }}"
else
echo "artifacts-found=false" >> $GITHUB_OUTPUT
echo "No coverage artifacts found for ${{ matrix.flag }}"
fi
- name: Get total coverage of current branch
shell: bash -x -e -u -o pipefail {0}
if: ${{ steps.check-artifacts.outputs.artifacts-found == 'true' }}
run: |
pip install coverage
find coverage-downloads -name ".coverage" -type f
coverage combine --keep $(find coverage-downloads -name ".coverage" -type f)
coverage report -i --show-missing
rm -rf coverage-downloads
- name: Skip coverage processing
if: ${{ steps.check-artifacts.outputs.artifacts-found == 'false' }}
run: |
echo "No coverage artifacts found for ${{ matrix.flag }}, skipping coverage processing"
- name: Upload coverage reports to Codecov
if: ${{ steps.check-artifacts.outputs.artifacts-found == 'true' }}
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true
flags: ${{ matrix.flag }}
base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').base.sha }}
- name: Upload artifacts
if: ${{ steps.check-artifacts.outputs.artifacts-found == 'true' }}
uses: actions/upload-artifact@v6
with:
name: coverage-${{ matrix.flag }}-aggregated
path: |
.coverage
include-hidden-files: true
DCO_merge_group:
name: DCO
if: github.event_name == 'merge_group'
runs-on: ubuntu-latest
steps:
- run: echo "The real DCO check happens on PRs only. This is a placeholder for the merge queue to keep the DCO check as a required status check."