CICD NeMo RL #13663
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: "CICD NeMo RL" | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - "pull-request/[0-9]+" | |
| schedule: | |
| - cron: "0 9 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| test_to_run: | |
| required: false | |
| default: L2 | |
| type: choice | |
| options: | |
| - docs | |
| - Lfast | |
| - L0 | |
| - L1 | |
| - L2 | |
| description: Test level to run. docs = doc tests only (reuses main container), Lfast = fast subset (reuses main container), L0 = unit/docs/lint, L1 = L0 + functional, L2 = L1 + convergence | |
| image_tag: | |
| description: "Override container image tag (e.g. 'main'). Skips container build." | |
| required: false | |
| default: "" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name || 'main' }}-${{ github.event_name }} | |
| cancel-in-progress: true | |
| env: | |
| GB200_CONTAINER_REGISTRY: ${{ vars.GB200_CONTAINER_REGISTRY }} | |
| jobs: | |
| pre-flight: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| test_level: ${{ steps.evaluate.outputs.test_level }} | |
| image_tag: ${{ steps.evaluate.outputs.image_tag }} | |
| base_ref: ${{ steps.base-head-ref.outputs.base_ref }} | |
| base_sha: ${{ steps.base-head-ref.outputs.base_sha }} | |
| head_ref: ${{ steps.base-head-ref.outputs.head_ref }} | |
| head_sha: ${{ steps.base-head-ref.outputs.head_sha }} | |
| head_label: ${{ steps.base-head-ref.outputs.head_label }} | |
| has_skip_cicd: ${{ steps.base-head-ref.outputs.has_skip_cicd }} | |
| test_sha: ${{ steps.base-head-ref.outputs.test_sha }} | |
| steps: | |
| - name: Get PR info | |
| id: get-pr-info | |
| if: startsWith(github.ref, 'refs/heads/pull-request/') | |
| uses: nv-gha-runners/get-pr-info@main | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Determine base and head references | |
| id: base-head-ref | |
| env: | |
| IS_PULL_REQUEST_REF: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }} | |
| PR_INFO_JSON: ${{ steps.get-pr-info.outputs.pr-info }} | |
| run: | | |
| if [[ "$IS_PULL_REQUEST_REF" == "true" && -n "$PR_INFO_JSON" ]]; then | |
| base_ref=$(echo "$PR_INFO_JSON" | jq -r '.base.ref') | |
| base_sha=$(echo "$PR_INFO_JSON" | jq -r '.base.sha') | |
| head_ref=$(echo "$PR_INFO_JSON" | jq -r '.head.ref') | |
| head_sha=$(echo "$PR_INFO_JSON" | jq -r '.head.sha') | |
| test_sha=$(echo "$PR_INFO_JSON" | jq -r '.merge_commit_sha') | |
| head_label=$(echo "$PR_INFO_JSON" | jq -r '.head.label // empty') | |
| ci_label=$(echo "$PR_INFO_JSON" | jq -r '[.labels[]? | (if type == "string" then . else .name end) | select(startswith("CI:"))] | first // empty') | |
| has_skip_cicd=$(echo "$PR_INFO_JSON" | jq -r '[.labels[]? | (if type == "string" then . else .name end) | select(. == "Skip CICD")] | length > 0') | |
| else | |
| base_ref="HEAD~1" | |
| base_sha=$(git rev-parse HEAD~1) | |
| head_ref="HEAD" | |
| head_sha="${{ github.sha }}" | |
| test_sha="${{ github.sha }}" | |
| head_label="${{ github.ref_name }}" | |
| ci_label="" | |
| has_skip_cicd="false" | |
| fi | |
| [[ "$has_skip_cicd" != "true" ]] && has_skip_cicd="false" | |
| echo "base_ref=$base_ref" >> "$GITHUB_OUTPUT" | |
| echo "base_sha=$base_sha" >> "$GITHUB_OUTPUT" | |
| echo "head_ref=$head_ref" >> "$GITHUB_OUTPUT" | |
| echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT" | |
| echo "head_label=$head_label" >> "$GITHUB_OUTPUT" | |
| echo "ci_label=$ci_label" >> "$GITHUB_OUTPUT" | |
| echo "has_skip_cicd=$has_skip_cicd" >> "$GITHUB_OUTPUT" | |
| - name: Get changed files | |
| id: changed-files | |
| if: startsWith(github.ref, 'refs/heads/pull-request/') | |
| uses: step-security/changed-files@v45.0.1 | |
| with: | |
| base_sha: ${{ steps.base-head-ref.outputs.base_sha }} | |
| files_yaml: | | |
| doc: | |
| - '**.md' | |
| - docs/** | |
| src: | |
| - '!**.md' | |
| - '!docs/**' | |
| - name: Evaluate conditions | |
| id: evaluate | |
| env: | |
| DOCS_ONLY: ${{ steps.changed-files.outputs.doc_any_changed == 'true' && steps.changed-files.outputs.src_any_changed == 'false' }} | |
| CHANGED_DOCS: ${{ steps.changed-files.outputs.doc_all_changed_files }} | |
| CHANGED_SRC: ${{ steps.changed-files.outputs.src_all_changed_files }} | |
| IS_PULLREQUEST: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }} | |
| LABEL: ${{ steps.base-head-ref.outputs.ci_label }} | |
| MERGE_GROUP: ${{ github.event_name == 'merge_group' }} | |
| run: | | |
| # Some output that's helpful for debugging | |
| echo "Docs changed: $CHANGED_DOCS" | |
| echo "Src changed: $CHANGED_SRC" | |
| echo "LABEL: $LABEL" | |
| echo "IS_PULLREQUEST: $IS_PULLREQUEST" | |
| echo "DOCS_ONLY: $DOCS_ONLY" | |
| # Run CI only (on main or if label is attached) and if it's not only docs | |
| # Determine test level based on conditions | |
| if [[ "$DOCS_ONLY" == "true" || "$LABEL" == "CI:docs" ]]; then | |
| # For doc-only changes, run only doc tests | |
| TEST_LEVEL="docs" | |
| elif [[ "$LABEL" == "CI:Lfast" ]]; then | |
| TEST_LEVEL="Lfast" | |
| elif [[ "$LABEL" == "CI:L0" ]]; then | |
| TEST_LEVEL="L0" | |
| elif [[ "$LABEL" == "CI:L1" || "$IS_PULLREQUEST" == "false" || "$MERGE_GROUP" == "true" ]]; then | |
| # For labeled PRs, pushes to main (IS_PULL_REQUEST=false), or merge group events, run L1 by default | |
| TEST_LEVEL="L1" | |
| elif [[ "$LABEL" == "CI:L2" ]]; then | |
| TEST_LEVEL="L2" | |
| else | |
| # Skip tests by default for non-labeled PRs | |
| TEST_LEVEL="none" | |
| fi | |
| if [[ "${{ github.event_name }}" == "schedule" ]]; then | |
| echo "Setting test level to L1 for nightly scheduled run" | |
| TEST_LEVEL="L1" | |
| fi | |
| # Override test level if specified in workflow_dispatch | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| echo "Overriding test level from $TEST_LEVEL to ${{ inputs.test_to_run }}" | |
| TEST_LEVEL="${{ inputs.test_to_run }}" | |
| fi | |
| echo "test_level=$TEST_LEVEL" | tee -a "$GITHUB_OUTPUT" | |
| # Determine image tag: Lfast and docs reuse the main container, workflow_dispatch can override | |
| IMAGE_TAG="" | |
| if [[ "$TEST_LEVEL" == "Lfast" || "$TEST_LEVEL" == "docs" ]]; then | |
| IMAGE_TAG="main" | |
| fi | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ inputs.image_tag }}" ]]; then | |
| IMAGE_TAG="${{ inputs.image_tag }}" | |
| fi | |
| echo "image_tag=$IMAGE_TAG" | tee -a "$GITHUB_OUTPUT" | |
| org-member-pre-flight: | |
| uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.80.1 | |
| with: | |
| default_runner_prefix: ${{ vars.DEFAULT_H100_RUNNER }} | |
| non_nvidia_runner_prefix: ${{ vars.NON_NVIDIA_H100_RUNNER }} | |
| default_test_data_path: ${{ vars.DEFAULT_H100_TEST_DATA_PATH }} | |
| non_nvidia_test_data_path: ${{ vars.NON_NVIDIA_H100_TEST_DATA_PATH }} | |
| default_registry: ${{ vars.DEFAULT_H100_CONTAINER_REGISTRY }} | |
| non_nvidia_registry: ${{ vars.NON_NVIDIA_H100_CONTAINER_REGISTRY }} | |
| sso_users_filename: ${{ vars.SSO_USERS_FILENAME }} | |
| secrets: | |
| NVIDIA_MANAGEMENT_ORG_PAT: ${{ secrets.NVIDIA_MANAGEMENT_ORG_PAT }} | |
| gb200-config: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| registry: ${{ steps.config.outputs.registry }} | |
| steps: | |
| - name: Configure GB200 registry | |
| id: config | |
| env: | |
| GB200_REGISTRY: ${{ env.GB200_CONTAINER_REGISTRY }} | |
| run: echo "registry=$GB200_REGISTRY" | tee -a "$GITHUB_OUTPUT" | |
| pr-branch-up-to-date-check: | |
| name: Check if PR branch is up to date | |
| needs: [pre-flight] | |
| if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }} | |
| runs-on: ubuntu-latest | |
| env: | |
| MAX_COMMITS_BEHIND: 10 | |
| steps: | |
| - name: Check how many commits behind target branch | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| REPO: ${{ github.repository }} | |
| BASE_SHA: ${{ needs.pre-flight.outputs.base_sha }} | |
| HEAD_SHA: ${{ needs.pre-flight.outputs.head_sha }} | |
| BASE_REF: ${{ needs.pre-flight.outputs.base_ref }} | |
| HEAD_LABEL: ${{ needs.pre-flight.outputs.head_label }} | |
| run: | | |
| echo "Repository: $REPO" | |
| echo "Base branch: $BASE_REF (SHA: $BASE_SHA)" | |
| echo "PR head: $HEAD_LABEL (SHA: $HEAD_SHA)" | |
| echo "Maximum commits behind allowed: $MAX_COMMITS_BEHIND" | |
| API_RESPONSE=$(gh api "repos/$REPO/compare/$HEAD_SHA...$BASE_REF" --jq '{behind_by: .behind_by, ahead_by: .ahead_by, status: .status}') | |
| COMMITS_BEHIND=$(echo "$API_RESPONSE" | jq -r '.ahead_by') | |
| COMMITS_AHEAD=$(echo "$API_RESPONSE" | jq -r '.behind_by') | |
| STATUS=$(echo "$API_RESPONSE" | jq -r '.status') | |
| echo "Comparison status: $STATUS" | |
| echo "PR is $COMMITS_BEHIND commits behind and $COMMITS_AHEAD commits ahead of $BASE_REF" | |
| # Check if we're behind by more than the allowed number | |
| if [ "$COMMITS_BEHIND" -gt "$MAX_COMMITS_BEHIND" ]; then | |
| echo "❌ ERROR: This PR is $COMMITS_BEHIND commits behind $BASE_REF, which exceeds the maximum allowed ($MAX_COMMITS_BEHIND commits)." | |
| echo "Please rebase or merge the latest changes from $BASE_REF into your PR branch." | |
| exit 1 | |
| else | |
| echo "✅ PR is acceptably fresh ($COMMITS_BEHIND commits behind, limit is $MAX_COMMITS_BEHIND)" | |
| fi | |
| lint-check: | |
| name: Lint check | |
| needs: [pre-flight] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Free up disk space | |
| run: | | |
| # Remove unnecessary packages and files on Ubuntu | |
| sudo apt-get clean | |
| sudo rm -rf /usr/local/lib/android || true | |
| sudo rm -rf /opt/ghc || true | |
| sudo rm -rf /usr/local/.ghcup || true | |
| sudo rm -rf /usr/share/dotnet || true | |
| sudo rm -rf /opt/az || true | |
| # Clear pip and npm caches | |
| pip cache purge || true | |
| sudo npm cache clean --force || true | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| with: | |
| submodules: "recursive" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| version: "0.11.18" | |
| enable-cache: true | |
| prune-cache: false | |
| # Faster than uv python install since it caches python alongside runner | |
| - name: "Set up Python" | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version-file: ".python-version" | |
| - name: Check lint | |
| run: | | |
| uv venv | |
| uv run --group dev pre-commit install | |
| uv run --group dev pre-commit run --all-files --show-diff-on-failure --color=always | |
| # TODO: this is a temporary check and should be removed once we have 100% correctness | |
| - name: Check if any files with zero errors not in whitelist | |
| run: | | |
| missing_count=0 | |
| for file in $(uv run --group dev pyrefly check $(git ls-files 'nemo_rl/**/*.py' 'examples/**/*.py' 'docs/*.py' 'tools/**/*.py') --output-format json | jq -r --slurpfile all_files <(git ls-files 'nemo_rl/**/*.py' 'examples/**/*.py' 'docs/*.py' 'tools/**/*.py' | jq -R -s 'split("\n")[:-1]') --arg pwd "$(pwd)/" '(.errors | group_by(.path) | map({(.[0].path | sub($pwd; "")): length}) | add // {}) as $error_counts | $all_files[0][] | . as $file | if ($error_counts[$file] // 0) == 0 then $file else empty end'); do | |
| if ! fgrep -q "$file" pyrefly.toml; then | |
| echo "File $file has zero errors but is not in pyrefly.toml in the 'project-includes' list. Please add it to this whitelist." | |
| ((missing_count++)) | |
| fi | |
| done | |
| exit $missing_count | |
| - name: Minimize uv cache | |
| run: uv cache prune --ci | |
| cicd-wait-in-queue: | |
| name: Wait in test approval queue | |
| needs: [pre-flight, lint-check] | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: test | |
| deployment: false | |
| if: >- | |
| ${{ | |
| always() && | |
| startsWith(github.ref, 'refs/heads/pull-request/') && | |
| contains('Lfast L0 L1 L2', needs.pre-flight.outputs.test_level) && | |
| needs.pre-flight.result == 'success' && | |
| needs.lint-check.result == 'success' && | |
| !cancelled() | |
| }} | |
| steps: | |
| - name: Approved | |
| run: echo "Approved to run CI tests." | |
| sphinx-build: | |
| needs: [pre-flight, cicd-wait-in-queue] | |
| if: >- | |
| ${{ | |
| always() && | |
| needs.pre-flight.result == 'success' && | |
| needs.pre-flight.outputs.test_level != 'none' && | |
| ( | |
| needs.cicd-wait-in-queue.result == 'success' || | |
| needs.pre-flight.outputs.test_level == 'docs' || | |
| !startsWith(github.ref, 'refs/heads/pull-request/') | |
| ) && | |
| !cancelled() | |
| }} | |
| uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_docs.yml@v0.57.0 | |
| build-container: | |
| name: Build H100 container | |
| if: >- | |
| ${{ | |
| always() && | |
| needs.pre-flight.result == 'success' && | |
| needs.org-member-pre-flight.result == 'success' && | |
| needs.pre-flight.outputs.test_level != 'none' && | |
| needs.pre-flight.outputs.image_tag == '' && | |
| ( | |
| needs.cicd-wait-in-queue.result == 'success' || | |
| !startsWith(github.ref, 'refs/heads/pull-request/') | |
| ) && | |
| !cancelled() | |
| }} | |
| needs: [pre-flight, org-member-pre-flight, cicd-wait-in-queue] | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| uses: ./.github/workflows/_build_container.yml | |
| with: | |
| build-ref: ${{ needs.pre-flight.outputs.test_sha }} | |
| image-name: ${{ vars.CI_CONTAINER_NAME }} | |
| dockerfile: docker/Dockerfile | |
| platform: linux/amd64 | |
| registry: ${{ needs.org-member-pre-flight.outputs.registry }} | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| target: release | |
| build-contexts: | | |
| nemo-rl=. | |
| ${{ vars.UV_BUILD_CACHE == 'enabled' && format('uv-cache-seed=docker-image://{0}/{1}:uv-cache', needs.org-member-pre-flight.outputs.registry, vars.CI_CONTAINER_NAME) || '' }} | |
| trtllm-ccache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-ccache' || '' }} | |
| trtllm-wheel-cache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-wheel-cache' || '' }} | |
| build-args: | | |
| MAX_JOBS=4 | |
| TRTLLM_BUILD_JOBS=24 | |
| NEMO_RL_COMMIT=${{ needs.pre-flight.outputs.test_sha }} | |
| build-container-gb200: | |
| name: Build GB200/GCP container | |
| if: >- | |
| ${{ | |
| always() && | |
| needs.pre-flight.result == 'success' && | |
| needs.org-member-pre-flight.result == 'success' && | |
| needs.gb200-config.result == 'success' && | |
| needs.pre-flight.outputs.test_level != 'none' && | |
| needs.pre-flight.outputs.image_tag == '' && | |
| needs.org-member-pre-flight.outputs.is_member == 'true' && | |
| vars.DISABLE_GB200_TESTS != 'true' && | |
| contains('L1 L2', needs.pre-flight.outputs.test_level) && | |
| ( | |
| needs.cicd-wait-in-queue.result == 'success' || | |
| !startsWith(github.ref, 'refs/heads/pull-request/') | |
| ) && | |
| !cancelled() | |
| }} | |
| needs: [pre-flight, org-member-pre-flight, gb200-config, cicd-wait-in-queue] | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| uses: ./.github/workflows/_build_container.yml | |
| with: | |
| build-ref: ${{ needs.pre-flight.outputs.test_sha }} | |
| image-name: ${{ vars.CI_CONTAINER_NAME }} | |
| dockerfile: docker/Dockerfile | |
| platform: linux/arm64 | |
| registry: ${{ needs.gb200-config.outputs.registry }} | |
| runner: ${{ vars.GB200_RUNNER }} | |
| target: release | |
| build-contexts: | | |
| nemo-rl=. | |
| ${{ vars.UV_BUILD_CACHE == 'enabled' && format('uv-cache-seed=docker-image://{0}/{1}:uv-cache', needs.gb200-config.outputs.registry, vars.CI_CONTAINER_NAME) || '' }} | |
| trtllm-ccache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-ccache' || '' }} | |
| trtllm-wheel-cache-tag: ${{ vars.TRTLLM_BUILD_CACHE == 'enabled' && 'trtllm-wheel-cache' || '' }} | |
| build-args: | | |
| MAX_JOBS=4 | |
| TRTLLM_BUILD_JOBS=8 | |
| NEMO_RL_COMMIT=${{ needs.pre-flight.outputs.test_sha }} | |
| update-uv-cache: | |
| name: Update uv build cache | |
| needs: [build-container, org-member-pre-flight] | |
| if: >- | |
| ${{ | |
| github.ref == 'refs/heads/main' && | |
| vars.UV_BUILD_CACHE == 'enabled' && | |
| needs.build-container.result == 'success' | |
| }} | |
| runs-on: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| env: | |
| REGISTRY: ${{ needs.org-member-pre-flight.outputs.registry }} | |
| IMAGE_NAME: ${{ vars.CI_CONTAINER_NAME }} | |
| steps: | |
| - name: Extract and push uv cache image | |
| run: | | |
| set -euo pipefail | |
| SRC="${REGISTRY}/${IMAGE_NAME}:${{ github.run_id }}" | |
| DST="${REGISTRY}/${IMAGE_NAME}:uv-cache" | |
| docker pull "${SRC}" | |
| CID=$(docker create "${SRC}" true) | |
| mkdir -p /tmp/uv-cache | |
| docker cp "${CID}:/root/.cache/uv/." /tmp/uv-cache/ | |
| docker rm "${CID}" | |
| printf 'FROM scratch\nCOPY uv-cache/ /\n' > /tmp/Dockerfile.uv-cache | |
| docker build -t "${DST}" -f /tmp/Dockerfile.uv-cache /tmp | |
| docker push "${DST}" | |
| docker rmi "${SRC}" "${DST}" 2>/dev/null || true | |
| rm -rf /tmp/uv-cache /tmp/Dockerfile.uv-cache | |
| update-uv-cache-gb200: | |
| name: Update GB200 uv build cache | |
| needs: [build-container-gb200, gb200-config] | |
| if: >- | |
| ${{ | |
| github.ref == 'refs/heads/main' && | |
| vars.UV_BUILD_CACHE == 'enabled' && | |
| needs.build-container-gb200.result == 'success' | |
| }} | |
| runs-on: ${{ vars.GB200_RUNNER }} | |
| env: | |
| REGISTRY: ${{ needs.gb200-config.outputs.registry }} | |
| IMAGE_NAME: ${{ vars.CI_CONTAINER_NAME }} | |
| steps: | |
| - name: Extract and push uv cache image | |
| run: | | |
| set -euo pipefail | |
| SRC="${REGISTRY}/${IMAGE_NAME}:${{ github.run_id }}" | |
| DST="${REGISTRY}/${IMAGE_NAME}:uv-cache" | |
| docker pull "${SRC}" | |
| CID=$(docker create "${SRC}" true) | |
| mkdir -p /tmp/uv-cache | |
| docker cp "${CID}:/root/.cache/uv/." /tmp/uv-cache/ | |
| docker rm "${CID}" | |
| printf 'FROM scratch\nCOPY uv-cache/ /\n' > /tmp/Dockerfile.uv-cache | |
| docker build -t "${DST}" -f /tmp/Dockerfile.uv-cache /tmp | |
| docker push "${DST}" | |
| docker rmi "${SRC}" "${DST}" 2>/dev/null || true | |
| rm -rf /tmp/uv-cache /tmp/Dockerfile.uv-cache | |
| cicd-doc-tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - script: Docs_Tests | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| needs: [pre-flight, build-container, org-member-pre-flight, cicd-wait-in-queue] | |
| if: >- | |
| ${{ | |
| ( | |
| always() && | |
| contains('docs Lfast L0 L1 L2', needs.pre-flight.outputs.test_level) && | |
| needs.pre-flight.result == 'success' && | |
| needs.org-member-pre-flight.result == 'success' && | |
| ( | |
| needs.cicd-wait-in-queue.result == 'success' || | |
| needs.pre-flight.outputs.test_level == 'docs' || | |
| !startsWith(github.ref, 'refs/heads/pull-request/') | |
| ) && | |
| (needs.build-container.result == 'success' || needs.build-container.result == 'skipped') | |
| ) && !cancelled() | |
| }} | |
| runs-on: ${{ matrix.runner }} | |
| name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: main | |
| uses: ./.github/actions/test-template | |
| with: | |
| runner: ${{ matrix.runner }} | |
| registry: ${{ needs.org-member-pre-flight.outputs.registry }} | |
| image: ${{ vars.CI_CONTAINER_NAME }} | |
| image-tag: ${{ needs.pre-flight.outputs.image_tag }} | |
| test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }} | |
| script: ${{ matrix.script }} | |
| is_doc_test: "true" | |
| test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }} | |
| cicd-unit-tests: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 16 | |
| matrix: | |
| include: | |
| - script: L0_Unit_Tests_Vllm_1 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Vllm_2 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Vllm_3 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Sglang | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Trtllm | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Mcore | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Mcore_Policy_1 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Mcore_Policy_2 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Mcore_Policy_3 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Automodel | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Automodel_Policy_1 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Automodel_Policy_2 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Automodel_Policy_3 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Models_1 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Models_2 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Models_3 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Models_4 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Environments | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Nemo_Gym | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Algorithms | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Data | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Distributed | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L0_Unit_Tests_Other | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| needs: [pre-flight, build-container, cicd-doc-tests, org-member-pre-flight, cicd-wait-in-queue] | |
| if: >- | |
| ${{ | |
| ( | |
| always() && | |
| contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) && | |
| needs.pre-flight.result == 'success' && | |
| needs.org-member-pre-flight.result == 'success' && | |
| ( | |
| needs.cicd-wait-in-queue.result == 'success' || | |
| !startsWith(github.ref, 'refs/heads/pull-request/') | |
| ) && | |
| (needs.build-container.result == 'success' || needs.build-container.result == 'skipped') && | |
| (needs.cicd-doc-tests.result == 'success' || needs.cicd-doc-tests.result == 'skipped') | |
| ) && !cancelled() | |
| }} | |
| runs-on: ${{ matrix.runner }} | |
| name: ${{ matrix.script }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: main | |
| # Lfast reuses the main image. Until TRT-LLM lands on main, running this | |
| # shard would make prefetch_venvs compile the cp313 wheel from source. | |
| if: ${{ needs.pre-flight.outputs.test_level != 'Lfast' || matrix.script != 'L0_Unit_Tests_Trtllm' }} | |
| uses: ./.github/actions/test-template | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| with: | |
| runner: ${{ matrix.runner }} | |
| script: ${{ matrix.script }} | |
| registry: ${{ needs.org-member-pre-flight.outputs.registry }} | |
| test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }} | |
| image: ${{ vars.CI_CONTAINER_NAME }} | |
| image-tag: ${{ needs.pre-flight.outputs.image_tag }} | |
| is_unit_test: "true" | |
| cpu-only: ${{ matrix.cpu-only || false }} | |
| test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }} | |
| unit-test-script-check: | |
| name: Check unit test script coverage | |
| needs: [pre-flight, cicd-wait-in-queue] | |
| if: >- | |
| ${{ | |
| always() && | |
| contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) && | |
| needs.pre-flight.result == 'success' && | |
| ( | |
| needs.cicd-wait-in-queue.result == 'success' || | |
| !startsWith(github.ref, 'refs/heads/pull-request/') | |
| ) && | |
| !cancelled() | |
| }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ needs.pre-flight.outputs.test_sha }} | |
| - name: Verify L0 unit scripts are in the workflow | |
| run: | | |
| set -euo pipefail | |
| expected=$(mktemp) | |
| configured=$(mktemp) | |
| find tests/unit -maxdepth 1 -type f -name 'L0_Unit*.sh' \ | |
| -exec basename {} .sh \; | sort -u > "$expected" | |
| { | |
| grep -E '^[[:space:]]*-[[:space:]]*script:[[:space:]]*L0_Unit' .github/workflows/cicd-main.yml || true | |
| } | sed -E 's/^[[:space:]]*-[[:space:]]*script:[[:space:]]*//' | sort -u > "$configured" | |
| missing=$(comm -23 "$expected" "$configured") | |
| if [[ -n "$missing" ]]; then | |
| echo "The following tests/unit/L0_Unit*.sh scripts are missing from .github/workflows/cicd-main.yml:" | |
| printf '%s\n' "$missing" | |
| exit 1 | |
| fi | |
| echo "All L0 unit scripts are included in .github/workflows/cicd-main.yml." | |
| functional-test-script-check: | |
| name: Check functional test script coverage | |
| needs: [pre-flight, cicd-wait-in-queue] | |
| if: >- | |
| ${{ | |
| always() && | |
| contains('L1 L2 Lfast', needs.pre-flight.outputs.test_level) && | |
| needs.pre-flight.result == 'success' && | |
| ( | |
| needs.cicd-wait-in-queue.result == 'success' || | |
| !startsWith(github.ref, 'refs/heads/pull-request/') | |
| ) && | |
| !cancelled() | |
| }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ needs.pre-flight.outputs.test_sha }} | |
| - name: Verify L1 functional scripts are in the workflow | |
| run: | | |
| set -euo pipefail | |
| expected=$(mktemp) | |
| configured=$(mktemp) | |
| find tests/functional -maxdepth 1 -type f -name 'L1_Functional*.sh' \ | |
| -exec basename {} .sh \; | sort -u > "$expected" | |
| { | |
| grep -E '^[[:space:]]*-[[:space:]]*script:[[:space:]]*L1_Functional' .github/workflows/cicd-main.yml || true | |
| } | sed -E 's/^[[:space:]]*-[[:space:]]*script:[[:space:]]*//' | sort -u > "$configured" | |
| missing=$(comm -23 "$expected" "$configured") | |
| if [[ -n "$missing" ]]; then | |
| echo "The following tests/functional/L1_Functional*.sh scripts are missing from .github/workflows/cicd-main.yml:" | |
| printf '%s\n' "$missing" | |
| exit 1 | |
| fi | |
| echo "All L1 functional scripts are included in .github/workflows/cicd-main.yml." | |
| cicd-functional-tests: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 16 | |
| matrix: | |
| include: | |
| - script: L1_Functional_Tests_Megatron_1 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Megatron_2 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Megatron_3 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Megatron_4 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_AutoModel | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_SGLang | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Trtllm | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Gym | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_GRPO_1 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_GRPO_2 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_GRPO_3 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_SFT | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_PPO | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Eval | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Other_1 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Other_2 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| needs: [pre-flight, build-container, cicd-unit-tests, functional-test-script-check, org-member-pre-flight, cicd-wait-in-queue] | |
| runs-on: ${{ matrix.runner }} | |
| if: >- | |
| ${{ | |
| always() && | |
| contains('L1 L2', needs.pre-flight.outputs.test_level) && | |
| needs.pre-flight.result == 'success' && | |
| needs.org-member-pre-flight.result == 'success' && | |
| ( | |
| needs.cicd-wait-in-queue.result == 'success' || | |
| !startsWith(github.ref, 'refs/heads/pull-request/') | |
| ) && | |
| (needs.build-container.result == 'success' || needs.build-container.result == 'skipped') && | |
| needs.cicd-unit-tests.result == 'success' && | |
| needs.functional-test-script-check.result == 'success' && | |
| !cancelled() | |
| }} | |
| name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: main | |
| uses: ./.github/actions/test-template | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| with: | |
| runner: ${{ matrix.runner }} | |
| registry: ${{ needs.org-member-pre-flight.outputs.registry }} | |
| image: ${{ vars.CI_CONTAINER_NAME }} | |
| test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }} | |
| script: ${{ matrix.script }} | |
| test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }} | |
| cicd-functional-tests-gb200: | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 16 | |
| matrix: | |
| include: | |
| - script: L1_Functional_Tests_Megatron_1 | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_Megatron_2 | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_Megatron_3 | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_Megatron_4 | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_AutoModel | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_SGLang | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_Gym | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_GRPO_1 | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_GRPO_2 | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_GRPO_3 | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_SFT | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_PPO | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_Eval | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_Other_1 | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_Other_2 | |
| runner: ${{ vars.GB200_RUNNER }} | |
| - script: L1_Functional_Tests_GB200_MXFP8 | |
| runner: ${{ vars.GB200_RUNNER }} | |
| needs: [pre-flight, build-container-gb200, cicd-unit-tests, functional-test-script-check, org-member-pre-flight, gb200-config, cicd-wait-in-queue] | |
| runs-on: ${{ matrix.runner }} | |
| if: >- | |
| ${{ | |
| always() && | |
| contains('L1 L2', needs.pre-flight.outputs.test_level) && | |
| needs.org-member-pre-flight.outputs.is_member == 'true' && | |
| vars.DISABLE_GB200_TESTS != 'true' && | |
| needs.pre-flight.result == 'success' && | |
| needs.org-member-pre-flight.result == 'success' && | |
| ( | |
| needs.cicd-wait-in-queue.result == 'success' || | |
| !startsWith(github.ref, 'refs/heads/pull-request/') | |
| ) && | |
| (needs.build-container-gb200.result == 'success' || needs.build-container-gb200.result == 'skipped') && | |
| needs.cicd-unit-tests.result == 'success' && | |
| needs.functional-test-script-check.result == 'success' && | |
| !cancelled() | |
| }} | |
| name: gb200_${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: main | |
| uses: ./.github/actions/test-template | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| with: | |
| runner: ${{ matrix.runner }} | |
| registry: ${{ needs.gb200-config.outputs.registry }} | |
| image: ${{ vars.CI_CONTAINER_NAME }} | |
| test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }} | |
| image-tag: ${{ needs.pre-flight.outputs.image_tag }} | |
| script: ${{ matrix.script }} | |
| test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }} | |
| cicd-fast-functional-tests: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - script: L1_Functional_Tests_Megatron_1 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Megatron_2 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Megatron_3 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Megatron_4 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_AutoModel | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_SGLang | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Gym | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_GRPO_1 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_GRPO_2 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_GRPO_3 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_SFT | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_PPO | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Eval | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Other_1 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| - script: L1_Functional_Tests_Other_2 | |
| runner: ${{ needs.org-member-pre-flight.outputs.runner_prefix }} | |
| needs: [pre-flight, functional-test-script-check, org-member-pre-flight, cicd-wait-in-queue] | |
| if: >- | |
| ${{ | |
| always() && | |
| contains('Lfast', needs.pre-flight.outputs.test_level) && | |
| needs.pre-flight.result == 'success' && | |
| needs.org-member-pre-flight.result == 'success' && | |
| ( | |
| needs.cicd-wait-in-queue.result == 'success' || | |
| !startsWith(github.ref, 'refs/heads/pull-request/') | |
| ) && | |
| needs.functional-test-script-check.result == 'success' && | |
| !cancelled() | |
| }} | |
| runs-on: ${{ matrix.runner }} | |
| name: fast_${{ matrix.script }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: main | |
| uses: ./.github/actions/test-template | |
| env: | |
| HF_TOKEN: ${{ secrets.HF_TOKEN }} | |
| with: | |
| runner: ${{ matrix.runner }} | |
| script: ${{ matrix.script }} | |
| image-tag: ${{ needs.pre-flight.outputs.image_tag }} | |
| registry: ${{ needs.org-member-pre-flight.outputs.registry }} | |
| image: ${{ vars.CI_CONTAINER_NAME }} | |
| test_data_path: ${{ needs.org-member-pre-flight.outputs.test_data_path }} | |
| test-commit-sha: ${{ needs.pre-flight.outputs.test_sha }} | |
| CI_QA_Gate: | |
| name: "CI quality check${{ needs.pre-flight.outputs.test_level == 'none' && ' (No tests run: Label CI:L*)' || '' }}" | |
| if: always() | |
| runs-on: ubuntu-latest | |
| needs: | |
| - pre-flight | |
| - org-member-pre-flight | |
| - pr-branch-up-to-date-check | |
| - lint-check | |
| - cicd-wait-in-queue | |
| - sphinx-build | |
| - build-container | |
| - build-container-gb200 | |
| - cicd-doc-tests | |
| - cicd-unit-tests | |
| - unit-test-script-check | |
| - functional-test-script-check | |
| - cicd-functional-tests | |
| - cicd-functional-tests-gb200 | |
| - cicd-fast-functional-tests | |
| steps: | |
| - name: main | |
| env: | |
| JOB_RESULTS: ${{ toJSON(needs) }} | |
| # Job is considered successful if nothing was run, or if all jobs were successful (the tests run even if only docs were run b/c doctests are selected) | |
| ALL_SUCCESS: >- | |
| ${{ | |
| needs.lint-check.result == 'success' && | |
| (needs.cicd-wait-in-queue.result == 'success' || needs.cicd-wait-in-queue.result == 'skipped') && | |
| (needs.pr-branch-up-to-date-check.result == 'success' || needs.pr-branch-up-to-date-check.result == 'skipped') && | |
| ( | |
| needs.pre-flight.outputs.test_level != 'none' && | |
| needs.sphinx-build.result == 'success' && | |
| (needs.build-container.result == 'success' || needs.build-container.result == 'skipped') && | |
| (needs.build-container-gb200.result == 'success' || needs.build-container-gb200.result == 'skipped') && | |
| ( | |
| ( | |
| (needs.cicd-doc-tests.result == 'success' || needs.cicd-doc-tests.result == 'skipped') && | |
| ( | |
| !contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) || | |
| needs.cicd-unit-tests.result == 'success' | |
| ) && | |
| ( | |
| !contains('L0 L1 L2 Lfast', needs.pre-flight.outputs.test_level) || | |
| needs.unit-test-script-check.result == 'success' | |
| ) && | |
| ( | |
| !contains('L1 L2 Lfast', needs.pre-flight.outputs.test_level) || | |
| needs.functional-test-script-check.result == 'success' | |
| ) && | |
| ( | |
| !contains('L1 L2', needs.pre-flight.outputs.test_level) || | |
| needs.cicd-functional-tests.result == 'success' | |
| ) && | |
| ( | |
| needs.org-member-pre-flight.outputs.is_member != 'true' || | |
| vars.DISABLE_GB200_TESTS == 'true' || | |
| !contains('L1 L2', needs.pre-flight.outputs.test_level) || | |
| needs.cicd-functional-tests-gb200.result == 'success' | |
| ) && | |
| ( | |
| !contains('Lfast', needs.pre-flight.outputs.test_level) || | |
| needs.cicd-fast-functional-tests.result == 'success' | |
| ) | |
| ) | |
| ) | |
| ) | |
| }} | |
| CI_SKIP: ${{ needs.pre-flight.outputs.has_skip_cicd }} | |
| TEST_LEVEL: ${{ needs.pre-flight.outputs.test_level }} | |
| run: | | |
| SUMMARY=$(echo $JOB_RESULTS | jq 'to_entries[] | .key + ": " + .value.result' | tr -d '"') | |
| echo '🤖: CICD Result for test level: ${{ needs.pre-flight.outputs.test_level }}' >> $GITHUB_STEP_SUMMARY | |
| echo "$SUMMARY" >> $GITHUB_STEP_SUMMARY | |
| if [[ "$TEST_LEVEL" == "none" ]]; then | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "---" >> $GITHUB_STEP_SUMMARY | |
| echo "⚠️ **No tests were run.** This PR does not have a CI label." >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "To trigger tests, add one of the following labels to your PR:" >> $GITHUB_STEP_SUMMARY | |
| echo "| Label | What it runs |" >> $GITHUB_STEP_SUMMARY | |
| echo "|-------|-------------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| \`CI:docs\` | Doc tests only |" >> $GITHUB_STEP_SUMMARY | |
| echo "| \`CI:Lfast\` | Fast subset (reuses main container) |" >> $GITHUB_STEP_SUMMARY | |
| echo "| \`CI:L0\` | Unit tests + docs + lint |" >> $GITHUB_STEP_SUMMARY | |
| echo "| \`CI:L1\` | L0 + functional tests |" >> $GITHUB_STEP_SUMMARY | |
| echo "| \`CI:L2\` | L1 + convergence tests |" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "This check will remain failed until a CI label is added." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| test "$ALL_SUCCESS" = "true" || test "$CI_SKIP" = "true" | |
| notify-nightly-failure: | |
| name: Notify nightly test failure | |
| runs-on: ubuntu-latest | |
| needs: [CI_QA_Gate] | |
| environment: | |
| name: main | |
| deployment: false | |
| if: ${{ always() && github.event_name == 'schedule' && needs.CI_QA_Gate.result == 'failure' }} | |
| steps: | |
| - name: Send Slack notification | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_TEAM_CHANNEL_WEBHOOK }} | |
| run: | | |
| MESSAGE='{ | |
| "blocks": [ | |
| { | |
| "type": "section", | |
| "text": { | |
| "type": "mrkdwn", | |
| "text": "🚨 Nightly GitHub CI test failed on main branch\n\n• Repository: ${{ github.repository }}\n• Commit: `${{ github.sha }}`\n• Workflow: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>" | |
| } | |
| } | |
| ] | |
| }' | |
| curl -X POST -H "Content-type: application/json" --data "$MESSAGE" "$SLACK_WEBHOOK" | |
| Coverage: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - CI_QA_Gate | |
| - cicd-doc-tests | |
| - cicd-unit-tests | |
| - cicd-functional-tests | |
| if: always() | |
| strategy: | |
| matrix: | |
| flag: [doc-test, unit-test, e2e] | |
| steps: | |
| - name: Get PR info | |
| id: get-pr-info | |
| if: startsWith(github.ref, 'refs/heads/pull-request/') | |
| uses: nv-gha-runners/get-pr-info@main | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Download coverage reports of current branch | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: coverage-${{ matrix.flag }}-* | |
| path: coverage-downloads | |
| - name: Check if artifacts were downloaded | |
| id: check-artifacts | |
| run: | | |
| # Check if any .coverage files were downloaded | |
| if find coverage-downloads -name ".coverage" -type f 2>/dev/null | grep -q .; then | |
| echo "artifacts-found=true" >> $GITHUB_OUTPUT | |
| echo "Found coverage artifacts for ${{ matrix.flag }}" | |
| else | |
| echo "artifacts-found=false" >> $GITHUB_OUTPUT | |
| echo "No coverage artifacts found for ${{ matrix.flag }}" | |
| fi | |
| - name: Get total coverage of current branch | |
| shell: bash -x -e -u -o pipefail {0} | |
| if: ${{ steps.check-artifacts.outputs.artifacts-found == 'true' }} | |
| run: | | |
| pip install coverage | |
| find coverage-downloads -name ".coverage" -type f | |
| coverage combine --keep $(find coverage-downloads -name ".coverage" -type f) | |
| coverage report -i --show-missing | |
| rm -rf coverage-downloads | |
| - name: Skip coverage processing | |
| if: ${{ steps.check-artifacts.outputs.artifacts-found == 'false' }} | |
| run: | | |
| echo "No coverage artifacts found for ${{ matrix.flag }}, skipping coverage processing" | |
| - name: Upload coverage reports to Codecov | |
| if: ${{ steps.check-artifacts.outputs.artifacts-found == 'true' }} | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| verbose: true | |
| flags: ${{ matrix.flag }} | |
| base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').base.sha }} | |
| - name: Upload artifacts | |
| if: ${{ steps.check-artifacts.outputs.artifacts-found == 'true' }} | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: coverage-${{ matrix.flag }}-aggregated | |
| path: | | |
| .coverage | |
| include-hidden-files: true | |
| DCO_merge_group: | |
| name: DCO | |
| if: github.event_name == 'merge_group' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo "The real DCO check happens on PRs only. This is a placeholder for the merge queue to keep the DCO check as a required status check." |