Merge branch 'main' into fix/warn-ignored-dataset-config-keys

Author: RayenTian

.github/workflows/cicd-main.yml

@@ -374,6 +374,7 @@ jobs:
         needs.pre-flight.outputs.test_level != 'none' &&
         needs.pre-flight.outputs.image_tag == '' &&
         needs.org-member-pre-flight.outputs.is_member == 'true' &&
+        vars.DISABLE_GB200_TESTS != 'true' &&
         contains('L1 L2', needs.pre-flight.outputs.test_level) &&
         (
           needs.cicd-wait-in-queue.result == 'success' ||
@@ -801,6 +802,7 @@ jobs:
         always() &&
         contains('L1 L2', needs.pre-flight.outputs.test_level) &&
         needs.org-member-pre-flight.outputs.is_member == 'true' &&
+        vars.DISABLE_GB200_TESTS != 'true' &&
         needs.pre-flight.result == 'success' &&
         needs.org-member-pre-flight.result == 'success' &&
         (
@@ -952,6 +954,7 @@ jobs:
                     ) &&
                     (
                       needs.org-member-pre-flight.outputs.is_member != 'true' ||
+                      vars.DISABLE_GB200_TESTS == 'true' ||
                       !contains('L1 L2', needs.pre-flight.outputs.test_level) ||
                       needs.cicd-functional-tests-gb200.result == 'success'
                     ) &&

3rdparty/Megatron-Bridge-workspace/Megatron-Bridge

@@ -41,13 +41,11 @@ install_arm64_from_source() {
     local build_dir="/tmp/apptainer-build"
     local source_tarball="/tmp/apptainer-${APPTAINER_VERSION}.tar.gz"
     local source_url="https://github.com/apptainer/apptainer/releases/download/v${APPTAINER_VERSION}/apptainer-${APPTAINER_VERSION}.tar.gz"
+    # curl, git, and wget are installed by docker/Dockerfile and must remain in the final image.
     local build_packages=(
         autoconf
         automake
-        build-essential
-        curl
         dh-apparmor
-        git
         libfuse3-dev
         liblzo2-dev
         liblz4-dev
@@ -57,10 +55,10 @@ install_arm64_from_source() {
         libtool
         libzstd-dev
         pkg-config
-        wget
         zlib1g-dev
     )
     local runtime_packages=(
+        build-essential
         ca-certificates
         cryptsetup
         fakeroot
@@ -77,6 +75,13 @@ install_arm64_from_source() {
         uidmap
         zlib1g
     )
+    local new_build_packages=()
+
+    for package in "${build_packages[@]}"; do
+        if ! dpkg-query -W -f='${db:Status-Abbrev}' "${package}" 2>/dev/null | grep -q '^ii '; then
+            new_build_packages+=("${package}")
+        fi
+    done
 
     apt-get update
     apt-get install -y --no-install-recommends "${runtime_packages[@]}" "${build_packages[@]}"
@@ -100,7 +105,9 @@ install_arm64_from_source() {
     rm -rf "${build_dir}"
 
     apt-get install -y --no-install-recommends "${runtime_packages[@]}"
-    apt-get purge -y --auto-remove "${build_packages[@]}"
+    if ((${#new_build_packages[@]} > 0)); then
+        apt-get purge -y --auto-remove "${new_build_packages[@]}"
+    fi
     rm -rf /usr/local/go /root/go /root/.cache/go-build
 }
 

docker/install_apptainer.sh

@@ -18,3 +18,6 @@ NeMo RL supports multiple generation/rollout backends to accommodate different m
 
 For detailed information on backend selection, configuration, and examples, see the [Generation Backends documentation](../design-docs/generation.md).
 
+See [Weight Refit: Choosing a Transport](../guides/refit.md) to select colocated
+IPC, NCCL, sparse delta, or NIXL checkpoint-engine refit for the configured
+training and generation backends.