Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 61 additions & 1 deletion .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,72 @@ jobs:

echo "should_skip=${SHOULD_SKIP}" >> "$GITHUB_OUTPUT"
##############################################################################
# ENVIRONMENT DRIFT GATE
##############################################################################
# The nightly builds the fvdb-core wheel in fvdb-core's env/build_environment.yml
# and installs it into the benchmark env defined in this repo. If those pins
# disagree, the wheel is compiled against one libtorch and loaded against
# another, and the run dies with an undefined-symbol ImportError at `import
# fvdb` -- but only after the GPU runners are up, and in the comparative job
# only after ~1h of build time. Fail here instead, on ubuntu-latest, before
# any EC2 instance is provisioned.
#
# The fvdb-core env is fetched at the exact commit this run builds from rather
# than read off disk: the build job only clones fvdb-core when no cached wheel
# artifact exists, so an on-disk check would silently skip on cache hits.
check-env-drift:
if: ${{ github.repository == 'openvdb/fvdb-reality-capture' && needs.get-fvdb-core-commit-hash.outputs.should_skip != 'true' }}
name: Check env drift vs fvdb-core
needs: get-fvdb-core-commit-hash
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
# Must match the ref the build and benchmark jobs check out, or a
# workflow_dispatch run would validate main's pins while building and
# benchmarking a different branch's.
ref: ${{ github.event.inputs.branch || 'main' }}
persist-credentials: false
- name: Compare pinned build/runtime environments
env:
FVDB_CORE_SHA: ${{ needs.get-fvdb-core-commit-hash.outputs.fvdb_core_commit_hash }}
BENCH_ENV: tests/benchmarks/comparative/docker/benchmark_environment.yml
run: |
set -euo pipefail
curl -fsSL \
"https://raw.githubusercontent.com/openvdb/fvdb-core/${FVDB_CORE_SHA}/env/build_environment.yml" \
-o core_env.yml
echo "fvdb-core @ ${FVDB_CORE_SHA}"
rc=0
for key in pytorch-gpu cuda-version python; do
core=$(grep -oP "^\s*-\s*${key}=\K\S+" core_env.yml || true)
bench=$(grep -oP "^\s*-\s*${key}=\K\S+" "${BENCH_ENV}" || true)
# Treat an unreadable pin as failure: a renamed or reformatted line
# would otherwise compare empty-to-empty and pass silently.
if [ -z "${core}" ] || [ -z "${bench}" ]; then
echo "::error::Could not read '${key}' from both environments (fvdb-core='${core}', benchmark='${bench}'). Update this check if the pin format changed."
rc=1
continue
fi
printf '%-14s fvdb-core=%-10s benchmark=%s\n' "${key}" "${core}" "${bench}"
if [ "${core}" != "${bench}" ]; then
echo "::error::${key} drift: fvdb-core builds the wheel against ${core} but ${BENCH_ENV} installs ${bench}. Sync ${BENCH_ENV} to match fvdb-core."
rc=1
fi
done
exit "${rc}"

##############################################################################
# BUILD FVDB
##############################################################################
start-build-runner:
if: ${{ github.repository == 'openvdb/fvdb-reality-capture' && needs.get-fvdb-core-commit-hash.outputs.should_skip != 'true' }}
name: Start CPU-only EC2 runner for build
needs: get-fvdb-core-commit-hash
# check-env-drift gates the whole EC2 chain: every runner-provisioning job
# descends from this one, so a drift failure costs no instance time.
needs:
- get-fvdb-core-commit-hash
- check-env-drift
runs-on: ubuntu-latest
outputs:
label: ${{ steps.start-build-runner.outputs.label }}
Expand Down
Loading