Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Turn linux conda builds into no-ops #5917

Merged
merged 2 commits into from
Nov 14, 2024
Merged

Turn linux conda builds into no-ops #5917

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0b33cd8
Disable conda builds
malfet Nov 14, 2024
7314083
And use alpine linux, it's faster to checkout
malfet Nov 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 3 additions & 158 deletions .github/workflows/build_conda_linux.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,163 +88,8 @@ jobs:
container:
image: ${{ matrix.container_image }}
options: ${{ matrix.gpu_arch_type == 'cuda' && '--gpus all' || ' ' }}
# If a build is taking longer than 60 minutes on these runners we need
# to have a conversation
timeout-minutes: ${{ inputs.timeout }}
timeout-minutes: 3
steps:
- uses: actions/checkout@v4
with:
# Support the use case where we need to checkout someone's fork
repository: ${{ inputs.test-infra-repository }}
ref: ${{ inputs.test-infra-ref }}
path: test-infra
- uses: ./test-infra/.github/actions/set-channel
- name: Set PYTORCH_VERSION
if: ${{ env.CHANNEL == 'test' }}
- name: Conda builds are deprecated, see https://github.com/pytorch/pytorch/issues/138506
run: |
# When building RC, set the version to be the current candidate version,
# otherwise, leave it alone so nightly will pick up the latest
echo "PYTORCH_VERSION=${{ matrix.stable_version }}" >> "${GITHUB_ENV}"
- uses: ./test-infra/.github/actions/setup-binary-builds
with:
repository: ${{ inputs.repository }}
ref: ${{ inputs.ref }}
setup-miniconda: true
python-version: ${{ env.PYTHON_VERSION }}
cuda-version: ${{ env.CU_VERSION }}
- name: Combine Env Var and Build Env Files
if: ${{ inputs.env-var-script != '' }}
working-directory: ${{ inputs.repository }}
run: |
set -euxo pipefail
cat "${{ inputs.env-var-script }}" >> "${BUILD_ENV_FILE}"
- name: Run Pre-Script with Caching
if: ${{ inputs.pre-script != '' }}
uses: ./test-infra/.github/actions/run-script-with-cache
with:
cache-path: ${{ inputs.cache-path }}
cache-key: ${{ inputs.cache-key }}
repository: ${{ inputs.repository }}
script: ${{ inputs.pre-script }}
- name: Setup base environment variables
run: |
set -euxo pipefail
echo "SOURCE_ROOT_DIR=${GITHUB_WORKSPACE}/${REPOSITORY}" >> "${GITHUB_ENV}"
- name: Build the conda (conda-build)
working-directory: ${{ inputs.repository }}
env:
CUDATOOLKIT_CHANNEL: ${{ env.CUDATOOLKIT_CHANNEL }}
run: |
set -euxo pipefail
# shellcheck disable=SC1090
source "${BUILD_ENV_FILE}"

# Please note these are required changes to make sure our conda environment take precedence before
# any other environments https://github.com/conda-incubator/setup-miniconda#conda-incubatorsetup-miniconda
# such as test environment created by default.
export PATH=$CONDA_ENV/bin:$CONDA_ENV:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH
export Python3_ROOT_DIR=${CONDA_ENV}
export Python3_EXECUTABLE="${CONDA_ENV}"

${CONDA_RUN} conda build \
-c defaults \
-c "${CUDATOOLKIT_CHANNEL}" \
-c "pytorch-${CHANNEL}" \
--no-anaconda-upload \
--python "${PYTHON_VERSION}" \
--output-folder distr/ \
"${CONDA_PACKAGE_DIRECTORY}"
- name: Run Post-Script
if: ${{ inputs.post-script != '' }}
uses: ./test-infra/.github/actions/run-script-with-cache
with:
repository: ${{ inputs.repository }}
script: ${{ inputs.post-script }}
- name: Smoke Test
env:
PACKAGE_NAME: ${{ inputs.package-name }}
SMOKE_TEST_SCRIPT: ${{ inputs.smoke-test-script }}
GPU_ARCH: ${{ matrix.gpu_arch_type }}
DESIRED_CUDA: ${{ matrix.desired_cuda }}
GPU_ARCH_VERSION: ${{ matrix.gpu_arch_version }}
run: |
set -euxo pipefail
# shellcheck disable=SC1090
source "${BUILD_ENV_FILE}"
CONDA_LOCAL_CHANNEL="file://$(readlink -f ${{ inputs.repository }}/distr)"

# Please note these are required changes to make sure our conda environment take precedence before
# any other environments https://github.com/conda-incubator/setup-miniconda#conda-incubatorsetup-miniconda
# such as test environment created by default.
export PATH=$CONDA_ENV/bin:$CONDA_ENV:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH
export Python3_ROOT_DIR=${CONDA_ENV}
export Python3_EXECUTABLE="${CONDA_ENV}"

export CONDA_ENV_SMOKE="${RUNNER_TEMP}/pytorch_pkg_helpers_${GITHUB_RUN_ID}_smoke"
conda create --yes --quiet --prefix "${CONDA_ENV_SMOKE}" python="${PYTHON_VERSION}"
export CONDA_RUN_SMOKE="conda run -p ${CONDA_ENV_SMOKE}"

if [[ "${GPU_ARCH}" = "cuda" ]]; then
CONSTRAINTS="pytorch-cuda=${GPU_ARCH_VERSION}"
else
CONSTRAINTS="cpuonly"
fi

CONSTRAINT_BUILD=""
if [[ "${CHANNEL}" = "test" ]]; then
CONSTRAINT_BUILD="=${BUILD_VERSION}"
fi

${CONDA_RUN_SMOKE} conda install \
--yes \
--quiet \
-c "${CONDA_LOCAL_CHANNEL}" \
-c pytorch-"${CHANNEL}" \
-c nvidia \
-c defaults \
"${PACKAGE_NAME}${CONSTRAINT_BUILD}" \
${CONSTRAINTS} \
--override-channels

# check for installed package channel
CONDA_PKG=$(${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}" | grep "${CONDA_LOCAL_CHANNEL}")
if [[ -z "${CONDA_PKG}" ]]; then
${CONDA_RUN_SMOKE} conda list | grep "${PACKAGE_NAME}"
echo "${PACKAGE_NAME} is not installed from local channel"
exit 1
fi

if [[ ! -f "${{ inputs.repository }}"/${SMOKE_TEST_SCRIPT} ]]; then
echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} not found"
${CONDA_RUN_SMOKE} python3 -c "import ${PACKAGE_NAME}; print('package version is ', ${PACKAGE_NAME}.__version__)"
else
echo "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT} found"
${CONDA_RUN_SMOKE} python "${{ inputs.repository }}/${SMOKE_TEST_SCRIPT}"
fi
conda env remove -p "${CONDA_ENV_SMOKE}"
# NB: Only upload to GitHub after passing smoke tests
- name: Upload artifact to GitHub
continue-on-error: true
uses: actions/upload-artifact@v3
with:
name: ${{ env.ARTIFACT_NAME }}
path: ${{ inputs.repository }}/distr/

upload:
needs: build
uses: ./.github/workflows/_binary_conda_upload.yml
if: always()
with:
repository: ${{ inputs.repository }}
ref: ${{ inputs.ref }}
test-infra-repository: ${{ inputs.test-infra-repository }}
test-infra-ref: ${{ inputs.test-infra-ref }}
build-matrix: ${{ inputs.build-matrix }}
trigger-event: ${{ inputs.trigger-event }}
secrets:
CONDA_PYTORCHBOT_TOKEN: ${{ secrets.CONDA_PYTORCHBOT_TOKEN }}
CONDA_PYTORCHBOT_TOKEN_TEST: ${{ secrets.CONDA_PYTORCHBOT_TOKEN_TEST }}

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ inputs.repository }}-${{ github.event_name == 'workflow_dispatch' }}
cancel-in-progress: true
echo "No more conda builds, see https://github.com/pytorch/pytorch/issues/138506"
Loading