Start migration off defaults to conda-forge channel (facebookresearch…

…#4126) Summary: Good resource on overriding channels to make sure we aren't using `defaults`:https://stackoverflow.com/questions/67695893/how-do-i-completely-purge-and-disable-the-default-channel-in-anaconda-and-switch Explanation of changes: - - Python upgraded to 3.12 because otherwise we fail with the below error, and I thought we might want to keep telemetry? Let me know otherwise ``` LibMambaUnsatisfiableError: Encountered problems while solving: - package conda-anaconda-telemetry-0.1.1-py312h06a4308_0 requires python >=3.12,<3.13.0a0, but none of the providers can be installed Could not solve for environment specs The following packages are incompatible ├─ conda-anaconda-telemetry is installable and it requires │ └─ python >=3.12,<3.13.0a0 , which can be installed; └─ python 3.11** is not installable because it conflicts with any installable versions previously reported. ``` - Solver changes: conda/conda-libmamba-solver#283 (comment) We need to manually install libmamba from conda-forge instead of defaults, then use it. If it and libarchive come from different channels, build fails with: ``` Error while loading conda entry point: conda-libmamba-solver (libarchive.so.20: cannot open shared object file: No such file or directory) CondaValueError: You have chosen a non-default solver backend (libmamba) but it was not recognized. Choose one of: classic ``` or the very cryptic: (it refers to glibc version...) ``` Your installed version is: 2.35 ``` - Why not use the classic solver, why do we need the above change to keep using libmamba? classic is not able to resolve dependencies for us. Others suggest just to use libmamba: conda/conda-build#5516 (comment). Classic fails with ``` DependencyNeedsBuildingError: Unsatisfiable dependencies for platform linux-64: {'__glibc', '__archspec', '__unix', '__conda', '__linux'} ``` - cpuinfo.py copied in because Python 3.12 removes distutils, and setuptools does not have feature parity. People have complained that they can't use FAISS with Python > 3.11 before and this is the dependency that I see. facebookresearch#3936 - cuda-toolkit upgraded to 12.5.1 because of this error on 12.4.1: ``` nvlink fatal : Input file '/home/runner/miniconda3/bin/../lib/libcudadevrt.a:cuda_device_runtime.o' newer than toolkit (126 vs 124) ``` - Differential Revision: D68043874
mnorris11 · Jan 14, 2025 · 8dcf96d · 8dcf96d
1 parent b9fe1dc
commit 8dcf96d
Show file tree

Hide file tree

Showing 7 changed files with 765 additions and 34 deletions.
diff --git a/.github/actions/build_cmake/action.yml b/.github/actions/build_cmake/action.yml
@@ -22,49 +22,82 @@ runs:
     - name: Setup miniconda
       uses: conda-incubator/setup-miniconda@v3
       with:
+        channels: conda-forge
+        conda-remove-defaults: 'true'
         python-version: '3.11'
         miniconda-version: latest
+        conda-solver: "classic"
+    - name: Fix CI failure
+      shell: bash
+      run: |
+        # TODO: do we need to remove everything not installed from conda-forge?
+        conda remove -c conda-forge --override-channels anaconda-anon-usage conda-anaconda-telemetry
     - name: Configure build environment
       shell: bash
       run: |
         # initialize Conda
+        conda list --show-channel-urls
+        echo "*****1*****"
+        conda config --show channels
+        echo "*****2*****"
+        conda config --show-sources
+        # echo "*****2.1*****"
+        # conda remove conda-libmamba-solver
+        echo "*****2.2*****"
+        conda install -c conda-forge --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive --override-channels
+        echo "*****2.3*****"
+        conda list --show-channel-urls
+        echo "*****2.4*****"
         conda config --set solver libmamba
-        conda update -y -q conda
+        echo "*****2.5*****"
+        conda update -c conda-forge -y -q conda --override-channels
         echo "$CONDA/bin" >> $GITHUB_PATH
+        echo "*****3*****"
+        conda config --show channels
+
+        echo "*****4.1*****"
 
-        conda install -y -q python=3.11 cmake=3.26 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2
+        conda install -c conda-forge -y -q python=3.11 cmake=3.26 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2 --override-channels
 
+        echo "*****5*****"
+        conda config --show channels
+        echo "*****6*****"
         # install base packages for ARM64
         if [ "${{ runner.arch }}" = "ARM64" ]; then
-          conda install -y -q -c conda-forge openblas=0.3 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
+          conda install -c conda-forge -y -q openblas=0.3 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17 --override-channels
         fi
+        echo "*****6.1*****"
 
         # install base packages for X86_64
         if [ "${{ runner.arch }}" = "X64" ]; then
           # TODO: merge this with ARM64
-          conda install -y -q -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17
-          conda install -y -q mkl=2023 mkl-devel=2023
+          conda install -c conda-forge -y -q gxx_linux-64=14.2 sysroot_linux-64=2.17 --override-channels
+          conda install -c conda-forge -y -q mkl=2023 mkl-devel=2023 --override-channels
         fi
+        echo "*****6.2*****"
 
         # no CUDA needed for ROCm so skip this
         if [ "${{ inputs.rocm }}" = "ON" ]; then
           :
         # regular CUDA for GPU builds
         elif [ "${{ inputs.gpu }}" = "ON" ] && [ "${{ inputs.cuvs }}" = "OFF" ]; then
-          conda install -y -q cuda-toolkit=12.4 -c "nvidia/label/cuda-12.4.0"
+          conda install -c conda-forge -y -q cuda-toolkit=12.4 -c "nvidia/label/cuda-12.4.0" --override-channels
         # and CUDA from cuVS channel for cuVS builds
         elif [ "${{ inputs.cuvs }}" = "ON" ]; then
-          conda install -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c conda-forge
+          conda install -c conda-forge -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai --override-channels
         fi
+        echo "*****6.3*****"
         # install test packages
         if [ "${{ inputs.rocm }}" = "ON" ]; then
           : # skip torch install via conda, we need to install via pip to get
             #  ROCm-enabled version until it's supported in conda by PyTorch
         elif [ "${{ inputs.gpu }}" = "ON" ]; then
-          conda install -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c "nvidia/label/cuda-12.4.0"
+          conda install -c conda-forge -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c "nvidia/label/cuda-12.4.1" --override-channels
         else
-          conda install -y -q "pytorch<2.5" -c pytorch
+          conda install -c conda-forge -y -q "pytorch<2.5" -c pytorch --override-channels
         fi
+        echo "*****7*****"
+        conda config --show channels
     - name: ROCm - Install dependencies
       if: inputs.rocm == 'ON'
       shell: bash

diff --git a/.github/actions/build_conda/action.yml b/.github/actions/build_conda/action.yml
@@ -31,66 +31,87 @@ runs:
       with:
         python-version: '3.11'
         miniconda-version:  latest
-    - name: Install conda build tools
-      shell: ${{ steps.choose_shell.outputs.shell }}
-      run: |
-        conda install -y -q "conda!=24.11.0"
-        conda install -y -q "conda-build!=24.11.0"
+        channels: conda-forge
+        conda-remove-defaults: "true"
+        conda-solver: "classic"
     - name: Fix CI failure
       shell: ${{ steps.choose_shell.outputs.shell }}
       if: runner.os != 'Windows'
       run: conda remove conda-anaconda-telemetry
+    - name: Install conda build tools
+      shell: ${{ steps.choose_shell.outputs.shell }}
+      run: |
+        echo "*****1*****"
+        conda list --show-channel-urls
+        # echo "*****2.1*****"
+        # conda remove conda-libmamba-solver
+        echo "*****2.2*****"
+        conda install -c conda-forge --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive --override-channels
+        echo "*****2.2*****"
+        conda list --show-channel-urls
+        echo "*****2.3*****"
+        conda config --set solver libmamba
+        echo "*****2.4*****"
+        # TODO: try without zlib
+        conda install -c conda-forge -y -q "conda!=24.11.0" --override-channels
+        conda install -c conda-forge -y -q "conda-build!=24.11.0" --override-channels
     - name: Enable anaconda uploads
       if: inputs.label != ''
       shell: ${{ steps.choose_shell.outputs.shell }}
       env:
         PACKAGE_TYPE: ${{ inputs.label }}
       run: |
-        conda install -y -q anaconda-client
-        conda config --set anaconda_upload yes
+        conda install -c conda-forge -y -q anaconda-client --override-channels
+        conda config -c conda-forge --set anaconda_upload yes --override-channels
     - name: Conda build (CPU)
       if: inputs.label == '' && inputs.cuda == ''
       shell: ${{ steps.choose_shell.outputs.shell }}
       working-directory: conda
       run: |
-        conda build faiss --python 3.11 -c pytorch
+        # TODO remove
+        echo "*****3.1*****"
+        conda list --show-channel-urls
+        echo "*****3.2*****"
+        conda build -c conda-forge faiss --python 3.11 -c pytorch --override-channels
     - name: Conda build (CPU) w/ anaconda upload
       if: inputs.label != '' && inputs.cuda == ''
       shell: ${{ steps.choose_shell.outputs.shell }}
       working-directory: conda
       env:
         PACKAGE_TYPE: ${{ inputs.label }}
       run: |
-        conda build faiss --user pytorch --label ${{ inputs.label }} -c pytorch
+        conda build -c conda-forge faiss --user pytorch --label ${{ inputs.label }} -c pytorch --override-channels
     - name: Conda build (GPU)
       if: inputs.label == '' && inputs.cuda != '' && inputs.cuvs == ''
       shell: ${{ steps.choose_shell.outputs.shell }}
       working-directory: conda
       run: |
-        conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
-            -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia
+        conda build -c conda-forge faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
+            -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia --override-channels
     - name: Conda build (GPU) w/ anaconda upload
       if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs == ''
       shell: ${{ steps.choose_shell.outputs.shell }}
       working-directory: conda
       env:
         PACKAGE_TYPE: ${{ inputs.label }}
       run: |
-        conda build faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
-            --user pytorch --label ${{ inputs.label }} -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} -c nvidia
+        conda build -c conda-forge faiss-gpu --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
+            --user pytorch --label ${{ inputs.label }} -c pytorch -c nvidia/label/cuda-${{ inputs.cuda }} \
+            -c nvidia --override-channels
     - name: Conda build (GPU w/ cuVS)
       if: inputs.label == '' && inputs.cuda != '' && inputs.cuvs != ''
       shell: ${{ steps.choose_shell.outputs.shell }}
       working-directory: conda
       run: |
-        conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
-            -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia
+        conda build -c conda-forge faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
+            -c pytorch -c rapidsai -c rapidsai-nightly -c nvidia --override-channels
     - name: Conda build (GPU w/ cuVS) w/ anaconda upload
       if: inputs.label != '' && inputs.cuda != '' && inputs.cuvs != ''
       shell: ${{ steps.choose_shell.outputs.shell }}
       working-directory: conda
       env:
         PACKAGE_TYPE: ${{ inputs.label }}
       run: |
-        conda build faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
-            --user pytorch --label ${{ inputs.label }} -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia
+        conda build -c conda-forge faiss-gpu-cuvs --variants '{ "cudatoolkit": "${{ inputs.cuda }}" }' \
+            --user pytorch --label ${{ inputs.label }} -c pytorch -c rapidsai -c rapidsai-nightly \
+            -c nvidia --override-channels
diff --git a/faiss/python/CMakeLists.txt b/faiss/python/CMakeLists.txt
@@ -313,6 +313,7 @@ configure_file(class_wrappers.py class_wrappers.py COPYONLY)
 configure_file(gpu_wrappers.py gpu_wrappers.py COPYONLY)
 configure_file(extra_wrappers.py extra_wrappers.py COPYONLY)
 configure_file(array_conversions.py array_conversions.py COPYONLY)
+configure_file(cpuinfo.py cpuinfo.py COPYONLY)
 
 # file(GLOB files "${PROJECT_SOURCE_DIR}/../../contrib/*.py")
 file(COPY ${PROJECT_SOURCE_DIR}/../../contrib  DESTINATION .)
diff --git a/faiss/python/__init__.py b/faiss/python/__init__.py
@@ -20,6 +20,7 @@
 from faiss import class_wrappers
 from faiss.gpu_wrappers import *
 from faiss.array_conversions import *
+from faiss.cpuinfo import *
 from faiss.extra_wrappers import kmin, kmax, pairwise_distances, rand, randint, \
     lrand, randn, rand_smooth_vectors, eval_intersection, normalize_L2, \
     ResultHeap, knn, Kmeans, checksum, matrix_bucket_sort_inplace, bucket_sort, \