Start migration off defaults to conda-forge channel (facebookresearch…

…#4126)

Summary:

Good resource on overriding channels to make sure we aren't using `defaults`:https://stackoverflow.com/questions/67695893/how-do-i-completely-purge-and-disable-the-default-channel-in-anaconda-and-switch

Explanation of changes:
-
- Python upgraded to 3.12 because otherwise we fail with the below error, and I thought we might want to keep telemetry? Let me know otherwise
```
LibMambaUnsatisfiableError: Encountered problems while solving:
  - package conda-anaconda-telemetry-0.1.1-py312h06a4308_0 requires python >=3.12,<3.13.0a0, but none of the providers can be installed

Could not solve for environment specs
The following packages are incompatible
├─ conda-anaconda-telemetry is installable and it requires
│  └─ python >=3.12,<3.13.0a0 , which can be installed;
└─ python 3.11**  is not installable because it conflicts with any installable versions previously reported.
```
- Solver changes: conda/conda-libmamba-solver#283 (comment) We need to manually install libmamba from conda-forge instead of defaults, then use it. If it and libarchive come from different channels, build fails with:
```
Error while loading conda entry point: conda-libmamba-solver (libarchive.so.20: cannot open shared object file: No such file or directory)

CondaValueError: You have chosen a non-default solver backend (libmamba) but it was not recognized. Choose one of: classic
```
or the very cryptic: (it refers to glibc version...)
```
Your installed version is: 2.35
```
- Why not use the classic solver, why do we need the above change to keep using libmamba? classic is not able to resolve dependencies for us. Others suggest just to use libmamba: conda/conda-build#5516 (comment). Classic fails with
```
DependencyNeedsBuildingError: Unsatisfiable dependencies for platform linux-64: {'__glibc', '__archspec', '__unix', '__conda', '__linux'}
```
- cpuinfo.py copied in because Python 3.12 removes distutils, and setuptools does not have feature parity. People have complained that they can't use FAISS with Python > 3.11 before and this is the dependency that I see. facebookresearch#3936
- cuda-toolkit upgraded to 12.5.1 because of this error on 12.4.1:
```
nvlink fatal : Input file
  '/home/runner/miniconda3/bin/../lib/libcudadevrt.a:cuda_device_runtime.o'
  newer than toolkit (126 vs 124)
```
-

Differential Revision: D68043874

.github/actions/build_cmake/action.yml

@@ -23,28 +23,59 @@ runs:
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.11'
-        miniconda-version: latest
+        miniforge-version: latest # ensures conda-forge channel is used.
+        channels: conda-forge
+        conda-remove-defaults: 'true'
+        # conda-solver: "classic"
+    # - name: Fix CI failure
+    #   shell: bash
+    #   run: |
+    #     # TODO: do we need to remove everything not installed from conda-forge?
+    #     conda remove -c conda-forge anaconda-anon-usage conda-anaconda-telemetry
     - name: Configure build environment
       shell: bash
       run: |
         # initialize Conda
         conda config --set solver libmamba
+        conda list --show-channel-urls
+        echo "*****1*****"
+        conda config --show channels
+        echo "*****2*****"
+        conda config --show-sources
+        # echo "*****2.1*****"
+        # conda remove conda-libmamba-solver
+        # echo "*****2.2*****"
+        # conda install -c conda-forge --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive
+        # echo "*****2.3*****"
+        # conda list --show-channel-urls
+        # echo "*****2.4*****"
+        # conda config --set solver libmamba
+        echo "*****2.5*****"
         conda update -y -q conda
         echo "$CONDA/bin" >> $GITHUB_PATH
+        echo "*****3*****"
+        conda config --show channels
+
+        echo "*****4.1*****"
 
         conda install -y -q python=3.11 cmake=3.26 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2
 
+        echo "*****5*****"
+        conda config --show channels
+        echo "*****6*****"
         # install base packages for ARM64
         if [ "${{ runner.arch }}" = "ARM64" ]; then
           conda install -y -q -c conda-forge openblas=0.3 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
         fi
+        echo "*****6.1*****"
 
         # install base packages for X86_64
         if [ "${{ runner.arch }}" = "X64" ]; then
           # TODO: merge this with ARM64
           conda install -y -q -c conda-forge gxx_linux-64=14.2 sysroot_linux-64=2.17
           conda install -y -q mkl=2023 mkl-devel=2023
         fi
+        echo "*****6.2*****"
 
         # no CUDA needed for ROCm so skip this
         if [ "${{ inputs.rocm }}" = "ON" ]; then
@@ -56,15 +87,18 @@ runs:
         elif [ "${{ inputs.cuvs }}" = "ON" ]; then
           conda install -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c conda-forge
         fi
+        echo "*****6.3*****"
         # install test packages
         if [ "${{ inputs.rocm }}" = "ON" ]; then
           : # skip torch install via conda, we need to install via pip to get
             #  ROCm-enabled version until it's supported in conda by PyTorch
         elif [ "${{ inputs.gpu }}" = "ON" ]; then
-          conda install -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c "nvidia/label/cuda-12.4.0"
+          conda install -y -q "pytorch<2.5" pytorch-cuda=12.4 -c pytorch -c "nvidia/label/cuda-12.4.1"
         else
           conda install -y -q "pytorch<2.5" -c pytorch
         fi
+        echo "*****7*****"
+        conda config --show channels
     - name: ROCm - Install dependencies
       if: inputs.rocm == 'ON'
       shell: bash

.github/actions/build_conda/action.yml

@@ -30,16 +30,31 @@ runs:
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.11'
-        miniconda-version:  latest
+        miniforge-version: latest # ensures conda-forge channel is used.
+        channels: conda-forge
+        conda-remove-defaults: 'true'
+        # conda-solver: "classic"
     - name: Install conda build tools
       shell: ${{ steps.choose_shell.outputs.shell }}
       run: |
+        echo "*****1*****"
+        conda list --show-channel-urls
+        # echo "*****2.1*****"
+        # conda remove conda-libmamba-solver
+        # echo "*****2.2*****"
+        # conda install -c conda-forge --solver=classic conda-forge::conda-libmamba-solver conda-forge::libmamba conda-forge::libmambapy conda-forge::libarchive
+        # echo "*****2.2*****"
+        # conda list --show-channel-urls
+        # echo "*****2.3*****"
+        # conda config --set solver libmamba
+        echo "*****2.4*****"
+        # Lower zlib version to allow intel-openmp=2023 installation  (was "zlib<1.3.0", try without this)
         conda install -y -q "conda!=24.11.0"
         conda install -y -q "conda-build!=24.11.0"
-    - name: Fix CI failure
-      shell: ${{ steps.choose_shell.outputs.shell }}
-      if: runner.os != 'Windows'
-      run: conda remove conda-anaconda-telemetry
+    # - name: Fix CI failure
+    #   shell: ${{ steps.choose_shell.outputs.shell }}
+    #   if: runner.os != 'Windows'
+    #   run: conda remove conda-anaconda-telemetry
     - name: Enable anaconda uploads
       if: inputs.label != ''
       shell: ${{ steps.choose_shell.outputs.shell }}
@@ -53,6 +68,10 @@ runs:
       shell: ${{ steps.choose_shell.outputs.shell }}
       working-directory: conda
       run: |
+        # TODO remove
+        echo "*****3.1*****"
+        conda list --show-channel-urls
+        echo "*****3.2*****"
         conda build faiss --python 3.11 -c pytorch
     - name: Conda build (CPU) w/ anaconda upload
       if: inputs.label != '' && inputs.cuda == ''

.github/workflows/build-pull-request.yml

@@ -117,8 +117,8 @@ jobs:
         with:
           gpu: ON
           rocm: ON
-  linux-arm64-SVE-cmake:
-    name: Linux arm64 SVE (cmake)
+  linux-aarch64-SVE-cmake:
+    name: Linux aarch64 SVE (cmake)
     needs: linux-x86_64-cmake
     runs-on: faiss-aws-r8g.large
     steps:
@@ -155,8 +155,8 @@ jobs:
           fetch-tags: true
       - name: Build and Package (conda)
         uses: ./.github/actions/build_conda
-  linux-arm64-conda:
-    name: Linux arm64 (conda)
+  linux-aarch64-conda:
+    name: Linux aarch64 (conda)
     needs: linux-x86_64-cmake
     runs-on: 2-core-ubuntu-arm
     steps:

faiss/python/CMakeLists.txt

@@ -313,6 +313,7 @@ configure_file(class_wrappers.py class_wrappers.py COPYONLY)
 configure_file(gpu_wrappers.py gpu_wrappers.py COPYONLY)
 configure_file(extra_wrappers.py extra_wrappers.py COPYONLY)
 configure_file(array_conversions.py array_conversions.py COPYONLY)
+configure_file(cpuinfo.py cpuinfo.py COPYONLY)
 
 # file(GLOB files "${PROJECT_SOURCE_DIR}/../../contrib/*.py")
 file(COPY ${PROJECT_SOURCE_DIR}/../../contrib  DESTINATION .)

faiss/python/__init__.py

@@ -20,6 +20,7 @@
 from faiss import class_wrappers
 from faiss.gpu_wrappers import *
 from faiss.array_conversions import *
+from faiss.cpuinfo import *
 from faiss.extra_wrappers import kmin, kmax, pairwise_distances, rand, randint, \
     lrand, randn, rand_smooth_vectors, eval_intersection, normalize_L2, \
     ResultHeap, knn, Kmeans, checksum, matrix_bucket_sort_inplace, bucket_sort, \