ci: Constrain all dependencies

Signed-off-by: Ihar Hrachyshka <ihar.hrachyshka@gmail.com>

Author: booxter

.github/actions/run-smoke/action.yml

@@ -34,7 +34,7 @@ runs:
       run: |
         python${{ inputs.python-version }} -m venv --upgrade-deps venv
         . venv/bin/activate
-        pip install tox
+        pip install tox -c constraints-dev.txt
 
     # flash-attn has a bug in the setup.py that causes pip to attempt
     # installing it before torch is installed. This is a bug because their
@@ -46,7 +46,7 @@ runs:
       run: |
         source venv/bin/activate
         # The list is taken from the pull request linked above
-        pip install torch packaging setuptools wheel psutil ninja
+        pip install torch packaging setuptools wheel psutil ninja -c constraints-dev.txt
 
     - name: "Install tox-current-env to reuse the venv with pre-installed build dependencies"
       shell: bash
@@ -59,8 +59,9 @@ runs:
       run: |
         source venv/bin/activate
         tox -e py3-smoke --print-deps-to-file=./deps.txt
-        pip install -r ./deps.txt --no-build-isolation
-        pip install .
+        pip_install="pip install -c constraints-dev.txt"
+        $pip_install -r ./deps.txt --no-build-isolation
+        $pip_install .
 
     - name: "Show disk utilization BEFORE tests"
       shell: bash

.github/workflows/constraints-update.yml

@@ -0,0 +1,47 @@
+name: Update constraints-dev.txt
+
+on:
+  schedule:
+    - cron: '0 3 * * 1'  # Every Monday at 03:00 UTC
+  workflow_dispatch:
+
+jobs:
+  update-constraints:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.11'
+
+      - name: Install tox
+        run: python -m pip install --upgrade pip tox -c constraints-dev.txt
+
+      - name: Run tox constraints
+        run: tox -e constraints
+
+      - name: git diff
+        run: |
+          git diff constraints-dev.txt
+
+      - name: set release date
+        run: |
+          echo "date=$(date --rfc-3339=date)" >> "${GITHUB_ENV}"
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8
+        with:
+          signoff: true
+          commit-message: "chore: update constraints-dev.txt (${{ env.date }})"
+          title: "chore: update constraints-dev.txt (${{ env.date }})"
+          body: |
+            This PR updates the constraints-dev.txt file using `tox -e constraints`.
+          branch: update-constraints
+          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

.github/workflows/e2e-nvidia-l40s-x4.yml

@@ -177,27 +177,15 @@ jobs:
       - name: Install ilab
         working-directory: ./instructlab
         run: |
-          export CUDA_HOME="/usr/local/cuda"
-          export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64"
-          export PATH="$PATH:$CUDA_HOME/bin"
-          python3.11 -m venv --upgrade-deps venv
-          . venv/bin/activate
-          nvidia-smi
-          python3.11 -m pip cache remove llama_cpp_python
-
-          CMAKE_ARGS="-DLLAMA_CUDA=on" python3.11 -m pip install . -c constraints-dev.txt
-
-          # https://github.com/instructlab/instructlab/issues/1821
-          # install with Torch and build dependencies installed
-          python3.11 -m pip install packaging wheel setuptools-scm
-          python3.11 -m pip install .[cuda] -r requirements-vllm-cuda.txt
+          ./scripts/install-ilab-with-cuda.sh
 
       - name: Update instructlab-training library
         working-directory: ./training
         run: |
           . ../instructlab/venv/bin/activate
-          pip install .
-          pip install .[cuda]
+          pip_install="pip install -c constraints-dev.txt"
+          $pip_install .
+          $pip_install .[cuda]
 
       - name: Check disk before tests
         if: always()
@@ -380,7 +368,7 @@ jobs:
         working-directory: ./training
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt
+          pip install -r requirements-dev.txt -c constraints-dev.txt
 
       - name: Try to upload Phase 1 to s3
         id: phase-1-upload-s3

.github/workflows/lint.yml

@@ -80,7 +80,7 @@ jobs:
       - name: Install tox
         run: |
           python -m pip install --upgrade pip
-          python -m pip install tox tox-gh
+          python -m pip install tox tox-gh -c constraints-dev.txt
 
       - name: "${{ matrix.lint.name }}"
         run: |

.github/workflows/unit.yaml

@@ -72,7 +72,7 @@ jobs:
         run: |
           python -m venv --upgrade-deps venv
           . venv/bin/activate
-          pip install tox
+          pip install tox -c constraints-dev.txt
 
       - name: "Show disk utilization BEFORE tests"
         if: always()

constraints-dev.txt.in

@@ -2,7 +2,7 @@ flash-attn>=2.4.0
 bitsandbytes>=0.43.1
 
 # required for FSDP updates
-accelerate>=0.34.2,<1.1.0
+accelerate>=0.34.2
 
 # first version that includes Granite kernels
 liger-kernel>=0.5.4

requirements-cuda.txt

@@ -1,3 +1,3 @@
 # SPDX-License-Identifier: Apache-2.0
 
-deepspeed>=0.14.3
+deepspeed>=0.14.3

requirements-deepspeed.txt

@@ -2,12 +2,15 @@
 
 -r requirements.txt
 
+isort
 matplotlib
 numpy
-pre-commit>=3.0.4,<5.0
-pylint>=2.16.2,<4.0
+pre-commit>=3.0.4
+pylint>=2.16.2
 pylint-pydantic
-tox>=4.4.2,<5
+ruff
+tox>=4.4.2
+tox-current-env
 
 ipython
 ipykernel

requirements-dev.txt

@@ -9,9 +9,7 @@ transformers>=4.45.2
 
 datasets>=2.15.0
 numba
-# Note: numpy ranges copied from instructlab/instructlab
-#
-numpy>=1.26.4,<2.0.0
+numpy>=1.26.4
 rich
 instructlab-dolomite>=0.2.0
 trl>=0.9.4