Drop python 3.8, add 3.11 & 3.12 to testing (#1143)

* Initial python version bump in CI

* Add libprotobuf to GPU CI environments

* Replace mentions of old env files

* Remove strict channel priority to try to unblock env solves?

* Establish minimum version for mlflow

* Revert "Remove strict channel priority to try to unblock env solves?"

This reverts commit e454833.

* Try strict channel priority without nodefaults

* Bump mlflow min version to fix windows failures

* Build python 3.11 wheels

* Run wheel builds in PR test

* Try protoc action in wheels build to unblock

* Skip hive testing on 3.11 for now

* Fix workflow syntax errors

* Stop running wheel CI

* Bump pyo3 abi minor version

* Initial run of pyupgrade to py39

* Continue marking test_dask_fsql as flaky

* More places to drop 3.8

* Try running tests on python 3.12

* Add environment file

* Skip sasl installation

* Drop protoc build dep

* Drop mlflow constraint

* Set min version for mlflow

* Drop mlflow from 3.12 tests for now

* Relocate docker/server files to continuous_integration

* Unpin dask/distributed

* unpin 3.9 gpu environment

* add 3.12 to classifiers

* unpin dask in gpuci 3.9

Author: charlesbluca

.github/workflows/conda.yml

@@ -31,7 +31,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
         arch: ["linux-64", "linux-aarch64"]
     steps:
       - name: Manage disk space
@@ -72,7 +72,7 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          python-version: "3.8"
+          python-version: "3.9"
           channel-priority: strict
       - name: Install dependencies
         run: |

.github/workflows/docker.yml

@@ -11,7 +11,7 @@ on:
       - Cargo.toml
       - Cargo.lock
       - pyproject.toml
-      - docker/**
+      - continuous_integration/docker/**
       - .github/workflows/docker.yml
 
 # When this workflow is queued, automatically cancel any previous running
@@ -47,7 +47,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/main.dockerfile
+          file: ./continuous_integration/docker/main.dockerfile
           build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }}
           platforms: ${{ matrix.platform }}
           tags: ${{ steps.docker_meta_main.outputs.tags }}
@@ -68,7 +68,7 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./docker/cloud.dockerfile
+          file: ./continuous_integration/docker/cloud.dockerfile
           build-args: DOCKER_META_VERSION=${{ steps.docker_meta_main.outputs.version }}
           platforms: ${{ matrix.platform }}
           tags: ${{ steps.docker_meta_cloud.outputs.tags }}

.github/workflows/test-upstream.yml

@@ -36,21 +36,21 @@ jobs:
     name: "Test upstream dev (${{ matrix.os }}, python: ${{ matrix.python }}, distributed: ${{ matrix.distributed }})"
     runs-on: ${{ matrix.os }}
     env:
-      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml
+      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}.yaml
       DASK_SQL_DISTRIBUTED_TESTS: ${{ matrix.distributed }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
         distributed: [false]
         include:
           # run tests on a distributed client
           - os: "ubuntu-latest"
-            python: "3.8"
+            python: "3.9"
             distributed: true
           - os: "ubuntu-latest"
-            python: "3.10"
+            python: "3.11"
             distributed: true
     steps:
       - uses: actions/checkout@v4
@@ -75,7 +75,6 @@ jobs:
       - name: Install hive testing dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
-          mamba install -c conda-forge "sasl>=0.3.1"
           docker pull bde2020/hive:2.3.2-postgresql-metastore
           docker pull bde2020/hive-metastore-postgresql:2.3.0
       - name: Install upstream dev Dask
@@ -109,8 +108,6 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          # TODO: drop support for python 3.8, add support for python 3.11
-          # https://github.com/dask-contrib/dask-sql/pull/1143
           python-version: "3.9"
           channel-priority: strict
       - name: Optionally update upstream cargo dependencies

.github/workflows/test.yml

@@ -37,21 +37,21 @@ jobs:
     needs: [detect-ci-trigger]
     runs-on: ${{ matrix.os }}
     env:
-      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}-dev.yaml
+      CONDA_FILE: continuous_integration/environment-${{ matrix.python }}.yaml
       DASK_SQL_DISTRIBUTED_TESTS: ${{ matrix.distributed }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python: ["3.8", "3.9", "3.10"]
+        python: ["3.9", "3.10", "3.11", "3.12"]
         distributed: [false]
         include:
           # run tests on a distributed client
           - os: "ubuntu-latest"
-            python: "3.8"
+            python: "3.9"
             distributed: true
           - os: "ubuntu-latest"
-            python: "3.10"
+            python: "3.11"
             distributed: true
     steps:
       - uses: actions/checkout@v4
@@ -76,7 +76,6 @@ jobs:
       - name: Install hive testing dependencies
         if: matrix.os == 'ubuntu-latest'
         run: |
-          mamba install -c conda-forge "sasl>=0.3.1"
           docker pull bde2020/hive:2.3.2-postgresql-metastore
           docker pull bde2020/hive-metastore-postgresql:2.3.0
       - name: Optionally install upstream dev Dask
@@ -107,9 +106,7 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           use-mamba: true
-          # TODO: drop support for python 3.8, add support for python 3.11
-          # https://github.com/dask-contrib/dask-sql/pull/1143
-          python-version: ${{ needs.detect-ci-trigger.outputs.triggered == 'true' && '3.9' || '3.8' }}
+          python-version: "3.9"
           channel-priority: strict
       - name: Install dependencies and nothing else
         run: |

CONTRIBUTING.md

@@ -19,7 +19,7 @@ rustup update
 To initialize and activate the conda environment for a given Python version:
 
 ```
-conda env create -f dask-sql/continuous_integration/environment-{$PYTHON_VER}-dev.yaml
+conda env create -f dask-sql/continuous_integration/environment-{$PYTHON_VER}.yaml
 conda activate dask-sql
 ```
 

Cargo.toml

@@ -14,7 +14,7 @@ async-trait = "0.1.74"
 datafusion-python = { git = "https://github.com/apache/arrow-datafusion-python.git", ref = "da6c183" }
 env_logger = "0.10"
 log = "^0.4"
-pyo3 = { version = "0.19.2", features = ["extension-module", "abi3", "abi3-py38"] }
+pyo3 = { version = "0.19.2", features = ["extension-module", "abi3", "abi3-py39"] }
 pyo3-log = "0.9.0"
 
 [build-dependencies]

README.md

@@ -101,7 +101,7 @@ If you want to have the newest (unreleased) `dask-sql` version or if you plan to
 
 Create a new conda environment and install the development environment:
 
-    conda env create -f continuous_integration/environment-3.9-dev.yaml
+    conda env create -f continuous_integration/environment-3.9.yaml
 
 It is not recommended to use `pip` instead of `conda` for the environment setup.
 

docker/cloud.dockerfile renamed to continuous_integration/docker/cloud.dockerfile

@@ -1,5 +1,5 @@
-python>=3.8
-dask>=2022.3.0,<=2023.11.0
+python>=3.9
+dask>=2022.3.0
 pandas>=1.4.0
 jpype1>=1.0.2
 openjdk>=8

docker/conda.txt renamed to continuous_integration/docker/conda.txt

@@ -11,12 +11,12 @@ RUN sh /rustup-init.sh -y --default-toolchain=stable --profile=minimal \
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 # Install conda dependencies for dask-sql
-COPY docker/conda.txt /opt/dask_sql/
+COPY continuous_integration/docker/conda.txt /opt/dask_sql/
 RUN mamba install -y \
     # build requirements
     "maturin>=1.3,<1.4" \
     # core dependencies
-    "dask>=2022.3.0,<=2023.11.0" \
+    "dask>=2022.3.0" \
     "pandas>=1.4.0" \
     "fastapi>=0.92.0" \
     "httpx>=0.24.1" \
@@ -44,7 +44,7 @@ RUN cd /opt/dask_sql/ \
     && CONDA_PREFIX="/opt/conda/" maturin develop
 
 # Set the script to execute
-COPY scripts/startup_script.py /opt/dask_sql/startup_script.py
+COPY continuous_integration/scripts/startup_script.py /opt/dask_sql/startup_script.py
 
 EXPOSE 8080
 ENTRYPOINT [ "/usr/bin/prepare.sh", "/opt/conda/bin/python", "/opt/dask_sql/startup_script.py" ]