tests: Generate part of the Tasks tox entries by a script (#3976)

- remove hardcoded entries for `celery`, `spark`, `huey`, `dramatiq` from the tox template - remove them from the ignore list in `populate_tox.py` - run `populate_tox.py` to fill in entries for them - run `split_gh_tox_actions.py` to generate the CI yaml files so that they correspond to the new tox.ini The remaining integrations in this group are not trivial to switch over to the script, I'll do this step by step in follow-up PRs. --------- Co-authored-by: Daniel Szoke <[email protected]>
getsentry · Feb 14, 2025 · 8f22def · 8f22def
1 parent 25ddbca
commit 8f22def
Show file tree

Hide file tree

Showing 6 changed files with 56 additions and 128 deletions.
diff --git a/.github/workflows/test-integrations-ai.yml b/.github/workflows/test-integrations-ai.yml
@@ -101,7 +101,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8","3.9","3.10","3.11","3.12","3.13"]
+        python-version: ["3.8","3.9","3.11","3.12"]
         # python3.6 reached EOL and is no longer being supported on
         # new versions of hosted runners on Github Actions
         # ubuntu-20.04 is the last version that supported python3.6

diff --git a/.github/workflows/test-integrations-tasks.yml b/.github/workflows/test-integrations-tasks.yml
@@ -29,7 +29,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6","3.7","3.8","3.10","3.11","3.12","3.13"]
+        python-version: ["3.7","3.8","3.10","3.11","3.12","3.13"]
         # python3.6 reached EOL and is no longer being supported on
         # new versions of hosted runners on Github Actions
         # ubuntu-20.04 is the last version that supported python3.6
@@ -115,7 +115,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6","3.7","3.8","3.9","3.10","3.11","3.12"]
+        python-version: ["3.6","3.7","3.8","3.9","3.10","3.11","3.12","3.13"]
         # python3.6 reached EOL and is no longer being supported on
         # new versions of hosted runners on Github Actions
         # ubuntu-20.04 is the last version that supported python3.6

diff --git a/scripts/populate_tox/config.py b/scripts/populate_tox/config.py
@@ -13,9 +13,19 @@
         },
         "python": ">=3.8",
     },
+    "celery": {
+        "package": "celery",
+        "deps": {
+            "*": ["newrelic", "redis"],
+            "py3.7": ["importlib-metadata<5.0"],
+        },
+    },
     "clickhouse_driver": {
         "package": "clickhouse-driver",
     },
+    "dramatiq": {
+        "package": "dramatiq",
+    },
     "flask": {
         "package": "flask",
         "deps": {
@@ -33,6 +43,9 @@
             "py3.6": ["aiocontextvars"],
         },
     },
+    "huey": {
+        "package": "huey",
+    },
     "huggingface_hub": {
         "package": "huggingface_hub",
     },
@@ -54,6 +67,10 @@
     "redis_py_cluster_legacy": {
         "package": "redis-py-cluster",
     },
+    "spark": {
+        "package": "pyspark",
+        "python": ">=3.8",
+    },
     "sqlalchemy": {
         "package": "sqlalchemy",
     },

diff --git a/scripts/populate_tox/populate_tox.py b/scripts/populate_tox/populate_tox.py
@@ -59,19 +59,18 @@
     "beam",
     "boto3",
     "bottle",
-    "celery",
     "chalice",
     "cohere",
     "cloud_resource_context",
     "cohere",
     "django",
-    "dramatiq",
     "falcon",
     "fastapi",
     "gcp",
     "grpc",
     "httpx",
     "huey",
+    "huggingface_hub",
     "langchain",
     "langchain_notiktoken",
     "litestar",
@@ -85,7 +84,6 @@
     "requests",
     "rq",
     "sanic",
-    "spark",
     "starlite",
     "tornado",
 }

diff --git a/scripts/populate_tox/tox.jinja b/scripts/populate_tox/tox.jinja
@@ -74,13 +74,6 @@ envlist =
     {py3.6,py3.9}-bottle-v{0.12}
     {py3.6,py3.12,py3.13}-bottle-latest
 
-    # Celery
-    {py3.6,py3.8}-celery-v{4}
-    {py3.6,py3.8}-celery-v{5.0}
-    {py3.7,py3.10}-celery-v{5.1,5.2}
-    {py3.8,py3.11,py3.12}-celery-v{5.3,5.4,5.5}
-    {py3.8,py3.12,py3.13}-celery-latest
-
     # Chalice
     {py3.6,py3.9}-chalice-v{1.16}
     {py3.8,py3.12,py3.13}-chalice-latest
@@ -107,12 +100,6 @@ envlist =
     {py3.10,py3.11,py3.12}-django-v{5.0,5.1}
     {py3.10,py3.12,py3.13}-django-latest
 
-    # dramatiq
-    {py3.6,py3.9}-dramatiq-v{1.13}
-    {py3.7,py3.10,py3.11}-dramatiq-v{1.15}
-    {py3.8,py3.11,py3.12}-dramatiq-v{1.17}
-    {py3.8,py3.11,py3.12}-dramatiq-latest
-
     # Falcon
     {py3.6,py3.7}-falcon-v{1,1.4,2}
     {py3.6,py3.11,py3.12}-falcon-v{3}
@@ -139,10 +126,6 @@ envlist =
     {py3.9,py3.11,py3.12}-httpx-v{0.25,0.27}
     {py3.9,py3.12,py3.13}-httpx-latest
 
-    # Huey
-    {py3.6,py3.11,py3.12}-huey-v{2.0}
-    {py3.6,py3.12,py3.13}-huey-latest
-
     # Langchain
     {py3.9,py3.11,py3.12}-langchain-v0.1
     {py3.9,py3.11,py3.12}-langchain-v0.3
@@ -208,10 +191,6 @@ envlist =
     {py3.8,py3.11,py3.12}-sanic-v{24.6}
     {py3.9,py3.12,py3.13}-sanic-latest
 
-    # Spark
-    {py3.8,py3.10,py3.11}-spark-v{3.1,3.3,3.5}
-    {py3.8,py3.10,py3.11,py3.12}-spark-latest
-
     # Starlite
     {py3.8,py3.11}-starlite-v{1.48,1.51}
     # 1.51.14 is the last starlite version; the project continues as litestar
@@ -317,21 +296,6 @@ deps =
     bottle-v0.12: bottle~=0.12.0
     bottle-latest: bottle
 
-    # Celery
-    celery: redis
-    celery-v4: Celery~=4.0
-    celery-v5.0: Celery~=5.0.0
-    celery-v5.1: Celery~=5.1.0
-    celery-v5.2: Celery~=5.2.0
-    celery-v5.3: Celery~=5.3.0
-    celery-v5.4: Celery~=5.4.0
-    # TODO: update when stable is out
-    celery-v5.5: Celery==5.5.0rc4
-    celery-latest: Celery
-
-    celery: newrelic
-    {py3.7}-celery: importlib-metadata<5.0
-
     # Chalice
     chalice: pytest-chalice==0.0.5
     chalice-v1.16: chalice~=1.16.0
@@ -370,12 +334,6 @@ deps =
     django-v5.1: Django==5.1rc1
     django-latest: Django
 
-    # dramatiq
-    dramatiq-v1.13: dramatiq>=1.13,<1.14
-    dramatiq-v1.15: dramatiq>=1.15,<1.16
-    dramatiq-v1.17: dramatiq>=1.17,<1.18
-    dramatiq-latest: dramatiq
-
     # Falcon
     falcon-v1.4: falcon~=1.4.0
     falcon-v1: falcon~=1.0
@@ -425,10 +383,6 @@ deps =
     httpx-v0.27: httpx~=0.27.0
     httpx-latest: httpx
 
-    # Huey
-    huey-v2.0: huey~=2.0.0
-    huey-latest: huey
-
     # Langchain
     langchain-v0.1: openai~=1.0.0
     langchain-v0.1: langchain~=0.1.11
@@ -543,14 +497,6 @@ deps =
     sanic-v24.6: sanic~=24.6.0
     sanic-latest: sanic
 
-    # Spark
-    spark-v3.1: pyspark~=3.1.0
-    spark-v3.3: pyspark~=3.3.0
-    spark-v3.5: pyspark~=3.5.0
-    # TODO: update to ~=4.0.0 once stable is out
-    spark-v4.0: pyspark==4.0.0.dev2
-    spark-latest: pyspark
-
     # Starlite
     starlite: pytest-asyncio
     starlite: python-multipart

diff --git a/tox.ini b/tox.ini
@@ -74,13 +74,6 @@ envlist =
     {py3.6,py3.9}-bottle-v{0.12}
     {py3.6,py3.12,py3.13}-bottle-latest
 
-    # Celery
-    {py3.6,py3.8}-celery-v{4}
-    {py3.6,py3.8}-celery-v{5.0}
-    {py3.7,py3.10}-celery-v{5.1,5.2}
-    {py3.8,py3.11,py3.12}-celery-v{5.3,5.4,5.5}
-    {py3.8,py3.12,py3.13}-celery-latest
-
     # Chalice
     {py3.6,py3.9}-chalice-v{1.16}
     {py3.8,py3.12,py3.13}-chalice-latest
@@ -107,12 +100,6 @@ envlist =
     {py3.10,py3.11,py3.12}-django-v{5.0,5.1}
     {py3.10,py3.12,py3.13}-django-latest
 
-    # dramatiq
-    {py3.6,py3.9}-dramatiq-v{1.13}
-    {py3.7,py3.10,py3.11}-dramatiq-v{1.15}
-    {py3.8,py3.11,py3.12}-dramatiq-v{1.17}
-    {py3.8,py3.11,py3.12}-dramatiq-latest
-
     # Falcon
     {py3.6,py3.7}-falcon-v{1,1.4,2}
     {py3.6,py3.11,py3.12}-falcon-v{3}
@@ -139,10 +126,6 @@ envlist =
     {py3.9,py3.11,py3.12}-httpx-v{0.25,0.27}
     {py3.9,py3.12,py3.13}-httpx-latest
 
-    # Huey
-    {py3.6,py3.11,py3.12}-huey-v{2.0}
-    {py3.6,py3.12,py3.13}-huey-latest
-
     # Langchain
     {py3.9,py3.11,py3.12}-langchain-v0.1
     {py3.9,py3.11,py3.12}-langchain-v0.3
@@ -208,10 +191,6 @@ envlist =
     {py3.8,py3.11,py3.12}-sanic-v{24.6}
     {py3.9,py3.12,py3.13}-sanic-latest
 
-    # Spark
-    {py3.8,py3.10,py3.11}-spark-v{3.1,3.3,3.5}
-    {py3.8,py3.10,py3.11,py3.12}-spark-latest
-
     # Starlite
     {py3.8,py3.11}-starlite-v{1.48,1.51}
     # 1.51.14 is the last starlite version; the project continues as litestar
@@ -225,13 +204,6 @@ envlist =
     # These come from the populate_tox.py script. Eventually we should move all
     # integration tests there.
 
-    # ~~~ AI ~~~
-    {py3.8,py3.10,py3.11}-huggingface_hub-v0.22.2
-    {py3.8,py3.10,py3.11}-huggingface_hub-v0.24.7
-    {py3.8,py3.11,py3.12}-huggingface_hub-v0.26.5
-    {py3.8,py3.12,py3.13}-huggingface_hub-v0.28.1
-
-
     # ~~~ DBs ~~~
     {py3.7,py3.11,py3.12}-clickhouse_driver-v0.2.9
 
@@ -282,6 +254,22 @@ envlist =
     {py3.9,py3.12,py3.13}-strawberry-v0.260.2
 
 
+    # ~~~ Tasks ~~~
+    {py3.6,py3.7,py3.8}-celery-v4.4.7
+    {py3.6,py3.7,py3.8}-celery-v5.0.5
+    {py3.8,py3.11,py3.12}-celery-v5.4.0
+
+    {py3.6,py3.7}-dramatiq-v1.9.0
+    {py3.6,py3.8,py3.9}-dramatiq-v1.12.3
+    {py3.7,py3.10,py3.11}-dramatiq-v1.15.0
+    {py3.8,py3.12,py3.13}-dramatiq-v1.17.1
+
+    {py3.8,py3.9}-spark-v3.0.3
+    {py3.8,py3.9}-spark-v3.2.4
+    {py3.8,py3.10,py3.11}-spark-v3.4.4
+    {py3.8,py3.10,py3.11}-spark-v3.5.4
+
+
     # ~~~ Web 1 ~~~
     {py3.6,py3.7,py3.8}-flask-v1.1.4
     {py3.8,py3.12,py3.13}-flask-v2.3.3
@@ -389,21 +377,6 @@ deps =
     bottle-v0.12: bottle~=0.12.0
     bottle-latest: bottle
 
-    # Celery
-    celery: redis
-    celery-v4: Celery~=4.0
-    celery-v5.0: Celery~=5.0.0
-    celery-v5.1: Celery~=5.1.0
-    celery-v5.2: Celery~=5.2.0
-    celery-v5.3: Celery~=5.3.0
-    celery-v5.4: Celery~=5.4.0
-    # TODO: update when stable is out
-    celery-v5.5: Celery==5.5.0rc4
-    celery-latest: Celery
-
-    celery: newrelic
-    {py3.7}-celery: importlib-metadata<5.0
-
     # Chalice
     chalice: pytest-chalice==0.0.5
     chalice-v1.16: chalice~=1.16.0
@@ -442,12 +415,6 @@ deps =
     django-v5.1: Django==5.1rc1
     django-latest: Django
 
-    # dramatiq
-    dramatiq-v1.13: dramatiq>=1.13,<1.14
-    dramatiq-v1.15: dramatiq>=1.15,<1.16
-    dramatiq-v1.17: dramatiq>=1.17,<1.18
-    dramatiq-latest: dramatiq
-
     # Falcon
     falcon-v1.4: falcon~=1.4.0
     falcon-v1: falcon~=1.0
@@ -497,10 +464,6 @@ deps =
     httpx-v0.27: httpx~=0.27.0
     httpx-latest: httpx
 
-    # Huey
-    huey-v2.0: huey~=2.0.0
-    huey-latest: huey
-
     # Langchain
     langchain-v0.1: openai~=1.0.0
     langchain-v0.1: langchain~=0.1.11
@@ -615,14 +578,6 @@ deps =
     sanic-v24.6: sanic~=24.6.0
     sanic-latest: sanic
 
-    # Spark
-    spark-v3.1: pyspark~=3.1.0
-    spark-v3.3: pyspark~=3.3.0
-    spark-v3.5: pyspark~=3.5.0
-    # TODO: update to ~=4.0.0 once stable is out
-    spark-v4.0: pyspark==4.0.0.dev2
-    spark-latest: pyspark
-
     # Starlite
     starlite: pytest-asyncio
     starlite: python-multipart
@@ -645,13 +600,6 @@ deps =
     # These come from the populate_tox.py script. Eventually we should move all
     # integration tests there.
 
-    # ~~~ AI ~~~
-    huggingface_hub-v0.22.2: huggingface_hub==0.22.2
-    huggingface_hub-v0.24.7: huggingface_hub==0.24.7
-    huggingface_hub-v0.26.5: huggingface_hub==0.26.5
-    huggingface_hub-v0.28.1: huggingface_hub==0.28.1
-
-
     # ~~~ DBs ~~~
     clickhouse_driver-v0.2.9: clickhouse-driver==0.2.9
 
@@ -713,6 +661,25 @@ deps =
     strawberry: httpx
 
 
+    # ~~~ Tasks ~~~
+    celery-v4.4.7: celery==4.4.7
+    celery-v5.0.5: celery==5.0.5
+    celery-v5.4.0: celery==5.4.0
+    celery: newrelic
+    celery: redis
+    py3.7-celery: importlib-metadata<5.0
+
+    dramatiq-v1.9.0: dramatiq==1.9.0
+    dramatiq-v1.12.3: dramatiq==1.12.3
+    dramatiq-v1.15.0: dramatiq==1.15.0
+    dramatiq-v1.17.1: dramatiq==1.17.1
+
+    spark-v3.0.3: pyspark==3.0.3
+    spark-v3.2.4: pyspark==3.2.4
+    spark-v3.4.4: pyspark==3.4.4
+    spark-v3.5.4: pyspark==3.5.4
+
+
     # ~~~ Web 1 ~~~
     flask-v1.1.4: flask==1.1.4
     flask-v2.3.3: flask==2.3.3