feat: add support for Python 3.13 & deprecate Python 3.8 (#3045)

* update layer build scripts to support python 3.13

* update dependencies

* deprecate python 3.8

* update workflows

* typing

* pin ray

* regenerate poetry.lock

* update poetry lock

Author: kukushking

.github/workflows/minimal-tests.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         platform: [ubuntu-latest, macos-latest, windows-latest]
 
     env:

awswrangler/opensearch/_write.py

@@ -608,10 +608,10 @@ def index_documents(
     try:
         if progressbar:
             widgets = [
-                progressbar.Percentage(),
-                progressbar.SimpleProgress(format=" (%(value_s)s/%(max_value_s)s)"),
-                progressbar.Bar(),
-                progressbar.Timer(),
+                progressbar.Percentage(),  # type: ignore[no-untyped-call]
+                progressbar.SimpleProgress(format=" (%(value_s)s/%(max_value_s)s)"),  # type: ignore[no-untyped-call]
+                progressbar.Bar(),  # type: ignore[no-untyped-call]
+                progressbar.Timer(),  # type: ignore[no-untyped-call]
             ]
             progress_bar = progressbar.ProgressBar(
                 widgets=widgets, max_value=total_documents, prefix="Indexing: "

building/build-lambda-layers.sh

@@ -21,17 +21,6 @@ pushd lambda
 # Building all related docker images
 ./build-docker-images.sh $PYTHON_VERSION
 
-# Python 3.8
-if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.8" ]]
-then
-  docker run \
-    --volume "$DIR_NAME":/aws-sdk-pandas/ \
-    --workdir /aws-sdk-pandas/building/lambda \
-    --rm \
-    awswrangler-build-py38 \
-    build-lambda-layer.sh "${VERSION}-py3.8${ARCH_SUFFIX}" "ninja-build"
-fi
-
 # Python 3.9
 if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.9" ]]
 then
@@ -75,3 +64,14 @@ then
     awswrangler-build-py312 \
     build-lambda-layer.sh "${VERSION}-py3.12${ARCH_SUFFIX}" "ninja-build"
 fi
+
+# Python 3.13
+if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.13" ]]
+then
+  docker run \
+    --volume "$DIR_NAME":/aws-sdk-pandas/ \
+    --workdir /aws-sdk-pandas/building/lambda \
+    --rm \
+    awswrangler-build-py313 \
+    build-lambda-layer.sh "${VERSION}-py3.13${ARCH_SUFFIX}" "ninja-build"
+fi

building/lambda/build-docker-images.sh

@@ -8,16 +8,6 @@ export DOCKER_BUILDKIT=1
 
 PYTHON_VERSION=${1:-ALL}
 
-# Python 3.8
-if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.8" ]]
-then
-  docker build \
-    --pull \
-    --tag awswrangler-build-py38 \
-    --build-arg base_image=public.ecr.aws/lambda/python:3.8 \
-    .
-fi
-
 # Python 3.9
 if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.9" ]]
 then
@@ -59,4 +49,15 @@ then
     .
 fi
 
+# Python 3.13
+if [[ $PYTHON_VERSION == "ALL" || $PYTHON_VERSION == "3.13" ]]
+then
+  docker build \
+    --pull \
+    --tag awswrangler-build-py313 \
+    --build-arg base_image=public.ecr.aws/lambda/python:3.13 \
+    --file Dockerfile.al2023 \
+    .
+fi
+
 rm -rf pyproject.toml poetry.lock

poetry.lock

@@ -18,28 +18,25 @@ documentation = "https://aws-sdk-pandas.readthedocs.io/"
 keywords = ["pandas", "aws"]
 
 classifiers = [
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.8, <4.0"
+python = ">=3.9, <4.0"
 
 # Required
 boto3 = "^1.20.32"
 botocore = "^1.23.32"
-pandas = [
-    { version = ">=1.2.0,<2.1.0", markers = "python_version < \"3.9\"" },
-    { version = ">=1.2.0,<3.0.0", markers = "python_version >= \"3.9\"" },
+pandas = { version = ">=1.2.0,<3.0.0" }
+numpy = {  version = ">=1.26,<3.0" }
+pyarrow = [
+    { version = ">=8.0.0,<18.0.0", markers = "python_version < \"3.13\"" },
+    { version = ">=18.0.0", markers = "python_version >= \"3.13\"" }
 ]
-numpy = [
-    { version = ">=1.18,<2.0", markers = "python_version < \"3.9\"" },
-    { version = ">=1.26,<3.0", markers = "python_version >= \"3.9\"" }
-]
-pyarrow = ">=8.0.0"
 typing-extensions = "^4.4.0"
 packaging = ">=21.1,<25.0"
 
@@ -75,11 +72,8 @@ geopandas = [
 ]
 
 # Distributed
-modin = [
-    { version = "0.23.1post0", markers = "python_version < \"3.9\"", optional = true },
-    { version = "^0.31.0", markers = "python_version >= \"3.9\"", optional = true }
-]
-ray = { version = "^2.30.0", extras = ["default", "data"], optional = true }
+modin = { version = "^0.31.0", optional = true }
+ray = { version = ">=2.30.0,<2.38.0", extras = ["default", "data"], optional = true }
 
 [tool.poetry.extras]
 redshift = ["redshift-connector"]
@@ -182,7 +176,7 @@ max-returns=6  # Maximum number of return / yield for function / method body.
 max-statements=50  # Maximum number of statements in function / method body.
 
 [tool.mypy]
-python_version = "3.8"
+python_version = "3.9"
 strict = true
 ignore_missing_imports = true
 warn_unused_ignores = true

pyproject.toml

@@ -6,7 +6,7 @@ authors = ["Amazon Web Services"]
 license = "Apache License 2.0"
 
 [tool.poetry.dependencies]
-python = ">=3.8, <4.0"
+python = ">=3.9, <4.0"
 "aws-cdk-lib" = "^2.128.0"
 "aws-cdk.aws-glue-alpha" = "^2.128.0a0"
 "aws-cdk.aws-neptune-alpha" = "^2.128.0a0"

test_infra/poetry.lock

@@ -1,5 +1,5 @@
 [tox]
-envlist = py{37,38,39,310,311,312}
+envlist = py{39,310,311,312,313}
 isolated_build = True
 
 [testenv]
@@ -11,7 +11,9 @@ passenv =
        AWS_SESSION_TOKEN
 setenv =
        COV_FAIL_UNDER = 87.00
-allowlist_externals = poetry
+allowlist_externals = 
+       pytest
+       poetry
 commands_pre =
        poetry install --no-root --sync --extras "deltalake gremlin mysql opencypher opensearch oracle postgres redshift sparql sqlserver geopandas"
 commands =
@@ -21,7 +23,7 @@ commands =
               --dist load --maxschedchunk 2 \
               --junitxml=test-reports/junit.xml --log-file=test-reports/logs.txt tests/unit
 
-[testenv:py{37,38,39,310,311}-distributed]
+[testenv:py{39,310,311,312,313}-distributed]
 passenv =
        AWS_PROFILE
        AWS_DEFAULT_REGION