feat: add python3.10 support (#468)

mndeveci · briensea · web-flow · commit d68a6b5ed674 · 2023-03-20T21:17:04.000Z
* feat: add python3.10 support (#32) * Add python3.10 support. * add support for py3.10 * add ci jobs for py3.10 * Fix python3.10 workflows. --------- Co-authored-by: Mehmet Nuri Deveci <5735811+mndeveci@users.noreply.github.com> * Update DESIGN.md * update test cases with py310 differences --------- Co-authored-by: Sean O Brien <sean.obrien.2016@mumail.ie>
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -58,6 +58,7 @@ jobs:
           - "3.7"
           - "3.8"
           - "3.9"
+          - "3.10"
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -82,6 +83,7 @@ jobs:
           - "3.7"
           - "3.8"
           - "3.9"
+          - "3.10"
         npm:
           - 8
           - 9
@@ -113,6 +115,7 @@ jobs:
           - "3.7"
           - "3.8"
           - "3.9"
+          - "3.10"
         npm:
           - 8
           - 9
@@ -144,6 +147,7 @@ jobs:
           - "3.7"
           - "3.8"
           - "3.9"
+          - "3.10"
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -166,9 +170,10 @@ jobs:
           - ubuntu-latest
           - windows-latest
         python:
-          - "3.9"
-          - "3.8"
           - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -194,9 +199,10 @@ jobs:
           - ubuntu-latest
           - windows-latest
         python:
-          - "3.9"
-          - "3.8"
           - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -220,9 +226,10 @@ jobs:
           - ubuntu-latest
           - windows-latest
         python:
-          - "3.9"
-          - "3.8"
           - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -242,9 +249,10 @@ jobs:
           - ubuntu-latest
           - windows-latest
         python:
-          - "3.9"
-          - "3.8"
           - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -264,9 +272,10 @@ jobs:
           - ubuntu-latest
           - windows-latest
         python:
-          - "3.9"
-          - "3.8"
           - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
         ruby:
           - "3.1"
           - "2.7"
@@ -292,9 +301,10 @@ jobs:
           - ubuntu-latest
           - windows-latest
         python:
-          - "3.9"
-          - "3.8"
           - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
diff --git a/aws_lambda_builders/validator.py b/aws_lambda_builders/validator.py
@@ -17,6 +17,7 @@
     "python3.7": [X86_64],
     "python3.8": [ARM64, X86_64],
     "python3.9": [ARM64, X86_64],
+    "python3.10": [ARM64, X86_64],
     "ruby2.7": [ARM64, X86_64],
     "java8": [ARM64, X86_64],
     "java11": [ARM64, X86_64],
diff --git a/aws_lambda_builders/workflows/python_pip/DESIGN.md b/aws_lambda_builders/workflows/python_pip/DESIGN.md
@@ -49,7 +49,7 @@ def build_dependencies(artifacts_dir_path,
 
     :type runtime: str
     :param runtime: Python version to build dependencies for. This can
-        either be python3.6, python3.7, python3.8 or python3.9. These are 
+        either be python3.7, python3.8, python3.9 or python3.10. These are
         currently the only supported values.
 
     :type ui: :class:`lambda_builders.actions.python_pip.utils.UI`
diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py
@@ -84,6 +84,7 @@ def get_lambda_abi(runtime):
         "python3.7": "cp37m",
         "python3.8": "cp38",
         "python3.9": "cp39",
+        "python3.10": "cp310",
     }
 
     if runtime not in supported:
@@ -98,7 +99,7 @@ def __init__(self, runtime, osutils=None, dependency_builder=None, architecture=
 
         :type runtime: str
         :param runtime: Python version to build dependencies for. This can
-            either be python3.7, python3.8 or python3.9. These are currently the
+            either be python3.7, python3.8, python3.9 or python3.10. These are currently the
             only supported values.
 
         :type osutils: :class:`lambda_builders.utils.OSUtils`
diff --git a/setup.py b/setup.py
@@ -61,6 +61,7 @@ def read_version():
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
         "Topic :: Internet",
         "Topic :: Software Development :: Build Tools",
         "Topic :: Utilities",
diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py
@@ -18,7 +18,7 @@
 logger = logging.getLogger("aws_lambda_builders.workflows.python_pip.workflow")
 IS_WINDOWS = platform.system().lower() == "windows"
 NOT_ARM = platform.processor() != "aarch64"
-ARM_RUNTIMES = {"python3.8", "python3.9"}
+ARM_RUNTIMES = {"python3.8", "python3.9", "python3.10"}
 
 
 @parameterized_class(("experimental_flags",), [([]), ([EXPERIMENTAL_FLAG_BUILD_PERFORMANCE])])
@@ -60,6 +60,7 @@ def setUp(self):
             "python3.7": "python3.8",
             "python3.8": "python3.9",
             "python3.9": "python3.7",
+            "python3.10": "python3.9",
         }
 
     def tearDown(self):
@@ -92,33 +93,43 @@ def test_must_build_python_project(self):
             experimental_flags=self.experimental_flags,
         )
 
-        self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"])
-        expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"})
+        if self.runtime == "python3.10":
+            self.check_architecture_in("numpy-1.23.5.dist-info", ["manylinux2014_x86_64", "manylinux1_x86_64"])
+            expected_files = self.test_data_files.union({"numpy", "numpy-1.23.5.dist-info", "numpy.libs"})
+        else:
+            self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"])
+            expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"})
 
         output_files = set(os.listdir(self.artifacts_dir))
         self.assertEqual(expected_files, output_files)
 
     def test_must_build_python_project_python3_binary(self):
         python_paths = which("python")
-        executable_dir = pathlib.Path(tempfile.gettempdir())
-        new_python_path = executable_dir.joinpath("python3")
-        os.symlink(python_paths[0], new_python_path)
-        # Build with access to the newly symlinked python3 binary.
-        self.builder.build(
-            self.source_dir,
-            self.artifacts_dir,
-            self.scratch_dir,
-            self.manifest_path_valid,
-            runtime=self.runtime,
-            experimental_flags=self.experimental_flags,
-            executable_search_paths=[executable_dir],
-        )
-        self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"])
-        expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"})
+        with tempfile.TemporaryDirectory() as executable_dir_str:
+            executable_dir = pathlib.Path(executable_dir_str)
+            new_python_path = executable_dir.joinpath("python3")
+            os.symlink(python_paths[0], new_python_path)
+            # Build with access to the newly symlinked python3 binary.
+            self.builder.build(
+                self.source_dir,
+                self.artifacts_dir,
+                self.scratch_dir,
+                self.manifest_path_valid,
+                runtime=self.runtime,
+                experimental_flags=self.experimental_flags,
+                executable_search_paths=[executable_dir],
+            )
 
-        output_files = set(os.listdir(self.artifacts_dir))
-        self.assertEqual(expected_files, output_files)
-        os.unlink(new_python_path)
+            if self.runtime == "python3.10":
+                self.check_architecture_in("numpy-1.23.5.dist-info", ["manylinux2014_x86_64", "manylinux1_x86_64"])
+                expected_files = self.test_data_files.union({"numpy", "numpy-1.23.5.dist-info", "numpy.libs"})
+            else:
+                self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2010_x86_64", "manylinux1_x86_64"])
+                expected_files = self.test_data_files.union({"numpy", "numpy-1.20.3.dist-info", "numpy.libs"})
+
+            output_files = set(os.listdir(self.artifacts_dir))
+            self.assertEqual(expected_files, output_files)
+            os.unlink(new_python_path)
 
     @skipIf(NOT_ARM, "Skip if not running on ARM64")
     def test_must_build_python_project_from_sdist_with_arm(self):
@@ -154,10 +165,15 @@ def test_must_build_python_project_with_arm_architecture(self):
             experimental_flags=self.experimental_flags,
         )
         expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-1.20.3.dist-info"})
+        if self.runtime == "python3.10":
+            expected_files = self.test_data_files.union({"numpy", "numpy.libs", "numpy-1.23.5.dist-info"})
         output_files = set(os.listdir(self.artifacts_dir))
         self.assertEqual(expected_files, output_files)
 
-        self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2014_aarch64"])
+        if self.runtime == "python3.10":
+            self.check_architecture_in("numpy-1.23.5.dist-info", ["manylinux2014_aarch64"])
+        else:
+            self.check_architecture_in("numpy-1.20.3.dist-info", ["manylinux2014_aarch64"])
 
     def test_mismatch_runtime_python_project(self):
         # NOTE : Build still works if other versions of python are accessible on the path. eg: /usr/bin/python3.7
diff --git a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt
@@ -1,3 +1,6 @@
 numpy==1.15.4; python_version == '2.7'
 numpy==1.17.4; python_version == '3.6'
-numpy==1.20.3; python_version >= '3.7'
+numpy==1.20.3; python_version == '3.7'
+numpy==1.20.3; python_version == '3.8'
+numpy==1.20.3; python_version == '3.9'
+numpy==1.23.5; python_version == '3.10'
diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py
@@ -98,6 +98,9 @@ def test_get_lambda_abi_python38(self):
     def test_get_lambda_abi_python39(self):
         assert "cp39" == get_lambda_abi("python3.9")
 
+    def test_get_lambda_abi_python310(self):
+        assert "cp310" == get_lambda_abi("python3.10")
+
 
 class TestPythonPipDependencyBuilder(object):
     def test_can_call_dependency_builder(self, osutils):
