[Python] Support python3.11/12 (#1064)

chaokunyang · web-flow · commit c806cbaa5ae0 · 2023-11-21T22:13:38.000+08:00
diff --git a/.bazelrc b/.bazelrc
@@ -13,9 +13,6 @@ build:macos --force_pic
 build:clang-cl --compiler=clang-cl
 build:msvc --compiler=msvc-cl
 
-# for abi issue
-build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0"
-
 # test config
 test --build_tests_only
 test --cache_test_results=no
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -36,6 +36,8 @@ jobs:
   java:
     name: Java CI
     runs-on: ubuntu-latest
+    env:
+      MY_VAR: "PATH"
     strategy:
       matrix:
         java-version: ["8", "11", "17"]
@@ -53,7 +55,7 @@ jobs:
       - name: Install pyfury
         run: ./ci/run_ci.sh install_pyfury
       - name: Run CI with Maven
-        run: PATH="$HOME/miniconda/bin:$PATH" ./ci/run_ci.sh java${{ matrix.java-version }}
+        run: ./ci/run_ci.sh java${{ matrix.java-version }}
   java21:
     name: Java CI
     runs-on: ubuntu-latest
@@ -74,7 +76,7 @@ jobs:
       - name: Install pyfury
         run: ./ci/run_ci.sh install_pyfury
       - name: Run CI with Maven
-        run: PATH="$HOME/miniconda/bin:$PATH" ./ci/run_ci.sh java${{ matrix.java-version }}
+        run: ./ci/run_ci.sh java${{ matrix.java-version }}
   scala:
     name: Scala CI
     runs-on: ubuntu-latest
@@ -143,7 +145,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.6, 3.9]
+        python-version: [3.7, 3.12]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
@@ -175,7 +177,7 @@ jobs:
       - name: Install pyfury
         run: ./ci/run_ci.sh install_pyfury
       - name: Run Golang CI
-        run: PATH="$HOME/miniconda/bin:$PATH" ./ci/run_ci.sh go
+        run: ./ci/run_ci.sh go
   lint:
     name: Code Style Check
     runs-on: ubuntu-latest
diff --git a/.github/workflows/release-java.yaml b/.github/workflows/release-java.yaml
@@ -33,7 +33,7 @@ jobs:
           server-username: MAVEN_USERNAME
           server-password: MAVEN_PASSWORD
       - name: Publish java packages
-        run: export PATH="$HOME/miniconda/bin:$PATH" && cd java && mvn --batch-mode -q deploy -Dgpg.skip -DskipTests -Prelease
+        run: cd java && mvn --batch-mode -q deploy -Dgpg.skip -DskipTests -Prelease
         env:
           MAVEN_USERNAME: ${{ secrets.OSSRH_ID }}
           MAVEN_PASSWORD: ${{ secrets.OSSRH_KEY }}
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -28,7 +28,7 @@ jobs:
       url: https://pypi.org/project/pyfury
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8, 3.9, 3.10.12]
+        python-version: [3.7, 3.8, 3.9, 3.10.12, 3.11, 3.12]
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
@@ -47,7 +47,8 @@ jobs:
           ci/deploy.sh bump_py_version $version
       - name: Build a binary wheel
         run: |
-          pip install pyarrow==6.0.1 Cython wheel numpy pytest
+          ../ci/deploy.sh install_pyarrow
+          pip install setuptools wheel Cython numpy pytest
           cd python
           pip list
           export PATH=~/bin:$PATH
diff --git a/bazel/arrow/pyarrow_configure.bzl b/bazel/arrow/pyarrow_configure.bzl
@@ -186,7 +186,7 @@ def _get_pyarrow_include(repository_ctx, python_bin="python"):
 def _get_pyarrow_shared_library(repository_ctx, library_name, python_bin="python"):
     """Gets the pyarrow shared library path."""
     code = """import pyarrow, os, glob;print(glob.glob(os.path.join(""" +\
-        """os.path.dirname(pyarrow.__file__), 'lib{}.*.*'))[0])""".format(library_name)
+        """os.path.dirname(pyarrow.__file__), 'lib{}.*'))[0])""".format(library_name)
     result = _execute(
         repository_ctx, [
             python_bin, "-c", code
diff --git a/bazel/fury.bzl b/bazel/fury.bzl
@@ -1 +1 @@
-COPTS = ["-pthread","-std=c++17", "-D_GLIBCXX_USE_CXX11_ABI=0"]
+COPTS = ["-pthread","-std=c++17"]
diff --git a/bazel/fury_deps_setup.bzl b/bazel/fury_deps_setup.bzl
@@ -113,11 +113,14 @@ def auto_http_archive(
     )
 
 def setup_deps():
-    auto_http_archive(
+    # Fix @platforms error, see https://groups.google.com/g/bazel-discuss/c/iQyt08ZaNek
+    http_archive(
         name = "bazel_skylib",
-        strip_prefix = None,
-        url = "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
-        sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
+        sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506",
+        urls = [
+            "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
+            "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz",
+        ],
     )
     http_archive(
         name = "rules_python",
@@ -150,6 +153,9 @@ def setup_deps():
     )
     auto_http_archive(
         name = "com_google_absl",
-        url = "https://github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.tar.gz",
-        sha256 = "dcf71b9cba8dc0ca9940c4b316a0c796be8fab42b070bb6b7cab62b48f0e66c4",
-    )
+        sha256 = "5366d7e7fa7ba0d915014d387b66d0d002c03236448e1ba9ef98122c13b35c36",
+        strip_prefix = "abseil-cpp-20230125.3",
+        urls = [
+            "https://github.com/abseil/abseil-cpp/archive/20230125.3.tar.gz",
+        ],
+    )
diff --git a/ci/deploy.sh b/ci/deploy.sh
@@ -30,15 +30,19 @@ ROOT="$(git rev-parse --show-toplevel)"
 cd "$ROOT"
 WHEEL_DIR="$ROOT/.whl"
 
-PYTHONS=("cp36-cp36m"
-         "cp37-cp37m"
+PYTHONS=("cp37-cp37m"
          "cp38-cp38"
-         "cp39-cp39")
+         "cp39-cp39"
+         "cp310-cp310"
+         "cp310-cp311"
+         "cp312-cp312")
 
-VERSIONS=("3.6"
-          "3.7"
+VERSIONS=("3.7"
           "3.8"
-          "3.9")
+          "3.9"
+          "3.10"
+          "3.11"
+          "3.12")
 
 source $(conda info --base)/etc/profile.d/conda.sh
 
@@ -128,11 +132,12 @@ deploy_python() {
     git clean -f -f -x -d -e .whl
     # Ensure bazel select the right version of python
     bazel clean --expunge
-    pip install --ignore-installed twine cython pyarrow==6.0.1 numpy
+    install_pyarrow
+    pip install --ignore-installed twine setuptools cython numpy
     pyarrow_dir=$(python -c "import importlib.util; import os; print(os.path.dirname(importlib.util.find_spec('pyarrow').origin))")
     # ensure pyarrow is clean
     rm -rf "$pyarrow_dir"
-    pip install --ignore-installed pyarrow==6.0.1
+    pip install --ignore-installed pyarrow==$pyarrow_version
     python setup.py clean
     python setup.py bdist_wheel
     mv dist/pyfury*.whl "$WHEEL_DIR"
@@ -144,6 +149,17 @@ deploy_python() {
   twine upload -r pypi "$WHEEL_DIR"/pyfury*.whl
 }
 
+install_pyarrow() {
+  pyversion=$(python -V | cut -d' ' -f2)
+  if [[ $pyversion  ==  3.7* ]]; then
+    pyarrow_version=12.0.0
+    sed -i -E "s/pyarrow_version = .*/pyarrow_version = \"12.0.0\"/" "$ROOT"/python/setup.py
+  else
+    pyarrow_version=14.0.0
+  fi
+  pip install pyarrow==$pyarrow_version
+}
+
 case "$1" in
 java) # Deploy jars to maven repository.
   deploy_jars
diff --git a/ci/run_ci.sh b/ci/run_ci.sh
@@ -25,27 +25,30 @@ install_python() {
   wget -q https://repo.anaconda.com/miniconda/Miniconda3-py38_23.5.2-0-Linux-x86_64.sh -O Miniconda3.sh
   bash Miniconda3.sh -b -p $HOME/miniconda && rm -f miniconda.*
   which python
-  echo "Python version $(python -V)"
+  echo "Python version $(python -V), path $(which python)"
 }
 
 install_pyfury() {
-  export PATH="$HOME/miniconda/bin:$PATH"
-  echo "Python version $(python -V)"
-  pip install pyarrow==6.0.1 Cython wheel numpy pytest
+  echo "Python version $(python -V), path $(which python)"
+  "$ROOT"/ci/deploy.sh install_pyarrow
+  pip install Cython wheel numpy pytest
   pushd "$ROOT/python"
   pip list
   export PATH=~/bin:$PATH
   echo "Install pyfury"
+  # Fix strange installed deps not found
+  pip install setuptools
   pip install -v -e .
   popd
 }
 
 install_bazel() {
-  if command -v java >/dev/null; then
+  if command -v bazel >/dev/null; then
     echo "existing bazel location $(which bazel)"
     echo "existing bazel version $(bazel version)"
   fi
-  URL="https://github.com/bazelbuild/bazel/releases/download/4.2.0/bazel-4.2.0-installer-linux-x86_64.sh"
+  # GRPC support bazel 6.3.2 https://grpc.github.io/grpc/core/md_doc_bazel_support.html
+  URL="https://github.com/bazelbuild/bazel/releases/download/6.3.2/bazel-6.3.2-installer-linux-x86_64.sh"
   wget -q -O install.sh $URL
   chmod +x install.sh
   set +x
@@ -188,7 +191,7 @@ case $1 in
     ;;
     cpp)
       echo "Install pyarrow"
-      pip install pyarrow==6.0.1
+      "$ROOT"/ci/deploy.sh install_pyarrow
       export PATH=~/bin:$PATH
       echo "bazel version: $(bazel version)"
       set +e
diff --git a/docs/guide/DEVELOPMENT.md b/docs/guide/DEVELOPMENT.md
@@ -26,7 +26,7 @@ mvn clean compile -DskipTests
 
 ```bash
 cd python
-pip install pyarrow==6.0.1 Cython wheel numpy pytest
+pip install pyarrow==14.0.0 Cython wheel numpy pytest
 pip install -v -e .
 ```
 
@@ -45,7 +45,7 @@ bazel build //src/fury/util:fury_util
 Build fury row format：
 
 ```bash
-pip install pyarrow==6.0.1
+pip install pyarrow==14.0.0
 bazel build //src/fury/row:fury_row_format
 ```
 
diff --git a/java/pom.xml b/java/pom.xml
@@ -73,7 +73,7 @@
     <maven.compiler.target>1.8</maven.compiler.target>
     <guava.version>32.1.2-jre</guava.version>
     <janino.version>3.1.10</janino.version>
-    <arrow.version>5.0.0</arrow.version>
+    <arrow.version>14.0.0</arrow.version>
     <jackson-bom.version>2.13.2.20220328</jackson-bom.version>
     <commons_codec.version>1.13</commons_codec.version>
     <fury.java.rootdir>${basedir}</fury.java.rootdir>
diff --git a/python/pyfury/type.py b/python/pyfury/type.py
@@ -16,6 +16,7 @@
 import dataclasses
 import enum
 import importlib
+import inspect
 
 import typing
 from typing import TypeVar
@@ -313,10 +314,18 @@ def infer_field(field_name, type_, visitor: TypeVisitor, types_path=None):
                 f"Collection types should be {list, dict} instead of {type_}"
             )
     else:
-        if hasattr(origin, "__annotations__"):
-            return visitor.visit_customized(field_name, type_, types_path=types_path)
-        else:
+        if is_function(origin) or not hasattr(origin, "__annotations__"):
             return visitor.visit_other(field_name, type_, types_path=types_path)
+        else:
+            return visitor.visit_customized(field_name, type_, types_path=types_path)
+
+
+def is_function(func):
+    return inspect.isfunction(func) or is_cython_function(func)
+
+
+def is_cython_function(func):
+    return getattr(func, "func_name", None) is not None
 
 
 def compute_string_hash(string):
diff --git a/python/setup.py b/python/setup.py
@@ -35,7 +35,7 @@
 except FileExistsError:
     pass
 
-pyarrow_version = "6.0.1"
+pyarrow_version = "14.0.0"
 # Check if we're running 64-bit Python
 if not sys.maxsize > 2**32:
     raise RuntimeError("Not supported on 32-bit")
@@ -151,12 +151,10 @@ def parse_version():
         "all": [f"pyarrow == {pyarrow_version}"],
     },
     setup_requires=[
-        "cython >= 0.29.14",
+        "Cython",
         "wheel",
         f"pyarrow == {pyarrow_version}",
         "numpy" 'dataclasses; python_version<"3.7"',
-        'pickle5; python_version<"3.8"',
-        "cloudpickle",
     ],
     distclass=BinaryDistribution,
     ext_modules=ext_modules,

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-COPTS = ["-pthread","-std=c++17", "-D_GLIBCXX_USE_CXX11_ABI=0"]`
	`1`	`+COPTS = ["-pthread","-std=c++17"]`