chore(sdk): Bump up pyfunc package versions (#649)

deadlycoconuts · web-flow · commit fbbdd3ae9bd3 · 2025-05-19T12:43:44.000+08:00
# Description Given that the current release of Merlin is `v0.48.3`, we need to update the default version of the `merlin-batch-predictor` and `merlin-pyfunc-server` Python packages. ## Additional Unrelated Changes 🚨 Due to the upgrading of `numpy` in this PR [here](#641 (comment)), Python 3.8 is no longer supported. The CICD pipeline fails because Python 3.8 is currently still being used. This draft (at the time of writing) PR #639 is supposed to include changes to remove support for Python 3.8 but is temporarily paused due to other changes. @naufalandika To fix the CICD pipeline for the moment, this PR copies the existing changes in that PR to remove support entirely for Python 3.8 (and includes other changes to other tests/examples that mention Python 3.8). # Modifications - `python/sdk/merlin/model.py` - Bumped up max version of the `merlin-batch-predictor` and `merlin-pyfunc-server` Python packages # Tests  # Checklist - [x] Added PR label - [ ] Added unit test, integration, and/or e2e tests - [x] Tested locally - [ ] Updated documentation - [ ] Update Swagger spec if the PR introduce API changes - [ ] Regenerated Golang and Python client if the PR introduces API changes # Release Notes ```release-note NONE ```
diff --git a/.github/workflows/merlin.yml b/.github/workflows/merlin.yml
@@ -43,7 +43,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10"]
     env:
       PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
       PIPENV_VENV_IN_PROJECT: true
@@ -78,7 +78,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10"]
     env:
       PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }}
       PIPENV_VENV_IN_PROJECT: true
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -24,7 +24,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: 3.9
           cache: pip
       - name: Install dependencies
         working-directory: ./python/sdk
@@ -51,7 +51,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: 3.9
           cache: pip
       - name: Install dependencies
         working-directory: ./python/pyfunc-server
@@ -79,7 +79,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: 3.9
           cache: pip
       - name: Install dependencies
         working-directory: ./python/batch-predictor
@@ -101,7 +101,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9", "3.10"]
     needs:
       - publish-python-sdk
     steps:
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -11,6 +11,6 @@ sphinx:
 
 # Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.8
+  version: 3.9
   install:
     - requirements: python/sdk/docs/requirements_docs.txt
diff --git a/Makefile b/Makefile
@@ -12,7 +12,7 @@ TEST_TAGS?=
 GOLANGCI_LINT_VERSION="v1.58.1"
 PROTOC_GEN_GO_JSON_VERSION="v1.1.0"
 PROTOC_GEN_GO_VERSION="v1.26"
-PYTHON_VERSION ?= "39"	#set as 38 39 310 for 3.8-3.10 respectively
+PYTHON_VERSION ?= "39"	#set as 39 310 for 3.9-3.10 respectively
 
 all: setup init-dep lint test clean build run
 
diff --git a/api/api/versions_api.go b/api/api/versions_api.go
@@ -28,7 +28,7 @@ import (
 	"gorm.io/gorm"
 )
 
-const DEFAULT_PYTHON_VERSION = "3.8.*"
+const DEFAULT_PYTHON_VERSION = "3.9.*"
 
 type VersionsController struct {
 	*AppContext
diff --git a/api/config/config.go b/api/config/config.go
@@ -230,7 +230,7 @@ type ImageBuilderConfig struct {
 	MaximumRetry  int32                 `validate:"required" default:"3"`
 	K8sConfig     *mlpcluster.K8sConfig `validate:"required" default:"-"`
 	SafeToEvict   bool                  `default:"false"`
-	// Supported Python versions for image builder. Example: ["3.8.*", "3.9.*", "3.10.*"]
+	// Supported Python versions for image builder. Example: ["3.9.*", "3.10.*"]
 	SupportedPythonVersions []string
 }
 
diff --git a/api/config/config_test.go b/api/config/config_test.go
@@ -449,7 +449,7 @@ func TestLoad(t *testing.T) {
 						Name: "dev-server",
 					},
 					SafeToEvict:             false,
-					SupportedPythonVersions: []string{"3.8.*", "3.9.*", "3.10.*"},
+					SupportedPythonVersions: []string{"3.9.*", "3.10.*"},
 				},
 				BatchConfig: BatchConfig{
 					Tolerations: []v1.Toleration{
diff --git a/api/config/testdata/base-configs-1.yaml b/api/config/testdata/base-configs-1.yaml
@@ -81,7 +81,6 @@ ImageBuilderConfig:
         provideClusterInfo: true
   SafeToEvict: false
   SupportedPythonVersions:
-    - 3.8.*
     - 3.9.*
     - 3.10.*
 BatchConfig:
diff --git a/api/pkg/imagebuilder/imagebuilder_test.go b/api/pkg/imagebuilder/imagebuilder_test.go
@@ -56,7 +56,7 @@ const (
 	testArtifactURISuffix = "://bucket-name/mlflow/11/68eb8538374c4053b3ecad99a44170bd/artifacts"
 	testCondaEnvUrlSuffix = testArtifactURISuffix + "/model/conda.yaml"
 	testCondaEnvContent   = `dependencies:
-- python=3.8.*
+- python=3.9.*
 - pip:
 	- mlflow
 	- joblib
@@ -121,7 +121,7 @@ var (
 		"--use-new-run",
 	}
 
-	defaultSupportedPythonVersions = []string{"3.8.*", "3.9.*", "3.10.*"}
+	defaultSupportedPythonVersions = []string{"3.9.*", "3.10.*"}
 
 	configWithGCRPushRegistry = Config{
 		BuildNamespace: testBuildNamespace,
diff --git a/docs/developer/e2e-test.md b/docs/developer/e2e-test.md
@@ -11,7 +11,7 @@ Run E2E test as part of github actions workflow. This E2E test will be triggered
 - Pull merlin repository
 - Pull mlp repository
 - Setup go
-- Setup python 3.8
+- Setup python 3.9
 - Setup cluster
 - Setup mlp namespace
 - Deploy mlp
diff --git a/examples/metrics/env.yaml b/examples/metrics/env.yaml
@@ -1,2 +1,2 @@
 dependencies:
-  - python=3.8
+  - python=3.9
diff --git a/examples/pyfunc-upi/env.yaml b/examples/pyfunc-upi/env.yaml
@@ -1,5 +1,5 @@
 dependencies:
-  - python=3.8
+  - python=3.9
   - pip:
     - numpy
     - xgboost==1.6.2
diff --git a/examples/pyfunc/env.yaml b/examples/pyfunc/env.yaml
@@ -1,5 +1,5 @@
 dependencies:
-  - python=3.8
+  - python=3.9
   - pip:
       - joblib>=0.13.0,<1.2.0 # >=1.2.0 upon upgrade of kserve's version
       - scikit-learn>=1.1.2
diff --git a/examples/transformer/custom-transformer/env.yaml b/examples/transformer/custom-transformer/env.yaml
@@ -1,5 +1,5 @@
 dependencies:
-  - python=3.8
+  - python=3.9
   - pip:
     - torch==1.3.1
     - torchvision==0.4.2
diff --git a/examples/transformer/feast-enricher-transformer/env.yaml b/examples/transformer/feast-enricher-transformer/env.yaml
@@ -1,2 +1,2 @@
 dependencies:
-  - python=3.8
+  - python=3.9
diff --git a/examples/transformer/standard-transformer/env.yaml b/examples/transformer/standard-transformer/env.yaml
@@ -1,2 +1,2 @@
 dependencies:
-  - python=3.8
+  - python=3.9
diff --git a/examples/transformer/upi/env.yaml b/examples/transformer/upi/env.yaml
@@ -1,2 +1,2 @@
 dependencies:
-  - python=3.8
+  - python=3.9
diff --git a/python/batch-predictor/README.md b/python/batch-predictor/README.md
@@ -69,7 +69,7 @@ OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES python main.py --job-name iris-predictio
 
 ### Requirements
 
-- python >= 3.8.0
+- python >= 3.9.0
 - pipenv (install using `pip install pipenv`)
 - protoc (see [installation instruction](http://google.github.io/proto-lens/installing-protoc.html))
 - gcloud (see [installation instruction](https://cloud.google.com/sdk/install))
diff --git a/python/batch-predictor/setup.py b/python/batch-predictor/setup.py
@@ -34,7 +34,7 @@
     description="Base PySpark application for running Merlin prediction batch job",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
-    python_requires=">=3.8,<3.11",
+    python_requires=">=3.9,<3.11",
     packages=find_packages(exclude=["test"]),
     install_requires=REQUIRE,
     tests_require=TESTS_REQUIRE,
diff --git a/python/batch-predictor/test-model/MLmodel b/python/batch-predictor/test-model/MLmodel
@@ -9,7 +9,7 @@ flavors:
     env: conda.yaml
     loader_module: mlflow.pyfunc.model
     python_model: python_model.pkl
-    python_version: 3.8.18
+    python_version: 3.9.22
 mlflow_version: 1.26.1
 model_uuid: 49ff290d6d854ee3ad83f6233ad891b9
 run_id: d65892b5f2f44cbaae64c682c61bccd6
diff --git a/python/batch-predictor/test-model/conda.yaml b/python/batch-predictor/test-model/conda.yaml
@@ -1,5 +1,5 @@
 dependencies:
-  - python=3.8.*
+  - python=3.9.*
   - pip:
       - mlflow
       - joblib>=0.13.0,<1.2.0
diff --git a/python/batch-predictor/test-model/python_env.yaml b/python/batch-predictor/test-model/python_env.yaml
@@ -1,4 +1,4 @@
-python: 3.8.18
+python: 3.9.22
 build_dependencies:
 - pip==24.0
 - setuptools==69.0.3
diff --git a/python/observation-publisher/requirements.txt b/python/observation-publisher/requirements.txt
@@ -30,7 +30,7 @@ botocore==1.35.39
     #   s3transfer
 cachetools==5.3.2
     # via google-auth
-caraml-auth-google==0.0.0.post7
+caraml-auth-google==0.0.0.post9
     # via merlin-sdk
 caraml-upi-protos==1.0.0
     # via
@@ -112,9 +112,13 @@ googleapis-common-protos==1.62.0
     #   caraml-upi-protos
     #   google-api-core
 grpcio==1.60.1
-    # via grpcio-tools
+    # via
+    #   grpcio-tools
+    #   merlin-sdk
 grpcio-tools==1.60.1
-    # via caraml-upi-protos
+    # via
+    #   caraml-upi-protos
+    #   merlin-sdk
 gunicorn==21.2.0
     # via mlflow
 hydra-core==1.3.2
@@ -150,7 +154,7 @@ mlflow==1.26.1
     # via merlin-sdk
 mypy-extensions==1.0.0
     # via typing-inspect
-numpy==1.23.5
+numpy==1.26.4
     # via
     #   merlin-sdk
     #   mlflow
@@ -189,7 +193,9 @@ protobuf==4.25.3
     #   merlin-sdk
     #   mlflow
 pyarrow==15.0.0
-    # via arize
+    # via
+    #   arize
+    #   pyodps
 pyasn1==0.5.1
     # via
     #   pyasn1-modules
@@ -204,6 +210,8 @@ pygments==2.17.2
     # via rich
 pyjwt==2.8.0
     # via databricks-cli
+pyodps==0.12.2
+    # via -r requirements.in
 pyprind==2.11.3
     # via merlin-sdk
 python-dateutil==2.8.2
@@ -236,6 +244,7 @@ requests==2.31.0
     #   google-cloud-bigquery
     #   google-cloud-storage
     #   mlflow
+    #   pyodps
     #   requests-futures
 requests-futures==1.0.0
     # via arize
@@ -275,7 +284,6 @@ typing-extensions==4.9.0
     #   alembic
     #   pydantic
     #   pydantic-core
-    #   sqlalchemy
     #   typing-inspect
 typing-inspect==0.9.0
     # via dataclasses-json
@@ -294,7 +302,6 @@ werkzeug==3.0.1
     # via flask
 zipp==3.17.0
     # via importlib-metadata
-pyodps==0.12.2
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools
+# setuptools
diff --git a/python/pyfunc-scaffolding/{{cookiecutter.model_slug}}/Pipfile b/python/pyfunc-scaffolding/{{cookiecutter.model_slug}}/Pipfile
@@ -7,4 +7,4 @@ verify_ssl = true
 merlin-sdk = "=={{ cookiecutter.merlin_sdk_version }}"
 
 [requires]
-python_version = "3.8"
+python_version = "3.9"
diff --git a/python/pyfunc-scaffolding/{{cookiecutter.model_slug}}/env/conda.yaml b/python/pyfunc-scaffolding/{{cookiecutter.model_slug}}/env/conda.yaml
@@ -1,2 +1,2 @@
 dependencies:
-  - python=3.8
+  - python=3.9
diff --git a/python/pyfunc-server/pyfuncserver/utils/contants.py b/python/pyfunc-server/pyfuncserver/utils/contants.py
@@ -11,14 +11,14 @@
 conda env.yaml that you pass to log_pyfunc method, e.g.:
 
 dependencies:
-- python=3.8
+- python=3.9
 - pip:
   - my-missing-package==1.0
   
 OR if the package is not available in PyPI distribution
 
 dependencies:
-- python=3.8
+- python=3.9
 - my-missing-package=1.0
 
 See: https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#creating-an-environment-from-an-environment-yml-file
diff --git a/python/pyfunc-server/setup.py b/python/pyfunc-server/setup.py
@@ -44,7 +44,7 @@
     description="Model Server implementation for Merlin PyFunc model",
     long_description=open("README.md").read(),
     long_description_content_type="text/markdown",
-    python_requires=">=3.8,<3.11",
+    python_requires=">=3.9,<3.11",
     packages=find_packages(exclude=["test"]),
     install_requires=REQUIRE,
     tests_require=TESTS_REQUIRE,
diff --git a/python/pyfunc-server/test/local-artifacts/MLmodel b/python/pyfunc-server/test/local-artifacts/MLmodel
@@ -12,7 +12,7 @@ flavors:
     env: conda.yaml
     loader_module: mlflow.pyfunc.model
     python_model: python_model.pkl
-    python_version: 3.8.18
+    python_version: 3.9.22
 mlflow_version: 1.26.1
 model_uuid: 126ccf3f97024b43a011b5aee072010c
 run_id: 74232b10e8064ccda29611c265921034
diff --git a/python/pyfunc-server/test/local-artifacts/conda.yaml b/python/pyfunc-server/test/local-artifacts/conda.yaml
@@ -1,5 +1,5 @@
 dependencies:
-  - python=3.8.*
+  - python=3.9.*
   - pip:
       - mlflow
       - joblib>=0.13.0,<1.2.0
diff --git a/python/pyfunc-server/test/local-artifacts/python_env.yaml b/python/pyfunc-server/test/local-artifacts/python_env.yaml
@@ -1,4 +1,4 @@
-python: 3.8.18
+python: 3.9.22
 build_dependencies:
 - pip==24.0
 - setuptools==69.0.3
diff --git a/python/sdk/Dockerfile b/python/sdk/Dockerfile
@@ -1,4 +1,4 @@
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.9
 FROM python:${PYTHON_VERSION}-slim-buster
 
 LABEL org.opencontainers.image.source https://github.com/caraml-dev/merlin
diff --git a/python/sdk/merlin/model.py b/python/sdk/merlin/model.py
@@ -1007,9 +1007,9 @@ def log_pyfunc_model(
                 "log_pyfunc_model is only for PyFunc, PyFuncV2 and PyFuncV3 model"
             )
 
-        merlin_requirements = ["merlin-pyfunc-server<0.46.0"]
+        merlin_requirements = ["merlin-pyfunc-server<0.49.0"]
         if self._model.type == ModelType.PYFUNC_V2:
-            merlin_requirements = ["merlin-batch-predictor<0.46.0"]
+            merlin_requirements = ["merlin-batch-predictor<0.49.0"]
 
         # add/replace python version in conda to match that used to create model version
         conda_env = process_conda_env(
diff --git a/python/sdk/requirements.txt b/python/sdk/requirements.txt
@@ -16,7 +16,7 @@ PyYAML>=5.4
 six>=1.10
 urllib3>=1.26
 numpy>=1.26.4
-caraml-auth-google==0.0.0.post16.dev0
+caraml-auth-google==0.0.0.post9
 pydantic==2.5.3
 grpcio-tools>=1.50.0,<1.63
 grpcio>=1.60.1

Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@ import (`
`28`	`28`	`"gorm.io/gorm"`
`29`	`29`	`)`
`30`	`30`
`31`		`-const DEFAULT_PYTHON_VERSION = "3.8.*"`
	`31`	`+const DEFAULT_PYTHON_VERSION = "3.9.*"`
`32`	`32`
`33`	`33`	`type VersionsController struct {`
`34`	`34`	`*AppContext`
Original file line number	Diff line number	Diff line change
`@@ -230,7 +230,7 @@ type ImageBuilderConfig struct {`
`230`	`230`	MaximumRetry int32 `validate:"required" default:"3"`
`231`	`231`	K8sConfig *mlpcluster.K8sConfig `validate:"required" default:"-"`
`232`	`232`	SafeToEvict bool `default:"false"`
`233`		`- // Supported Python versions for image builder. Example: ["3.8.", "3.9.", "3.10.*"]`
	`233`	`+ // Supported Python versions for image builder. Example: ["3.9.", "3.10."]`
`234`	`234`	`SupportedPythonVersions []string`
`235`	`235`	`}`
`236`	`236`