Merge branch 'main' into patch-1

microsoft · Jan 28, 2025 · 94c2485 · 94c2485
2 parents 4f0dcbb + 7d4287b
commit 94c2485
Show file tree

Hide file tree

Showing 50 changed files with 227 additions and 94 deletions.
diff --git a/.github/workflows/promptflow-core-test.yml b/.github/workflows/promptflow-core-test.yml
@@ -12,6 +12,7 @@ on:
 
 env:
   IS_IN_CI_PIPELINE: "true"
+  PF_DISABLE_TRACING: "false"
   TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
   WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
   RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
@@ -28,7 +29,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:
@@ -87,7 +88,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:

diff --git a/.github/workflows/promptflow-evals-e2e-test-azure.yml b/.github/workflows/promptflow-evals-e2e-test-azure.yml
@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:

diff --git a/.github/workflows/promptflow-evals-e2e-test-local.yml b/.github/workflows/promptflow-evals-e2e-test-local.yml
@@ -22,7 +22,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:

diff --git a/.github/workflows/promptflow-evals-installation-test.yml b/.github/workflows/promptflow-evals-installation-test.yml
@@ -33,7 +33,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:

diff --git a/.github/workflows/promptflow-evals-performance-test.yml b/.github/workflows/promptflow-evals-performance-test.yml
@@ -23,7 +23,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:

diff --git a/.github/workflows/promptflow-evals-unit-test.yml b/.github/workflows/promptflow-evals-unit-test.yml
@@ -18,7 +18,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:

diff --git a/.github/workflows/promptflow-import-linter.yml b/.github/workflows/promptflow-import-linter.yml
@@ -26,33 +26,28 @@ jobs:
     - name: Install all packages
       run: |
         touch src/promptflow-tracing/promptflow/__init__.py
-        poetry install --with dev -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-tracing
+        poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-tracing --with dev
         touch src/promptflow-core/promptflow/__init__.py
-        poetry install --with dev -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-core
+        poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-core --with dev
         touch src/promptflow-devkit/promptflow/__init__.py
-        poetry install --with dev -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-devkit
+        poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-devkit --with dev
         touch src/promptflow-azure/promptflow/__init__.py
-        poetry install --with dev -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure
+        poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure --with dev
         touch src/promptflow-evals/promptflow/__init__.py
-        poetry install --with dev -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-evals
+        poetry install -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-evals --with dev
       working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: import lint
       run: |
         echo "=== Running import lint in promptflow-tracing ==="
-        cd ${{ env.WORKING_DIRECTORY }}/src/promptflow-tracing
-        poetry run lint-imports
+        poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-tracing run lint-imports
         echo "=== Running import lint in promptflow-core ==="
-        cd ${{ env.WORKING_DIRECTORY }}/src/promptflow-core
-        poetry run lint-imports
+        poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-core run lint-imports
         echo "=== Running import lint in promptflow-devkit ==="
-        cd ${{ env.WORKING_DIRECTORY }}/src/promptflow-devkit
-        poetry run lint-imports
+        poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-devkit run lint-imports
         echo "=== Running import lint in promptflow-azure ==="
-        cd ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure
-        poetry run lint-imports
+        poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure run lint-imports
         echo "=== Running import lint in promptflow-evals ==="
-        cd ${{ env.WORKING_DIRECTORY }}/src/promptflow-evals
-        poetry run lint-imports
+        poetry -C ${{ env.WORKING_DIRECTORY }}/src/promptflow-evals run lint-imports
       working-directory: ${{ env.WORKING_DIRECTORY }}
     - name: import lint testing private imports from global
       working-directory: ${{ env.WORKING_DIRECTORY }}/src/promptflow-azure

diff --git a/.github/workflows/promptflow-parallel-e2e-test.yml b/.github/workflows/promptflow-parallel-e2e-test.yml
@@ -25,7 +25,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:

diff --git a/.github/workflows/promptflow-parallel-unit-test.yml b/.github/workflows/promptflow-parallel-unit-test.yml
@@ -25,7 +25,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:

diff --git a/.github/workflows/promptflow-release-testing-matrix.yml b/.github/workflows/promptflow-release-testing-matrix.yml
@@ -80,7 +80,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+        pythonVersion: ['3.9', '3.10', '3.11']
     defaults:
       run:
         shell: bash
@@ -136,7 +136,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+        pythonVersion: ['3.9', '3.10', '3.11']
     # snok/install-poetry need this to support Windows
     defaults:
       run:
@@ -193,7 +193,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+        pythonVersion: ['3.9', '3.10', '3.11']
     # snok/install-poetry need this to support Windows
     defaults:
       run:
@@ -243,7 +243,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+        pythonVersion: ['3.9', '3.10', '3.11']
     # snok/install-poetry need this to support Windows
     defaults:
       run:
@@ -301,7 +301,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+        pythonVersion: ['3.9', '3.10', '3.11']
     env:
       PROMPT_FLOW_TEST_MODE: "live"
       WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-azure
@@ -368,7 +368,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+        pythonVersion: ['3.9', '3.10', '3.11']
     runs-on: ${{ matrix.os }}
     steps:
     - name: checkout

diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml
@@ -23,6 +23,7 @@ permissions:
   contents: read
 env:
   IS_IN_CI_PIPELINE: "true"
+  PF_DISABLE_TRACING: "false"
   RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
   TRACING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
   CORE_DIRECTORY: ${{ github.workspace }}/src/promptflow-core
@@ -35,7 +36,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+        pythonVersion: ['3.9', '3.10', '3.11']
     environment:
       internal
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/promptflow-tracing-e2e-test.yml b/.github/workflows/promptflow-tracing-e2e-test.yml
@@ -11,6 +11,7 @@ on:
 
 env:
   IS_IN_CI_PIPELINE: "true"
+  PF_DISABLE_TRACING: "false"
   WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
   RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 
@@ -37,7 +38,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:

diff --git a/.github/workflows/promptflow-tracing-unit-test.yml b/.github/workflows/promptflow-tracing-unit-test.yml
@@ -11,6 +11,7 @@ on:
 
 env:
   IS_IN_CI_PIPELINE: "true"
+  PF_DISABLE_TRACING: "false"
   WORKING_DIRECTORY: ${{ github.workspace }}/src/promptflow-tracing
   RECORD_DIRECTORY: ${{ github.workspace }}/src/promptflow-recording
 
@@ -33,7 +34,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-13]
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
       fail-fast: false
     # snok/install-poetry need this to support Windows
     defaults:

diff --git a/.github/workflows/sdk-cli-azure-test-production.yml b/.github/workflows/sdk-cli-azure-test-production.yml
@@ -43,7 +43,7 @@ jobs:
       matrix:
         # replay tests can cover more combinations
         os: [ubuntu-latest]
-        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+        pythonVersion: ['3.9', '3.10', '3.11']
     environment:
       internal
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/sdk-cli-azure-test-pull-request.yml b/.github/workflows/sdk-cli-azure-test-pull-request.yml
@@ -36,7 +36,7 @@ jobs:
       matrix:
         # replay tests can cover more combinations
         os: [ubuntu-latest]
-        pythonVersion: ['3.8', '3.9', '3.10', '3.11']
+        pythonVersion: ['3.9', '3.10', '3.11']
 
     runs-on: ${{ matrix.os }}
     steps:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -30,7 +30,7 @@ repos:
       - id: cspell
         args: ['--config', '.cspell.json', "--no-must-find-files"]
     - repo: https://github.com/hadialqattan/pycln
-      rev: v2.1.2 # Possible releases: https://github.com/hadialqattan/pycln/tags
+      rev: v2.5.0 # Possible releases: https://github.com/hadialqattan/pycln/tags
       hooks:
           - id: pycln
             name: "Clean unused python imports"
@@ -44,9 +44,10 @@ repos:
           # Use black profile for isort to avoid conflicts
           # see https://github.com/PyCQA/isort/issues/1518
           args: ["--profile", "black", --line-length=120]
-    - repo: https://github.com/pylint-dev/pylint
-      rev: v3.0.3
-      hooks:
-        - id: pylint
-          name: pylint
-          args: [ --rcfile=pylintrc, --output-format=parseable ]
+    # Commented out for now since it's not working locally but works on CI
+    # - repo: https://github.com/pylint-dev/pylint
+    #   rev: v3.0.3
+    #   hooks:
+    #     - id: pylint
+    #       name: pylint
+    #       args: [ --rcfile=pylintrc, --output-format=parseable ]
diff --git a/docs/dev/dev_setup.md b/docs/dev/dev_setup.md
@@ -5,13 +5,13 @@
 Select either Conda or Poetry to set up your development environment.
 
 1. Conda environment setup
-  - First create a new [conda](https://conda.io/projects/conda/en/latest/user-guide/getting-started.html) environment. Please specify python version as 3.8/3.9/3.10/3.11.
+  - First create a new [conda](https://conda.io/projects/conda/en/latest/user-guide/getting-started.html) environment. Please specify python version as 3.9/3.10/3.11.
     `conda create -n <env_name> python=3.9`.
   - Activate the env you created.
   - In root folder, run `python scripts/dev-setup/main.py` to install the packages and dependencies; if you are using Visual Studio Code, it is recommended to add `--vscode` (which is `python scripts/dev-setup/main.py --vscode`) to enable VS Code to recognize the packages.
 
 2. Poetry environment setup
-  - Install [poetry](https://python-poetry.org/docs/). Please specify python version as 3.8/3.9/3.10/3.11.
+  - Install [poetry](https://python-poetry.org/docs/). Please specify python version as 3.9/3.10/3.11.
   - Each folder under [src](../../src/) (except the promptflow folder) is a separate package, so you need to install the dependencies for each package.
     - `poetry install -C promptflow-core -E <extra> --with dev,test`
     - `poetry install -C promptflow-devkit -E <extra> --with dev,test`

diff --git a/docs/how-to-guides/installation/index.md b/docs/how-to-guides/installation/index.md
@@ -9,7 +9,7 @@ This guide outlines the Promptflow SDK and CLI installation process.
 ## Create a virtual environment (optional)
 When installing prompt flow locally, we recommend using a virtual environment for the installation. This ensures
 that the dependencies for prompt flow are isolated from the rest of your system. Please ensure you have a working 
-python environment (python>=3.8,<4.0), a new virtual environment is preferred.
+python environment (python>=3.9,<4.0), a new virtual environment is preferred.
 
 ::::{tab-set}
 :::{tab-item} venv

diff --git a/pylintrc b/pylintrc
@@ -1,7 +1,7 @@
 [MASTER]
 py-version=3.9
 ignore-patterns=test_*,conftest,setup,.*_utils\.py
-ignore-paths=src\promptflow-evals\tests,src\promptflow-evals\samples,promptflow\evals\evaluate\_telemetry,promptflow\evals\evaluate\_batch_run_client\code_client.py,promptflow\evals\evaluate\_batch_run_client\proxy_client.py
+ignore-paths=src\promptflow-evals\tests,src\promptflow-evals\samples,promptflow\evals\evaluate\_telemetry,promptflow\evals\evaluate\_batch_run_client\code_client.py,promptflow\evals\evaluate\_batch_run_client\proxy_client.py,src/promptflow-devkit/tests/sdk_pfs_test/utils.py
 reports=no
 
 [MESSAGES CONTROL]

diff --git a/scripts/installer/curl_install_pypi/install.py b/scripts/installer/curl_install_pypi/install.py
@@ -218,7 +218,7 @@ def verify_python_version():
     print_status('Verifying Python version.')
     v = sys.version_info
     if v < (3, 8):
-        raise CLIInstallError('The promptflow does not support Python versions less than 3.8.')
+        raise CLIInstallError('The promptflow does not support Python versions less than 3.9.')
     if 'conda' in sys.version:
         raise CLIInstallError("This script does not support the Python Anaconda environment. "
                               "Create an Anaconda virtual environment and install with 'pip'")

diff --git a/src/promptflow-azure/CHANGELOG.md b/src/promptflow-azure/CHANGELOG.md
@@ -1,5 +1,10 @@
 # promptflow-azure package
 
+## v1.17.0 (2025.1.8)
+
+### Improvements
+- Dropped Python 3.8 support for security reasons.
+
 ## v1.16.0 (2024.09.30)
 
 ## v1.15.0 (2024.08.15)

diff --git a/src/promptflow-azure/pyproject.toml b/src/promptflow-azure/pyproject.toml
@@ -22,7 +22,6 @@ classifiers = [
   "Programming Language :: Python",
   "Programming Language :: Python :: 3",
   "Programming Language :: Python :: 3 :: Only",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
@@ -38,7 +37,7 @@ packages = [
 "Bug Reports" = "https://github.com/microsoft/promptflow/issues"
 
 [tool.poetry.dependencies]
-python = ">=3.8,<3.9.7 || >3.9.7,<3.13"
+python = ">=3.9,<3.9.7 || >3.9.7,<3.13"
 azure-core = ">=1.26.4,<2.0.0"
 azure-storage-blob = {extras = ["aio"], version = ">=12.17.0,<13.0.0"}  # add [aio] for async run download feature
 azure-identity = ">=1.12.0,<2.0.0"
@@ -58,7 +57,7 @@ promptflow-tools = {path = "../promptflow-tools"}
 promptflow-recording =  {path = "../promptflow-recording", develop = true}
 
 [tool.poetry.group.ci.dependencies]
-azure-ai-ml = ">=1.14.0,<1.19.0"  # 1.19.0 will break azure replay test
+azure-ai-ml = ">=1.14.0,<2.0.0"  # 1.19.0 will break azure replay test however stick with newer ver for marshmallow bug
 import-linter = "*"
 promptflow-core = {path = "../promptflow-core", extras = ["azureml-serving"]}
 promptflow-devkit = {path = "../promptflow-devkit"}

diff --git a/src/promptflow-core/CHANGELOG.md b/src/promptflow-core/CHANGELOG.md
@@ -1,5 +1,20 @@
 # promptflow-core package
 
+## v1.17.2 (2025.1.23)
+
+### Bugs fixed
+- Jinja template is going to use Sandbox Environment at rendering. With `PF_USE_SANDBOX_FOR_JINJA` set to false, sanbox environment is not used.
+
+## v1.17.1 (2025.1.13)
+
+### Others
+- Promptflow Tracing feature is now disabled by default, with `PF_DISABLE_TRACING` set to true by default.
+
+## v1.17.0 (2025.1.8)
+
+### Improvements
+- Dropped Python 3.8 support for security reasons.
+
 ## v1.16.0 (2024.09.30)
 ### Bugs fixed
 - Fix promptflow serving app logged inputs out with default logging level.