Merge branch 'main' into bandit_precommit

apache · Nov 9, 2023 · 35553cb · 35553cb
2 parents 21feea8 + 9990443
commit 35553cb
Show file tree

Hide file tree

Showing 680 changed files with 10,901 additions and 3,179 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -77,7 +77,6 @@ github:
     - utkarsharma2
     - Lee-W
     - sunank200
-    - jens-scheffler-bosch
     - nathadfield
     - RNHTTR
 

diff --git a/.github/ISSUE_TEMPLATE/airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_bug_report.yml
@@ -25,7 +25,7 @@ body:
         the latest release or main to see if the issue is fixed before reporting it.
       multiple: false
       options:
-        - "2.7.2"
+        - "2.7.3"
         - "main (development)"
         - "Other Airflow 2 version (please specify below)"
     validations:

diff --git a/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml b/.github/ISSUE_TEMPLATE/airflow_providers_bug_report.yml
@@ -48,6 +48,7 @@ body:
         - celery
         - cloudant
         - cncf-kubernetes
+        - cohere
         - common-io
         - common-sql
         - daskexecutor
@@ -81,11 +82,14 @@ body:
         - odbc
         - openfaas
         - openlineage
+        - openai
         - opensearch
         - opsgenie
         - oracle
         - pagerduty
         - papermill
+        - pgvector
+        - pinecone
         - plexus
         - postgres
         - presto
@@ -107,6 +111,7 @@ body:
         - telegram
         - trino
         - vertica
+        - weaviate
         - yandex
         - zendesk
     validations:

diff --git a/.github/actions/build-prod-images/action.yml b/.github/actions/build-prod-images/action.yml
@@ -25,19 +25,12 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: "Install Breeze"
-      uses: ./.github/actions/breeze
     - name: "Regenerate dependencies in case they was modified manually so that we can build an image"
       shell: bash
       run: |
         pip install rich>=12.4.4 pyyaml
         python scripts/ci/pre_commit/pre_commit_update_providers_dependencies.py
       if: env.UPGRADE_TO_NEWER_DEPENDENCIES != 'false'
-    - name: "Pull CI image for PROD build: ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}"
-      shell: bash
-      run: breeze ci-image pull --tag-as-latest
-      env:
-        PYTHON_MAJOR_MINOR_VERSION: "3.8"
     - name: "Cleanup dist and context file"
       shell: bash
       run: rm -fv ./dist/* ./docker-context-files/*

diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
@@ -153,7 +153,6 @@ jobs:
           path: "target-airflow"
           ref: ${{ github.base_ref }}
           persist-credentials: false
-          submodules: recursive
       - name: >
           Override "scripts/ci", "dev" and ".github/actions" with the target branch
           so that the PR does not override it
@@ -209,14 +208,12 @@ jobs:
         with:
           ref: ${{ needs.build-info.outputs.target-commit-sha }}
           persist-credentials: false
-          submodules: recursive
       - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there.
         uses: actions/checkout@v4
         with:
           path: "target-airflow"
           ref: ${{ github.base_ref }}
           persist-credentials: false
-          submodules: recursive
       - name: >
           Override "scripts/ci", "dev" and ".github/actions" with the target branch
           so that the PR does not override it
@@ -282,14 +279,12 @@ jobs:
         with:
           ref: ${{ needs.build-info.outputs.target-commit-sha }}
           persist-credentials: false
-          submodules: recursive
       - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there.
         uses: actions/checkout@v4
         with:
           path: "target-airflow"
           ref: ${{ github.base_ref }}
           persist-credentials: false
-          submodules: recursive
       - name: >
           Override "scripts/ci", "dev" and ".github/actions" with the target branch
           so that the PR does not override it
@@ -304,6 +299,15 @@ jobs:
           mv -v "target-airflow/dev" "."
           rm -rfv ".github/actions"
           mv -v "target-airflow/.github/actions" ".github"
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+      - name: >
+          Pull CI image for PROD build:
+          ${{ steps.selective-checks.outputs.default-python-version }}:${{ env.IMAGE_TAG }}"
+        shell: bash
+        run: breeze ci-image pull --tag-as-latest
+        env:
+          PYTHON_MAJOR_MINOR_VERSION: ${{ steps.selective-checks.outputs.default-python-version }}
       - name: >
           Build PROD Images
           ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
@@ -343,14 +347,12 @@ jobs:
         with:
           ref: ${{ needs.build-info.outputs.target-commit-sha }}
           persist-credentials: false
-          submodules: recursive
       - name: Checkout target branch to 'target-airflow' folder to use ci/scripts and breeze from there.
         uses: actions/checkout@v4
         with:
           path: "target-airflow"
           ref: ${{ github.base_ref }}
           persist-credentials: false
-          submodules: recursive
       - name: >
           Override "scripts/ci", "dev" and ".github/actions" with the target branch
           so that the PR does not override it

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -135,7 +135,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-          submodules: recursive
       - name: Fetch incoming commit ${{ github.sha }} with its parent
         uses: actions/checkout@v4
         with:
@@ -318,7 +317,6 @@ jobs:
         with:
           ref: ${{ needs.build-info.outputs.targetCommitSha }}
           persist-credentials: false
-          submodules: recursive
         if: needs.build-info.outputs.in-workflow-build == 'true'
       - name: >
           Build CI Images
@@ -353,7 +351,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-          submodules: recursive
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
       - name: Pull CI images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG }}
@@ -417,7 +414,17 @@ jobs:
         with:
           ref: ${{ needs.build-info.outputs.targetCommitSha }}
           persist-credentials: false
-          submodules: recursive
+        if: needs.build-info.outputs.in-workflow-build == 'true'
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+        if: needs.build-info.outputs.in-workflow-build == 'true'
+      - name: >
+          Pull CI image for PROD build:
+          ${{ steps.selective-checks.outputs.default-python-version }}:${{ env.IMAGE_TAG }}"
+        shell: bash
+        run: breeze ci-image pull --tag-as-latest
+        env:
+          PYTHON_MAJOR_MINOR_VERSION: ${{ steps.selective-checks.outputs.default-python-version }}
         if: needs.build-info.outputs.in-workflow-build == 'true'
       - name: >
           Build PROD Images
@@ -432,6 +439,57 @@ jobs:
           PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
           DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
 
+  build-prod-images-bullseye:
+    timeout-minutes: 80
+    name: >
+      ${{needs.build-info.outputs.build-job-description}} Bullseye PROD images
+      ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+    runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}}
+    needs: [build-info, build-ci-images, generate-constraints]
+    if: needs.build-info.outputs.canary-run == 'true'
+    env:
+      DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
+      DEFAULT_CONSTRAINTS_BRANCH: ${{ needs.build-info.outputs.default-constraints-branch }}
+      RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+      BACKEND: sqlite
+      DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
+      VERSION_SUFFIX_FOR_PYPI: "dev0"
+      DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}}
+      # Force more parallelism for build even on public images
+      PARALLELISM: 6
+    steps:
+      - name: Cleanup repo
+        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ needs.build-info.outputs.targetCommitSha }}
+          persist-credentials: false
+          submodules: recursive
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+      - name: >
+          Pull CI image for PROD build:
+          ${{ steps.selective-checks.outputs.default-python-version }}:${{ env.IMAGE_TAG }}"
+        shell: bash
+        run: breeze ci-image pull --tag-as-latest
+        env:
+          PYTHON_MAJOR_MINOR_VERSION: ${{ steps.selective-checks.outputs.default-python-version }}
+      - name: >
+          Build Bullseye PROD Images
+          ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}}
+        uses: ./.github/actions/build-prod-images
+        with:
+          build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }}
+        env:
+          UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}
+          DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }}
+          PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}}
+          DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }}
+          DEBIAN_VERSION: "bullseye"
+          # Do not override the "bookworm" image - just push a new bullseye image
+          # TODO: improve caching for that build
+          IMAGE_TAG: "bullseye-${{ github.event.pull_request.head.sha || github.sha }}"
+
   run-breeze-tests:
     timeout-minutes: 10
     name: Breeze unit tests
@@ -538,7 +596,6 @@ jobs:
         with:
           fetch-depth: 2
           persist-credentials: false
-
   wait-for-ci-images:
     timeout-minutes: 120
     name: "Wait for CI images"
@@ -595,9 +652,9 @@ jobs:
         with:
           path: ~/.cache/pre-commit
           # yamllint disable-line rule:line-length
-          key: "pre-commit-full-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
+          key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
           restore-keys: |
-            pre-commit-full-${{steps.breeze.outputs.host-python-version}}-
+            pre-commit-${{steps.breeze.outputs.host-python-version}}-
       - name: "Static checks"
         run: breeze static-checks --all-files --show-diff-on-failure --color always --initialize-environment
         env:
@@ -643,12 +700,11 @@ jobs:
         with:
           path: ~/.cache/pre-commit
           # yamllint disable-line rule:line-length
-          key: "pre-commit-basic-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
+          key: "pre-commit-${{steps.breeze.outputs.host-python-version}}-${{ hashFiles('.pre-commit-config.yaml') }}"
           restore-keys: "\
-            pre-commit-full-${{steps.breeze.outputs.host-python-version}}-\
+            pre-commit-${{steps.breeze.outputs.host-python-version}}-\
             ${{ hashFiles('.pre-commit-config.yaml') }}\n
-            pre-commit-basic-${{steps.breeze.outputs.host-python-version}}-\n
-            pre-commit-full-${{steps.breeze.outputs.host-python-version}}-"
+            pre-commit-${{steps.breeze.outputs.host-python-version}}-"
       - name: Fetch incoming commit ${{ github.sha }} with its parent
         uses: actions/checkout@v4
         with:
@@ -684,7 +740,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-          submodules: recursive
       - name: >
           Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}}
         uses: ./.github/actions/prepare_breeze_and_image
@@ -703,32 +758,34 @@ jobs:
         run: >
           git clone https://github.com/apache/airflow-site.git ${GITHUB_WORKSPACE}/airflow-site &&
           echo "AIRFLOW_SITE_DIRECTORY=${GITHUB_WORKSPACE}/airflow-site" >> "$GITHUB_ENV"
+        if: needs.build-info.outputs.full-tests-needed == 'true'
       - name: "Publish docs"
         run: >
           breeze release-management publish-docs
           --override-versioned --run-in-parallel
           ${{ needs.build-info.outputs.docs-list-as-string }}
+        if: needs.build-info.outputs.full-tests-needed == 'true'
       - name: "Generate back references for providers"
         run: breeze release-management add-back-references all-providers
+        if: needs.build-info.outputs.full-tests-needed == 'true'
       - name: "Generate back references for apache-airflow"
         run: breeze release-management add-back-references apache-airflow
+        if: needs.build-info.outputs.full-tests-needed == 'true'
       - name: "Generate back references for docker-stack"
         run: breeze release-management add-back-references docker-stack
+        if: needs.build-info.outputs.full-tests-needed == 'true'
       - name: "Generate back references for helm-chart"
         run: breeze release-management add-back-references helm-chart
+        if: needs.build-info.outputs.full-tests-needed == 'true'
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a  # v4.0.1
-        if: >
-          github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
-          github.event_name == 'push'
+        if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
         with:
           aws-access-key-id: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
           aws-region: eu-central-1
       - name: "Upload documentation to AWS S3"
-        if: >
-          github.ref == 'refs/heads/main' && github.repository == 'apache/airflow' &&
-          github.event_name == 'push'
+        if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main'
         run: aws s3 sync --delete ./files/documentation s3://apache-airflow-docs
       - name: "Fix ownership"
         run: breeze ci fix-ownership
@@ -1596,7 +1653,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-          submodules: recursive
       - name: "Download all artifacts from the current build"
         uses: actions/download-artifact@v3
         with:
@@ -1783,7 +1839,6 @@ jobs:
         with:
           # Needed to perform push action
           persist-credentials: false
-          submodules: recursive
       - name: "Set constraints branch name"
         id: constraints-branch
         run: ./scripts/ci/constraints/ci_branch_constraints.sh >> ${GITHUB_OUTPUT}
@@ -1944,7 +1999,6 @@ jobs:
         with:
           ref: ${{ needs.build-info.outputs.targetCommitSha }}
           persist-credentials: false
-          submodules: recursive
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
       - name: "Start ARM instance"

diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml
@@ -58,7 +58,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-          submodules: recursive
       - name: "Install Breeze"
         uses: ./.github/actions/breeze
       - name: Selective checks

diff --git a/.gitignore b/.gitignore
@@ -98,6 +98,9 @@ target/
 # pyenv
 .python-version
 
+# nvm (Node Version Manager)
+.nvmrc
+
 # celery beat schedule file
 celerybeat-schedule
 

diff --git a/.gitmodules b/.gitmodules
diff --git a/.gitpod.yml b/.gitpod.yml
@@ -19,6 +19,9 @@
 # Reference: https://www.gitpod.io/docs/references/gitpod-yml
 # Docs: https://www.gitpod.io/docs/config-gitpod-file/
 
+# default "gitpod/workspace-full" python version > 3.11
+image: gitpod/workspace-python-3.11
+
 tasks:
   - init: ./scripts/ci/install_breeze.sh
   - name: Install pre-commit