Merge branch 'main' into explain-oom

Author: dqhl76

.github/actions/pack_binaries/action.yml

@@ -5,7 +5,7 @@ inputs:
     description: "Release target"
     required: true
   category:
-    description: "Release default/hdfs/udf/testsuite"
+    description: "Release default/hdfs/udf/testsuite/dbg"
     required: false
     default: default
   version:
@@ -15,14 +15,24 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Download artifact
+    - name: Download artifact for dbg
+      if: inputs.category == 'dbg'
       uses: ./.github/actions/artifact_download
       with:
         sha: ${{ github.sha }}
         target: ${{ inputs.target }}
-        category: ${{ inputs.category }}
+        category: default
         path: distro/bin
         artifacts: metactl,meta,query,query.debug
+    - name: Download artifact for others
+      if: inputs.category != 'dbg'
+      uses: ./.github/actions/artifact_download
+      with:
+        sha: ${{ github.sha }}
+        target: ${{ inputs.target }}
+        category: ${{ inputs.category }}
+        path: distro/bin
+        artifacts: metactl,meta,query
     - name: Get Latest BendSQL
       id: bendsql
       uses: pozetroninc/github-action-get-latest-release@master
@@ -53,7 +63,7 @@ runs:
         cp ./scripts/distribution/configs/databend-* distro/configs/
         cp ./scripts/distribution/release-readme.txt distro/readme.txt
         cp -r ./scripts/distribution/package-scripts/* distro/scripts/
-        tar -C ./distro --exclude='*.debug' -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
+        tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
         sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
         echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
     - name: post sha256
@@ -68,37 +78,3 @@ runs:
         name: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
         path: ${{ steps.pack_binaries.outputs.pkg_name }}.tar.gz
         retention-days: 1
-    - name: Pack DBG Binaries
-      id: pack_dbg_binaries
-      shell: bash
-      run: |
-        target=${{ inputs.target }}
-        version=${{ inputs.version }}
-        case ${{ inputs.category }} in
-          default)
-            pkg_name="databend-dbg-${version}-${target}"
-            ;;
-          *)
-          pkg_name="databend-dbg-${{ inputs.category }}-${version}-${target}"
-            ;;
-        esac
-        mkdir -p distro/{bin,configs,systemd,scripts}
-        cp ./scripts/distribution/systemd/databend-* distro/systemd/
-        cp ./scripts/distribution/configs/databend-* distro/configs/
-        cp ./scripts/distribution/release-readme.txt distro/readme.txt
-        cp -r ./scripts/distribution/package-scripts/* distro/scripts/
-        tar -C ./distro -czvf ${pkg_name}.tar.gz bin configs systemd scripts readme.txt
-        sha256sum ${pkg_name}.tar.gz >> sha256-${pkg_name}.txt
-        echo "pkg_name=$pkg_name" >> $GITHUB_OUTPUT
-    - name: post dbg sha256
-      uses: actions/upload-artifact@v4
-      with:
-        name: sha256sums-${{ inputs.category }}-${{ inputs.target }}-gdb
-        path: sha256-${{ steps.pack_dbg_binaries.outputs.pkg_name }}.txt
-        retention-days: 1
-    - name: post dbg binaries
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
-        path: ${{ steps.pack_dbg_binaries.outputs.pkg_name }}.tar.gz
-        retention-days: 1

.github/actions/pack_deb/action.yml

@@ -64,7 +64,7 @@ runs:
         export deb_version="${{ steps.info.outputs.deb_version }}"
         export deb_arch="${{ steps.info.outputs.deb_arch }}"
         pkg_name="databend-query_${deb_version}_${deb_arch}.deb"
-        nfpm pkg --packager deb -t "$pkg_name" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-query.yaml)
+        nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-query.yaml)
 
     - name: Build Databend Meta Package
       shell: bash
@@ -74,7 +74,7 @@ runs:
         export deb_version="${{ steps.info.outputs.deb_version }}"
         export deb_arch="${{ steps.info.outputs.deb_arch }}"
         pkg_name="databend-meta_${deb_version}_${deb_arch}.deb"
-        nfpm pkg --packager deb -t "$pkg_name" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-meta.yaml)
+        nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-meta.yaml)
 
     - name: Build Databend Debug Package
       shell: bash
@@ -84,7 +84,7 @@ runs:
         export deb_version="${{ steps.info.outputs.deb_version }}"
         export deb_arch="${{ steps.info.outputs.deb_arch }}"
         pkg_name="databend-query-dbg_${deb_version}_${deb_arch}.deb"
-        nfpm pkg --packager deb -t "$pkg_name" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-query-dbg.yaml)
+        nfpm pkg --packager deb -t "${path}/${pkg_name}" -f <(envsubst '${version} ${path} ${arch}' < scripts/distribution/nfpm-query-dbg.yaml)
 
     - name: Update release to github
       shell: bash

.github/actions/publish_binary/action.yml

@@ -23,7 +23,7 @@ runs:
           default)
             publish_name="databend-${{ inputs.version }}-${{ inputs.target }}"
             ;;
-          hdfs|udf|testsuite)
+          hdfs|udf|testsuite|dbg)
             publish_name="databend-${{ inputs.category }}-${{ inputs.version }}-${{ inputs.target }}"
             ;;
           *)
@@ -38,18 +38,12 @@ runs:
       # Reference: https://cli.github.com/manual/gh_release_upload
       run: |
         gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}.tar.gz --clobber
-        if [ -f ${{ steps.name.outputs.name }}-dbg.tar.gz ]; then
-          gh release upload ${{ inputs.version }} ${{ steps.name.outputs.name }}-dbg.tar.gz --clobber
-        fi
 
     - name: Sync normal release to R2
       shell: bash
       if: inputs.category == 'default'
       run: |
         aws s3 cp ${{ steps.name.outputs.name }}.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}.tar.gz --no-progress --checksum-algorithm=CRC32
-        if [ -f ${{ steps.name.outputs.name }}-dbg.tar.gz ]; then
-          aws s3 cp ${{ steps.name.outputs.name }}-dbg.tar.gz s3://repo/databend/${{ inputs.version }}/${{ steps.name.outputs.name }}-dbg.tar.gz --no-progress --checksum-algorithm=CRC32
-        fi
         gh api /repos/databendlabs/databend/tags > tags.json
         aws s3 cp ./tags.json s3://repo/databend/tags.json --no-progress --checksum-algorithm=CRC32
         gh api /repos/databendlabs/databend/releases > releases.json

.github/actions/test_sqllogic_cluster_linux/action.yml

@@ -9,6 +9,11 @@ inputs:
     description: "logic test handlers, mysql,http,clickhouse"
     required: false
     default: ""
+  parallel:
+    description: "logic test parallel"
+    required: false
+    default: ""
+
 runs:
   using: "composite"
   steps:
@@ -20,4 +25,6 @@ runs:
       shell: bash
       env:
         TEST_HANDLERS: ${{ inputs.handlers }}
+        TEST_PARALLEL: ${{ inputs.parallel }}
+        TEST_EXT_ARGS: '--skip_file tpcds_spill_1.test,tpcds_spill_2.test,tpcds_spill_3.test'
       run: bash ./scripts/ci/ci-run-sqllogic-tests-cluster.sh ${{ inputs.dirs }}

.github/actions/test_sqllogic_iceberg_tpch/action.yml

@@ -32,6 +32,7 @@ runs:
         pip install pyspark
         python3 tests/sqllogictests/scripts/prepare_iceberg_tpch_data.py
 
+
     - name: Run sqllogic Tests with Standalone lib
       shell: bash
       env:

.github/actions/test_sqllogic_stage/action.yml

@@ -13,7 +13,7 @@ inputs:
     description: "storage backend for stage, choices: s3,fs"
     required: true
     default: ""
-  deducp:
+  dedup:
     description: "path type for dedup when copy, choices: full_path,sub_path"
     required: true
     default: ""

.github/actions/test_sqllogic_standalone_linux/action.yml

@@ -29,12 +29,20 @@ runs:
       with:
         artifacts: sqllogictests,meta,query
 
+    - uses: actions/github-script@v7
+      id: ext-args
+      env:
+        DIRS: ${{ inputs.dirs }}
+      with:
+        script: require('.github/actions/test_sqllogic_standalone_linux/script.js')(core)
+
     - name: Run sqllogic Tests with Standalone mode
       if: inputs.storage-format == 'all' || inputs.storage-format == 'parquet'
       shell: bash
       env:
         TEST_HANDLERS: ${{ inputs.handlers }}
         TEST_PARALLEL: ${{ inputs.parallel }}
+        TEST_EXT_ARGS: ${{ steps.ext-args.outputs.parquet }}
         CACHE_ENABLE_TABLE_META_CACHE: ${{ inputs.enable_table_meta_cache}}
       run: bash ./scripts/ci/ci-run-sqllogic-tests.sh ${{ inputs.dirs }}
 
@@ -44,5 +52,6 @@ runs:
       env:
         TEST_HANDLERS: ${{ inputs.handlers }}
         TEST_PARALLEL: ${{ inputs.parallel }}
+        TEST_EXT_ARGS: '--skip_file tpcds_spill_1.test,tpcds_spill_2.test,tpcds_spill_3.test'
         CACHE_ENABLE_TABLE_META_CACHE: ${{ inputs.enable_table_meta_cache}}
       run: bash ./scripts/ci/ci-run-sqllogic-tests-native.sh ${{ inputs.dirs }}

.github/actions/test_sqllogic_standalone_linux/script.js

@@ -0,0 +1,12 @@
+module.exports = (core) => {
+    switch (process.env.DIRS) {
+        case 'tpcds':
+            const parquet = [
+                '--skip_file tpcds_spill_2.test,tpcds_spill_3.test',
+                '--skip_file tpcds_spill_1.test,tpcds_spill_3.test',
+                '--skip_file tpcds_spill_1.test,tpcds_spill_2.test',
+            ][Date.now() % 3];
+            core.setOutput('parquet', parquet)
+            return
+    }
+}

.github/workflows/dev.yml

@@ -24,7 +24,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Check Source File Changes
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         id: src
         with:
           files_ignore: |

.github/workflows/main.yml

@@ -16,7 +16,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Check Source File Changes
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         id: src
         with:
           files_ignore: |

.github/workflows/merge_group.yml

@@ -27,7 +27,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Check Source File Changes
-        uses: tj-actions/changed-files@v45
+        uses: tj-actions/changed-files@v46
         id: src
         with:
           files_ignore: |

.github/workflows/release.yml

@@ -178,11 +178,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - category: default
-            target: x86_64-unknown-linux-gnu
-          - category: default
-            target: aarch64-unknown-linux-gnu
+        category:
+          - default
+          - dbg
+        target:
+          - x86_64-unknown-linux-gnu
+          - aarch64-unknown-linux-gnu
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/reuse.benchmark.yml

@@ -206,7 +206,7 @@ jobs:
           for p in results/*; do
             dataset=$(basename $p)
             aws s3 sync results/$dataset/ ${REPORT_S3_PREFIX}/ --include "*.json" --no-progress --checksum-algorithm=CRC32
-            aws s3 sync "s3://benchmark/clickbench/release/${dataset}/latest/" ./results/${dataset}/ --exclude "*" --include "*.json" --no-progress --checksum-algorithm=CRC32
+            aws s3 sync "s3://benchmark/clickbench/release/${dataset}/latest/" ./results/${dataset}/ --exclude "*" --include "*.json"
             ./update_results.py --dataset $dataset --pr ${{ inputs.source_id }}
             aws s3 cp ./results/${dataset}.html ${REPORT_S3_PREFIX}/${dataset}.html --no-progress --checksum-algorithm=CRC32
             echo "* **${dataset}**: https://benchmark.databend.com/clickbench/pr/${{ inputs.source_id }}/${{ inputs.run_id }}/${dataset}.html" >> /tmp/body
@@ -254,8 +254,8 @@ jobs:
             mv $result results/${dataset}/$(basename $result)
           done
 
-          aws s3 sync s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date --date='-1 month' -u +%Y)/$(date --date='-1 month' -u +%m)/ ./results/${{ matrix.dataset }}/ --no-progress --checksum-algorithm=CRC32
-          aws s3 sync s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date -u +%Y)/$(date -u +%m)/ ./results/${{ matrix.dataset }}/ --no-progress --checksum-algorithm=CRC32
+          aws s3 sync s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date --date='-1 month' -u +%Y)/$(date --date='-1 month' -u +%m)/ ./results/${{ matrix.dataset }}/
+          aws s3 sync s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date -u +%Y)/$(date -u +%m)/ ./results/${{ matrix.dataset }}/
           ./update_results.py --dataset ${{ matrix.dataset }} --release ${{ inputs.source_id }}
 
           RESULT_PREFIX="s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date -u +%Y)/$(date -u +%m)/$(date -u +%Y-%m-%d)/${{ inputs.source_id }}"

.github/workflows/reuse.sqllogic.yml

@@ -50,7 +50,7 @@ jobs:
           - { dirs: "crdb", runner: "2c8g" }
           - { dirs: "base", runner: "2c8g" }
           - { dirs: "ydb", runner: "2c8g" }
-          - { dirs: "tpcds", runner: "2c8g" }
+          - { dirs: "tpcds", runner: "4c16g", parallel: "1" }
           - { dirs: "tpch", runner: "2c8g" }
           - { dirs: "standalone", runner: "2c8g" }
         handler:
@@ -59,10 +59,11 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test_sqllogic_standalone_linux
-        timeout-minutes: 15
+        timeout-minutes: 20
         with:
           dirs: ${{ matrix.tests.dirs }}
           handlers: ${{ matrix.handler }}
+          parallel: ${{ matrix.tests.parallel }}
           storage-format: all
       - name: Upload failure
         if: failure()
@@ -192,6 +193,7 @@ jobs:
         with:
           dirs: ${{ matrix.tests.dirs }}
           handlers: ${{ matrix.handler }}
+          parallel: ${{ matrix.tests.parallel }}
       - name: Upload failure
         if: failure()
         uses: ./.github/actions/artifact_failure

.typos.toml

@@ -13,6 +13,7 @@
 "Ser" = "Ser"
 "flate" = "flate"
 "Tke" = "Tke"
+"typ" = "typ"
 
 [files]
 extend-exclude = [