From 655a056cfa4ba5819e2419ae4459aa16b06d3184 Mon Sep 17 00:00:00 2001 From: logan-keede Date: Sun, 18 May 2025 05:27:25 +0530 Subject: [PATCH 1/6] review ready --- .github/workflows/semver_check.yml | 107 +++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 .github/workflows/semver_check.yml diff --git a/.github/workflows/semver_check.yml b/.github/workflows/semver_check.yml new file mode 100644 index 000000000000..5cacd95acedb --- /dev/null +++ b/.github/workflows/semver_check.yml @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: API Changes + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +on: + push: + paths-ignore: + - "docs/**" + - "**.md" + - ".github/ISSUE_TEMPLATE/**" + - ".github/pull_request_template.md" + pull_request: + paths-ignore: + - "docs/**" + - "**.md" + - ".github/ISSUE_TEMPLATE/**" + - ".github/pull_request_template.md" + +jobs: + semver-checks: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Install semver-checks + run: cargo install cargo-semver-checks --locked + - name: Run semver-check datasource + run: cargo semver-checks check-release -p datafusion-datasource -p datafusion-datasource-avro -p datafusion-datasource-csv -p datafusion-datasource-json -p datafusion-datasource-parquet + continue-on-error: true + - name: Run semver-check common + run: cargo semver-checks check-release -p datafusion-common -p datafusion-common-runtime + continue-on-error: true + - name: Run semver-check catalog + run: cargo semver-checks check-release -p datafusion-catalog -p datafusion-catalog-listing + continue-on-error: true + - name: Run semver-check execution + run: cargo semver-checks check-release -p datafusion-execution -p datafusion-session + continue-on-error: true + - name: Run semver-check expr + run: cargo semver-checks check-release -p datafusion-expr -p datafusion-expr-common -p datafusion-physical-expr + continue-on-error: true + - name : Run semver-check functions + run: cargo semver-checks check-release -p datafusion-functions -p datafusion-functions-aggregate -p datafusion-functions-aggregate-common -p datafusion-functions-nested -p datafusion-functions-table -p datafusion-functions-window -p datafusion-functions-window-common + continue-on-error: true + - name: Run semver-check macros + run: cargo semver-checks check-release -p datafusion-macros + continue-on-error: true + - name: Run semver-check optimizer + run: cargo semver-checks check-release -p datafusion-optimizer -p datafusion-physicl-optimizer + continue-on-error: true + + semver-checks-2: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Install semver-checks + run: cargo install cargo-semver-checks --locked + - name: Run semver-check physical-plan + run: cargo semver-checks check-release -p datafusion-physical-plan + continue-on-error: true + - name: Run semver-check proto + run: cargo semver-checks check-release -p datafusion-proto -p datafusion-proto-common + continue-on-error: true + - name: Run semver-check substrait + run: cargo semver-checks check-release -p datafusion-substrait + continue-on-error: true + - name: Run semver-check sql + run: cargo semver-checks check-release -p datafusion-sql + continue-on-error: true + - name: Run semver-check spark + run: cargo semver-checks check-release -p datafusion-spark + continue-on-error: true + - name: Run semver-check sqllogictest + run: cargo semver-checks check-release -p datafusion-sqllogictest + continue-on-error: true + - name: Run semver-check datafusion-cli + run: cargo semver-checks check-release -p datafusion-cli + continue-on-error: true + - name: Run semver-check datafusion + run: cargo semver-checks check-release -p datafusion + continue-on-error: true From bd3d218f72ba7f44b3b75ccde37055f04ad95cbb Mon Sep 17 00:00:00 2001 From: logan-keede Date: Mon, 19 May 2025 12:08:47 +0530 Subject: [PATCH 2/6] Tweak rust setup for datafusion --- .github/workflows/semver_check.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/semver_check.yml b/.github/workflows/semver_check.yml index 5cacd95acedb..062e3b33c44d 100644 --- a/.github/workflows/semver_check.yml +++ b/.github/workflows/semver_check.yml @@ -39,11 +39,14 @@ jobs: semver-checks: runs-on: ubuntu-latest continue-on-error: true + container: + image: amd64/rust steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Rust - uses: actions-rust-lang/setup-rust-toolchain@v1 + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable - name: Install semver-checks run: cargo install cargo-semver-checks --locked - name: Run semver-check datasource @@ -74,11 +77,14 @@ jobs: semver-checks-2: runs-on: ubuntu-latest continue-on-error: true + container: + image: amd64/rust steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Rust - uses: actions-rust-lang/setup-rust-toolchain@v1 + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable - name: Install semver-checks run: cargo install cargo-semver-checks --locked - name: Run semver-check physical-plan From 16e4690aad32bd0aac250033ca00ca56fa129056 Mon Sep 17 00:00:00 2001 From: logan-keede Date: Mon, 19 May 2025 15:17:09 +0530 Subject: [PATCH 3/6] Tweak rust setup to avoid overloading disk space --- .github/workflows/semver_check.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/semver_check.yml b/.github/workflows/semver_check.yml index 062e3b33c44d..087a9222b4d3 100644 --- a/.github/workflows/semver_check.yml +++ b/.github/workflows/semver_check.yml @@ -71,7 +71,10 @@ jobs: run: cargo semver-checks check-release -p datafusion-macros continue-on-error: true - name: Run semver-check optimizer - run: cargo semver-checks check-release -p datafusion-optimizer -p datafusion-physicl-optimizer + run: cargo semver-checks check-release -p datafusion-optimizer -p datafusion-physical-optimizer + continue-on-error: true + - name: Run semver-check physical-plan + run: cargo semver-checks check-release -p datafusion-physical-plan continue-on-error: true semver-checks-2: @@ -87,15 +90,12 @@ jobs: rust-version: stable - name: Install semver-checks run: cargo install cargo-semver-checks --locked - - name: Run semver-check physical-plan - run: cargo semver-checks check-release -p datafusion-physical-plan - continue-on-error: true - name: Run semver-check proto run: cargo semver-checks check-release -p datafusion-proto -p datafusion-proto-common continue-on-error: true - - name: Run semver-check substrait - run: cargo semver-checks check-release -p datafusion-substrait - continue-on-error: true + # - name: Run semver-check substrait #not working for some reason, TB Triaged + # run: cargo semver-checks check-release -p datafusion-substrait + # continue-on-error: true - name: Run semver-check sql run: cargo semver-checks check-release -p datafusion-sql continue-on-error: true From ed301672d26e4b61f89a2e2c32b959acbbae1a98 Mon Sep 17 00:00:00 2001 From: logan-keede Date: Mon, 19 May 2025 15:52:28 +0530 Subject: [PATCH 4/6] Tweak semver-check setup to avoid overloading disk space 2 --- .github/workflows/semver_check.yml | 33 ++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/.github/workflows/semver_check.yml b/.github/workflows/semver_check.yml index 087a9222b4d3..7fac16f6a72d 100644 --- a/.github/workflows/semver_check.yml +++ b/.github/workflows/semver_check.yml @@ -67,15 +67,7 @@ jobs: - name : Run semver-check functions run: cargo semver-checks check-release -p datafusion-functions -p datafusion-functions-aggregate -p datafusion-functions-aggregate-common -p datafusion-functions-nested -p datafusion-functions-table -p datafusion-functions-window -p datafusion-functions-window-common continue-on-error: true - - name: Run semver-check macros - run: cargo semver-checks check-release -p datafusion-macros - continue-on-error: true - - name: Run semver-check optimizer - run: cargo semver-checks check-release -p datafusion-optimizer -p datafusion-physical-optimizer - continue-on-error: true - - name: Run semver-check physical-plan - run: cargo semver-checks check-release -p datafusion-physical-plan - continue-on-error: true + semver-checks-2: runs-on: ubuntu-latest @@ -105,9 +97,32 @@ jobs: - name: Run semver-check sqllogictest run: cargo semver-checks check-release -p datafusion-sqllogictest continue-on-error: true + + semver-checks-3: + runs-on: ubuntu-latest + continue-on-error: true + container: + image: amd64/rust + steps: + - uses: actions/checkout@v4 + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + - name: Install semver-checks + run: cargo install cargo-semver-checks --locked - name: Run semver-check datafusion-cli run: cargo semver-checks check-release -p datafusion-cli continue-on-error: true - name: Run semver-check datafusion run: cargo semver-checks check-release -p datafusion continue-on-error: true + - name: Run semver-check macros + run: cargo semver-checks check-release -p datafusion-macros + continue-on-error: true + - name: Run semver-check optimizer + run: cargo semver-checks check-release -p datafusion-optimizer -p datafusion-physical-optimizer + continue-on-error: true + - name: Run semver-check physical-plan + run: cargo semver-checks check-release -p datafusion-physical-plan + continue-on-error: true \ No newline at end of file From 90d68a0d7daa6cfe5546eb09939ec47b7e50a92b Mon Sep 17 00:00:00 2001 From: logan-keede Date: Mon, 19 May 2025 23:26:51 +0530 Subject: [PATCH 5/6] Add substrait back --- .github/workflows/semver_check.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/semver_check.yml b/.github/workflows/semver_check.yml index 7fac16f6a72d..43e178b04c59 100644 --- a/.github/workflows/semver_check.yml +++ b/.github/workflows/semver_check.yml @@ -85,9 +85,9 @@ jobs: - name: Run semver-check proto run: cargo semver-checks check-release -p datafusion-proto -p datafusion-proto-common continue-on-error: true - # - name: Run semver-check substrait #not working for some reason, TB Triaged - # run: cargo semver-checks check-release -p datafusion-substrait - # continue-on-error: true + - name: Run semver-check substrait #not working for some reason, TB Triaged + run: apt-get install -y cmake && cargo semver-checks check-release -p datafusion-substrait + continue-on-error: true - name: Run semver-check sql run: cargo semver-checks check-release -p datafusion-sql continue-on-error: true From 5566125fa4f5ab6bef2d6f54f92f634c104b4313 Mon Sep 17 00:00:00 2001 From: logan-keede Date: Tue, 20 May 2025 13:13:39 +0530 Subject: [PATCH 6/6] disk space management --- .github/workflows/semver_check.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/semver_check.yml b/.github/workflows/semver_check.yml index 43e178b04c59..dd930342dd93 100644 --- a/.github/workflows/semver_check.yml +++ b/.github/workflows/semver_check.yml @@ -91,12 +91,6 @@ jobs: - name: Run semver-check sql run: cargo semver-checks check-release -p datafusion-sql continue-on-error: true - - name: Run semver-check spark - run: cargo semver-checks check-release -p datafusion-spark - continue-on-error: true - - name: Run semver-check sqllogictest - run: cargo semver-checks check-release -p datafusion-sqllogictest - continue-on-error: true semver-checks-3: runs-on: ubuntu-latest @@ -125,4 +119,10 @@ jobs: continue-on-error: true - name: Run semver-check physical-plan run: cargo semver-checks check-release -p datafusion-physical-plan - continue-on-error: true \ No newline at end of file + continue-on-error: true + - name: Run semver-check spark + run: cargo semver-checks check-release -p datafusion-spark + continue-on-error: true + - name: Run semver-check sqllogictest + run: cargo semver-checks check-release -p datafusion-sqllogictest + continue-on-error: true