Skip to content

Commit 64b37b3

Browse files
Omega359Omega359
committed
Merge branch 'main' into feature/scalar_func_args_session_config
# Conflicts: # Cargo.lock # datafusion/functions/benches/date_trunc.rs # datafusion/physical-expr/src/scalar_function.rs
2 parents 057e7bc + 592fe6a commit 64b37b3

File tree

91 files changed

+3686
-2914
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

91 files changed

+3686
-2914
lines changed

‎.github/workflows/extended.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ jobs:
124124
rust-version: stable
125125
- name: Run sqllogictest
126126
run: |
127-
cargo test --profile release-nonlto --test sqllogictests -- --include-sqlite
127+
cargo test --features backtrace --profile release-nonlto --test sqllogictests -- --include-sqlite
128128
cargo clean
129129
130130

‎.github/workflows/rust.yml

Lines changed: 105 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,12 @@ jobs:
6666
# the changes to `Cargo.lock` after building with the updated manifest.
6767
cargo check --profile ci --workspace --all-targets --features integration-tests --locked
6868
69-
# cargo check common, functions and substrait with no default features
70-
linux-cargo-check-no-default-features:
71-
name: cargo check no default features
69+
# Check datafusion-common features
70+
#
71+
# Ensure via `cargo check` that the crate can be built with a
72+
# subset of the features packages enabled.
73+
linux-datafusion-common-features:
74+
name: cargo check datafusion-common features
7275
needs: linux-build-lib
7376
runs-on: ubuntu-latest
7477
container:
@@ -79,28 +82,68 @@ jobs:
7982
uses: ./.github/actions/setup-builder
8083
with:
8184
rust-version: stable
82-
- name: Check datafusion without default features
83-
# Some of the test binaries require the parquet feature still
84-
#run: cargo check --all-targets --no-default-features -p datafusion
85-
run: cargo check --profile ci --no-default-features -p datafusion
86-
87-
- name: Check datafusion-common without default features
85+
- name: Check datafusion-common (no-default-features)
8886
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-common
87+
# Note: don't check other feature flags as datafusion-common is not typically used standalone
8988

90-
- name: Check datafusion-functions without default features
91-
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-functions
92-
93-
- name: Check datafusion-substrait without default features
89+
# Check datafusion-substrait features
90+
#
91+
# Ensure via `cargo check` that the crate can be built with a
92+
# subset of the features packages enabled.
93+
linux-datafusion-substrait-features:
94+
name: cargo check datafusion-substrait features
95+
needs: linux-build-lib
96+
runs-on: ubuntu-latest
97+
container:
98+
image: amd64/rust
99+
steps:
100+
- uses: actions/checkout@v4
101+
- name: Setup Rust toolchain
102+
uses: ./.github/actions/setup-builder
103+
with:
104+
rust-version: stable
105+
- name: Check datafusion-substrait (no-default-features)
94106
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait
107+
- name: Check datafusion-substrait (physical)
108+
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=physical
109+
- name: Install cmake
110+
run: |
111+
# note the builder setup runs apt-get update / installs protobuf compiler
112+
apt-get install -y cmake
113+
- name: Check datafusion-substrait (protoc)
114+
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-substrait --features=protoc
95115

96-
- name: Check workspace in debug mode
97-
run: cargo check --profile ci --all-targets --workspace
98-
99-
- name: Check workspace with additional features
100-
run: cargo check --profile ci --workspace --benches --features avro,json,integration-tests
101-
102-
# cargo check datafusion to ensure that the datafusion crate can be built with only a
103-
# subset of the function packages enabled.
116+
# Check datafusion-proto features
117+
#
118+
# Ensure via `cargo check` that the crate can be built with a
119+
# subset of the features packages enabled.
120+
linux-datafusion-proto-features:
121+
name: cargo check datafusion-proto features
122+
needs: linux-build-lib
123+
runs-on: ubuntu-latest
124+
container:
125+
image: amd64/rust
126+
steps:
127+
- uses: actions/checkout@v4
128+
- name: Setup Rust toolchain
129+
uses: ./.github/actions/setup-builder
130+
with:
131+
rust-version: stable
132+
- name: Check datafusion-proto (no-default-features)
133+
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto
134+
# fails due to https://github.com/apache/datafusion/issues/15157
135+
#- name: Check datafusion-proto (json)
136+
# run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=json
137+
- name: Check datafusion-proto (parquet)
138+
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=parquet
139+
- name: Check datafusion-proto (avro)
140+
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-proto --features=avro
141+
142+
143+
# Check datafusion crate features
144+
#
145+
# Ensure via `cargo check` that the crate can be built with a
146+
# subset of the features packages enabled.
104147
linux-cargo-check-datafusion:
105148
name: cargo check datafusion
106149
needs: linux-build-lib
@@ -113,6 +156,11 @@ jobs:
113156
uses: ./.github/actions/setup-builder
114157
with:
115158
rust-version: stable
159+
- name: Check datafusion (no-default-features)
160+
# Some of the test binaries require the parquet feature still
161+
#run: cargo check --all-targets --no-default-features -p datafusion
162+
run: cargo check --profile ci --no-default-features -p datafusion
163+
116164
- name: Check datafusion (nested_expressions)
117165
run: cargo check --profile ci --no-default-features --features=nested_expressions -p datafusion
118166

@@ -134,8 +182,10 @@ jobs:
134182
- name: Check datafusion (string_expressions)
135183
run: cargo check --profile ci --no-default-features --features=string_expressions -p datafusion
136184

137-
# cargo check datafusion-functions to ensure that the datafusion-functions crate can be built with
138-
# only a subset of the function packages enabled.
185+
# Check datafusion-functions crate features
186+
#
187+
# Ensure via `cargo check` that the crate can be built with a
188+
# subset of the features packages enabled.
139189
linux-cargo-check-datafusion-functions:
140190
name: cargo check functions
141191
needs: linux-build-lib
@@ -148,6 +198,9 @@ jobs:
148198
uses: ./.github/actions/setup-builder
149199
with:
150200
rust-version: stable
201+
- name: Check datafusion-functions (no-default-features)
202+
run: cargo check --profile ci --all-targets --no-default-features -p datafusion-functions
203+
151204
- name: Check datafusion-functions (crypto)
152205
run: cargo check --profile ci --all-targets --no-default-features --features=crypto_expressions -p datafusion-functions
153206

@@ -171,21 +224,41 @@ jobs:
171224
name: cargo test (amd64)
172225
needs: linux-build-lib
173226
runs-on: ubuntu-latest
174-
container:
175-
image: amd64/rust
176227
steps:
177228
- uses: actions/checkout@v4
178229
with:
179230
submodules: true
180231
fetch-depth: 1
181232
- name: Setup Rust toolchain
182-
uses: ./.github/actions/setup-builder
183-
with:
184-
rust-version: stable
233+
run: rustup toolchain install stable
234+
- name: Install Protobuf Compiler
235+
run: sudo apt-get install -y protobuf-compiler
236+
- name: Setup Minio - S3-compatible storage
237+
run: |
238+
docker run -d --name minio-container \
239+
-p 9000:9000 \
240+
-e MINIO_ROOT_USER=TEST-DataFusionLogin -e MINIO_ROOT_PASSWORD=TEST-DataFusionPassword \
241+
-v $(pwd)/datafusion/core/tests/data:/source quay.io/minio/minio \
242+
server /data
243+
docker exec minio-container /bin/sh -c "\
244+
mc ready local
245+
mc alias set localminio http://localhost:9000 TEST-DataFusionLogin TEST-DataFusionPassword && \
246+
mc mb localminio/data && \
247+
mc cp -r /source/* localminio/data"
185248
- name: Run tests (excluding doctests)
249+
env:
250+
RUST_BACKTRACE: 1
251+
AWS_ENDPOINT: http://127.0.0.1:9000
252+
AWS_ACCESS_KEY_ID: TEST-DataFusionLogin
253+
AWS_SECRET_ACCESS_KEY: TEST-DataFusionPassword
254+
TEST_STORAGE_INTEGRATION: 1
255+
AWS_ALLOW_HTTP: true
186256
run: cargo test --profile ci --exclude datafusion-examples --exclude ffi_example_table_provider --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests
187257
- name: Verify Working Directory Clean
188258
run: git diff --exit-code
259+
- name: Minio Output
260+
if: ${{ !cancelled() }}
261+
run: docker logs minio-container
189262

190263
linux-test-example:
191264
name: cargo examples (amd64)
@@ -259,6 +332,10 @@ jobs:
259332
uses: ./.github/actions/setup-builder
260333
with:
261334
rust-version: stable
335+
- name: Install dependencies
336+
run: |
337+
apt-get update -qq
338+
apt-get install -y -qq clang
262339
- name: Install wasm-pack
263340
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
264341
- name: Build with wasm-pack

0 commit comments

Comments
 (0)