Bug/deltalake s3 (datafusion-contrib#345)

Author: matthewmturner

.github/workflows/test.yml

@@ -179,24 +179,29 @@ jobs:
       matrix:
         arch: [amd64]
     steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-      - name: Cache Cargo
-        uses: actions/cache@v4
-        with:
-          path: /home/runner/.cargo
-          key: cargo-dft-cache-
-      - name: Cache Rust dependencies
-        uses: actions/cache@v4
-        with:
-          path: target
-          key: ${{ runner.os }}-cargo-target-${{ hashFiles('Cargo.lock') }}
+      - uses: actions/checkout@v3
       - name: Setup Rust Toolchain
         uses: ./.github/actions/setup-rust
+      - name: Start LocalStack
+        uses: LocalStack/setup-localstack@v0.2.3
+        with:
+          image-tag: 'latest'
+          install-awslocal: 'true'
+          configuration: DEBUG=1
+      - name: Upload Delta Lake data to LocalStack
+        run: |
+          awslocal s3 mb s3://test
+          awslocal s3 sync data/deltalake/simple_table s3://test/deltalake/simple_table
+          echo "Delta Lake data uploaded to LocalStack"
       - name: Run Deltalake tests
         run: |
-          cargo test --features=deltalake extension_cases::deltalake
+          cargo test --features="deltalake s3" extension_cases::deltalake
+        env:
+          AWS_ACCESS_KEY_ID: LSIAQAAAAAAVNCBMPNSG
+          AWS_SECRET_ACCESS_KEY: 5555555555555555555555555555555555555555
+          AWS_ENDPOINT_URL: http://localhost:4566
+          AWS_REGION: us-east-1
+          AWS_ALLOW_HTTP: true
   test-udfs-wasm:
     name: Extension / UDFs-WASM
     runs-on: ubuntu-latest

CLAUDE.md

@@ -52,6 +52,7 @@ cargo test --features=flightsql extension_cases::flightsql -- --test-threads=1
 cargo test --features=s3 extension_cases::s3
 cargo test --features=functions-json extension_cases::functions_json
 cargo test --features=deltalake extension_cases::deltalake
+cargo test --features="deltalake s3" extension_cases::deltalake::test_deltalake_s3  # Requires LocalStack
 cargo test --features=udfs-wasm extension_cases::udfs_wasm
 cargo test --features=vortex extension_cases::vortex
 cargo test --features=vortex cli_cases::basic::test_output_vortex
@@ -189,16 +190,20 @@ The main runtime in `src/main.rs` uses a single-threaded Tokio runtime optimized
 
 ### Testing Against LocalStack
 
-Some tests (S3, TUI, CLI) require LocalStack for S3 testing. The CI workflow shows the setup:
+Some tests (S3, TUI, CLI, Delta Lake + S3) require LocalStack for S3 testing. The CI workflow shows the setup:
 
 ```bash
 # Start LocalStack
 localstack start -d
-awslocal s3api create-bucket --bucket tmp --acl public-read
-awslocal s3 mv data/aggregate_test_100.csv s3://tmp/
+awslocal s3api create-bucket --bucket test --acl public-read
+awslocal s3 mv data/aggregate_test_100.csv s3://test/
 
-# Run tests
+# Run S3 tests
 cargo test --features=s3 extension_cases::s3
+
+# For Delta Lake + S3 tests, also sync the delta lake data
+awslocal s3 sync data/deltalake/simple_table s3://test/deltalake/simple_table
+cargo test --features="deltalake s3" extension_cases::deltalake::test_deltalake_s3
 ```
 
 ### Benchmarking