[RFC] feat!: kernel-based log replay

.github/actions/load-dat/action.yaml

@@ -0,0 +1,26 @@
+name: Delta Acceptance Tests
+description: Load Delta Lake acceptance test data
+
+inputs:
+  version:
+    description: "The Python version to set up"
+    required: false
+    default: "0.0.3"
+
+  target-directory:
+    description: target directory for acceptance test data
+    required: false
+    default: ${{ github.workspace }}/dat
+
+runs:
+  using: composite
+
+  steps:
+    - name: load DAT
+      shell: bash
+      run: |
+        rm -rf ${{ inputs.target-directory }}
+        curl -OL https://github.com/delta-incubator/dat/releases/download/v${{ inputs.version }}/deltalake-dat-v${{ inputs.version }}.tar.gz
+        mkdir -p ${{ inputs.target-directory }}
+        tar --no-same-permissions -xzf deltalake-dat-v${{ inputs.version }}.tar.gz --directory ${{ inputs.target-directory }}
+        rm deltalake-dat-v${{ inputs.version }}.tar.gz

.github/actions/setup-env/action.yml

@@ -4,12 +4,12 @@ description: "Set up Python, virtual environment, and Rust toolchain"
 inputs:
   python-version:
     description: "The Python version to set up"
-    required: true
+    required: false
     default: "3.10"
 
   rust-toolchain:
     description: "The Rust toolchain to set up"
-    required: true
+    required: false
     default: "stable"
 
 runs:

.github/workflows/build.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.82'
+          toolchain: "1.82"
           override: true
 
       - name: Build
@@ -40,7 +40,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.82'
+          toolchain: "1.82"
           override: true
 
       - name: Format
@@ -62,7 +62,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.82'
+          toolchain: "1.82"
           override: true
 
       - name: build and lint with clippy
@@ -92,9 +92,12 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.82'
+          toolchain: "1.82"
           override: true
 
+      - name: Load DAT data
+        uses: ./.github/actions/load-dat
+
       - name: Run tests
         run: cargo test --verbose --features ${{ env.DEFAULT_FEATURES }}
 
@@ -121,7 +124,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.82'
+          toolchain: "1.82"
           override: true
 
       # Install Java and Hadoop for HDFS integration tests
@@ -136,6 +139,9 @@ jobs:
           tar -xf hadoop-3.4.0.tar.gz -C $GITHUB_WORKSPACE
           echo "$GITHUB_WORKSPACE/hadoop-3.4.0/bin" >> $GITHUB_PATH
 
+      - name: Load DAT data
+        uses: ./.github/actions/load-dat
+
       - name: Start emulated services
         run: docker compose up -d
 
@@ -162,7 +168,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: '1.82'
+          toolchain: "1.82"
           override: true
 
       - name: Download Lakectl
@@ -177,4 +183,3 @@ jobs:
       - name: Run tests with rustls (default)
         run: |
           cargo test --features integration_test_lakefs,lakefs,datafusion
-

.github/workflows/codecov.yml

@@ -16,17 +16,25 @@ jobs:
       CARGO_TERM_COLOR: always
     steps:
       - uses: actions/checkout@v4
+
       - name: Install rust
         uses: actions-rs/toolchain@v1
         with:
           profile: default
           toolchain: '1.82'
           override: true
+
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
+
       - uses: Swatinem/rust-cache@v2
+
+      - name: Load DAT data
+        uses: ./.github/actions/load-dat
+
       - name: Generate code coverage
         run: cargo llvm-cov --features ${DEFAULT_FEATURES} --workspace --codecov --output-path codecov.json -- --skip read_table_version_hdfs --skip test_read_tables_lakefs
+
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:

.gitignore

@@ -22,7 +22,7 @@ __blobstorage__
 .githubchangeloggenerator.cache.log
 .githubchangeloggenerator.cache/
 .githubchangeloggenerator*
-data
+.zed/
 
 # Add all Cargo.lock files except for those in binary crates
 Cargo.lock
@@ -33,5 +33,6 @@ Cargo.lock
 justfile
 site
 __pycache__
+dat/
 .zed
 .zed/

Cargo.toml

@@ -26,11 +26,16 @@ debug = true
 debug = "line-tables-only"
 
 [workspace.dependencies]
-delta_kernel = { version = "0.9.0", features = [
+delta_kernel = { git = "https://github.com/roeap/delta-kernel-rs", rev = "b0cd12264ae4ada8d51cff02b25864258568eb88", features = [
     "arrow_54",
     "developer-visibility",
     "default-engine-rustls",
 ] }
+# delta_kernel = { path = "../delta-kernel-rs/kernel", features = [
+#     "arrow_54",
+#     "developer-visibility",
+#     "default-engine-rustls",
+# ] }
 
 # arrow
 arrow = { version = "54" }

crates/core/Cargo.toml

@@ -15,7 +15,7 @@ rust-version.workspace = true
 features = ["datafusion", "json", "unity-experimental"]
 
 [dependencies]
-delta_kernel.workspace = true
+delta_kernel = { workspace = true }
 
 # arrow
 arrow = { workspace = true }
@@ -29,10 +29,7 @@ arrow-ord = { workspace = true }
 arrow-row = { workspace = true }
 arrow-schema = { workspace = true, features = ["serde"] }
 arrow-select = { workspace = true }
-parquet = { workspace = true, features = [
-    "async",
-    "object_store",
-] }
+parquet = { workspace = true, features = ["async", "object_store"] }
 pin-project-lite = "^0.2.7"
 
 # datafusion
@@ -49,7 +46,7 @@ datafusion-functions-aggregate = { workspace = true, optional = true }
 # serde
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
-strum = { workspace =  true}
+strum = { workspace = true }
 
 # "stdlib"
 bytes = { workspace = true }
@@ -75,6 +72,9 @@ tokio = { workspace = true, features = [
     "parking_lot",
 ] }
 
+# cahce
+quick_cache = { version = "0.6.9" }
+
 # other deps (these should be organized and pulled into workspace.dependencies as necessary)
 cfg-if = "1"
 dashmap = "6"
@@ -100,6 +100,7 @@ humantime = { version = "2.1.0" }
 [dev-dependencies]
 criterion = "0.5"
 ctor = "0"
+datatest-stable = "0.2"
 deltalake-test = { path = "../test", features = ["datafusion"] }
 dotenvy = "0"
 fs_extra = "1.2.0"
@@ -130,3 +131,8 @@ python = ["arrow/pyarrow"]
 native-tls = ["delta_kernel/default-engine"]
 rustls = ["delta_kernel/default-engine-rustls"]
 cloud = ["object_store/cloud"]
+
+[[test]]
+name = "dat"
+harness = false
+

crates/core/src/kernel/mod.rs

@@ -10,6 +10,7 @@ pub mod error;
 pub mod models;
 pub mod scalars;
 mod snapshot;
+pub mod snapshot_next;
 pub mod transaction;
 
 pub use error::*;