Merge branch 'next' into santiagopittella-merge-main-to-next

Author: Mirko-von-Leipzig

.github/workflows/ci.yml

@@ -139,6 +139,30 @@ jobs:
       - name: Doc tests
         run: cargo test --doc --workspace --all-features
 
+  # Temporary Diesel schema drift checks. Delete this job once Diesel is removed.
+  diesel-schema:
+    name: diesel schema
+    runs-on: warp-ubuntu-latest-x64-8x
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          persist-credentials: false
+      - name: Rustup
+        run: rustup toolchain install --no-self-update
+      - uses: WarpBuilds/rust-cache@9d0cc3090d9c87de74ea67617b246e978735b1a1 # v2.9.1
+        with:
+          shared-key: ${{ github.job }}
+          prefix-key: ${{ env.CACHE_PREFIX }}
+          save-if: ${{ env.SAVE_CACHE }}
+      - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18
+        with:
+          tool: diesel_cli@2.3.9
+      - name: Check Diesel schemas
+        run: |
+          cargo test --locked --workspace --all-features \
+            diesel_schema_is_in_sync_with_migrations -- --ignored
+
   doc:
     runs-on: warp-ubuntu-latest-x64-8x
     steps:

Cargo.lock

@@ -84,7 +84,6 @@ deadpool          = { default-features = false, version = "0.12" }
 deadpool-diesel   = { version = "0.6" }
 deadpool-sync     = { default-features = false, version = "0.1" }
 diesel            = { version = "2.3" }
-diesel_migrations = { version = "2.3" }
 fs-err            = { version = "3" }
 futures           = { version = "0.3" }
 hex               = { version = "0.4" }

Cargo.toml

@@ -22,7 +22,6 @@ anyhow                     = { workspace = true }
 backon                     = { workspace = true }
 clap                       = { features = ["env", "string"], workspace = true }
 diesel                     = { features = ["numeric", "sqlite"], workspace = true }
-diesel_migrations          = { features = ["sqlite"], workspace = true }
 futures                    = { workspace = true }
 humantime                  = { workspace = true }
 libsqlite3-sys             = { workspace = true }
@@ -44,7 +43,8 @@ tracing                    = { workspace = true }
 url                        = { workspace = true }
 
 [build-dependencies]
-build-rs = { workspace = true }
+build-rs      = { workspace = true }
+miden-node-db = { workspace = true }
 
 [dev-dependencies]
 miden-node-utils = { features = ["testing"], workspace = true }

bin/ntx-builder/Cargo.toml

@@ -1,11 +1,9 @@
-// This build.rs is required to trigger the `diesel_migrations::embed_migrations!` proc-macro in
-// `src/db/migrations.rs` to include the latest version of the migrations into the binary, see
-// <https://docs.rs/diesel_migrations/latest/diesel_migrations/macro.embed_migrations.html#automatic-rebuilds>.
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    miden_node_db::migration::Migrator::generate("src/db/migrations")?;
 
-fn main() {
-    build_rs::output::rerun_if_changed("src/db/migrations");
     // If we do one re-write, the default rules are disabled,
     // hence we need to trigger explicitly on `Cargo.toml`.
     // <https://doc.rust-lang.org/cargo/reference/build-scripts.html#rerun-if-changed>
     build_rs::output::rerun_if_changed("Cargo.toml");
+    Ok(())
 }

bin/ntx-builder/build.rs

@@ -1,29 +1,73 @@
-use diesel::SqliteConnection;
-use diesel_migrations::{EmbeddedMigrations, MigrationHarness, embed_migrations};
+use std::path::Path;
+
 use miden_node_db::DatabaseError;
 use tracing::instrument;
 
 use crate::COMPONENT;
-use crate::db::schema_hash::verify_schema;
 
-// The rebuild is automatically triggered by `build.rs` as described in
-// <https://docs.rs/diesel_migrations/latest/diesel_migrations/macro.embed_migrations.html#automatic-rebuilds>.
-pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("src/db/migrations");
+include!(concat!(env!("OUT_DIR"), "/db_migrator.rs"));
 
 #[instrument(level = "debug", target = COMPONENT, skip_all, err)]
-pub fn apply_migrations(conn: &mut SqliteConnection) -> Result<(), DatabaseError> {
-    let migrations = conn.pending_migrations(MIGRATIONS).expect("In memory migrations never fail");
-    tracing::info!(target: COMPONENT, migrations = migrations.len(), "Applying pending migrations");
-
-    let Err(e) = conn.run_pending_migrations(MIGRATIONS) else {
-        // Migrations applied successfully, verify schema hash.
-        verify_schema(conn)?;
-        return Ok(());
-    };
-    tracing::warn!(target: COMPONENT, "Failed to apply migration: {e:?}");
-    // Something went wrong; revert the last migration.
-    conn.revert_last_migration(MIGRATIONS)
-        .expect("Duality is maintained by the developer");
+pub fn apply_migrations(database_filepath: &Path) -> Result<(), DatabaseError> {
+    let migrator = migrator().map_err(DatabaseError::migration)?;
+    tracing::info!(
+        target: COMPONENT,
+        migration_count = migrator.schema_hashes().len(),
+        "Applying database migrations"
+    );
 
+    migrator.migrate(database_filepath).map_err(DatabaseError::migration)?;
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use std::process::Command;
+
+    use anyhow::{Context, Result, ensure};
+    use miden_node_db::migration::SchemaHash;
+
+    use super::*;
+
+    const EXPECTED_SCHEMA_HASHES: [SchemaHash; 1] = [SchemaHash::from_hex(
+        "c6434bc6a142cd96dd4072bea641546d99788b1495cb0e52c2d98b9138f9c30d",
+    )];
+
+    #[test]
+    fn migration_schema_hashes_are_stable() -> Result<()> {
+        let migrator = migrator()?;
+
+        assert_eq!(migrator.schema_hashes(), &EXPECTED_SCHEMA_HASHES);
+        Ok(())
+    }
+
+    #[test]
+    #[ignore = "requires diesel CLI; CI runs this in the diesel-schema job"]
+    fn diesel_schema_is_in_sync_with_migrations() -> Result<()> {
+        let temp_dir = tempfile::tempdir()?;
+        let database_filepath = temp_dir.path().join("ntx-builder.sqlite3");
+        apply_migrations(&database_filepath)?;
+
+        let output = Command::new("diesel")
+            .arg("print-schema")
+            .arg("--database-url")
+            .arg(&database_filepath)
+            .current_dir(env!("CARGO_MANIFEST_DIR"))
+            .output()
+            .context(
+                "failed to run diesel CLI; install it with \
+                 `cargo install diesel_cli --no-default-features --features sqlite`",
+            )?;
+
+        ensure!(
+            output.status.success(),
+            "diesel print-schema failed: {}",
+            String::from_utf8_lossy(&output.stderr)
+        );
+
+        let generated =
+            String::from_utf8(output.stdout).context("diesel CLI output is not UTF-8")?;
+        assert_eq!(generated, include_str!("schema.rs"));
+        Ok(())
+    }
+}

bin/ntx-builder/src/db/migrations.rs

@@ -4,7 +4,7 @@ CREATE TABLE chain_state (
     -- Singleton constraint: only one row allowed.
     id              INTEGER NOT NULL PRIMARY KEY CHECK (id = 0),
     -- Block number of the chain tip.
-    block_num       INTEGER NOT NULL,
+    block_num       BIGINT NOT NULL,
     -- Serialized BlockHeader.
     block_header    BLOB    NOT NULL,
 
@@ -33,7 +33,7 @@ CREATE UNIQUE INDEX idx_accounts_inflight ON accounts(account_id, transaction_id
 CREATE INDEX idx_accounts_account ON accounts(account_id);
 CREATE INDEX idx_accounts_tx ON accounts(transaction_id) WHERE transaction_id IS NOT NULL;
 
--- Notes: committed, inflight, and nullified — all in one table.
+-- Notes: committed, inflight, and nullified - all in one table.
 -- created_by = NULL means committed note; non-NULL means created by inflight tx.
 -- consumed_by = NULL means unconsumed; non-NULL means consumed by inflight tx.
 -- committed_at = block number when the consuming transaction was committed on-chain.
@@ -49,7 +49,7 @@ CREATE TABLE notes (
     -- Backoff tracking: number of failed execution attempts.
     attempt_count   INTEGER NOT NULL DEFAULT 0,
     -- Backoff tracking: block number of the last failed attempt. NULL if never attempted.
-    last_attempt    INTEGER,
+    last_attempt    BIGINT,
     -- Latest execution error message. NULL if no error recorded.
     last_error      TEXT,
     -- NULL if the note came from a committed block; transaction ID if created by inflight tx.
@@ -58,7 +58,7 @@ CREATE TABLE notes (
     consumed_by     BLOB,
     -- Block number at which the note's consuming transaction was committed.
     -- NULL while the note is still pending or in-flight; set on block commit.
-    committed_at    INTEGER,
+    committed_at    BIGINT,
 
     CONSTRAINT notes_attempt_count_non_negative CHECK (attempt_count >= 0),
     CONSTRAINT notes_last_attempt_is_u32 CHECK (last_attempt BETWEEN 0 AND 0xFFFFFFFF),