Matrix test

.github/workflows/ci_client_runtime_test.yml

@@ -0,0 +1,119 @@
+name: Zombienet Client-Runtime Compatibility Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      runtime_tags:
+        description: 'Comma-separated list of 3 runtime version tags (e.g., v2.0.4,v2.0.3,v2.0.2). Leave empty for auto-discovery of 3 latest.'
+        required: false
+        type: string
+      binary_tag:
+        description: 'Client binary tag (e.g., polkadot-stable2503). Leave empty for auto-discovery of latest.'
+        required: false
+        type: string
+
+env:
+  RUST_LOG: "zombienet_orchestrator=debug,zombienet_provider=debug"
+
+jobs:
+  prepare-matrix:
+    runs-on: parity-default
+    container:
+      image: docker.io/paritytech/ci-unified:bullseye-1.88.0-2025-06-27-v202506301118
+    outputs:
+      runtime_matrix: ${{ steps.set-matrix.outputs.runtime_matrix }}
+      binary_tag: ${{ steps.set-matrix.outputs.binary_tag }}
+    steps:
+      - name: Fetch Versions
+        id: set-matrix
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INPUT_RUNTIME_TAGS: ${{ inputs.runtime_tags }}
+          INPUT_BINARY_TAG: ${{ inputs.binary_tag }}
+        run: |
+          # Use provided runtime tags or fetch 3 latest from fellows/runtimes (v2.0.4, v2.0.3, etc)
+          if [[ -n "$INPUT_RUNTIME_TAGS" ]]; then
+            # Convert comma-separated input to JSON array
+            RUNTIMES=$(echo "$INPUT_RUNTIME_TAGS" | tr ',' '\n' | jq -R . | jq -s -c .)
+            echo "Using provided runtime tags: $RUNTIMES"
+          else
+            RUNTIMES=$(gh api graphql -f query='
+              query {
+                repository(owner: "polkadot-fellows", name: "runtimes") {
+                  releases(first: 3, orderBy: {field: CREATED_AT, direction: DESC}) {
+                    nodes { tagName }
+                  }
+                }
+              }' --jq '.data.repository.releases.nodes[].tagName' | grep -E "^v[0-9]+" | head -n 3 | jq -R . | jq -s -c .)
+            echo "Auto-discovered runtime tags: $RUNTIMES"
+          fi
+
+          # Use provided binary tag or fetch latest stable from paritytech/polkadot-sdk
+          if [[ -n "$INPUT_BINARY_TAG" ]]; then
+            BINARY_TAG="$INPUT_BINARY_TAG"
+            echo "Using provided binary tag: $BINARY_TAG"
+          else
+            BINARY_TAG=$(gh api graphql -f query='
+              query {
+                repository(owner: "paritytech", name: "polkadot-sdk") {
+                  releases(first: 20, orderBy: {field: CREATED_AT, direction: DESC}) {
+                    nodes { tagName }
+                  }
+                }
+              }' --jq '.data.repository.releases.nodes[].tagName' | grep -E "^polkadot-stable[0-9]+" | grep -v "\-rc" | sort -Vr | head -n 1)
+            echo "Auto-discovered binary tag: $BINARY_TAG"
+          fi
+
+          echo "runtime_matrix=$RUNTIMES" >> $GITHUB_OUTPUT
+          echo "binary_tag=$BINARY_TAG" >> $GITHUB_OUTPUT
+
+  test-runtime:
+    needs: prepare-matrix
+    runs-on: parity-default
+    name: "Test Runtime: ${{ matrix.runtime_tag }} with binary: ${{ needs.prepare-matrix.outputs.binary_tag }}"
+    container:
+      image: docker.io/paritytech/ci-unified:bullseye-1.88.0-2025-06-27-v202506301118
+    strategy:
+      fail-fast: false
+      matrix:
+        runtime_tag: ${{ fromJson(needs.prepare-matrix.outputs.runtime_matrix) }}
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Download Latest Binaries
+        shell: bash
+        run: |
+          BIN_TAG="${{ needs.prepare-matrix.outputs.binary_tag }}"
+          mkdir -p ./bin
+          NODE_BINS=("polkadot" "polkadot-execute-worker" "polkadot-prepare-worker")
+          for bin in "${NODE_BINS[@]}"; do
+            echo "Downloading $bin @ $BIN_TAG"
+            curl -fL "https://github.com/paritytech/polkadot-sdk/releases/download/$BIN_TAG/$bin" -o "./bin/$bin"
+            chmod +x "./bin/$bin"
+          done
+          echo "$PWD/bin" >> $GITHUB_PATH
+
+      - name: Download Specific Runtime
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p artifacts
+          TAG="${{ matrix.runtime_tag }}"
+
+          echo "Downloading Runtimes for $TAG..."
+
+          gh release download "$TAG" -R polkadot-fellows/runtimes \
+            -p "polkadot_runtime-*.compact.compressed.wasm" \
+            --output artifacts/relay_runtime.wasm
+
+      - name: Run Compatibility Test
+        env:
+          RELAY_RUNTIME_PATH: ${{ github.workspace }}/artifacts/relay_runtime.wasm
+          ZOMBIE_PROVIDER: native
+          PATH: ${{ github.workspace }}/bin:${{ env.PATH }}
+        run: |
+          echo "Testing Binary ${{ needs.prepare-matrix.outputs.binary_tag }} against Runtime ${{ matrix.runtime_tag }}"
+          cargo test --test client-runtime -- --nocapture

.github/workflows/ci_runtime_client_test.yml

@@ -0,0 +1,136 @@
+name: Zombienet Runtime-Client Compatibility Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      runtime_tag:
+        description: 'Runtime version tag (e.g., v2.0.4). Leave empty for auto-discovery of latest.'
+        required: false
+        type: string
+      binary_tags:
+        description: 'Comma-separated list of 3 client binary tags (e.g., polkadot-stable2503,polkadot-stable2502,polkadot-stable2501). Leave empty for auto-discovery of 3 latest.'
+        required: false
+        type: string
+
+env:
+  RUST_LOG: "zombienet_orchestrator=debug,zombienet_provider=debug"
+
+jobs:
+  prepare-matrix:
+    runs-on: parity-default
+    container:
+      image: docker.io/paritytech/ci-unified:bullseye-1.88.0-2025-06-27-v202506301118
+    outputs:
+      binary_matrix: ${{ steps.set-matrix.outputs.binary_matrix }}
+      runtime_tag: ${{ steps.set-matrix.outputs.runtime_tag }}
+    steps:
+      - name: Fetch Versions
+        id: set-matrix
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          INPUT_RUNTIME_TAG: ${{ inputs.runtime_tag }}
+          INPUT_BINARY_TAGS: ${{ inputs.binary_tags }}
+        run: |
+          # 1. Use provided runtime tag or fetch the single latest from fellows/runtimes (e.g., v2.0.4)
+          if [[ -n "$INPUT_RUNTIME_TAG" ]]; then
+            RUNTIME_TAG="$INPUT_RUNTIME_TAG"
+            echo "Using provided runtime tag: $RUNTIME_TAG"
+          else
+            RUNTIME_TAG=$(gh api graphql -f query='
+              query {
+                repository(owner: "polkadot-fellows", name: "runtimes") {
+                  releases(first: 5, orderBy: {field: CREATED_AT, direction: DESC}) {
+                    nodes { tagName }
+                  }
+                }
+              }' --jq '.data.repository.releases.nodes[].tagName' | grep -E "^v[0-9]+" | head -n 1)
+            echo "Auto-discovered runtime tag: $RUNTIME_TAG"
+          fi
+
+          # 2. Use provided binary tags or fetch 3 latest stable binary tags from paritytech/polkadot-sdk
+          if [[ -n "$INPUT_BINARY_TAGS" ]]; then
+            # Convert comma-separated input to JSON array
+            BINARIES=$(echo "$INPUT_BINARY_TAGS" | tr ',' '\n' | jq -R . | jq -s -c .)
+            echo "Using provided binary tags: $BINARIES"
+          else
+            BINARIES=$(gh api graphql -f query='
+              query {
+                repository(owner: "paritytech", name: "polkadot-sdk") {
+                  releases(first: 20, orderBy: {field: CREATED_AT, direction: DESC}) {
+                    nodes { tagName }
+                  }
+                }
+              }' --jq '.data.repository.releases.nodes[].tagName' | grep -E "^polkadot-stable[0-9]+" | grep -v "\-rc" | sort -Vr | head -n 3 | jq -R . | jq -s -c .)
+            echo "Auto-discovered binary tags: $BINARIES"
+          fi
+
+          echo "runtime_tag=$RUNTIME_TAG" >> $GITHUB_OUTPUT
+          echo "binary_matrix=$BINARIES" >> $GITHUB_OUTPUT
+
+  test-mixed-binaries:
+    needs: prepare-matrix
+    runs-on: parity-default
+    name: "Mixed binaries compatibility test"
+    container:
+      image: docker.io/paritytech/ci-unified:bullseye-1.88.0-2025-06-27-v202506301118
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: Download Binaries
+        shell: bash
+        run: |
+          BIN_TAGS=($(echo '${{ needs.prepare-matrix.outputs.binary_matrix }}' | jq -r '.[]'))
+          mkdir -p ./bin
+          for i in "${!BIN_TAGS[@]}"; do
+            TAG="${BIN_TAGS[$i]}"
+            FILENAME="polkadot-${TAG}"
+            echo "Downloading Polkadot @ $TAG as $FILENAME"
+            curl -fL "https://github.com/paritytech/polkadot-sdk/releases/download/$TAG/polkadot" -o "./bin/$FILENAME"
+            chmod +x "./bin/$FILENAME"
+
+            echo "Downloading workers for $TAG"
+            mkdir -p "./bin/workers-${TAG}"
+            for worker in polkadot-execute-worker polkadot-prepare-worker; do
+              url="https://github.com/paritytech/polkadot-sdk/releases/download/$TAG/$worker"
+              echo "Downloading $worker"
+              curl -fL "$url" -o "./bin/workers-${TAG}/$worker"
+              chmod +x "./bin/workers-${TAG}/$worker"
+            done
+            ls -lh "./bin/workers-${TAG}"
+          done
+          echo "$PWD/bin" >> $GITHUB_PATH
+      - name: Download Latest Runtime
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p artifacts
+          gh release download "${{ needs.prepare-matrix.outputs.runtime_tag }}" \
+            -R polkadot-fellows/runtimes \
+            -p "polkadot_runtime-*.compact.compressed.wasm" \
+            --output artifacts/relay_runtime.wasm
+
+      - name: Run Mixed Version Test
+        shell: bash
+        env:
+          RELAY_RUNTIME_PATH: ${{ github.workspace }}/artifacts/relay_runtime.wasm
+          WORKERS_BASE_PATH: ${{ github.workspace }}/bin
+          ZOMBIE_PROVIDER: native
+          PATH: ${{ github.workspace }}/bin:${{ env.PATH }}
+        run: |
+          BIN_TAGS=($(echo '${{ needs.prepare-matrix.outputs.binary_matrix }}' | jq -r '.[]'))
+
+          export POLKADOT_BIN_LATEST="polkadot-${BIN_TAGS[0]}"
+          export POLKADOT_BIN_LATEST_1="polkadot-${BIN_TAGS[1]}"
+          export POLKADOT_BIN_LATEST_2="polkadot-${BIN_TAGS[2]}"
+          export WORKERS_PATH_LATEST="$WORKERS_BASE_PATH/workers-${BIN_TAGS[0]}"
+          export WORKERS_PATH_LATEST_1="$WORKERS_BASE_PATH/workers-${BIN_TAGS[1]}"
+          export WORKERS_PATH_LATEST_2="$WORKERS_BASE_PATH/workers-${BIN_TAGS[2]}"
+
+          echo "Running mixed network test with:"
+          echo "Alice (Latest)   : $POLKADOT_BIN_LATEST"
+          echo "Bob (Latest-1)   : $POLKADOT_BIN_LATEST_1"
+          echo "Charlie (Latest-2): $POLKADOT_BIN_LATEST_2"
+
+          echo "Running mixed network with 3 latest client versions"
+          cargo test --test client-runtime -- --nocapture

crates/orchestrator/src/network/node.rs

@@ -628,7 +628,7 @@ impl NetworkNode {
         let resolved_metric_name = if metric_name.contains("_bucket") {
             metric_name.to_string()
         } else {
-            format!("{}_bucket", metric_name)
+            format!("{metric_name}_bucket")
         };
 
         // First pass: collect all matching metrics with their label counts

crates/sdk/tests/client-runtime.rs

@@ -0,0 +1,85 @@
+use std::{env, path::PathBuf};
+
+use configuration::{NetworkConfig, NetworkConfigBuilder};
+use subxt::{ext::futures::StreamExt, OnlineClient, PolkadotConfig};
+use zombienet_sdk::NetworkConfigExt;
+
+fn small_network() -> NetworkConfig {
+    let relay_runtime_path = PathBuf::from(env::var("RELAY_RUNTIME_PATH").unwrap());
+    let polkadot_bin_latest = env::var("POLKADOT_BIN_LATEST").unwrap_or("polkadot".into());
+    let polkadot_bin_latest_1 = env::var("POLKADOT_BIN_LATEST_1").unwrap_or("polkadot".into());
+    let polkadot_bin_latest_2 = env::var("POLKADOT_BIN_LATEST_2").unwrap_or("polkadot".into());
+
+    let workers_path_latest = env::var("WORKERS_PATH_LATEST").ok();
+    let workers_path_latest_1 = env::var("WORKERS_PATH_LATEST_1").ok();
+    let workers_path_latest_2 = env::var("WORKERS_PATH_LATEST_2").ok();
+
+    NetworkConfigBuilder::new()
+        .with_relaychain(|r| {
+            let relaychain = r
+                .with_chain("polkadot-local")
+                .with_default_args(vec!["-lparachain=debug,runtime=debug".into()])
+                .with_chain_spec_runtime(relay_runtime_path, None)
+                .with_validator(|node| {
+                    let mut alice = node
+                        .with_name("alice")
+                        .with_command(polkadot_bin_latest.as_ref());
+                    if let Some(workers_path) = &workers_path_latest {
+                        alice =
+                            alice.with_args(vec![("--workers-path", workers_path.as_str()).into()]);
+                    }
+                    alice
+                })
+                .with_validator(|node| {
+                    let mut bob = node
+                        .with_name("bob")
+                        .with_command(polkadot_bin_latest_1.as_ref());
+                    if let Some(workers_path) = &workers_path_latest_1 {
+                        bob = bob.with_args(vec![("--workers-path", workers_path.as_str()).into()]);
+                    }
+                    bob
+                })
+                .with_validator(|node| {
+                    let mut charlie = node
+                        .with_name("charlie")
+                        .with_command(polkadot_bin_latest_2.as_ref());
+                    if let Some(workers_path) = &workers_path_latest_2 {
+                        charlie =
+                            charlie
+                                .with_args(vec![("--workers-path", workers_path.as_str()).into()]);
+                    }
+                    charlie
+                });
+            relaychain
+        })
+        .build()
+        .unwrap()
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn ci_native_smoke_should_works() {
+    tracing_subscriber::fmt::init();
+    let config = small_network();
+    let network = config.spawn_native().await.unwrap();
+
+    let alice = network.get_node("alice").unwrap();
+    let alice_client: OnlineClient<PolkadotConfig> = alice.wait_client().await.unwrap();
+
+    let bob = network.get_node("bob").unwrap();
+    let bob_client: OnlineClient<PolkadotConfig> = bob.wait_client().await.unwrap();
+
+    let charlie = network.get_node("charlie").unwrap();
+    let charlie_client: OnlineClient<PolkadotConfig> = charlie.wait_client().await.unwrap();
+
+    wait_n_blocks(&alice_client, 5, "alice").await;
+    wait_n_blocks(&bob_client, 5, "bob").await;
+    wait_n_blocks(&charlie_client, 5, "charlie").await;
+}
+
+async fn wait_n_blocks(client: &OnlineClient<PolkadotConfig>, n: usize, name: &str) {
+    let mut blocks = client.blocks().subscribe_finalized().await.unwrap().take(n);
+
+    while let Some(block) = blocks.next().await {
+        println!("{name} Block #{}", block.unwrap().header().number);
+    }
+}