diff --git a/.github/workflows/ci_client_runtime_test.yml b/.github/workflows/ci_client_runtime_test.yml new file mode 100644 index 000000000..45be8551c --- /dev/null +++ b/.github/workflows/ci_client_runtime_test.yml @@ -0,0 +1,119 @@ +name: Zombienet Client-Runtime Compatibility Test + +on: + workflow_dispatch: + inputs: + runtime_tags: + description: 'Comma-separated list of 3 runtime version tags (e.g., v2.0.4,v2.0.3,v2.0.2). Leave empty for auto-discovery of 3 latest.' + required: false + type: string + binary_tag: + description: 'Client binary tag (e.g., polkadot-stable2503). Leave empty for auto-discovery of latest.' + required: false + type: string + +env: + RUST_LOG: "zombienet_orchestrator=debug,zombienet_provider=debug" + +jobs: + prepare-matrix: + runs-on: parity-default + container: + image: docker.io/paritytech/ci-unified:bullseye-1.88.0-2025-06-27-v202506301118 + outputs: + runtime_matrix: ${{ steps.set-matrix.outputs.runtime_matrix }} + binary_tag: ${{ steps.set-matrix.outputs.binary_tag }} + steps: + - name: Fetch Versions + id: set-matrix + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUT_RUNTIME_TAGS: ${{ inputs.runtime_tags }} + INPUT_BINARY_TAG: ${{ inputs.binary_tag }} + run: | + # Use provided runtime tags or fetch 3 latest from fellows/runtimes (v2.0.4, v2.0.3, etc) + if [[ -n "$INPUT_RUNTIME_TAGS" ]]; then + # Convert comma-separated input to JSON array + RUNTIMES=$(echo "$INPUT_RUNTIME_TAGS" | tr ',' '\n' | jq -R . | jq -s -c .) + echo "Using provided runtime tags: $RUNTIMES" + else + RUNTIMES=$(gh api graphql -f query=' + query { + repository(owner: "polkadot-fellows", name: "runtimes") { + releases(first: 3, orderBy: {field: CREATED_AT, direction: DESC}) { + nodes { tagName } + } + } + }' --jq '.data.repository.releases.nodes[].tagName' | grep -E "^v[0-9]+" | head -n 3 | jq -R . | jq -s -c .) + echo "Auto-discovered runtime tags: $RUNTIMES" + fi + + # Use provided binary tag or fetch latest stable from paritytech/polkadot-sdk + if [[ -n "$INPUT_BINARY_TAG" ]]; then + BINARY_TAG="$INPUT_BINARY_TAG" + echo "Using provided binary tag: $BINARY_TAG" + else + BINARY_TAG=$(gh api graphql -f query=' + query { + repository(owner: "paritytech", name: "polkadot-sdk") { + releases(first: 20, orderBy: {field: CREATED_AT, direction: DESC}) { + nodes { tagName } + } + } + }' --jq '.data.repository.releases.nodes[].tagName' | grep -E "^polkadot-stable[0-9]+" | grep -v "\-rc" | sort -Vr | head -n 1) + echo "Auto-discovered binary tag: $BINARY_TAG" + fi + + echo "runtime_matrix=$RUNTIMES" >> $GITHUB_OUTPUT + echo "binary_tag=$BINARY_TAG" >> $GITHUB_OUTPUT + + test-runtime: + needs: prepare-matrix + runs-on: parity-default + name: "Test Runtime: ${{ matrix.runtime_tag }} with binary: ${{ needs.prepare-matrix.outputs.binary_tag }}" + container: + image: docker.io/paritytech/ci-unified:bullseye-1.88.0-2025-06-27-v202506301118 + strategy: + fail-fast: false + matrix: + runtime_tag: ${{ fromJson(needs.prepare-matrix.outputs.runtime_matrix) }} + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Download Latest Binaries + shell: bash + run: | + BIN_TAG="${{ needs.prepare-matrix.outputs.binary_tag }}" + mkdir -p ./bin + NODE_BINS=("polkadot" "polkadot-execute-worker" "polkadot-prepare-worker") + for bin in "${NODE_BINS[@]}"; do + echo "Downloading $bin @ $BIN_TAG" + curl -fL "https://github.com/paritytech/polkadot-sdk/releases/download/$BIN_TAG/$bin" -o "./bin/$bin" + chmod +x "./bin/$bin" + done + echo "$PWD/bin" >> $GITHUB_PATH + + - name: Download Specific Runtime + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p artifacts + TAG="${{ matrix.runtime_tag }}" + + echo "Downloading Runtimes for $TAG..." + + gh release download "$TAG" -R polkadot-fellows/runtimes \ + -p "polkadot_runtime-*.compact.compressed.wasm" \ + --output artifacts/relay_runtime.wasm + + - name: Run Compatibility Test + env: + RELAY_RUNTIME_PATH: ${{ github.workspace }}/artifacts/relay_runtime.wasm + ZOMBIE_PROVIDER: native + PATH: ${{ github.workspace }}/bin:${{ env.PATH }} + run: | + echo "Testing Binary ${{ needs.prepare-matrix.outputs.binary_tag }} against Runtime ${{ matrix.runtime_tag }}" + cargo test --test client-runtime -- --nocapture \ No newline at end of file diff --git a/.github/workflows/ci_runtime_client_test.yml b/.github/workflows/ci_runtime_client_test.yml new file mode 100644 index 000000000..b04ea8f11 --- /dev/null +++ b/.github/workflows/ci_runtime_client_test.yml @@ -0,0 +1,136 @@ +name: Zombienet Runtime-Client Compatibility Test + +on: + workflow_dispatch: + inputs: + runtime_tag: + description: 'Runtime version tag (e.g., v2.0.4). Leave empty for auto-discovery of latest.' + required: false + type: string + binary_tags: + description: 'Comma-separated list of 3 client binary tags (e.g., polkadot-stable2503,polkadot-stable2502,polkadot-stable2501). Leave empty for auto-discovery of 3 latest.' + required: false + type: string + +env: + RUST_LOG: "zombienet_orchestrator=debug,zombienet_provider=debug" + +jobs: + prepare-matrix: + runs-on: parity-default + container: + image: docker.io/paritytech/ci-unified:bullseye-1.88.0-2025-06-27-v202506301118 + outputs: + binary_matrix: ${{ steps.set-matrix.outputs.binary_matrix }} + runtime_tag: ${{ steps.set-matrix.outputs.runtime_tag }} + steps: + - name: Fetch Versions + id: set-matrix + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + INPUT_RUNTIME_TAG: ${{ inputs.runtime_tag }} + INPUT_BINARY_TAGS: ${{ inputs.binary_tags }} + run: | + # 1. Use provided runtime tag or fetch the single latest from fellows/runtimes (e.g., v2.0.4) + if [[ -n "$INPUT_RUNTIME_TAG" ]]; then + RUNTIME_TAG="$INPUT_RUNTIME_TAG" + echo "Using provided runtime tag: $RUNTIME_TAG" + else + RUNTIME_TAG=$(gh api graphql -f query=' + query { + repository(owner: "polkadot-fellows", name: "runtimes") { + releases(first: 5, orderBy: {field: CREATED_AT, direction: DESC}) { + nodes { tagName } + } + } + }' --jq '.data.repository.releases.nodes[].tagName' | grep -E "^v[0-9]+" | head -n 1) + echo "Auto-discovered runtime tag: $RUNTIME_TAG" + fi + + # 2. Use provided binary tags or fetch 3 latest stable binary tags from paritytech/polkadot-sdk + if [[ -n "$INPUT_BINARY_TAGS" ]]; then + # Convert comma-separated input to JSON array + BINARIES=$(echo "$INPUT_BINARY_TAGS" | tr ',' '\n' | jq -R . | jq -s -c .) + echo "Using provided binary tags: $BINARIES" + else + BINARIES=$(gh api graphql -f query=' + query { + repository(owner: "paritytech", name: "polkadot-sdk") { + releases(first: 20, orderBy: {field: CREATED_AT, direction: DESC}) { + nodes { tagName } + } + } + }' --jq '.data.repository.releases.nodes[].tagName' | grep -E "^polkadot-stable[0-9]+" | grep -v "\-rc" | sort -Vr | head -n 3 | jq -R . | jq -s -c .) + echo "Auto-discovered binary tags: $BINARIES" + fi + + echo "runtime_tag=$RUNTIME_TAG" >> $GITHUB_OUTPUT + echo "binary_matrix=$BINARIES" >> $GITHUB_OUTPUT + + test-mixed-binaries: + needs: prepare-matrix + runs-on: parity-default + name: "Mixed binaries compatibility test" + container: + image: docker.io/paritytech/ci-unified:bullseye-1.88.0-2025-06-27-v202506301118 + steps: + - name: Checkout Code + uses: actions/checkout@v4 + - name: Download Binaries + shell: bash + run: | + BIN_TAGS=($(echo '${{ needs.prepare-matrix.outputs.binary_matrix }}' | jq -r '.[]')) + mkdir -p ./bin + for i in "${!BIN_TAGS[@]}"; do + TAG="${BIN_TAGS[$i]}" + FILENAME="polkadot-${TAG}" + echo "Downloading Polkadot @ $TAG as $FILENAME" + curl -fL "https://github.com/paritytech/polkadot-sdk/releases/download/$TAG/polkadot" -o "./bin/$FILENAME" + chmod +x "./bin/$FILENAME" + + echo "Downloading workers for $TAG" + mkdir -p "./bin/workers-${TAG}" + for worker in polkadot-execute-worker polkadot-prepare-worker; do + url="https://github.com/paritytech/polkadot-sdk/releases/download/$TAG/$worker" + echo "Downloading $worker" + curl -fL "$url" -o "./bin/workers-${TAG}/$worker" + chmod +x "./bin/workers-${TAG}/$worker" + done + ls -lh "./bin/workers-${TAG}" + done + echo "$PWD/bin" >> $GITHUB_PATH + - name: Download Latest Runtime + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + mkdir -p artifacts + gh release download "${{ needs.prepare-matrix.outputs.runtime_tag }}" \ + -R polkadot-fellows/runtimes \ + -p "polkadot_runtime-*.compact.compressed.wasm" \ + --output artifacts/relay_runtime.wasm + + - name: Run Mixed Version Test + shell: bash + env: + RELAY_RUNTIME_PATH: ${{ github.workspace }}/artifacts/relay_runtime.wasm + WORKERS_BASE_PATH: ${{ github.workspace }}/bin + ZOMBIE_PROVIDER: native + PATH: ${{ github.workspace }}/bin:${{ env.PATH }} + run: | + BIN_TAGS=($(echo '${{ needs.prepare-matrix.outputs.binary_matrix }}' | jq -r '.[]')) + + export POLKADOT_BIN_LATEST="polkadot-${BIN_TAGS[0]}" + export POLKADOT_BIN_LATEST_1="polkadot-${BIN_TAGS[1]}" + export POLKADOT_BIN_LATEST_2="polkadot-${BIN_TAGS[2]}" + export WORKERS_PATH_LATEST="$WORKERS_BASE_PATH/workers-${BIN_TAGS[0]}" + export WORKERS_PATH_LATEST_1="$WORKERS_BASE_PATH/workers-${BIN_TAGS[1]}" + export WORKERS_PATH_LATEST_2="$WORKERS_BASE_PATH/workers-${BIN_TAGS[2]}" + + echo "Running mixed network test with:" + echo "Alice (Latest) : $POLKADOT_BIN_LATEST" + echo "Bob (Latest-1) : $POLKADOT_BIN_LATEST_1" + echo "Charlie (Latest-2): $POLKADOT_BIN_LATEST_2" + + echo "Running mixed network with 3 latest client versions" + cargo test --test client-runtime -- --nocapture \ No newline at end of file diff --git a/crates/orchestrator/src/network/node.rs b/crates/orchestrator/src/network/node.rs index fd8f0e8e9..e16eac6ca 100644 --- a/crates/orchestrator/src/network/node.rs +++ b/crates/orchestrator/src/network/node.rs @@ -628,7 +628,7 @@ impl NetworkNode { let resolved_metric_name = if metric_name.contains("_bucket") { metric_name.to_string() } else { - format!("{}_bucket", metric_name) + format!("{metric_name}_bucket") }; // First pass: collect all matching metrics with their label counts diff --git a/crates/sdk/tests/client-runtime.rs b/crates/sdk/tests/client-runtime.rs new file mode 100644 index 000000000..f168db32a --- /dev/null +++ b/crates/sdk/tests/client-runtime.rs @@ -0,0 +1,85 @@ +use std::{env, path::PathBuf}; + +use configuration::{NetworkConfig, NetworkConfigBuilder}; +use subxt::{ext::futures::StreamExt, OnlineClient, PolkadotConfig}; +use zombienet_sdk::NetworkConfigExt; + +fn small_network() -> NetworkConfig { + let relay_runtime_path = PathBuf::from(env::var("RELAY_RUNTIME_PATH").unwrap()); + let polkadot_bin_latest = env::var("POLKADOT_BIN_LATEST").unwrap_or("polkadot".into()); + let polkadot_bin_latest_1 = env::var("POLKADOT_BIN_LATEST_1").unwrap_or("polkadot".into()); + let polkadot_bin_latest_2 = env::var("POLKADOT_BIN_LATEST_2").unwrap_or("polkadot".into()); + + let workers_path_latest = env::var("WORKERS_PATH_LATEST").ok(); + let workers_path_latest_1 = env::var("WORKERS_PATH_LATEST_1").ok(); + let workers_path_latest_2 = env::var("WORKERS_PATH_LATEST_2").ok(); + + NetworkConfigBuilder::new() + .with_relaychain(|r| { + let relaychain = r + .with_chain("polkadot-local") + .with_default_args(vec!["-lparachain=debug,runtime=debug".into()]) + .with_chain_spec_runtime(relay_runtime_path, None) + .with_validator(|node| { + let mut alice = node + .with_name("alice") + .with_command(polkadot_bin_latest.as_ref()); + if let Some(workers_path) = &workers_path_latest { + alice = + alice.with_args(vec![("--workers-path", workers_path.as_str()).into()]); + } + alice + }) + .with_validator(|node| { + let mut bob = node + .with_name("bob") + .with_command(polkadot_bin_latest_1.as_ref()); + if let Some(workers_path) = &workers_path_latest_1 { + bob = bob.with_args(vec![("--workers-path", workers_path.as_str()).into()]); + } + bob + }) + .with_validator(|node| { + let mut charlie = node + .with_name("charlie") + .with_command(polkadot_bin_latest_2.as_ref()); + if let Some(workers_path) = &workers_path_latest_2 { + charlie = + charlie + .with_args(vec![("--workers-path", workers_path.as_str()).into()]); + } + charlie + }); + relaychain + }) + .build() + .unwrap() +} + +#[tokio::test(flavor = "multi_thread")] +async fn ci_native_smoke_should_works() { + tracing_subscriber::fmt::init(); + let config = small_network(); + let network = config.spawn_native().await.unwrap(); + + let alice = network.get_node("alice").unwrap(); + let alice_client: OnlineClient = alice.wait_client().await.unwrap(); + + let bob = network.get_node("bob").unwrap(); + let bob_client: OnlineClient = bob.wait_client().await.unwrap(); + + let charlie = network.get_node("charlie").unwrap(); + let charlie_client: OnlineClient = charlie.wait_client().await.unwrap(); + + wait_n_blocks(&alice_client, 5, "alice").await; + wait_n_blocks(&bob_client, 5, "bob").await; + wait_n_blocks(&charlie_client, 5, "charlie").await; +} + +async fn wait_n_blocks(client: &OnlineClient, n: usize, name: &str) { + let mut blocks = client.blocks().subscribe_finalized().await.unwrap().take(n); + + while let Some(block) = blocks.next().await { + println!("{name} Block #{}", block.unwrap().header().number); + } +}