chore: Improve canister snapshot download/upload

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 # UNRELEASED
 
+### feat: improved the canister snapshot download/upload feature
+
+Improved the canister snapshot download/upload feature by
+- adding progress bars to snapshot download/upload
+- streaming snapshot download/upload directly to/from disk.
+- supporting download/upload with resuming.
+- supporting download/upload with concurrency, default to 3 tasks in parallel.
+
 # 0.30.0
 
 ### feat: `dfx start --system-canisters` for bootstrapping system canisters

docs/cli-reference/dfx-canister.mdx

@@ -1028,11 +1028,13 @@ dfx canister snapshot download <canister> <snapshot> --dir <DIR>
 
 You can use the following arguments with the `dfx canister snapshot download` command.
 
-| Argument      | Description                                                                |
-|---------------|----------------------------------------------------------------------------|
-| `<canister>`  | The canister to download the snapshot from.                                |
-| `<snapshot>`  | The ID of the snapshot to download.                                        |
-| --dir `<dir>` | The directory to download the snapshot to. It should be created and empty. |
+| Argument                      | Description                                                                                |
+|-------------------------------|--------------------------------------------------------------------------------------------|
+| `<canister>`                  | The canister to download the snapshot from.                                                |
+| `<snapshot>`                  | The ID of the snapshot to download.                                                        |
+| --dir `<dir>`                 | The directory to download the snapshot to. It should be created and empty if not resuming. |
+| --resume                      | Whether to resume the download if the previous snapshot download failed.                   |
+| --concurrency `<concurrency>` | The number of concurrent downloads to perform [default: 3].                                |
 
 ### Examples
 
@@ -1056,11 +1058,13 @@ dfx canister snapshot upload <canister> --dir <DIR>
 
 You can use the following arguments with the `dfx canister snapshot upload` command.
 
-| Argument              | Description                                                                    |
-|-----------------------|--------------------------------------------------------------------------------|
-| `<canister>`          | The canister to upload the snapshot to.                                        |
-| --dir `<dir>`         | The directory to upload the snapshot from.                                     |
-| --replace `<replace>` | If a snapshot ID is specified, the snapshot identified by this ID will be deleted and a snapshot with a new ID will be returned. |
+| Argument                      | Description                                                                    |
+|-------------------------------|--------------------------------------------------------------------------------|
+| `<canister>`                  | The canister to upload the snapshot to.                                        |
+| --dir `<dir>`                 | The directory to upload the snapshot from.                                     |
+| --replace `<replace>`         | If a snapshot ID is specified, the snapshot identified by this ID will be deleted and a snapshot with a new ID will be returned. |
+| --resume `<resume>`           | The snapshot ID to resume uploading to.                                        |
+| --concurrency `<concurrency>` | The number of concurrent uploads to perform [default: 3].                      |
 
 ### Examples
 

e2e/tests-dfx/canister_extra.bash

@@ -1,14 +1,17 @@
 #!/usr/bin/env bats
 
 load ../utils/_
+load ../utils/toxiproxy
 
 setup() {
     standard_setup
     dfx_new hello
+    toxiproxy_start
 }
 
 teardown() {
     dfx_stop
+    toxiproxy_stop || true
     standard_teardown
 }
 
@@ -105,6 +108,146 @@ teardown() {
     assert_contains '(1 : nat)'
 }
 
+@test "canister snapshots download and upload via toxiproxy with high latency" {
+    # Start the dfx server on a random port.
+    dfx_port=$(get_ephemeral_port)
+    dfx_start --host "127.0.0.1:$dfx_port"
+
+    # Start toxiproxy and create a proxy.
+    proxy_port=$(get_ephemeral_port)
+    toxiproxy_create_proxy "127.0.0.1:$proxy_port" "127.0.0.1:$dfx_port" proxy_high_latency
+
+    install_asset counter
+    dfx deploy --no-wallet --network "http://127.0.0.1:$proxy_port"
+
+    assert_command dfx canister call hello_backend inc_read --network "http://127.0.0.1:$proxy_port"
+    assert_contains '(1 : nat)'
+
+    # Create a snapshot to download.
+    dfx canister stop hello_backend --network "http://127.0.0.1:$proxy_port"
+    assert_command dfx canister snapshot create hello_backend --network "http://127.0.0.1:$proxy_port"
+    assert_match 'Snapshot ID: ([0-9a-f]+)'
+    snapshot=${BASH_REMATCH[1]}
+
+    # Add latency to the proxy.
+    toxiproxy_add_latency 1500 300 proxy_high_latency
+
+    # Download through the proxy with latency.
+    OUTPUT_DIR="output"
+    mkdir -p "$OUTPUT_DIR"
+    assert_command dfx canister snapshot download hello_backend "$snapshot" --dir "$OUTPUT_DIR" --network "http://127.0.0.1:$proxy_port"
+    assert_contains "saved to '$OUTPUT_DIR'"
+
+    # Start the canister again.
+    dfx canister start hello_backend --network "http://127.0.0.1:$proxy_port"
+    assert_command dfx canister call hello_backend inc_read --network "http://127.0.0.1:$proxy_port"
+    assert_contains '(2 : nat)'
+
+    # Upload the snapshot to create a new snapshot.
+    assert_command dfx canister snapshot upload hello_backend --dir "$OUTPUT_DIR" --network "http://127.0.0.1:$proxy_port"
+    assert_match 'Snapshot ID: ([0-9a-f]+)'
+    snapshot_1=${BASH_REMATCH[1]}
+
+    # Stop the canister and load the new snapshot.
+    dfx canister stop hello_backend --network "http://127.0.0.1:$proxy_port"
+    assert_command dfx canister snapshot load hello_backend "$snapshot_1" --network "http://127.0.0.1:$proxy_port"
+
+    # Start the canister again and verify the loaded snapshot.
+    dfx canister start hello_backend --network "http://127.0.0.1:$proxy_port"
+    assert_command dfx canister call hello_backend read --network "http://127.0.0.1:$proxy_port"
+    assert_contains '(1 : nat)'
+
+    toxiproxy_delete_proxy proxy_high_latency
+}
+
+@test "canister snapshots download and upload via toxiproxy with network drop" {
+    # Start the dfx server on a random port.
+    dfx_port=$(get_ephemeral_port)
+    dfx_start --host "127.0.0.1:$dfx_port"
+
+    # Start toxiproxy and create a proxy.
+    proxy_port=$(get_ephemeral_port)
+    toxiproxy_create_proxy "127.0.0.1:$proxy_port" "127.0.0.1:$dfx_port" proxy_network_drop
+
+    install_asset counter
+    dfx deploy --no-wallet --network "http://127.0.0.1:$proxy_port"
+
+    assert_command dfx canister call hello_backend inc_read --network "http://127.0.0.1:$proxy_port"
+    assert_contains '(1 : nat)'
+
+    # Create a snapshot to download.
+    dfx canister stop hello_backend --network "http://127.0.0.1:$proxy_port"
+    assert_command dfx canister snapshot create hello_backend --network "http://127.0.0.1:$proxy_port"
+    assert_match 'Snapshot ID: ([0-9a-f]+)'
+    snapshot=${BASH_REMATCH[1]}
+
+    (
+        # Toxiproxy doesn't support disabling the proxy with certain amount of data being transferred.
+        # So we roughly wait for 0.5 seconds to disable the proxy and fail the snapshot download.
+        sleep 0.5
+        toxiproxy_toggle_proxy proxy_network_drop
+    ) &
+
+    # Download through the proxy should fail.
+    OUTPUT_DIR="output"
+    mkdir -p "$OUTPUT_DIR"
+    assert_command_fail timeout 10s dfx canister snapshot download hello_backend "$snapshot" --dir "$OUTPUT_DIR" --network "http://127.0.0.1:$proxy_port"
+
+    # Enable the proxy again.
+    toxiproxy_toggle_proxy proxy_network_drop
+
+    # Resume the download through the proxy.
+    assert_command dfx canister snapshot download hello_backend "$snapshot" --dir "$OUTPUT_DIR" -r --network "http://127.0.0.1:$proxy_port"
+    assert_contains "saved to '$OUTPUT_DIR'"
+
+    # Start the canister again.
+    dfx canister start hello_backend --network "http://127.0.0.1:$proxy_port"
+    assert_command dfx canister call hello_backend inc_read --network "http://127.0.0.1:$proxy_port"
+    assert_contains '(2 : nat)'
+
+    (
+        # Toxiproxy doesn't support disabling the proxy with certain amount of data being transferred.
+        # So we roughly wait for 0.5 seconds to disable the proxy and fail the snapshot upload.
+        sleep 0.5
+        toxiproxy_toggle_proxy proxy_network_drop
+    ) &
+
+    # Upload the snapshot should fail.
+    assert_command_fail timeout 10s dfx canister snapshot upload hello_backend --dir "$OUTPUT_DIR" --network "http://127.0.0.1:$proxy_port"
+
+    # Loop to find a .json filename in OUTPUT_DIR that matches ^[0-9a-f]+\.json$.
+    snapshot_1=""
+    while IFS= read -r json_file; do
+        [ -z "$json_file" ] && continue
+        if [[ "$json_file" =~ ^[0-9a-f]+\.json$ ]]; then
+            snapshot_1="${json_file%.json}"
+            break
+        fi
+    done < <(find "$OUTPUT_DIR" -maxdepth 1 -type f -name '*.json' -printf '%f\n')
+    if [ -z "$snapshot_1" ]; then
+        echo "No matching .json filename ([0-9a-f]+.json) found in $OUTPUT_DIR" >&2
+        false
+    fi
+
+    # Enable the proxy again.
+    toxiproxy_toggle_proxy proxy_network_drop
+
+    # Resume the upload through the proxy.
+    assert_command dfx canister snapshot upload hello_backend --dir "$OUTPUT_DIR" -r "$snapshot_1" --network "http://127.0.0.1:$proxy_port"
+    assert_contains "$snapshot_1"
+
+    # Stop the canister and load the new snapshot.
+    dfx canister stop hello_backend --network "http://127.0.0.1:$proxy_port"
+    assert_command dfx canister snapshot load hello_backend "$snapshot_1" --network "http://127.0.0.1:$proxy_port"
+
+    # Start the canister again and verify the loaded snapshot.
+    dfx canister start hello_backend --network "http://127.0.0.1:$proxy_port"
+    assert_command dfx canister call hello_backend read --network "http://127.0.0.1:$proxy_port"
+    assert_contains '(1 : nat)'
+
+    toxiproxy_delete_proxy proxy_network_drop
+}
+
 @test "can query a website" {
   dfx_start
 
@@ -116,4 +259,4 @@ teardown() {
   assert_command dfx canister call e2e_project_backend get_url '("www.githubstatus.com:443","https://www.githubstatus.com:443")'
   assert_contains "Git Operations"
   assert_contains "API Requests"
-}
+}

e2e/utils/toxiproxy.bash

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Helpers for toxiproxy to use toxiproxy-server and toxiproxy-cli
+
+: "${TOXIPROXY_HOST:=127.0.0.1}"
+: "${TOXIPROXY_PORT:=8474}"
+
+# Check if toxiproxy server is running
+toxiproxy_is_running() {
+  curl --silent --fail "http://${TOXIPROXY_HOST}:${TOXIPROXY_PORT}/version" >/dev/null 2>&1
+}
+
+# Start toxiproxy server
+toxiproxy_start() {
+  if toxiproxy_is_running; then
+    return 0
+  fi
+
+  if ! command -v toxiproxy-server >/dev/null 2>&1; then
+    echo "toxiproxy-server not found in PATH" >&2
+    return 1
+  fi
+
+  toxiproxy-server -host "$TOXIPROXY_HOST" -port "$TOXIPROXY_PORT" >/dev/null 2>&1 &
+  export E2E_TOXIPROXY_PID=$!
+
+  for _ in $(seq 1 50); do
+    if toxiproxy_is_running; then
+      return 0
+    fi
+    sleep 0.1
+  done
+
+  echo "Toxiproxy server did not become available on ${TOXIPROXY_HOST}:${TOXIPROXY_PORT}" >&2
+  return 1
+}
+
+# Stop toxiproxy server
+toxiproxy_stop() {
+  if [ -n "${E2E_TOXIPROXY_PID:-}" ]; then
+    kill "$E2E_TOXIPROXY_PID" >/dev/null 2>&1 || true
+    unset E2E_TOXIPROXY_PID
+  fi
+}
+
+# Create or replace a proxy
+toxiproxy_create_proxy() {
+  local listen=$1 upstream=$2 name=$3
+
+  # Ensure toxiproxy-cli is available
+  if ! command -v toxiproxy-cli >/dev/null 2>&1; then
+    echo "toxiproxy-cli not found in PATH" >&2
+    return 1
+  fi
+
+  toxiproxy-cli delete "$name" >/dev/null 2>&1 || true
+  toxiproxy-cli create --listen "$listen" --upstream "$upstream" "$name" >/dev/null 2>&1
+}
+
+# Delete a proxy
+toxiproxy_delete_proxy() {
+  local name=$1
+  toxiproxy-cli delete "$name" >/dev/null 2>&1 || true
+}
+
+# Set a proxy to enabled or disabled
+toxiproxy_toggle_proxy() {
+  local name=$1
+  toxiproxy-cli toggle "$name" >/dev/null 2>&1
+}
+
+# Add latency toxic (downstream)
+toxiproxy_add_latency() {
+  local latency=$1 jitter=$2 name=$3
+  toxiproxy-cli toxic add -t latency -a latency="$latency" -a jitter="$jitter" -d "$name" >/dev/null
+}

scripts/workflows/e2e-matrix.py

@@ -7,7 +7,7 @@
 SELECTED_TESTS = ["dfx/bitcoin", "dfx/canister_http_adapter", "dfx/start"]
 
 # Run these tests in serial
-SERIAL_TESTS = ["dfx/start", "dfx/bitcoin", "dfx/cycles-ledger", "dfx/ledger", "dfx/serial_misc"]
+SERIAL_TESTS = ["dfx/start", "dfx/bitcoin", "dfx/cycles-ledger", "dfx/ledger", "dfx/serial_misc", "dfx/canister_extra"]
 
 def test_scripts(prefix):
     all_files = os.listdir(f"e2e/tests-{prefix}")

scripts/workflows/provision-linux.sh

@@ -54,6 +54,22 @@ fi
 if [ "$E2E_TEST" = "tests-dfx/info.bash" ]; then
     sudo apt-get install --yes libarchive-zip-perl
 fi
+if [ "$E2E_TEST" = "tests-dfx/canister_extra.bash" ]; then
+    VERSION=v2.10.0
+    ARCH=$(uname -m)
+    case "$ARCH" in
+        x86_64|amd64)  ARCH_DL="amd64" ;;
+        arm64|aarch64) ARCH_DL="arm64" ;;
+        *) echo "Unsupported ARCH: $ARCH" >&2; exit 1 ;;
+    esac
+
+    BASE_URL="https://github.com/Shopify/toxiproxy/releases/download/$VERSION"
+    curl -fsSL "$BASE_URL/toxiproxy-server-linux-${ARCH_DL}" -o toxiproxy-server
+    curl -fsSL "$BASE_URL/toxiproxy-cli-linux-${ARCH_DL}" -o toxiproxy-cli
+    chmod +x toxiproxy-server toxiproxy-cli
+    sudo mv toxiproxy-server /usr/local/bin/
+    sudo mv toxiproxy-cli /usr/local/bin/
+fi
 
 # Set environment variables.
 echo "$HOME/bin" >> "$GITHUB_PATH"