feat(load): Connect load tests to prod-testnet (#766)

* feat(load): Add load metrics

* fix(lint)

* fix(e2e): Fix tests holding minting

* feat(e2e): Token Lane Refactor

* fix(tests): tests token + lint fix

* try fixing tokens holdings issue

* raise canton transfer amount

* fix setupSend first hold picking

* fix lint + unit tests

* fix load tests CI

* feat(tests): env agnostic boostrapper for e2e/load

* fix(load): remove unecessary variable override for devenv

* fix(load): add buffer to messages estimation

* feat(load): Connect load tests to prod-testnet

* fix(load): prod-testnet needs v1_0_0 bindings

* fix(load): fix prod-testnet receiver

* fix lint issues

* feat(load): enable load connectivity to proxy - tailscale

* fix(router): use v1_0_0 instead of ledger for router too

* fix router

* feat(load): pre-approve EVM tokens before load tests, not before every send

* refactor(load): move bindings affected code to a single layer

* fix(load): remove unecessary action params | use better defaults

* fix action

Author: rodrigombsoares

.github/actions/ccip-load-test/action.yml

@@ -16,7 +16,7 @@ inputs:
   message_rate:
     description: "CANTON_LOAD_MESSAGE_RATE"
     required: false
-    default: "1/45s"
+    default: "1/10s"
   load_duration:
     description: "CANTON_LOAD_DURATION"
     required: false
@@ -33,10 +33,6 @@ inputs:
     description: "Checkout path for chainlink-canton (`.` for PR root, `chainlink-canton` for nested devenv checkout)"
     required: false
     default: .
-  canton_ref:
-    description: "chainlink-canton git ref for devenv setup (empty = workflow SHA)"
-    required: false
-    default: ""
   skip_exec_confirm:
     description: "CANTON_LOAD_SKIP_EXEC_CONFIRM"
     required: false
@@ -49,10 +45,26 @@ inputs:
     description: "CANTON_PARTY_ID"
     required: false
     default: "u_d53a15c42af6::1220c250c23c55120f7c758bccc5cbc739629015ab921594e1c29656981f985bffa7"
+  user_id:
+    description: "CANTON_USER_ID"
+    required: false
+    default: "0oau1l03xrG9NQw9z5d7"
   grpc_url:
     description: "CANTON_GRPC_URL"
     required: false
     default: "testnet.cv1.bcy-v.metalhosts.com:443"
+  router_instance_id:
+    description: "CANTON_ROUTER_INSTANCE_ID (prod-testnet)"
+    required: false
+    default: "gha-load-router"
+  sender_instance_id:
+    description: "CANTON_SENDER_INSTANCE_ID (prod-testnet)"
+    required: false
+    default: "gha-load-sender"
+  receiver_instance_id:
+    description: "CANTON_RECEIVER_INSTANCE_ID (prod-testnet)"
+    required: false
+    default: "gha-load-receiver"
   CANTON_OKTA_AUTHORIZER_TESTNET:
     description: "GitHub secret CANTON_OKTA_AUTHORIZER_TESTNET"
     required: false
@@ -65,6 +77,12 @@ inputs:
   CCIP_PROD_TESTNET_PRIVATE_KEY:
     description: "GitHub secret CCIP_PROD_TESTNET_PRIVATE_KEY"
     required: false
+  TS_OAUTH_CLIENT_ID_CANTON:
+    description: "GitHub secret TS_OAUTH_CLIENT_ID_CANTON (prod-testnet Tailscale)"
+    required: false
+  TS_AUDIENCE_CANTON:
+    description: "GitHub secret TS_AUDIENCE_CANTON (prod-testnet Tailscale)"
+    required: false
   ccv-iam-role:
     description: "AWS IAM role for CCV ECR authentication (devenv only)"
     required: false
@@ -78,6 +96,23 @@ inputs:
 runs:
   using: composite
   steps:
+    - name: Setup CCIP devenv
+      if: inputs.ccip_env == 'devenv'
+      uses: ./.github/actions/setup-ccip-devenv
+      with:
+        canton-path: ${{ inputs.canton_path }}
+        ccv-iam-role: ${{ inputs.ccv-iam-role }}
+        jd-registry: ${{ inputs.jd-registry }}
+        jd-image: ${{ inputs.jd-image }}
+
+    - name: Connect to Tailscale (prod-testnet)
+      if: inputs.ccip_env == 'prod-testnet'
+      uses: tailscale/github-action@53acf823325fe9ca47f4cdaa951f90b4b0de5bb9 # v4.1.1
+      with:
+        oauth-client-id: ${{ inputs.TS_OAUTH_CLIENT_ID_CANTON }}
+        audience: ${{ inputs.TS_AUDIENCE_CANTON }}
+        tags: tag:bcy-validators-gha
+
     - name: Install Go (prod-testnet)
       if: inputs.ccip_env == 'prod-testnet'
       uses: actions/setup-go@v6
@@ -99,6 +134,7 @@ runs:
       env:
         CANTON_LOAD_MESSAGE_RATE: ${{ inputs.message_rate }}
         CANTON_LOAD_DURATION: ${{ inputs.load_duration }}
+        CANTON_LOAD_SKIP_EXEC_CONFIRM: ${{ inputs.skip_exec_confirm }}
       run: |
         case "${{ inputs.direction }}" in
           canton2evm) TEST_RUN='^TestCanton2EVM_Load$' ;;
@@ -121,12 +157,17 @@ runs:
         CANTON_CLIENT_ID: ${{ inputs.CANTON_OKTA_CLIENT_ID_TESTNET }}
         CANTON_CLIENT_SECRET: ${{ inputs.CANTON_OKTA_CLIENT_SECRET_TESTNET }}
         CANTON_PARTY_ID: ${{ inputs.party_id }}
+        CANTON_USER_ID: ${{ inputs.user_id }}
         CANTON_GRPC_URL: ${{ inputs.grpc_url }}
+        CANTON_ROUTER_INSTANCE_ID: ${{ inputs.router_instance_id }}
+        CANTON_SENDER_INSTANCE_ID: ${{ inputs.sender_instance_id }}
+        CANTON_RECEIVER_INSTANCE_ID: ${{ inputs.receiver_instance_id }}
         CANTON_LOAD_MESSAGE_RATE: ${{ inputs.message_rate }}
         CANTON_LOAD_DURATION: ${{ inputs.load_duration }}
         CANTON_CONFIRM_EXEC_TIMEOUT: ${{ inputs.confirm_exec_timeout }}
         CANTON_LOAD_SKIP_EXEC_CONFIRM: ${{ inputs.skip_exec_confirm }}
         PRIVATE_KEY: ${{ inputs.CCIP_PROD_TESTNET_PRIVATE_KEY }}
+        TEST_TIMEOUT: ${{ inputs.test_timeout }}
       run: |
         case "${{ inputs.direction }}" in
           canton2evm) TEST_RUN='^TestCanton2EVM_Load$' ;;
@@ -135,4 +176,13 @@ runs:
           evm2canton-token) TEST_RUN='^TestEVM2Canton_TokenLoad$' ;;
           *) echo "unknown direction: ${{ inputs.direction }}" >&2; exit 1 ;;
         esac
-        go test -timeout "${{ inputs.test_timeout }}" -v -count 1 -run "$TEST_RUN"
+        go test -tags=prodledger -timeout "$TEST_TIMEOUT" -v -count 1 -ccip-env=prod-testnet -run "$TEST_RUN"
+
+    - name: Upload devenv logs
+      if: always() && inputs.ccip_env == 'devenv'
+      uses: ./.github/actions/upload-ccip-devenv-logs
+      with:
+        canton-path: ${{ inputs.canton_path }}
+        test-package-dir: load
+        log-dump-suffix: ccip-load-tests
+        artifact-name: container-logs-ccip-load-tests

.github/actions/setup-ccip-devenv/action.yml

@@ -12,11 +12,6 @@ inputs:
     # that generates their configs. A mismatch across config-format changes (e.g. #1193
     # bootstrap monitoring.Config) makes the verifier container exit on startup.
     default: f20ea93af3005f17859ca1d7e6529e0121a0430c
-  canton-ref:
-    description: >-
-      chainlink-canton git ref. Leave empty to use the ref that triggered the workflow.
-    required: false
-    default: ''
   ccv-iam-role:
     description: AWS IAM role ARN for CCV ECR authentication
     required: true
@@ -41,7 +36,7 @@ inputs:
   free-disk-space:
     description: Run the free-disk-space action before builds
     required: false
-    default: 'true'
+    default: "true"
 
 runs:
   using: composite
@@ -58,7 +53,7 @@ runs:
       with:
         repository: smartcontractkit/chainlink-canton
         path: ${{ inputs.canton-path }}
-        ref: ${{ inputs.canton-ref != '' && inputs.canton-ref || github.sha }}
+        ref: ${{ github.sha }}
 
     - name: Check out chainlink-ccv
       uses: actions/checkout@v6

.github/workflows/ccip-load-command.yml

@@ -22,12 +22,12 @@ on:
           - canton2evm-token
           - evm2canton-token
       message_rate:
-        description: "CANTON_LOAD_MESSAGE_RATE (e.g. 1/1s, 1/45s)"
+        description: "CANTON_LOAD_MESSAGE_RATE (e.g. 1/45s)"
         required: false
-        default: "1/1s"
+        default: "1/10s"
         type: string
       load_duration:
-        description: "CANTON_LOAD_DURATION (Go duration, e.g. 90s, 2m)"
+        description: "CANTON_LOAD_DURATION (e.g. 90s, 2m)"
         required: false
         default: "90s"
         type: string
@@ -36,20 +36,15 @@ on:
         required: false
         default: "40m"
         type: string
-      canton_ref:
-        description: "chainlink-canton git ref (empty = workflow ref; devenv only)"
-        required: false
-        default: ""
-        type: string
       config_file:
         description: "CCIP_CONFIG_FILE basename under ccip/devenv (prod-testnet only)"
         required: false
         default: env-prod-testnet.ci.toml
         type: string
       skip_exec_confirm:
-        description: "CANTON_LOAD_SKIP_EXEC_CONFIRM (prod-testnet only)"
+        description: "CANTON_LOAD_SKIP_EXEC_CONFIRM"
         required: false
-        default: "true"
+        default: "false"
         type: string
       confirm_exec_timeout:
         description: "CANTON_CONFIRM_EXEC_TIMEOUT (prod-testnet only)"
@@ -78,25 +73,27 @@ jobs:
         run: |
           if [ "${{ inputs.ccip_env }}" = "prod-testnet" ]; then
             echo "canton_path=." >> "$GITHUB_OUTPUT"
-            if [ "${{ inputs.message_rate }}" = "1/1s" ]; then
-              echo "message_rate=1/45s" >> "$GITHUB_OUTPUT"
-            else
-              echo "message_rate=${{ inputs.message_rate }}" >> "$GITHUB_OUTPUT"
-            fi
+            echo "message_rate=${{ inputs.message_rate }}" >> "$GITHUB_OUTPUT"
             if [ "${{ inputs.load_duration }}" = "90s" ]; then
               echo "load_duration=2m" >> "$GITHUB_OUTPUT"
             else
               echo "load_duration=${{ inputs.load_duration }}" >> "$GITHUB_OUTPUT"
             fi
             if [ "${{ inputs.test_timeout }}" = "40m" ]; then
-              if [ "${{ inputs.direction }}" = "evm2canton" ]; then
+              if [ "${{ inputs.direction }}" = "evm2canton" ] || [ "${{ inputs.direction }}" = "evm2canton-token" ]; then
                 echo "test_timeout=45m" >> "$GITHUB_OUTPUT"
               else
                 echo "test_timeout=30m" >> "$GITHUB_OUTPUT"
               fi
             else
               echo "test_timeout=${{ inputs.test_timeout }}" >> "$GITHUB_OUTPUT"
             fi
+            direction="${{ inputs.direction }}"
+            if [ "$direction" = "canton2evm" ]; then
+              echo "skip_exec_confirm=true" >> "$GITHUB_OUTPUT"
+            else
+              echo "skip_exec_confirm=false" >> "$GITHUB_OUTPUT"
+            fi
           else
             echo "canton_path=chainlink-canton" >> "$GITHUB_OUTPUT"
             echo "message_rate=${{ inputs.message_rate }}" >> "$GITHUB_OUTPUT"
@@ -108,8 +105,21 @@ jobs:
         if: inputs.ccip_env == 'devenv'
         uses: ./.github/actions/setup-ccip-devenv
         with:
-          canton-ref: ${{ inputs.canton_ref }}
-          canton-path: ${{ steps.prod_defaults.outputs.canton_path }}
+          ccip_env: ${{ inputs.ccip_env }}
+          direction: ${{ inputs.direction }}
+          message_rate: ${{ steps.prod_defaults.outputs.message_rate }}
+          load_duration: ${{ steps.prod_defaults.outputs.load_duration }}
+          test_timeout: ${{ steps.prod_defaults.outputs.test_timeout }}
+          config_file: ${{ inputs.config_file }}
+          canton_path: ${{ steps.prod_defaults.outputs.canton_path }}
+          skip_exec_confirm: ${{ inputs.ccip_env == 'prod-testnet' && steps.prod_defaults.outputs.skip_exec_confirm || inputs.skip_exec_confirm }}
+          confirm_exec_timeout: ${{ inputs.confirm_exec_timeout }}
+          CANTON_OKTA_AUTHORIZER_TESTNET: ${{ secrets.CANTON_OKTA_AUTHORIZER_TESTNET }}
+          CANTON_OKTA_CLIENT_ID_TESTNET: ${{ secrets.CANTON_OKTA_CLIENT_ID_TESTNET }}
+          CANTON_OKTA_CLIENT_SECRET_TESTNET: ${{ secrets.CANTON_OKTA_CLIENT_SECRET_TESTNET }}
+          CCIP_PROD_TESTNET_PRIVATE_KEY: ${{ secrets.CCIP_PROD_TESTNET_PRIVATE_KEY }}
+          TS_OAUTH_CLIENT_ID_CANTON: ${{ secrets.TS_OAUTH_CLIENT_ID_CANTON }}
+          TS_AUDIENCE_CANTON: ${{ secrets.TS_AUDIENCE_CANTON }}
           ccv-iam-role: ${{ secrets.CCV_IAM_ROLE }}
           jd-registry: ${{ secrets.JD_REGISTRY }}
           jd-image: ${{ secrets.JD_IMAGE }}

.github/workflows/ccip-load-tests.yml

@@ -26,4 +26,3 @@ jobs:
           permission: write
           commands: |
             auto-fix
-            ccip-load