performance-testing.yml

##############################################################################
# Performance Testing Workflow
#
# This workflow runs performance tests for KMS in different deployment modes:
# - threshold: Standard threshold mode
# - thresholdWithEnclave: Threshold mode with Nitro Enclave security
#
# Triggers:
# - Manual dispatch with configurable parameters
##############################################################################
name: performance-testing

on:
  #=======================================================
  schedule:
    - cron: '0 0 * * 1-5' # Runs at midnight UTC (1 AM CET) Monday-Friday
  #=======================================================
  workflow_dispatch:
    inputs:
      build:
        description: 'Build new Docker images (if true, image tags below are ignored)'
        required: true
        default: false
        type: boolean
      deployment_type:
        description: 'Deployment type'
        required: true
        default: 'threshold'
        type: choice
        options:
          - 'threshold'
          - 'thresholdWithEnclave'
      client_logs:
        description: 'Enable core-client tracing logs during perf tests (off by default; logging skews perf numbers)'
        required: true
        default: false
        type: boolean
      fhe_params:
        description: 'FHE parameters for preprocessing and keygen'
        required: true
        default: 'Test'
        type: choice
        options:
          - 'Default'
          - 'Test'
      tls:
        description: 'TLS enabled. Only available for thresholdWithEnclave'
        required: true
        default: false
        type: boolean
      kms_branch:
        description: 'KMS chart source ref (optional; empty = Use workflow from ref)'
        required: false
        type: string
      kms_chart_version:
        description: 'KMS chart version (repository = use chart source ref)'
        required: true
        type: string
        default: 'repository'
      tkms_infra_chart_version:
        description: 'TKMS Infra chart version'
        required: true
        default: '0.3.2'
        type: string
      kms_core_image_tag:
        description: 'KMS Core image tag (ignored if build=true)'
        required: false
        default: ''
        type: string
      kms_core_client_image_tag:
        description: 'KMS Core client image tag (ignored if build=true)'
        required: false
        default: ''
        type: string
  #=======================================================

permissions: {}

concurrency:
  group: performance-testing-kms-ci
  cancel-in-progress: false

jobs:
  ############################################################################
  # Docker Build Job (Conditional)
  # Only runs when:
  # - workflow_dispatch with build=true
  # - schedule (nightly)
  ############################################################################
  docker-build:
    if: |
      (github.event_name == 'workflow_dispatch' && inputs.build == true) ||
      (github.event_name == 'schedule')
    name: performance-testing/docker-build
    permissions:
      actions: read # Required to read workflow run information
      contents: write # Required to checkout repository code
      id-token: write # Required for OIDC authentication
      pull-requests: read # Required to read pull requests information
      packages: write # Required to publish Docker images
      attestations: write # Required to create build attestations
    uses: ./.github/workflows/docker-build.yml
    with:
      # Perf only deploys the enclave image for thresholdWithEnclave runs (the scheduled
      # default). For a plain `threshold` run the enclave build + sign jobs are dead weight,
      # so skip them. core-service/core-client (and the cached golden image) are always needed.
      build-enclave: ${{ (inputs.deployment_type || 'thresholdWithEnclave') == 'thresholdWithEnclave' }}
    secrets:
      BLOCKCHAIN_ACTIONS_TOKEN: ${{ secrets.BLOCKCHAIN_ACTIONS_TOKEN }}
      AWS_ACCESS_KEY_S3_USER: ${{ secrets.AWS_ACCESS_KEY_S3_USER }}
      AWS_SECRET_KEY_S3_USER: ${{ secrets.AWS_SECRET_KEY_S3_USER }}
      CGR_USERNAME: ${{ secrets.CGR_USERNAME }}
      CGR_PASSWORD: ${{ secrets.CGR_PASSWORD }}

  ############################################################################
  # Performance Testing Job
  # Runs performance tests for KMS in different deployment modes
  ############################################################################
  performance-testing:
    needs: [docker-build]
    if: always() && !cancelled()
    name: performance-testing
    permissions:
      actions: read # Required to resolve the current job URL for Slack reports
    runs-on: "runs-on=${{ github.run_id }}/runner=2cpu-linux-x64/spot=false/extras=s3-cache"
    timeout-minutes: 1800
    env:
      DEPLOYMENT_TYPE: ${{ inputs.deployment_type || 'thresholdWithEnclave' }}
      TLS: ${{ github.event_name == 'workflow_dispatch' && format('{0}', inputs.tls) || 'true' }}
      CLIENT_LOGS: ${{ github.event_name == 'workflow_dispatch' && format('{0}', inputs.client_logs) || 'false' }}
      FHE_PARAMS: ${{ inputs.fhe_params || 'Test' }}
      KMS_BRANCH: ${{ inputs.kms_branch || github.ref }}
      NAMESPACE: 'kms-ci'
      KMS_CHART_VERSION: ${{ inputs.kms_chart_version || 'repository' }}
      TKMS_INFRA_CHART_VERSION: ${{ inputs.tkms_infra_chart_version || '0.3.2' }}
      # No dispatch input: the workflow is already at GitHub's 10-input
      # limit. Enable on both schedule and workflow_dispatch so a branch
      # run can verify scrape/remote-write without waiting for the nightly.
      ENABLE_KMS_METRICS: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
    steps:
      - name: Setup Runs-on
        uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # v2.0.3

      - name: Checkout Project
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          token: ${{ secrets.BLOCKCHAIN_ACTIONS_TOKEN }}
          persist-credentials: false

      - name: Preserve runner tools across chart checkout
        run: python3 ci/scripts/perf_runner.py stage-tools

      # ======================================================================
      # DETERMINE IMAGE TAGS
      # Use docker-build outputs if built, otherwise use provided inputs
      # ======================================================================
      - name: Determine image tags
        env:
          DOCKER_BUILD_TAG: ${{ needs.docker-build.outputs.image_tag }}
          INPUT_CORE_TAG: ${{ inputs.kms_core_image_tag }}
          INPUT_CLIENT_TAG: ${{ inputs.kms_core_client_image_tag }}
        run: python3 ci/scripts/perf_runner.py determine-tags

      - name: Login to zws GitHub Container Registry
        uses: ./.github/actions/docker-login
        with:
          registry: ${{ secrets.HARBOR_URL }}
          username: ${{ secrets.HARBOR_READ_LOGIN }}
          password: ${{ secrets.HARBOR_READ_TOKEN }}

      - name: Validate perf deployment selection
        run: python3 ci/scripts/perf_runner.py validate

      # VERIFY IMAGE TAGS EXIST IN REGISTRY
      # Fail fast if requested tags are missing, instead of waiting ~40min for
      # helm pre-install hooks to time out on ImagePullBackOff. Retry briefly
      # because freshly-pushed GHCR images can take a moment to appear through
      # the Harbor pull-through path used by the cluster.
      # ======================================================================
      - name: Verify image tags exist in registry
        env:
          IMAGE_REPO: hub.zama.org/ghcr/zama-ai/kms
        run: python3 ci/scripts/perf_runner.py verify-images

      # ======================================================================
      # TOOLING SETUP
      # ======================================================================
      - name: Setup tailscale
        uses: tailscale/github-action@84a3f23bb4d843bcf4da6cf824ec1be473daf4de # v3.2.3
        with:
          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
          tags: tag:kms-ci

      - name: Setup helm
        run: |
          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
          export PATH=$PATH:/usr/local/bin

      - name: Setup argo workflow CLI
        env:
          ARGO_OS: '${{ runner.os }}'
          ARGO_VERSION: 'v3.7.2'
        run: |
          # Detect OS
          if [[ "$(uname -s)" != "Darwin" ]]; then
            ARGO_OS="linux"
          fi
          # Download the binary
          curl -sLO "https://github.com/argoproj/argo-workflows/releases/download/${ARGO_VERSION}/argo-${ARGO_OS}-amd64.gz"
          # Unzip
          gunzip "argo-${ARGO_OS}-amd64.gz"
          # Make binary executable
          chmod +x "argo-${ARGO_OS}-amd64"
          # Move binary to path
          mv "./argo-${ARGO_OS}-amd64" /usr/local/bin/argo
          # Test installation
          argo version

      - name: setup kubectl
        uses: azure/setup-kubectl@776406bce94f63e41d621b960d78ee25c8b76ede

      # ======================================================================
      # KUBERNETES & HELM SETUP
      # ======================================================================
      - name: Setup kubeconfig
        run: |
          ###################################################################
          # Configure kubeconfig to connect to the Tailscale Kubernetes cluster
          ###################################################################
          echo "Configuring kubeconfig for Tailscale cluster..."
          tailscale configure kubeconfig tailscale-operator-zws-dev.diplodocus-boa.ts.net

      - name: Checkout Project KMS
        if: ${{ env.KMS_CHART_VERSION == 'repository' }}
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          ref: "${{ env.KMS_BRANCH }}"
          token: ${{ secrets.BLOCKCHAIN_ACTIONS_TOKEN }}
          persist-credentials: false

      - name: Set kubeconfig
        run: |
          ###################################################################
          # Verify and set the Kubernetes context
          ###################################################################
          echo "Available Kubernetes contexts:"
          kubectl config get-contexts
          echo "Setting context to Tailscale cluster..."
          kubectl config use-context tailscale-operator-zws-dev.diplodocus-boa.ts.net

      # ======================================================================
      # DEPLOYMENT
      # ======================================================================
      - name: Deploy KMS using unified script
        env:
          JQ_VERSION: 1.8.1
        run: |
          # Download JQ.
          wget https://github.com/jqlang/jq/releases/download/jq-"${JQ_VERSION}"/jq-linux-amd64 -P /tmp

          # Install JQ.
          sudo mv /tmp/jq-linux-amd64 /usr/local/bin/jq
          sudo chmod +x /usr/local/bin/jq
          echo "JQ version: $(jq --version)"

          METRICS_FLAG=()
          if [[ "${ENABLE_KMS_METRICS}" == "true" ]]; then
            METRICS_FLAG=(--enable-metrics)
          fi

          chmod +x ci/scripts/deploy.sh
          ./ci/scripts/deploy.sh \
            --target aws-perf \
            --namespace "${NAMESPACE}" \
            --deployment-type "${DEPLOYMENT_TYPE}" \
            --core-tag "${KMS_CORE_IMAGE_TAG}" \
            --client-tag "${KMS_CORE_CLIENT_IMAGE_TAG}" \
            --num-parties 13 \
            --kms-chart-version "${KMS_CHART_VERSION}" \
            --tkms-infra-version "${TKMS_INFRA_CHART_VERSION}" \
            "${METRICS_FLAG[@]}"

      - name: Capture network diagnostics before perf
        if: always()
        run: |
          python3 "${PERF_TOOLS_DIR:-ci/scripts}/collect_network_diagnostics.py" before-perf "${NAMESPACE}"

      # ======================================================================
      # PERFORMANCE TESTING
      # ======================================================================
      - name: Run performance testing
        env:
          SERVER_URL: ${{ github.server_url }}
          REPOSITORY: ${{ github.repository }}
          RUN_ID: ${{ github.run_id }}
          GH_TOKEN: ${{ github.token }}
        run: python3 "${PERF_TOOLS_DIR:-ci/scripts}/perf_runner.py" run

      - name: Report core CPU samples
        if: always()
        run: python3 "${PERF_TOOLS_DIR:-ci/scripts}/perf_runner.py" report-cpu

      - name: Capture network diagnostics after perf
        if: always()
        run: |
          python3 "${PERF_TOOLS_DIR:-ci/scripts}/collect_network_diagnostics.py" after-perf "${NAMESPACE}"

      - name: Report perf diagnostics
        if: always()
        run: python3 "${PERF_TOOLS_DIR:-ci/scripts}/perf_runner.py" report-diagnostics

      # ======================================================================
      # SAVE ARGO WORKFLOW LOGS
      # ======================================================================
      - name: Upload Argo workflow logs
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: argo-workflow-logs.txt
          path: argo-workflow-logs.txt
          retention-days: 30

      - name: Upload core CPU samples
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: core-cpu-samples.log
          path: core-cpu-samples.log
          retention-days: 30

      - name: Upload perf diagnostics
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: perf-diagnostics
          path: |
            perf-diagnostics/
            perf-diagnostics-controller.log
          retention-days: 30
          if-no-files-found: warn

      # ======================================================================
      # LOG COLLECTION
      # ======================================================================
      - name: Get logs from kms-core
        if: always()
        run: python3 "${PERF_TOOLS_DIR:-ci/scripts}/perf_runner.py" core-logs

      - name: Upload kms-core logs
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: kms-core-logs
          path: kms-core-*.txt
          retention-days: 30 # Keep logs for 30 days
          if-no-files-found: warn # Only warn if no log files are found

      # ======================================================================
      # CLEANUP
      # ======================================================================
      - name: Cleanup
        if: always()
        run: python3 "${PERF_TOOLS_DIR:-ci/scripts}/perf_runner.py" cleanup