Add fly:bats rake task for running BATs from a local branch

ci/fly-bats.yml

@@ -0,0 +1,116 @@
+---
+# Standalone pipeline for running the BOSH Acceptance Test suite (BATs)
+# against a specific branch.
+#
+# Intended to be driven by `bundle exec rake fly:bats` from src/, which
+# pushes the current branch and sets this pipeline automatically.
+#
+# Manual usage:
+#   fly -t bosh set-pipeline -p bats-local \
+#       -c ci/fly-bats.yml \
+#       --var bosh_repo=https://github.com/cloudfoundry/bosh.git \
+#       --var bosh_branch=my-feature-branch \
+#       --var env_name=bats-local \
+#       --var stemcell_name=bosh-google-kvm-ubuntu-noble \
+#       --var deploy_args="-o bosh-deployment/external-ip-not-recommended.yml" \
+#       --var bat_rspec_flags=""
+#   fly -t bosh unpause-pipeline -p bats-local
+#   fly -t bosh trigger-job -j bats-local/bats -w
+
+resources:
+  - name: bosh
+    type: git
+    source:
+      uri: ((bosh_repo))
+      branch: ((bosh_branch))
+
+  # bosh-ci is the same repo as bosh but filtered to ci/ paths so that
+  # scripts under ci/bats/ are available at bosh-ci/ in the task workspace.
+  - name: bosh-ci
+    type: git
+    source:
+      uri: ((bosh_repo))
+      branch: ((bosh_branch))
+      paths: [ci]
+
+  - name: bosh-cli
+    type: github-release
+    source:
+      owner: cloudfoundry
+      repository: bosh-cli
+      access_token: ((github_public_repo_token))
+
+  - name: stemcell
+    type: bosh-io-stemcell
+    source:
+      name: ((stemcell_name))
+
+  - name: bats
+    type: git
+    source:
+      uri: https://github.com/cloudfoundry/bosh-acceptance-tests.git
+      branch: master
+
+  - name: bosh-deployment
+    type: git
+    source:
+      uri: https://github.com/cloudfoundry/bosh-deployment
+      branch: master
+
+  - name: integration-image
+    type: registry-image
+    source:
+      repository: ghcr.io/cloudfoundry/bosh/integration
+      tag: main
+      username: ((github_read_write_packages.username))
+      password: ((github_read_write_packages.password))
+
+jobs:
+  - name: bats
+    serial: true
+    plan:
+      - do:
+          - in_parallel:
+              - get: bosh
+              - get: bosh-ci
+              - get: bosh-cli
+                params:
+                  globs: [bosh-cli-*-linux-amd64]
+              - get: stemcell
+              - get: bats
+              - get: bosh-deployment
+              - get: integration-image
+
+          - task: make-candidate
+            image: integration-image
+            file: bosh-ci/ci/tasks/make-candidate.yml
+
+          - task: compile-bosh-release
+            file: bosh-deployment/ci/tasks/shared/bosh-agent-compile.yml
+
+          - task: run-bats
+            image: integration-image
+            input_mapping:
+              bosh-release: compiled-release
+            config:
+              platform: linux
+              inputs:
+                - name: bosh
+                - name: bosh-ci
+                - name: bosh-cli
+                - name: bosh-deployment
+                - name: stemcell
+                - name: bats
+                - name: bosh-release
+              caches:
+                - path: cache-dot-bosh-dir
+              params:
+                BAT_INFRASTRUCTURE: gcp
+                GCP_JSON_KEY: ((gcp_json_key))
+                GCP_PROJECT_ID: ((gcp_project_id))
+                STEMCELL_NAME: ((stemcell_name))
+                ENV_NAME: ((env_name))
+                DEPLOY_ARGS: ((deploy_args))
+                BAT_RSPEC_FLAGS: ((bat_rspec_flags))
+              run:
+                path: bosh/ci/tasks/run-bats-pipeline.sh

ci/tasks/run-bats-pipeline.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+# run-bats-pipeline.sh
+#
+# Chains together the full BATs pipeline in a single Concourse task:
+#   terraform apply  → deploy-director → prepare-bats-config → run-bats
+#   terraform destroy ← destroy-director  ←  (EXIT trap, always runs)
+#
+# Required env vars (set via run-bats-pipeline.yml params):
+#   GCP_JSON_KEY      – GCP service account JSON (resolved from Concourse creds)
+#   GCP_PROJECT_ID    – GCP project ID
+#   ENV_NAME          – Unique terraform env name, e.g. "bats-local"
+#   BAT_INFRASTRUCTURE – "gcp"
+#   STEMCELL_NAME     – Stemcell name for bats-config.yml
+#   DEPLOY_ARGS       – Extra ops-file args for bosh create-env
+#   BAT_RSPEC_FLAGS   – Extra flags appended to the BAT run (optional)
+
+set -euo pipefail
+
+ROOT_DIR="$PWD"
+TERRAFORM_DIR="${ROOT_DIR}/bosh-ci/ci/bats/iaas/gcp/terraform"
+TERRAFORM_VERSION="1.9.8"
+
+# ── Shared working directories ───────────────────────────────────────────────
+mkdir -p director-state bats-config environment
+# cache-dot-bosh-dir is provided as a Concourse cache volume; create if absent
+mkdir -p cache-dot-bosh-dir/.bosh
+
+# prepare-bats-config.sh expects its terraform metadata at terraform/metadata
+# but deploy-director.sh expects it at environment/metadata.
+# Symlink terraform/ → environment/ so both scripts find what they need.
+ln -sf "${ROOT_DIR}/environment" "${ROOT_DIR}/terraform"
+
+# ── Install terraform ────────────────────────────────────────────────────────
+if ! command -v terraform &>/dev/null; then
+  echo "--- Installing terraform ${TERRAFORM_VERSION} ---"
+  TMP_TF_DIR="$(mktemp -d /tmp/terraform-install-XXXXXX)"
+  curl -sSL \
+    "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" \
+    -o "${TMP_TF_DIR}/terraform.zip"
+  curl -sSL \
+    "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" \
+    -o "${TMP_TF_DIR}/SHA256SUMS"
+  if command -v sha256sum &>/dev/null; then
+    (cd "${TMP_TF_DIR}" && grep "terraform_${TERRAFORM_VERSION}_linux_amd64.zip" SHA256SUMS | sha256sum -c -)
+  elif command -v shasum &>/dev/null; then
+    (cd "${TMP_TF_DIR}" && grep "terraform_${TERRAFORM_VERSION}_linux_amd64.zip" SHA256SUMS | shasum -a 256 -c -)
+  fi
+  unzip -qo "${TMP_TF_DIR}/terraform.zip" -d /usr/local/bin terraform
+  chmod +x /usr/local/bin/terraform
+  rm -rf "${TMP_TF_DIR}"
+fi
+
+# ── GCP credentials file (used by the GCS backend and the Google provider) ──
+GCP_CREDS_FILE="$(mktemp /tmp/gcp-creds-XXXXXX.json)"
+echo "${GCP_JSON_KEY}" > "${GCP_CREDS_FILE}"
+chmod 600 "${GCP_CREDS_FILE}"
+
+# ── Teardown trap (always runs on EXIT) ─────────────────────────────────────
+function collect_director_diagnostics {
+  # Only collect when we have a deployed director and bosh-cli is available.
+  [[ -f director-state/director-creds.yml ]] || return 0
+  [[ -f "${ROOT_DIR}/environment/metadata" ]] || return 0
+  command -v bosh-cli &>/dev/null || return 0
+
+  local director_ip
+  director_ip="$(jq -r '.director_public_ip // empty' "${ROOT_DIR}/environment/metadata")"
+  [[ -n "${director_ip}" ]] || return 0
+
+  echo "--- Collecting director diagnostics (IP: ${director_ip}) ---"
+
+  # Extract jumpbox SSH private key from the vars-store.
+  local jumpbox_key_file
+  jumpbox_key_file="$(mktemp /tmp/jumpbox-key-XXXXXX)"
+  bosh-cli interpolate director-state/director-creds.yml \
+    --path /jumpbox_ssh/private_key > "${jumpbox_key_file}" 2>/dev/null || { rm -f "${jumpbox_key_file}"; return 0; }
+  chmod 600 "${jumpbox_key_file}"
+
+  ssh -o StrictHostKeyChecking=no \
+      -o UserKnownHostsFile=/dev/null \
+      -o ConnectTimeout=10 \
+      -i "${jumpbox_key_file}" \
+      "jumpbox@${director_ip}" \
+    'echo "=== monit status ===" && sudo /var/vcap/bosh/bin/monit status;
+     echo "=== bosh_nats_sync log (last 100 lines) ===" && sudo tail -100 /var/vcap/sys/log/nats/bosh-nats-sync.log 2>/dev/null || true;
+     echo "=== bosh_nats_sync bpm stdout ===" && sudo cat /var/vcap/sys/log/bpm/nats/bosh_nats_sync.stdout.log 2>/dev/null || true;
+     echo "=== bosh_nats_sync bpm stderr ===" && sudo cat /var/vcap/sys/log/bpm/nats/bosh_nats_sync.stderr.log 2>/dev/null || true;
+     echo "=== nats log (last 50 lines) ===" && sudo tail -50 /var/vcap/sys/log/nats/nats.log 2>/dev/null || true;
+     echo "=== nats bpm stdout (last 50 lines) ===" && sudo tail -50 /var/vcap/sys/log/bpm/nats/nats.stdout.log 2>/dev/null || true;
+     echo "=== health_monitor log (last 100 lines) ===" && sudo tail -100 /var/vcap/sys/log/health_monitor/health_monitor.log 2>/dev/null || true;
+     echo "=== health_monitor bpm stdout ===" && sudo tail -50 /var/vcap/sys/log/bpm/health_monitor/health_monitor.stdout.log 2>/dev/null || true;
+     echo "=== health_monitor bpm stderr ===" && sudo tail -50 /var/vcap/sys/log/bpm/health_monitor/health_monitor.stderr.log 2>/dev/null || true' \
+    2>&1 || echo "(SSH diagnostics failed — VM may not be reachable)"
+
+  rm -f "${jumpbox_key_file}"
+}
+
+function teardown {
+  local exit_code=$?
+  set +e
+
+  # Always collect diagnostics – on success this helps correlate logs with
+  # passing runs; on failure it captures the state at the point of failure.
+  collect_director_diagnostics
+
+  echo "--- Tearing down BOSH director ---"
+  if [[ -f director-state/director-state.json ]]; then
+    # destroy-director.sh expects bosh-cli/bosh-cli-* to exist; restore it
+    # because deploy-director.sh already moved the original binary away.
+    cp /usr/local/bin/bosh-cli bosh-cli/bosh-cli-restore 2>/dev/null || true
+    bosh-ci/ci/bats/tasks/destroy-director.sh || true
+  fi
+
+  if [[ -n "${TERRAFORM_DIR:-}" ]] && [[ -d "${TERRAFORM_DIR}" ]]; then
+    echo "--- Destroying GCP environment (env: ${ENV_NAME}) ---"
+    pushd "${TERRAFORM_DIR}" >/dev/null
+    terraform destroy \
+      -input=false \
+      -auto-approve \
+      -var "project_id=${GCP_PROJECT_ID}" \
+      -var "gcp_credentials_json=${GCP_JSON_KEY}" \
+      -var "name=${ENV_NAME}" || true
+    popd >/dev/null
+  fi
+
+  if [[ -n "${GCP_CREDS_FILE:-}" ]]; then
+    rm -f "${GCP_CREDS_FILE}"
+  fi
+
+  exit "${exit_code}"
+}
+trap teardown EXIT
+
+# ── Provision GCP environment via terraform ──────────────────────────────────
+echo "--- Provisioning GCP environment (env: ${ENV_NAME}) ---"
+pushd "${TERRAFORM_DIR}" >/dev/null
+
+terraform init \
+  -input=false \
+  -reconfigure \
+  -backend-config="bucket=bosh-director-pipeline" \
+  -backend-config="prefix=bats-terraform/${ENV_NAME}" \
+  -backend-config="credentials=${GCP_CREDS_FILE}"
+
+terraform apply \
+  -input=false \
+  -auto-approve \
+  -var "project_id=${GCP_PROJECT_ID}" \
+  -var "gcp_credentials_json=${GCP_JSON_KEY}" \
+  -var "name=${ENV_NAME}"
+
+# Convert terraform outputs to the flat metadata JSON consumed by director-vars
+# and prepare-bats-config.sh.
+terraform output -json \
+  | jq 'with_entries(.value = .value.value)' \
+  > "${ROOT_DIR}/environment/metadata"
+
+popd >/dev/null
+
+# ── Deploy BOSH director ─────────────────────────────────────────────────────
+echo "--- Deploying BOSH director ---"
+# deploy-director.sh moves bosh-cli/bosh-cli-* to /usr/local/bin/bosh-cli.
+# After this call bosh-cli is installed system-wide as 'bosh-cli'.
+bosh-ci/ci/bats/tasks/deploy-director.sh
+
+# ── Prepare BATs config ──────────────────────────────────────────────────────
+echo "--- Preparing BATs config ---"
+bosh-ci/ci/bats/iaas/gcp/prepare-bats-config.sh
+
+# ── Run BATs ─────────────────────────────────────────────────────────────────
+echo "--- Running BATs ---"
+# Preserve user-supplied extra flags before sourcing the generated defaults.
+extra_bat_rspec_flags="${BAT_RSPEC_FLAGS:-}"
+
+# Source the environment file that prepare-bats-config.sh wrote; this exports
+# BOSH_ENVIRONMENT, BOSH_CLIENT, BOSH_CLIENT_SECRET, BOSH_CA_CERT,
+# BOSH_ALL_PROXY, and the default BAT_RSPEC_FLAGS.
+# shellcheck source=/dev/null
+source bats-config/bats.env
+
+# Allow the caller to append extra RSpec flags (e.g. "--tag wip").
+if [[ -n "${extra_bat_rspec_flags}" ]]; then
+  export BAT_RSPEC_FLAGS="${BAT_RSPEC_FLAGS:+${BAT_RSPEC_FLAGS} }${extra_bat_rspec_flags}"
+fi
+
+bats/ci/tasks/run-bats.sh

src/tasks/fly.rake

@@ -1,3 +1,5 @@
+require 'shellwords'
+
 namespace :fly do
   desc 'Fly unit specs'
   task :unit do
@@ -9,6 +11,57 @@ namespace :fly do
             COVERAGE: ENV.fetch('COVERAGE', false))
   end
 
+  desc 'Run BATs (BOSH Acceptance Tests) against the current branch via Concourse'
+  #
+  # Sets the ci/fly-bats.yml pipeline on Concourse, then triggers and watches
+  # the bats job.  The current branch is pushed to origin automatically so
+  # Concourse can fetch it.
+  #
+  # GCP credentials are resolved from the Concourse credential store
+  # ((gcp_json_key)) and ((gcp_project_id)).
+  #
+  # Useful env vars:
+  #   BATS_ENV_NAME    – terraform env name, must be unique per concurrent run
+  #                      (default: "bats-local")
+  #   STEMCELL_NAME    – GCP stemcell name override
+  #   DEPLOY_ARGS      – extra ops-files passed to bosh create-env
+  #   BAT_RSPEC_FLAGS  – extra flags appended to the RSpec BATs run
+  task :bats do
+    env_name = ENV.fetch('BATS_ENV_NAME', 'bats-local')
+
+    branch = `git -C .. rev-parse --abbrev-ref HEAD`.strip
+    if branch == 'HEAD' || branch.empty?
+      raise 'fly:bats must be run from a git branch (not in detached HEAD state).'
+    end
+
+    repo   = `git -C .. remote get-url origin`.strip
+                .sub(/\Agit@github\.com:/, 'https://github.com/')
+                .sub(/\.git\z/, '.git')
+
+    # Push the current branch so Concourse can check it out.
+    sh "git -C .. push origin HEAD"
+
+    # ── Set the pipeline ─────────────────────────────────────────────────────
+    sh [
+      "fly #{concourse_target}",
+      'set-pipeline',
+      '--non-interactive',
+      "--pipeline #{Shellwords.escape(env_name)}",
+      '--config ../ci/fly-bats.yml',
+      "--var bosh_repo=#{Shellwords.escape(repo)}",
+      "--var bosh_branch=#{Shellwords.escape(branch)}",
+      "--var env_name=#{Shellwords.escape(env_name)}",
+      "--var stemcell_name=#{Shellwords.escape(ENV.fetch('STEMCELL_NAME', 'bosh-google-kvm-ubuntu-noble'))}",
+      "--var deploy_args=#{Shellwords.escape(ENV.fetch('DEPLOY_ARGS', '-o bosh-deployment/external-ip-not-recommended.yml'))}",
+      "--var bat_rspec_flags=#{Shellwords.escape(ENV.fetch('BAT_RSPEC_FLAGS', ''))}",
+    ].compact.join(' ')
+
+    sh "fly #{concourse_target} unpause-pipeline --pipeline #{Shellwords.escape(env_name)}"
+
+    # ── Trigger and stream the job output ────────────────────────────────────
+    sh "fly #{concourse_target} trigger-job --job #{Shellwords.escape(env_name)}/bats --watch"
+  end
+
   desc 'Fly integration specs'
   task :integration, [:cli_dir] do |_, args|
     db, db_version = fetch_db_and_version('postgresql')