test: fix cloud functional tests (#585)

* test: skip missing cloud functional items

* ci: drop cloud functional soft fail

* test: read serverless list items for ids

* ci: ensure cloud qa fixtures exist

* ci: create hosted deployment fixture

* test: skip upgrade assistant before version

Author: margaretjgu

.buildkite/ensure-cloud-fixtures.sh

@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+# Copyright Elasticsearch B.V. and contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Idempotent QA fixtures so Cloud item GET tests have something to fetch.
+# Creates one serverless project per type, one serverless traffic filter,
+# one hosted traffic-filter ruleset, and one hosted deployment when the
+# matching list is empty. Skips extensions (need a plugin zip). Does not
+# print create responses (they can contain creds).
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+ELASTIC=(node "$REPO_ROOT/dist/cli.js" --json)
+REGION="${CLOUD_FIXTURE_REGION:-gcp-us-central1}"
+RULES='[{"source":"192.0.2.1"}]'
+
+first_id () {
+  jq -r '
+    if type == "array" then .[0].id // empty
+    else
+      .items[0].id
+      // .projects[0].id
+      // .filters[0].id
+      // .rulesets[0].id
+      // .deployments[0].id
+      // empty
+    end
+  '
+}
+
+ensure () {
+  local label="$1"
+  shift
+  local list_args=()
+  while [ "$1" != "--" ]; do
+    list_args+=("$1")
+    shift
+  done
+  shift
+  local id
+  id="$("${ELASTIC[@]}" "${list_args[@]}" | first_id)"
+  if [ -n "$id" ] && [ "$id" != "null" ]; then
+    echo "fixture exists: $label $id"
+    return 0
+  fi
+  echo "creating fixture: $label"
+  # stdout discarded so create responses never land in CI logs
+  "${ELASTIC[@]}" "$@" >/dev/null
+}
+
+ensure "search project" \
+  cloud serverless projects search list -- \
+  cloud serverless projects search create \
+  --name cli-functional-search \
+  --region-id "$REGION" \
+  --optimized-for general_purpose \
+  --yes
+
+ensure "observability project" \
+  cloud serverless projects observability list -- \
+  cloud serverless projects observability create \
+  --name cli-functional-o11y \
+  --region-id "$REGION" \
+  --product-tier logs_essentials \
+  --yes
+
+ensure "security project" \
+  cloud serverless projects security list -- \
+  cloud serverless projects security create \
+  --name cli-functional-security \
+  --region-id "$REGION" \
+  --yes
+
+ensure "serverless traffic filter" \
+  cloud serverless traffic-filters list-traffic-filters -- \
+  cloud serverless traffic-filters create-traffic-filter \
+  --name cli-functional-filter \
+  --type ip \
+  --region "$REGION" \
+  --rules "$RULES" \
+  --yes
+
+ensure "hosted traffic filter ruleset" \
+  cloud hosted traffic-filters get-traffic-filter-rulesets -- \
+  cloud hosted traffic-filters create-traffic-filter-ruleset \
+  --name cli-functional-ruleset \
+  --type ip \
+  --include-by-default false \
+  --region "$REGION" \
+  --rules "$RULES"
+
+ensure "hosted deployment" \
+  cloud hosted deployments list-deployments -- \
+  cloud hosted deployments create-deployment \
+  --name cli-functional \
+  --region "$REGION" \
+  --template-id gcp-general-purpose

.buildkite/pipeline.yml

@@ -65,4 +65,3 @@ steps:
         command: ".buildkite/run-cloud-tests.sh"
         artifact_paths:
           - "test/functional/cloud/*.log"
-        soft_fail: true

.buildkite/run-cloud-tests.sh

@@ -35,6 +35,9 @@ npm run build
 echo "--- Setting up Cloud credentials"
 source .buildkite/setup-env.sh
 
+echo "--- Ensuring Cloud fixtures"
+.buildkite/ensure-cloud-fixtures.sh
+
 echo "--- Generating Cloud functional tests"
 npm run codegen:functional:cloud
 

codegen/functional/cloud.ts

@@ -67,7 +67,9 @@ for (const file of yamlFiles) {
 
   const result = generateScript(testFile, apis, {
     clientArgs: ['cloud'],
-    preamble: CLOUD_PREAMBLE
+    preamble: CLOUD_PREAMBLE,
+    skipEmptySet: true,
+    skipNotFound: true
   })
 
   for (const action of result.skippedActions) allSkippedActions.add(action)

codegen/functional/generator.ts

@@ -46,6 +46,18 @@ export interface GenerateOptions {
   clientArgs?: string[]
   /** Preamble lines defining the `$ELASTIC` invocation and `$RESPONSE` (default: the `elastic` binary). */
   preamble?: string[]
+  /**
+   * If a `set` extraction is empty or `null`, try `.[0].id` (bare-array list
+   * responses) and skip the script when still empty. Used by Cloud tests
+   * against orgs that have no deployments or projects.
+   */
+  skipEmptySet?: boolean
+  /**
+   * If a do-step exits non-zero and the JSON error mentions 404 (or a
+   * hosted deployment with no version yet), skip the script. Used by Cloud
+   * tests for endpoints the org or public QA API does not expose.
+   */
+  skipNotFound?: boolean
 }
 
 const DEFAULT_PREAMBLE = ['exec < /dev/null', 'ELASTIC="elastic --json"', 'RESPONSE=""']
@@ -57,6 +69,8 @@ export function generateScript (
 ): GenerateResult {
   const clientArgs = opts.clientArgs ?? ['stack', 'es']
   const preamble = opts.preamble ?? DEFAULT_PREAMBLE
+  const skipEmptySet = opts.skipEmptySet === true
+  const skipNotFound = opts.skipNotFound === true
   const actionMap = buildActionMap(definitions)
   const skippedActions: string[] = []
   const lines: string[] = []
@@ -70,7 +84,7 @@ export function generateScript (
 
   if (testFile.teardown.length > 0) {
     const teardownBody: string[] = []
-    renderSteps(testFile.teardown, actionMap, clientArgs, teardownBody, skippedActions, '  ')
+    renderSteps(testFile.teardown, actionMap, clientArgs, teardownBody, skippedActions, '  ', false, skipEmptySet, skipNotFound)
     if (!hasExecutableLine(teardownBody)) {
       teardownBody.push('  :')
     }
@@ -98,13 +112,13 @@ export function generateScript (
 
   if (testFile.setup.length > 0) {
     lines.push('# --- Setup ---')
-    hadSkippedDo = renderSteps(testFile.setup, actionMap, clientArgs, lines, skippedActions, '')
+    hadSkippedDo = renderSteps(testFile.setup, actionMap, clientArgs, lines, skippedActions, '', false, skipEmptySet, skipNotFound)
     lines.push('')
   }
 
   for (const section of testFile.tests) {
     lines.push(`# --- Test: ${section.name} ---`)
-    renderSteps(section.steps, actionMap, clientArgs, lines, skippedActions, '', hadSkippedDo)
+    renderSteps(section.steps, actionMap, clientArgs, lines, skippedActions, '', hadSkippedDo, skipEmptySet, skipNotFound)
     lines.push('')
   }
 
@@ -255,7 +269,9 @@ function renderSteps (
   lines: string[],
   skippedActions: string[],
   indent: string,
-  initialHadSkippedDo = false
+  initialHadSkippedDo = false,
+  skipEmptySet = false,
+  skipNotFound = false
 ): boolean {
   // Assertions and set-steps read $RESPONSE, which is written by the most
   // recent successful `do`. If the last `do` was skipped (unmapped action,
@@ -279,7 +295,7 @@ function renderSteps (
       }
       // Only pass allowFailure from the setup→test propagation, not from
       // prior skipped steps within the same section (those are unrelated).
-      const result = renderDo(step, actionMap, clientArgs, lines, skippedActions, indent, initialHadSkippedDo)
+      const result = renderDo(step, actionMap, clientArgs, lines, skippedActions, indent, initialHadSkippedDo, skipNotFound)
       if (result === 'skipped') {
         responseFromLastDo = false
         hadSkippedDo = true
@@ -310,7 +326,7 @@ function renderSteps (
 
     switch (step.kind) {
       case 'set':
-        renderSet(step, lines, indent)
+        renderSet(step, lines, indent, skipEmptySet)
         // If a variable was previously unset, it's now set
         for (const varName of Object.values(step.assignments)) {
           unsetVars.delete(varName.toUpperCase().replace(/[^A-Z0-9]/g, '_'))
@@ -360,7 +376,8 @@ function renderDo (
   lines: string[],
   skippedActions: string[],
   indent: string,
-  allowFailure = false
+  allowFailure = false,
+  skipNotFound = false
 ): 'executed' | 'optional' | 'skipped' {
   if (step.catch != null) {
     lines.push(`${indent}# SKIPPED: catch not supported in MVP (catch: ${step.catch})`)
@@ -384,9 +401,25 @@ function renderDo (
   if (optional) {
     lines.push(`${indent}RESPONSE=$(${cmd}) || true`)
     return 'optional'
-  } else {
-    lines.push(`${indent}RESPONSE=$(${cmd})`)
   }
+  if (skipNotFound) {
+    // Handler errors go to stderr (`writeErr`); keep them out of $RESPONSE.
+    const errFile = '"${TMPDIR:-/tmp}/elastic-cli-do-err.$$"'
+    lines.push(`${indent}set +e`)
+    lines.push(`${indent}RESPONSE=$(${cmd} 2>${errFile})`)
+    lines.push(`${indent}_ec=$?`)
+    lines.push(`${indent}set -e`)
+    lines.push(`${indent}if [ "$_ec" -ne 0 ]; then`)
+    lines.push(`${indent}  if jq -e '(.error.message | tostring | test("404|Could not determine the version"))' ${errFile} >/dev/null 2>&1; then`)
+    lines.push(`${indent}    echo "SKIP: ${step.action} not available"`)
+    lines.push(`${indent}    exit 0`)
+    lines.push(`${indent}  fi`)
+    lines.push(`${indent}  cat ${errFile} >&2`)
+    lines.push(`${indent}  exit $_ec`)
+    lines.push(`${indent}fi`)
+    return 'executed'
+  }
+  lines.push(`${indent}RESPONSE=$(${cmd})`)
   return 'executed'
 }
 
@@ -510,11 +543,26 @@ function buildCommand (mapped: MappedAction, step: DoStep): string {
   return base
 }
 
-function renderSet (step: SetStep, lines: string[], indent: string): void {
+function renderSet (step: SetStep, lines: string[], indent: string, skipEmptySet = false): void {
   for (const [responsePath, varName] of Object.entries(step.assignments)) {
     const bashVar = varName.toUpperCase().replace(/[^A-Z0-9]/g, '_')
     const jqPath = toJqPath(responsePath)
-    lines.push(`${indent}${bashVar}=$(echo "$RESPONSE" | jq -r '${jqPath}')`)
+    if (skipEmptySet) {
+      // `try` so a nested path like `.regions[0].id` against a bare array
+      // yields empty instead of aborting the script (jq cannot index an
+      // array with a string).
+      lines.push(`${indent}${bashVar}=$(echo "$RESPONSE" | jq -r 'try (${jqPath} // empty) catch empty')`)
+      lines.push(`${indent}if [ -z "$${bashVar}" ] || [ "$${bashVar}" = "null" ]; then`)
+      // Serverless lists wrap items in `.items`; regions return a bare array.
+      lines.push(`${indent}  ${bashVar}=$(echo "$RESPONSE" | jq -r 'if type=="array" then .[0].id // empty else .items[0].id // empty end')`)
+      lines.push(`${indent}fi`)
+      lines.push(`${indent}if [ -z "$${bashVar}" ] || [ "$${bashVar}" = "null" ]; then`)
+      lines.push(`${indent}  echo "SKIP: no ${varName} in list response"`)
+      lines.push(`${indent}  exit 0`)
+      lines.push(`${indent}fi`)
+    } else {
+      lines.push(`${indent}${bashVar}=$(echo "$RESPONSE" | jq -r '${jqPath}')`)
+    }
   }
 }
 

codegen/functional/test/generator.test.ts

@@ -5,6 +5,7 @@
 
 import { describe, it } from 'node:test'
 import assert from 'node:assert/strict'
+import { execFileSync } from 'node:child_process'
 import { readFileSync } from 'node:fs'
 import { join } from 'node:path'
 import { parseTestFile } from '../parser.ts'
@@ -226,6 +227,87 @@ describe('generateScript', () => {
     assert.ok(result.script.includes('echo "PASS: get.yml"'))
   })
 
+  it('skipEmptySet retries bare array id then skips when empty', () => {
+    const testFile: TestFile = {
+      sourceFile: 'list.yml',
+      requires: { serverless: true, stack: true },
+      setup: [],
+      teardown: [],
+      tests: [{
+        name: 'get item',
+        steps: [
+          { kind: 'do', action: 'count', params: { index: 'x' }, body: undefined },
+          { kind: 'set', assignments: { 'regions.0.id': 'id' } }
+        ]
+      }]
+    }
+    const result = generateScript(testFile, testDefs, { skipEmptySet: true })
+    assert.ok(result.script.includes('try (.regions[0].id // empty) catch empty'))
+    assert.ok(result.script.includes('if type=="array" then .[0].id // empty else .items[0].id // empty end'))
+    assert.ok(result.script.includes('SKIP: no id in list response'))
+    assert.ok(result.script.includes('exit 0'))
+    assert.equal(
+      execFileSync('jq', ['-r', 'try (.regions[0].id // empty) catch empty'], {
+        input: '[{"id":"r1"}]',
+        encoding: 'utf8'
+      }).trim(),
+      ''
+    )
+    const fallback = 'if type=="array" then .[0].id // empty else .items[0].id // empty end'
+    assert.equal(
+      execFileSync('jq', ['-r', fallback], { input: '[{"id":"r1"}]', encoding: 'utf8' }).trim(),
+      'r1'
+    )
+    assert.equal(
+      execFileSync('jq', ['-r', fallback], { input: '{"items":[{"id":"p1"}]}', encoding: 'utf8' }).trim(),
+      'p1'
+    )
+    assert.equal(
+      execFileSync('jq', ['-r', 'try (.deployments[0].id // empty) catch empty'], {
+        input: '{"deployments":[]}',
+        encoding: 'utf8'
+      }).trim(),
+      ''
+    )
+  })
+
+  it('does not skip empty set extractions by default', () => {
+    const testFile: TestFile = {
+      sourceFile: 'list.yml',
+      requires: { serverless: true, stack: true },
+      setup: [],
+      teardown: [],
+      tests: [{
+        name: 'get item',
+        steps: [
+          { kind: 'do', action: 'count', params: { index: 'x' }, body: undefined },
+          { kind: 'set', assignments: { 'regions.0.id': 'id' } }
+        ]
+      }]
+    }
+    const result = generateScript(testFile, testDefs)
+    assert.equal(result.script.includes('SKIP: no id in list response'), false)
+    assert.equal(result.script.includes('.[0].id // empty'), false)
+  })
+
+  it('skipNotFound wraps do-steps to skip 404 errors', () => {
+    const content = readFileSync(join(fixturesDir, 'get.yml'), 'utf-8')
+    const testFile = parseTestFile(content, 'get.yml')
+    const result = generateScript(testFile, testDefs, { skipNotFound: true })
+    assert.ok(result.script.includes('set +e'))
+    assert.ok(result.script.includes('elastic-cli-do-err.$$'))
+    assert.ok(result.script.includes('not available'))
+    assert.ok(result.script.includes('Could not determine the version'))
+  })
+
+  it('does not wrap do-steps for 404 by default', () => {
+    const content = readFileSync(join(fixturesDir, 'get.yml'), 'utf-8')
+    const testFile = parseTestFile(content, 'get.yml')
+    const result = generateScript(testFile, testDefs)
+    assert.equal(result.script.includes('not available'), false)
+    assert.equal(result.script.includes('elastic-cli-do-err.$$'), false)
+  })
+
   it('tracks skipped actions for unregistered APIs', () => {
     const content = readFileSync(join(fixturesDir, 'get.yml'), 'utf-8')
     const testFile = parseTestFile(content, 'get.yml')