diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml index 6f2570f..9d808b7 100644 --- a/.github/workflows/acceptance.yml +++ b/.github/workflows/acceptance.yml @@ -1,17 +1,13 @@ # Acceptance test workflow — spins up a real OpenMetadata instance via # docker compose (official release image) and runs TF_ACC=1 tests against it. # -# The docker-compose.yml in docker/test/ is the verbatim official release file. -# docker/test/.env contains only the test-specific overrides (heap, log level, -# disable Airflow). To update to a new OM version, run: -# ./scripts/update-test-compose.sh -# then commit the updated docker-compose.yml and .env. +# To add a new version: add it to the matrix below and update docker/test/.env +# default if desired. # # Runs on pushes to main and PRs that touch the provider or its test harness — # doc and config changes do not need a live OpenMetadata stack. Because a # path-filtered workflow never reports its check, this one must NOT be a # required status check, or unrelated PRs would block forever. -# Can also be triggered manually to test a specific version. name: Acceptance Tests on: @@ -25,6 +21,7 @@ on: - 'go.sum' - 'docker/test/**' - 'scripts/testacc.sh' + - 'scripts/testacc-all.sh' - '.github/workflows/acceptance.yml' pull_request: paths: @@ -34,68 +31,60 @@ on: - 'go.sum' - 'docker/test/**' - 'scripts/testacc.sh' + - 'scripts/testacc-all.sh' - '.github/workflows/acceptance.yml' workflow_dispatch: - inputs: - openmetadata_version: - description: "OpenMetadata version (e.g. 1.12.4) — updates .env before running" - required: false - default: "" permissions: contents: read jobs: acceptance: - name: Acceptance Tests + name: "OM ${{ matrix.om_version }}" runs-on: ubuntu-latest timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + om_version: + - "1.12.4" + - "2.0.3" + env: OM_HOST: "http://localhost:8585" - # Health check is on the admin port (8586), not the API port (8585). OM_HEALTH_URL: "http://localhost:8586/healthcheck" OM_ADMIN_EMAIL: "admin@open-metadata.org" # Override via repo secret OM_ADMIN_PASSWORD if the default changes. - # Default is "admin" — the seeded password in the official OM DB image. OM_ADMIN_PASSWORD: ${{ secrets.OM_ADMIN_PASSWORD || 'admin' }} + # Passed to docker compose — overrides OPENMETADATA_VERSION in .env. + OPENMETADATA_VERSION: ${{ matrix.om_version }} steps: - # ── Checkout ──────────────────────────────────────────────────────────── + # ── Checkout ────────────────────────────────────────────────────────────── - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # ── Optionally override the version in .env ────────────────────────────── - - name: Override OPENMETADATA_VERSION - if: inputs.openmetadata_version != '' - # Via env, not direct interpolation — the input must not reach the shell as code. - env: - OM_VERSION: ${{ inputs.openmetadata_version }} - run: | - sed -i "s/^OPENMETADATA_VERSION=.*/OPENMETADATA_VERSION=${OM_VERSION}/" docker/test/.env - echo "Using OpenMetadata version: ${OM_VERSION}" - - # ── Go toolchain ──────────────────────────────────────────────────────── + # ── Go toolchain ────────────────────────────────────────────────────────── - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod cache: true - # ── Terraform CLI (required by terraform-plugin-testing) ───────────────── + # ── Terraform CLI (required by terraform-plugin-testing) ────────────────── - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 with: terraform_wrapper: false - # ── Tune kernel for Elasticsearch ──────────────────────────────────────── + # ── Tune kernel for Elasticsearch ───────────────────────────────────────── # Elasticsearch 9.x requires vm.max_map_count ≥ 262144. - name: Set vm.max_map_count for Elasticsearch run: sudo sysctl -w vm.max_map_count=262144 - # ── Start OpenMetadata test stack ──────────────────────────────────────── - # We start only the services needed for provider tests — mysql, elasticsearch, - # execute-migrate-all (DB migration), and openmetadata-server. - # The heavy `ingestion` (Airflow) service is intentionally excluded; - # PIPELINE_SERVICE_CLIENT_ENABLED=false in docker/test/.env means - # the OM server will not try to reach Airflow. + # ── Start OpenMetadata test stack ───────────────────────────────────────── + # OPENMETADATA_VERSION is set at the job level — docker compose picks it up + # from the shell environment, which takes precedence over the .env file. + # We skip the heavy `ingestion` (Airflow) service; PIPELINE_SERVICE_CLIENT_ENABLED=false + # in docker/test/.env means the OM server will not try to reach it. - name: Pull images run: | docker compose \ @@ -112,10 +101,10 @@ jobs: up -d \ mysql elasticsearch execute-migrate-all openmetadata-server - # ── Wait for OpenMetadata ──────────────────────────────────────────────── + # ── Wait for OpenMetadata ────────────────────────────────────────────────── - name: Wait for OpenMetadata to be healthy run: | - echo "Polling ${OM_HEALTH_URL} (up to 10 minutes)..." + echo "Polling ${OM_HEALTH_URL} for OpenMetadata ${OPENMETADATA_VERSION} (up to 10 minutes)..." for i in $(seq 1 60); do if curl -sf "${OM_HEALTH_URL}" > /dev/null 2>&1; then echo "OpenMetadata is ready after $((i * 10))s" @@ -131,11 +120,10 @@ jobs: logs openmetadata-server --tail=100 exit 1 - # ── Acquire JWT token ──────────────────────────────────────────────────── + # ── Acquire JWT token ────────────────────────────────────────────────────── - name: Acquire JWT token id: auth run: | - # OM requires the password to be base64-encoded in the login payload. OM_ADMIN_PASSWORD_B64="$(echo -n "${OM_ADMIN_PASSWORD}" | base64)" PAYLOAD_FILE="$(mktemp)" printf '{"email":"%s","password":"%s"}' "${OM_ADMIN_EMAIL}" "${OM_ADMIN_PASSWORD_B64}" > "${PAYLOAD_FILE}" @@ -170,7 +158,7 @@ jobs: echo "token=${TOKEN}" >> "${GITHUB_OUTPUT}" echo "JWT token acquired (length: ${#TOKEN})" - # ── Run acceptance tests ───────────────────────────────────────────────── + # ── Run acceptance tests ─────────────────────────────────────────────────── - name: Run acceptance tests env: TF_ACC: "1" @@ -183,7 +171,7 @@ jobs: -timeout 25m \ ./internal/provider/... - # ── Tear down (always runs) ────────────────────────────────────────────── + # ── Tear down (always runs) ──────────────────────────────────────────────── - name: Tear down test stack if: always() run: | diff --git a/.gitignore b/.gitignore index 4361bc2..c8d7482 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,9 @@ dist/ *.tfplan crash.log -# Docker — volume data written by docker compose (never commit runtime data) +# Docker — volume data written by docker compose (never commit runtime data). +# mysql-data is now a named volume (cleaned by `docker compose down -v`). +# The docker-volume dir may still contain other bind-mount leftovers. docker/test/docker-volume/ # Claude Code — local agent config, memory, and skills (dev-only, not project files) diff --git a/Makefile b/Makefile index a80c819..984e5db 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ ARCH := $(shell go env GOARCH 2>/dev/null || echo amd64) # Terraform plugin directory TF_PLUGIN_DIR := $(HOME)/.terraform.d/plugins/registry.terraform.io/codility/openmetadata/$(VERSION)/$(OS)_$(ARCH) -.PHONY: build install clean fmt lint test testacc testacc-external update-test-compose docs codegen deps help +.PHONY: build install clean fmt lint test testacc testacc-all testacc-external update-test-compose docs codegen deps help ## Build the provider binary build: @@ -35,9 +35,16 @@ test: go test -v -count=1 ./... ## Run acceptance tests against a local docker-compose OpenMetadata stack. -## Starts the stack, acquires a JWT, runs TF_ACC=1 tests, and tears down. +## Uses the default version from docker/test/.env, or override with OM_VERSION: +## make testacc +## make testacc OM_VERSION=2.0.3 testacc: - @bash scripts/testacc.sh + @OM_VERSION="$(OM_VERSION)" bash scripts/testacc.sh + +## Run acceptance tests against all versions listed in docker/test/versions. +## Versions are tested sequentially; exits 1 if any version fails. +testacc-all: + @bash scripts/testacc-all.sh ## Run acceptance tests against an already-running OpenMetadata instance. ## Requires OPENMETADATA_HOST and OPENMETADATA_TOKEN to be exported. @@ -47,11 +54,12 @@ testacc: testacc-external: TF_ACC=1 go test -v -count=1 -timeout 30m ./internal/provider/... -## Re-download the official OpenMetadata docker-compose.yml for the version -## pinned in docker/test/.env. Run this when bumping OPENMETADATA_VERSION, -## then commit both docker/test/docker-compose.yml and docker/test/.env. +## Re-download and re-parameterize the official OpenMetadata docker-compose.yml. +## Pass OM_VERSION to fetch a specific release (defaults to the version in .env): +## make update-test-compose +## make update-test-compose OM_VERSION=2.0.3 update-test-compose: - @bash scripts/update-test-compose.sh + @bash scripts/update-test-compose.sh $(OM_VERSION) ## Generate provider documentation (requires tfplugindocs) docs: diff --git a/docker/test/.env b/docker/test/.env index 48eac65..2e4ccf7 100644 --- a/docker/test/.env +++ b/docker/test/.env @@ -1,17 +1,23 @@ # OpenMetadata acceptance-test environment overrides. # -# This file is loaded by docker compose alongside the official OpenMetadata -# docker-compose.yml (which already uses ${VAR:-default} for every setting). -# Only values that differ from the upstream defaults are listed here. +# This file is loaded by docker compose alongside docker-compose.yml, which +# uses ${VAR:-default} for every setting. Only values that differ from the +# upstream defaults are listed here. # -# HOW TO UPGRADE TO A NEW OM VERSION -# ──────────────────────────────────── +# RUNNING AGAINST A DIFFERENT VERSION +# ───────────────────────────────────── +# Override OPENMETADATA_VERSION via an environment variable — it takes +# precedence over the value in this file: +# +# OPENMETADATA_VERSION=2.0.3 make testacc +# OM_VERSION=2.0.3 make testacc # make target maps OM_VERSION → env var +# +# HOW TO BUMP THE DEFAULT VERSION +# ───────────────────────────────── # 1. Update OPENMETADATA_VERSION below. -# 2. Run: ./scripts/update-test-compose.sh -# This re-downloads the official docker-compose.yml from that release. -# 3. Commit both docker/test/docker-compose.yml and docker/test/.env. +# 2. Commit. -# ── Version ─────────────────────────────────────────────────────────────────── +# ── Default version ──────────────────────────────────────────────────────────── OPENMETADATA_VERSION=1.12.4 # ── Heap reduction (saves ~1 GB on CI runners vs. the 2 GB upstream default) ── diff --git a/docker/test/docker-compose.yml b/docker/test/docker-compose.yml index acd4ff3..7031596 100644 --- a/docker/test/docker-compose.yml +++ b/docker/test/docker-compose.yml @@ -15,10 +15,11 @@ volumes: ingestion-volume-dags: ingestion-volume-tmp: es-data: + mysql-data: # named volume so `docker compose down -v` fully cleans the DB services: mysql: container_name: openmetadata_mysql - image: docker.getcollate.io/openmetadata/db:1.12.4 + image: docker.getcollate.io/openmetadata/db:${OPENMETADATA_VERSION} command: "--sort_buffer_size=10M" restart: always environment: @@ -28,7 +29,7 @@ services: ports: - "3306:3306" volumes: - - ./docker-volume/db-data:/var/lib/mysql + - mysql-data:/var/lib/mysql networks: - app_net healthcheck: @@ -59,7 +60,7 @@ services: execute-migrate-all: container_name: execute_migrate_all - image: docker.getcollate.io/openmetadata/server:1.12.4 + image: docker.getcollate.io/openmetadata/server:${OPENMETADATA_VERSION} command: "./bootstrap/openmetadata-ops.sh migrate" environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} @@ -284,7 +285,7 @@ services: openmetadata-server: container_name: openmetadata_server restart: always - image: docker.getcollate.io/openmetadata/server:1.12.4 + image: docker.getcollate.io/openmetadata/server:${OPENMETADATA_VERSION} environment: OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata} SERVER_PORT: ${SERVER_PORT:-8585} @@ -504,7 +505,7 @@ services: ingestion: container_name: openmetadata_ingestion - image: docker.getcollate.io/openmetadata/ingestion:1.12.4 + image: docker.getcollate.io/openmetadata/ingestion:${OPENMETADATA_VERSION} depends_on: elasticsearch: condition: service_started diff --git a/docker/test/versions b/docker/test/versions new file mode 100644 index 0000000..00dae80 --- /dev/null +++ b/docker/test/versions @@ -0,0 +1,11 @@ +# Supported OpenMetadata versions for acceptance tests. +# First non-comment line is the default used by `make testacc` (no OM_VERSION set). +# To add a new version: +# 1. Add the version here. +# 2. Run: make update-test-compose OM_VERSION= +# This downloads the official docker-compose.yml into docker/test//. +# 3. Copy docker/test//.env to docker/test//.env and update +# OPENMETADATA_VERSION. +# 4. Commit docker/test//.env and docker/test//docker-compose.yml. +1.12.4 +2.0.3 diff --git a/internal/provider/team_resource_test.go b/internal/provider/team_resource_test.go index c8935f0..0af01aa 100644 --- a/internal/provider/team_resource_test.go +++ b/internal/provider/team_resource_test.go @@ -78,6 +78,46 @@ func TestAccTeamResourceWithEmail(t *testing.T) { }) } +// TestAccTeamResourceWithParents verifies that a team can be placed under a +// parent team using the parents field. parents is preserved in state from the +// plan (not re-read from the API) so no drift occurs on subsequent plans. +// Import ignores parents because the API does not expose them in a form we can +// round-trip back to the user-supplied names. +func TestAccTeamResourceWithParents(t *testing.T) { + parentName := testRandName("tp") + childName := testRandName("tc") + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProtoV6ProviderFactories: testAccProtoV6ProviderFactories, + Steps: []resource.TestStep{ + // ── Create parent and child ──────────────────────────────────────── + { + Config: testAccTeamConfigWithParents(parentName, childName), + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr("openmetadata_team.parent", "name", parentName), + resource.TestCheckResourceAttr("openmetadata_team.child", "name", childName), + resource.TestCheckResourceAttr("openmetadata_team.child", "team_type", "Department"), + resource.TestCheckResourceAttr("openmetadata_team.child", "parents.#", "1"), + resource.TestCheckResourceAttr("openmetadata_team.child", "parents.0", parentName), + resource.TestCheckResourceAttrSet("openmetadata_team.child", "id"), + ), + }, + // ── Import child ──────────────────────────────────────────────── + { + ResourceName: "openmetadata_team.child", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"parents"}, + ImportStateIdFunc: func(s *terraform.State) (string, error) { + rs := s.RootModule().Resources["openmetadata_team.child"] + return rs.Primary.Attributes["name"], nil + }, + }, + }, + }) +} + func testAccTeamConfig(name, description, teamType string) string { return fmt.Sprintf(` %s @@ -100,3 +140,20 @@ resource "openmetadata_team" "test" { } `, testProviderBlock(), name, email) } + +func testAccTeamConfigWithParents(parentName, childName string) string { + return fmt.Sprintf(` +%s + +resource "openmetadata_team" "parent" { + name = %q + team_type = "Department" +} + +resource "openmetadata_team" "child" { + name = %q + team_type = "Department" + parents = [openmetadata_team.parent.name] +} +`, testProviderBlock(), parentName, childName) +} diff --git a/internal/resources/team.go b/internal/resources/team.go index 5329bd4..cadf38a 100644 --- a/internal/resources/team.go +++ b/internal/resources/team.go @@ -113,7 +113,11 @@ func (r *TeamResource) Create(ctx context.Context, req resource.CreateRequest, r return } - body := r.buildCreateBody(ctx, &plan) + body, err := r.buildCreateBody(ctx, &plan) + if err != nil { + resp.Diagnostics.AddError("Error building team request", err.Error()) + return + } raw, err := r.client.CreateOrUpdate(ctx, teamCollection, body) if err != nil { @@ -153,7 +157,11 @@ func (r *TeamResource) Update(ctx context.Context, req resource.UpdateRequest, r return } - body := r.buildCreateBody(ctx, &plan) + body, err := r.buildCreateBody(ctx, &plan) + if err != nil { + resp.Diagnostics.AddError("Error building team request", err.Error()) + return + } raw, err := r.client.CreateOrUpdate(ctx, teamCollection, body) if err != nil { @@ -192,13 +200,14 @@ func (r *TeamResource) ImportState(ctx context.Context, req resource.ImportState } var state TeamResourceModel + state.Parents = types.ListNull(types.StringType) // zero-value has no element type; initialize before readIntoState r.readIntoState(ctx, raw, &state, resp) resp.Diagnostics.Append(resp.State.Set(ctx, &state)...) } // --- internal helpers --- -func (r *TeamResource) buildCreateBody(ctx context.Context, plan *TeamResourceModel) map[string]interface{} { +func (r *TeamResource) buildCreateBody(ctx context.Context, plan *TeamResourceModel) (map[string]interface{}, error) { body := map[string]interface{}{ "name": plan.Name.ValueString(), "teamType": plan.TeamType.ValueString(), @@ -216,9 +225,27 @@ func (r *TeamResource) buildCreateBody(ctx context.Context, plan *TeamResourceMo body["isJoinable"] = plan.IsJoinable.ValueBool() } if !plan.Parents.IsNull() && !plan.Parents.IsUnknown() { - var parents []string - plan.Parents.ElementsAs(ctx, &parents, false) - body["parents"] = parents + var parentNames []string + plan.Parents.ElementsAs(ctx, &parentNames, false) + // The OM CreateTeam schema expects parents as a plain UUID array, + // not entity ref objects. Resolve each name to its UUID. + uuids := make([]string, 0, len(parentNames)) + for _, name := range parentNames { + raw, err := r.client.GetByName(ctx, teamCollection, name, nil) + if err != nil { + return nil, fmt.Errorf("resolving parent team %q: %w", name, err) + } + data, err := Unmarshal(raw) + if err != nil { + return nil, fmt.Errorf("parsing parent team %q: %w", name, err) + } + id, _ := data["id"].(string) + if id == "" { + return nil, fmt.Errorf("parent team %q returned no id", name) + } + uuids = append(uuids, id) + } + body["parents"] = uuids } if !plan.Policies.IsNull() && !plan.Policies.IsUnknown() { var policies []string @@ -233,7 +260,7 @@ func (r *TeamResource) buildCreateBody(ctx context.Context, plan *TeamResourceMo if !plan.Owners.IsNull() && !plan.Owners.IsUnknown() { body["owners"] = r.extractOwners(ctx, plan) } - return body + return body, nil } func (r *TeamResource) extractOwners(ctx context.Context, plan *TeamResourceModel) []EntityRef { @@ -265,12 +292,12 @@ func (r *TeamResource) readIntoState(ctx context.Context, raw []byte, state *Tea state.Email = StringVal(data, "email") state.IsJoinable = BoolVal(data, "isJoinable") state.FQN = StringVal(data, "fullyQualifiedName") - // parents is intentionally NOT read from the API response. OM always places - // teams under Organisation by default, which would override the null state - // and cause "Provider produced inconsistent result after apply". Parents - // provided by the user are sent on create/update but not reflected back. - // For import, parents is excluded via ImportStateVerifyIgnore. - state.Parents = types.ListNull(types.StringType) + // parents is intentionally NOT read from the API response. OM always + // returns the implicit Organisation parent even when the user didn't + // specify one, which would cause drift. We preserve whatever the caller + // already has on the model (the planned value on Create/Update, the + // existing state value on Read). On import the field is unknown, so + // callers must add "parents" to ImportStateVerifyIgnore. state.Policies = StringListVal(data, "policies") state.Domains = StringListVal(data, "domains") state.Owners = OwnersListNull() diff --git a/scripts/testacc-all.sh b/scripts/testacc-all.sh new file mode 100755 index 0000000..829a6dc --- /dev/null +++ b/scripts/testacc-all.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# scripts/testacc-all.sh — Run acceptance tests against every supported OM version. +# +# Reads the version list from docker/test/versions (one version per line, +# lines starting with # are comments) and calls scripts/testacc.sh for each. +# All extra arguments are forwarded to go test. +# +# Usage: +# ./scripts/testacc-all.sh # all versions +# ./scripts/testacc-all.sh -run TestAccTeam # filter tests, all versions +# +# Each version runs sequentially; the script reports a summary and exits 1 if +# any version fails. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +VERSIONS_FILE="${REPO_ROOT}/docker/test/versions" + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BOLD='\033[1m' +NC='\033[0m' + +[ -f "${VERSIONS_FILE}" ] || { echo "ERROR: ${VERSIONS_FILE} not found" >&2; exit 1; } + +VERSIONS=() +while IFS= read -r line; do + [[ "${line}" =~ ^[[:space:]]*# || -z "${line// }" ]] && continue + VERSIONS+=("${line}") +done < "${VERSIONS_FILE}" + +if [ ${#VERSIONS[@]} -eq 0 ]; then + echo "ERROR: no versions found in ${VERSIONS_FILE}" >&2 + exit 1 +fi + +echo -e "${BOLD}Testing ${#VERSIONS[@]} OpenMetadata version(s): ${VERSIONS[*]}${NC}" + +PASSED=() +FAILED=() + +for v in "${VERSIONS[@]}"; do + echo "" + echo -e "${BOLD}══════════════════════════════════════════════════${NC}" + echo -e "${BOLD} OpenMetadata ${v}${NC}" + echo -e "${BOLD}══════════════════════════════════════════════════${NC}" + + if OM_VERSION="${v}" bash "${SCRIPT_DIR}/testacc.sh" "$@"; then + PASSED+=("${v}") + echo -e "${GREEN}[PASS] ${v}${NC}" + else + FAILED+=("${v}") + echo -e "${RED}[FAIL] ${v}${NC}" + fi +done + +echo "" +echo -e "${BOLD}══════════════════════════════════════════════════${NC}" +echo -e "${BOLD} Summary${NC}" +echo -e "${BOLD}══════════════════════════════════════════════════${NC}" + +for v in "${PASSED[@]+"${PASSED[@]}"}"; do + echo -e " ${GREEN}✓ ${v}${NC}" +done +for v in "${FAILED[@]+"${FAILED[@]}"}"; do + echo -e " ${RED}✗ ${v}${NC}" +done + +if [ ${#FAILED[@]} -gt 0 ]; then + echo "" + echo -e "${RED}FAILED: ${FAILED[*]}${NC}" + exit 1 +fi + +echo -e "${GREEN}All versions passed.${NC}" diff --git a/scripts/testacc.sh b/scripts/testacc.sh index 99f7cca..21f502e 100755 --- a/scripts/testacc.sh +++ b/scripts/testacc.sh @@ -6,20 +6,23 @@ # # Usage: # ./scripts/testacc.sh [go test flags] +# OM_VERSION=2.0.3 ./scripts/testacc.sh [go test flags] # # Examples: -# ./scripts/testacc.sh # run all acceptance tests -# ./scripts/testacc.sh -run TestAccClassification # run one test -# ./scripts/testacc.sh -v -timeout 30m # verbose with custom timeout +# ./scripts/testacc.sh # default version (from .env) +# OM_VERSION=2.0.3 ./scripts/testacc.sh # specific version +# OM_VERSION=1.12.4 ./scripts/testacc.sh -run TestAccClassification +# +# To run all supported versions in sequence use scripts/testacc-all.sh (or make testacc-all). # # Prerequisites: docker (compose v2), go, python3, curl # # Environment overrides: -# OPENMETADATA_VERSION Image tag (read from docker/test/.env by default) -# OM_ADMIN_EMAIL Admin email (default: admin@open-metadata.org) -# OM_ADMIN_PASSWORD Admin password (default: admin) -# OM_HOST OM API base URL (default: http://localhost:8585) -# SKIP_DOCKER Set to 1 to skip compose lifecycle (use existing OM) +# OM_VERSION Passed as OPENMETADATA_VERSION to docker compose, overriding .env +# OM_ADMIN_EMAIL Admin email (default: admin@open-metadata.org) +# OM_ADMIN_PASSWORD Admin password (default: admin) +# OM_HOST OM API base URL (default: http://localhost:8585) +# SKIP_DOCKER Set to 1 to skip compose lifecycle (use an already-running OM) set -euo pipefail @@ -29,14 +32,13 @@ COMPOSE_FILE="${REPO_ROOT}/docker/test/docker-compose.yml" ENV_FILE="${REPO_ROOT}/docker/test/.env" OM_HOST="${OM_HOST:-http://localhost:8585}" -# OM exposes its health check on port 8586 (/healthcheck), not 8585. OM_HEALTH_URL="${OM_HEALTH_URL:-http://localhost:8586/healthcheck}" OM_ADMIN_EMAIL="${OM_ADMIN_EMAIL:-admin@open-metadata.org}" OM_ADMIN_PASSWORD="${OM_ADMIN_PASSWORD:-admin}" SKIP_DOCKER="${SKIP_DOCKER:-0}" -# Services to start — we intentionally skip the heavy `ingestion` (Airflow) -# service because PIPELINE_SERVICE_CLIENT_ENABLED=false makes it unnecessary. +# Services to start — intentionally excludes `ingestion` (Airflow) because +# PIPELINE_SERVICE_CLIENT_ENABLED=false makes it unnecessary for provider tests. OM_SERVICES="mysql elasticsearch execute-migrate-all openmetadata-server" RED='\033[0;31m' @@ -56,28 +58,46 @@ command -v curl >/dev/null 2>&1 || die "curl is not installed" command -v go >/dev/null 2>&1 || die "go is not installed" docker compose version >/dev/null 2>&1 || die "docker compose (v2) is not available" -[ -f "${COMPOSE_FILE}" ] || die "docker-compose.yml not found at ${COMPOSE_FILE}. Run: scripts/update-test-compose.sh" +[ -f "${COMPOSE_FILE}" ] || die "docker-compose.yml not found at ${COMPOSE_FILE}" [ -f "${ENV_FILE}" ] || die ".env not found at ${ENV_FILE}" +# ── resolve version (OM_VERSION overrides .env OPENMETADATA_VERSION) ────────── + +# When OM_VERSION is set, pass OPENMETADATA_VERSION as an environment variable. +# Docker Compose gives shell environment variables precedence over --env-file values. +if [ -n "${OM_VERSION:-}" ]; then + export OPENMETADATA_VERSION="${OM_VERSION}" +fi + +# Print the version that will actually be used. +EFFECTIVE_VERSION="${OPENMETADATA_VERSION:-$(grep -E '^OPENMETADATA_VERSION=' "${ENV_FILE}" | cut -d= -f2)}" +log "OpenMetadata version: ${EFFECTIVE_VERSION}" + +# ── docker compose helper ───────────────────────────────────────────────────── + +# Unique project name per version keeps named volumes (e.g. es-data) isolated so +# sequential runs against different versions don't share state. +PROJECT_NAME="openmetadata-testacc-${EFFECTIVE_VERSION//\./-}" + +dc() { docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" -p "${PROJECT_NAME}" "$@"; } + # ── docker compose lifecycle ─────────────────────────────────────────────────── cleanup() { if [ "${SKIP_DOCKER}" = "0" ]; then - log "Tearing down test stack..." - docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \ - down -v --remove-orphans 2>/dev/null || true + log "Tearing down test stack (${EFFECTIVE_VERSION})..." + dc down -v --remove-orphans 2>/dev/null || true fi } trap cleanup EXIT if [ "${SKIP_DOCKER}" = "0" ]; then log "Pulling images (this may take a while on first run)..." - docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" pull --quiet ${OM_SERVICES} + dc pull --quiet ${OM_SERVICES} log "Starting OpenMetadata test stack..." log " (DB migration runs first — expect 3-7 min on first boot)" - docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \ - up -d ${OM_SERVICES} + dc up -d ${OM_SERVICES} fi # ── wait for OpenMetadata ───────────────────────────────────────────────────── @@ -90,10 +110,9 @@ ELAPSED=0 until curl -sf "${OM_HEALTH_URL}" >/dev/null 2>&1; do if [ "${ELAPSED}" -ge "${MAX_WAIT}" ]; then echo "" - echo "OpenMetadata did not become healthy within ${MAX_WAIT}s." + echo "OpenMetadata ${EFFECTIVE_VERSION} did not become healthy within ${MAX_WAIT}s." echo "Server logs (last 50 lines):" - docker compose -f "${COMPOSE_FILE}" --env-file "${ENV_FILE}" \ - logs openmetadata-server --tail=50 + dc logs openmetadata-server --tail=50 exit 1 fi echo " ... still waiting (${ELAPSED}s / ${MAX_WAIT}s)" @@ -143,8 +162,8 @@ log "JWT token acquired (length: ${#OM_TOKEN})." # ── run acceptance tests ─────────────────────────────────────────────────────── log "Running acceptance tests..." -log " OPENMETADATA_HOST = ${OM_HOST}" -log " TF_ACC = 1" +log " OPENMETADATA_HOST = ${OM_HOST}" +log " TF_ACC = 1" cd "${REPO_ROOT}" TF_ACC=1 \ diff --git a/scripts/update-test-compose.sh b/scripts/update-test-compose.sh index da2973f..839f47a 100755 --- a/scripts/update-test-compose.sh +++ b/scripts/update-test-compose.sh @@ -1,41 +1,75 @@ #!/usr/bin/env bash # scripts/update-test-compose.sh # -# Downloads the official OpenMetadata docker-compose.yml for the version -# pinned in docker/test/.env and saves it to docker/test/docker-compose.yml. +# Downloads the official OpenMetadata docker-compose.yml for a given version, +# replaces the hardcoded image tags with ${OPENMETADATA_VERSION}, and saves the +# result to docker/test/docker-compose.yml. # -# Run this whenever you bump OPENMETADATA_VERSION in docker/test/.env, then -# commit the updated docker-compose.yml alongside the .env change. +# Usage: +# ./scripts/update-test-compose.sh +# ./scripts/update-test-compose.sh # uses OPENMETADATA_VERSION from .env # -# Requirements: gh (GitHub CLI), bash +# Examples: +# ./scripts/update-test-compose.sh 2.0.3 +# ./scripts/update-test-compose.sh 1.12.4 +# +# Run this when upgrading the compose template to pick up structural changes +# in the official file (new services, changed defaults, etc.). +# After running, review the diff and commit docker/test/docker-compose.yml. +# +# Requirements: gh (GitHub CLI), sed, bash set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" ENV_FILE="${REPO_ROOT}/docker/test/.env" -OUT_DIR="${REPO_ROOT}/docker/test" +OUT_FILE="${REPO_ROOT}/docker/test/docker-compose.yml" command -v gh >/dev/null 2>&1 || { echo "ERROR: gh (GitHub CLI) is required"; exit 1; } -# Read OPENMETADATA_VERSION from .env -VERSION="$(grep -E '^OPENMETADATA_VERSION=' "${ENV_FILE}" | cut -d= -f2 | tr -d ' ')" +# Resolve version: CLI arg > env var > .env file +if [ -n "${1:-}" ]; then + VERSION="${1}" +elif [ -n "${OPENMETADATA_VERSION:-}" ]; then + VERSION="${OPENMETADATA_VERSION}" +elif [ -n "${OM_VERSION:-}" ]; then + VERSION="${OM_VERSION}" +else + VERSION="$(grep -E '^OPENMETADATA_VERSION=' "${ENV_FILE}" | cut -d= -f2 | tr -d ' ')" +fi + if [ -z "${VERSION}" ]; then - echo "ERROR: OPENMETADATA_VERSION not found in ${ENV_FILE}" + echo "ERROR: version not specified. Pass it as an argument or set OPENMETADATA_VERSION." + echo "Usage: $0 e.g. $0 2.0.3" exit 1 fi TAG="${VERSION}-release" +TMPFILE="$(mktemp)" + echo "Downloading official docker-compose.yml for OpenMetadata ${VERSION} (tag: ${TAG})..." gh release download "${TAG}" \ --repo open-metadata/OpenMetadata \ --pattern "docker-compose.yml" \ - --dir "${OUT_DIR}" \ + --output "${TMPFILE}" \ --clobber -echo "Saved to ${OUT_DIR}/docker-compose.yml" +echo "Parameterizing image tags (replacing ${VERSION} → \${OPENMETADATA_VERSION})..." + +# Replace the hardcoded version in OM image tags with the variable so a single +# compose file works for all versions. +sed \ + -e "s|openmetadata/db:${VERSION}|openmetadata/db:\${OPENMETADATA_VERSION}|g" \ + -e "s|openmetadata/server:${VERSION}|openmetadata/server:\${OPENMETADATA_VERSION}|g" \ + -e "s|openmetadata/ingestion:${VERSION}|openmetadata/ingestion:\${OPENMETADATA_VERSION}|g" \ + "${TMPFILE}" > "${OUT_FILE}" + +rm -f "${TMPFILE}" + +echo "Saved to ${OUT_FILE}" echo "" echo "Next steps:" -echo " 1. Review the diff: git diff docker/test/docker-compose.yml" -echo " 2. Commit: git add docker/test/docker-compose.yml docker/test/.env && git commit" +echo " 1. Review: git diff docker/test/docker-compose.yml" +echo " 2. Commit: git add docker/test/docker-compose.yml && git commit -m 'chore: refresh compose template from ${VERSION}'"