From 724fd32ca2d4473da8bd28de04d04af839cb8782 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Mon, 3 Nov 2025 12:15:34 +0100 Subject: [PATCH 01/10] Automate Helm tests with Gradle This PR automates Helm tests and make them executable as part of the Gradle build. --- .github/workflows/gradle.yml | 5 +- .github/workflows/helm.yml | 2 - CONTRIBUTING.md | 78 +++++++- gradle/projects.main.properties | 2 + helm/polaris/README.md | 19 +- helm/polaris/README.md.gotmpl | 19 +- helm/polaris/build.gradle.kts | 245 +++++++++++++++++++++++++ site/content/in-dev/unreleased/helm.md | 19 +- 8 files changed, 367 insertions(+), 22 deletions(-) create mode 100644 helm/polaris/build.gradle.kts diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 8409190cd1..cb680f644c 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -66,7 +66,9 @@ jobs: ./gradlew check sourceTarball distTar distZip publishToMavenLocal \ -x :polaris-runtime-service:test \ -x :polaris-admin:test \ - -x intTest --continue + -x intTest \ + -x :polaris-helm:check \ + --continue - name: Save partial Gradle build cache uses: ./.github/actions/ci-incr-build-cache-save if: github.event_name == 'push' && github.ref == 'refs/heads/main' @@ -213,6 +215,7 @@ jobs: ./gradlew \ intTest \ -x :polaris-runtime-service:intTest \ + -x :polaris-helm:intTest \ --continue env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} diff --git a/.github/workflows/helm.yml b/.github/workflows/helm.yml index 4c31fb39f8..759494e4b7 100644 --- a/.github/workflows/helm.yml +++ b/.github/workflows/helm.yml @@ -98,8 +98,6 @@ jobs: ./gradlew \ :polaris-server:assemble \ :polaris-server:quarkusAppPartsBuild --rerun \ - :polaris-admin:assemble \ - :polaris-admin:quarkusAppPartsBuild --rerun \ -Dquarkus.container-image.build=true minikube image ls diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbacefaa59..780748ab58 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -119,12 +119,82 @@ Tips: * Keep in mind that the Git commit subject and message is going to be read by other people, potentially even after years. The Git commit subject and message will appear "as is" in release notes. * Make sure the subject and message are properly formatted and contains a concise description of the changes in way that someone who has no prior knowledge can understand the rationale of the change and the change itself. Remove information that's of no use for someone reading the Git commit log, for example single intermediate commit messages like `formatting` or `fix test`. -## Java version requirements +## Build Prerequisites and Required Tools -The Apache Polaris build currently requires Java 21 or later. There are a few tools that help you running the right Java version: +Apache Polaris (incubating) requires several tools to run a full build, including tests and documentation generation. -* [SDKMAN!](https://sdkman.io/) follow the installation instructions, then run `sdk list java` to see the available distributions and versions, then run `sdk install java ` using the identifier for the distribution and version (>= 21) of your choice. -* [jenv](https://www.jenv.be/) If on a Mac you can use jenv to set the appropriate SDK. +### Core Build Requirements + +These tools are required for basic building and testing: + +* **Git**: Required for cloning the repository and version control. + +* **Java 21+**: The build requires Java 21 or later. Suggested installers: + * [SDKMAN!](https://sdkman.io/) - Run `sdk list java` to see available distributions, then `sdk install java ` to install. + * [jEnv](https://www.jenv.be/) - You can also use jEnv to manage Java versions. + +* **Docker**: Required for integration tests and building container images. + +### Helm Chart Testing Requirements + +These tools are required to run Helm chart tests (part of `./gradlew test` and `./gradlew intTest`): + +* **Helm**: Kubernetes package manager, required for template validation and unit tests. + * macOS: `brew install helm` + * See [Helm installation instructions](https://helm.sh/docs/intro/install/) + +* **Helm Unittest Plugin**: Required for running Helm unit tests. + * Install: `helm plugin install https://github.com/helm-unittest/helm-unittest.git` + +* **helm-docs**: Required for generating Helm chart documentation. + * macOS: `brew install norwoodj/tap/helm-docs` + * See [helm-docs installation instructions](https://github.com/norwoodj/helm-docs) + +* **chart-testing (ct)**: Required for linting and testing Helm charts. + * macOS: `brew install chart-testing` + * See [chart-testing installation instructions](https://github.com/helm/chart-testing) + +* **Minikube**: Required for running Helm chart integration tests. + * macOS: `brew install minikube` + * See [Minikube installation instructions](https://minikube.sigs.k8s.io/docs/start/) + +* **kubectl**: Kubernetes command-line tool, required for Helm integration tests. + * macOS: `brew install kubectl` + * See [kubectl installation instructions](https://kubernetes.io/docs/tasks/tools/) + +### Other Tools + +These tools are helpful but not strictly required: + +* **jq**: JSON processor, useful for working with Polaris APIs and scripts. + * macOS: `brew install jq` + * See [jq installation instructions](https://jqlang.github.io/jq/download/) + +### Quick Installation (macOS) + +For macOS users with Homebrew, you can install all dependencies at once using the provided Makefile: + +```bash +make install-dependencies-brew +make install-optional-dependencies-brew +``` + +To check which dependencies are installed: +```bash +make check-dependencies +``` + +### Skipping Specific Tests + +If you don't have all the Helm-related tools installed, you can skip those tests: + +```bash +# Run checks excluding Helm tests +./gradlew check -x :polaris-helm:test + +# Run build excluding Helm integration tests +./gradlew build -x :polaris-helm:intTest +``` ## Code Contribution Guidelines diff --git a/gradle/projects.main.properties b/gradle/projects.main.properties index 5869bc5bc7..92264c6a0b 100644 --- a/gradle/projects.main.properties +++ b/gradle/projects.main.properties @@ -50,6 +50,8 @@ polaris-config-docs-annotations=tools/config-docs/annotations polaris-config-docs-generator=tools/config-docs/generator polaris-config-docs-site=tools/config-docs/site +polaris-helm=helm/polaris + # executor abstraction polaris-async-api=persistence/nosql/async/api polaris-async-java=persistence/nosql/async/java diff --git a/helm/polaris/README.md b/helm/polaris/README.md index bbfc8557df..9a8750d99f 100644 --- a/helm/polaris/README.md +++ b/helm/polaris/README.md @@ -28,6 +28,7 @@ weight: 675 Do not modify the README.md file directly, please modify README.md.gotmpl instead. To re-generate the README.md file, install helm-docs then run from the repo root: helm-docs --chart-search-root=helm + Alternatively, run ./gradlew helmDocs from the repo root. --> ![Version: 1.2.0-incubating-SNAPSHOT](https://img.shields.io/badge/Version-1.2.0--incubating--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0-incubating-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.2.0--incubating--SNAPSHOT-informational?style=flat-square) @@ -55,8 +56,6 @@ eval $(minikube docker-env) ./gradlew \ :polaris-server:assemble \ :polaris-server:quarkusAppPartsBuild --rerun \ - :polaris-admin:assemble \ - :polaris-admin:quarkusAppPartsBuild --rerun \ -Dquarkus.container-image.build=true ``` @@ -143,10 +142,11 @@ The following tools are required to run the tests: * [Helm Unit Test](https://github.com/helm-unittest/helm-unittest) * [Chart Testing](https://github.com/helm/chart-testing) -Quick installation instructions for these tools: +Quick installation of all required tools on macOS: + ```bash -helm plugin install https://github.com/helm-unittest/helm-unittest.git -brew install chart-testing +make install-dependencies-brew +make install-optional-dependencies-brew ``` The integration tests also require some fixtures to be deployed. The `ci/fixtures` directory @@ -183,6 +183,15 @@ Integration tests are run with the Chart Testing tool: ct install --namespace polaris --charts ./helm/polaris ``` +### Running tets with Gradle + +Both unit and integration tests can be run with Gradle. From the Polaris repo root: + +```bash +./gradlew :polaris-helm:test +./gradlew :polaris-helm:intTest +``` + ## Values | Key | Type | Default | Description | diff --git a/helm/polaris/README.md.gotmpl b/helm/polaris/README.md.gotmpl index 99feabb101..af0ecf7aa9 100644 --- a/helm/polaris/README.md.gotmpl +++ b/helm/polaris/README.md.gotmpl @@ -28,6 +28,7 @@ weight: 675 Do not modify the README.md file directly, please modify README.md.gotmpl instead. To re-generate the README.md file, install helm-docs then run from the repo root: helm-docs --chart-search-root=helm + Alternatively, run ./gradlew helmDocs from the repo root. --> {{ template "chart.deprecationWarning" . }} @@ -57,8 +58,6 @@ eval $(minikube docker-env) ./gradlew \ :polaris-server:assemble \ :polaris-server:quarkusAppPartsBuild --rerun \ - :polaris-admin:assemble \ - :polaris-admin:quarkusAppPartsBuild --rerun \ -Dquarkus.container-image.build=true ``` @@ -145,10 +144,11 @@ The following tools are required to run the tests: * [Helm Unit Test](https://github.com/helm-unittest/helm-unittest) * [Chart Testing](https://github.com/helm/chart-testing) -Quick installation instructions for these tools: +Quick installation of all required tools on macOS: + ```bash -helm plugin install https://github.com/helm-unittest/helm-unittest.git -brew install chart-testing +make install-dependencies-brew +make install-optional-dependencies-brew ``` The integration tests also require some fixtures to be deployed. The `ci/fixtures` directory @@ -185,4 +185,13 @@ Integration tests are run with the Chart Testing tool: ct install --namespace polaris --charts ./helm/polaris ``` +### Running tets with Gradle + +Both unit and integration tests can be run with Gradle. From the Polaris repo root: + +```bash +./gradlew :polaris-helm:test +./gradlew :polaris-helm:intTest +``` + {{ template "chart.valuesSection" . }} diff --git a/helm/polaris/build.gradle.kts b/helm/polaris/build.gradle.kts new file mode 100644 index 0000000000..02716f592a --- /dev/null +++ b/helm/polaris/build.gradle.kts @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +plugins { + id("base") + id("polaris-spotless") +} + +description = "Apache Polaris (incubating) Helm Chart" + +val helmTestReportsDir = layout.buildDirectory.dir("reports") + +val helmTemplateValidation by + tasks.registering(Exec::class) { + group = "verification" + description = "Run 'helm template' validation with all values files" + + workingDir = projectDir + + val outputFile = helmTestReportsDir.get().file("helm-template/validation.log") + + commandLine = + listOf( + "sh", + "-c", + """ + mkdir -p ${outputFile.asFile.parent} + for f in values.yaml ci/*.yaml; do + echo "Validating helm template with ${'$'}f" + helm template --debug --namespace polaris-ns --values ${'$'}f . + done > ${outputFile.asFile} 2>&1 + """ + .trimIndent(), + ) + + inputs.files( + fileTree(projectDir) { include("Chart.yaml", "values.yaml", "ci/*.yaml", "templates/**/*") } + ) + + outputs.file(outputFile) + } + +val helmUnitTest by + tasks.registering(Exec::class) { + group = "verification" + description = "Run Helm unit tests" + + workingDir = projectDir + + val outputFile = helmTestReportsDir.get().file("helm-unit/test.log") + + // Install the plugin if not already installed, then run tests + commandLine = + listOf( + "sh", + "-c", + """ + mkdir -p ${outputFile.asFile.parent} + helm plugin install https://github.com/helm-unittest/helm-unittest.git 2>/dev/null || true + helm unittest . > ${outputFile.asFile} 2>&1 + """ + .trimIndent(), + ) + + inputs.files( + fileTree(projectDir) { include("Chart.yaml", "values.yaml", "templates/**/*", "tests/**/*") } + ) + + outputs.file(outputFile) + } + +val chartTestingLint by + tasks.registering(Exec::class) { + group = "verification" + description = "Run chart-testing (lint)" + + workingDir = rootProject.projectDir + + val outputFile = helmTestReportsDir.get().file("ct/lint.log") + + commandLine = + listOf( + "sh", + "-c", + """ + mkdir -p ${outputFile.asFile.parent} + ct lint --debug --charts ./helm/polaris > ${outputFile.asFile} 2>&1 + """ + .trimIndent(), + ) + + inputs.files( + fileTree(projectDir) { include("Chart.yaml", "values.yaml", "ci/*.yaml", "templates/**/*") } + ) + + outputs.file(outputFile) + } + +// Task to build Docker images in minikube's Docker environment +val buildMinikubeImages by + tasks.registering(Exec::class) { + group = "build" + description = "Build Polaris Docker images in minikube's Docker environment" + + workingDir = rootProject.projectDir + + // Output: file containing SHA digest of built image + val outputFile = helmTestReportsDir.get().file("minikube-images.sha256") + + commandLine = + listOf( + "sh", + "-c", + """ + # Check if minikube is running + if ! minikube status >/dev/null 2>&1; then + echo "Minikube is not running. Starting minikube..." + minikube start + fi + + # Set up docker environment and build images + eval $(minikube -p minikube docker-env) + + # Build server image + docker build -t apache/polaris:latest \ + -f runtime/server/src/main/docker/Dockerfile.jvm \ + runtime/server + + # Capture image digest + mkdir -p ${outputFile.asFile.parent} + { + docker inspect --format='{{.Id}}' apache/polaris:latest + } > ${outputFile.asFile} + """ + .trimIndent(), + ) + + dependsOn(":polaris-server:quarkusBuild") + + inputs.dir(rootProject.file("runtime/server/build/quarkus-app")) + inputs.file(rootProject.file("runtime/server/src/main/docker/Dockerfile.jvm")) + + outputs.file(outputFile) + } + +// Task to run chart-testing install on minikube +val chartTestingInstall by + tasks.registering(Exec::class) { + group = "verification" + description = "Run chart-testing (install) on minikube" + + workingDir = rootProject.projectDir + + val outputFile = helmTestReportsDir.get().file("ct/install.log") + + commandLine = + listOf( + "sh", + "-c", + """ + mkdir -p ${outputFile.asFile.parent} + + # Create namespace if it doesn't exist + kubectl create namespace polaris-ns --dry-run=client -o yaml | kubectl apply -f - + + # Install fixtures + kubectl apply --namespace polaris-ns -f helm/polaris/ci/fixtures + + # Run chart-testing install + ct install \ + --namespace polaris-ns \ + --debug --charts ./helm/polaris > ${outputFile.asFile} 2>&1 + """ + .trimIndent(), + ) + + inputs.files( + fileTree(projectDir) { include("Chart.yaml", "values.yaml", "ci/**/*", "templates/**/*") } + ) + + outputs.file(outputFile) + + // Depend on the images being built in minikube + dependsOn(buildMinikubeImages) + } + +// Task to generate helm documentation +val helmDocs by + tasks.registering(Exec::class) { + group = "documentation" + description = "Generate Helm chart documentation using helm-docs" + + workingDir = rootProject.projectDir + + commandLine = + listOf( + "sh", + "-c", + """ + helm-docs --chart-search-root=helm + """ + .trimIndent(), + ) + + inputs.files(fileTree(projectDir) { include("Chart.yaml", "values.yaml", "README.md.gotmpl") }) + + outputs.file(projectDir.resolve("README.md")) + } + +val test by + tasks.registering { + group = "verification" + description = "Run all Helm chart tests" + dependsOn(helmTemplateValidation, helmUnitTest, chartTestingLint) + } + +val intTest by + tasks.registering { + group = "verification" + description = "Run Helm chart integration tests on minikube" + dependsOn(chartTestingInstall) + } + +tasks.named("check") { dependsOn(test) } + +tasks.named("build") { + dependsOn(intTest) + dependsOn(helmDocs) +} diff --git a/site/content/in-dev/unreleased/helm.md b/site/content/in-dev/unreleased/helm.md index ef82e8e675..3d5a8f660b 100644 --- a/site/content/in-dev/unreleased/helm.md +++ b/site/content/in-dev/unreleased/helm.md @@ -28,6 +28,7 @@ weight: 675 Do not modify the README.md file directly, please modify README.md.gotmpl instead. To re-generate the README.md file, install helm-docs then run from the repo root: helm-docs --chart-search-root=helm + Alternatively, run ./gradlew helmDocs from the repo root. --> ![Version: 1.2.0-incubating-SNAPSHOT](https://img.shields.io/badge/Version-1.2.0--incubating--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0-incubating-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.2.0--incubating--SNAPSHOT-informational?style=flat-square) @@ -55,8 +56,6 @@ eval $(minikube docker-env) ./gradlew \ :polaris-server:assemble \ :polaris-server:quarkusAppPartsBuild --rerun \ - :polaris-admin:assemble \ - :polaris-admin:quarkusAppPartsBuild --rerun \ -Dquarkus.container-image.build=true ``` @@ -145,10 +144,11 @@ The following tools are required to run the tests: * [Helm Unit Test](https://github.com/helm-unittest/helm-unittest) * [Chart Testing](https://github.com/helm/chart-testing) -Quick installation instructions for these tools: +Quick installation of all required tools on macOS: + ```bash -helm plugin install https://github.com/helm-unittest/helm-unittest.git -brew install chart-testing +make install-dependencies-brew +make install-optional-dependencies-brew ``` The integration tests also require some fixtures to be deployed. The `ci/fixtures` directory @@ -185,6 +185,15 @@ Integration tests are run with the Chart Testing tool: ct install --namespace polaris --charts ./helm/polaris ``` +### Running tets with Gradle + +Both unit and integration tests can be run with Gradle. From the Polaris repo root: + +```bash +./gradlew :polaris-helm:test +./gradlew :polaris-helm:intTest +``` + ## Values | Key | Type | Default | Description | From f7f9c73cc1f94de2cc42e10d52157cb31a847e59 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Mon, 3 Nov 2025 16:03:25 +0100 Subject: [PATCH 02/10] review --- .github/workflows/gradle.yml | 2 +- CONTRIBUTING.md | 6 +- helm/polaris/README.md | 12 +- helm/polaris/README.md.gotmpl | 12 +- helm/polaris/build.gradle.kts | 192 +++++++++++++++++-------- site/content/in-dev/unreleased/helm.md | 17 ++- 6 files changed, 170 insertions(+), 71 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index cb680f644c..7931b7e623 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -215,7 +215,7 @@ jobs: ./gradlew \ intTest \ -x :polaris-runtime-service:intTest \ - -x :polaris-helm:intTest \ + -x :polaris-helm:check \ --continue env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 780748ab58..273f886335 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -133,7 +133,11 @@ These tools are required for basic building and testing: * [SDKMAN!](https://sdkman.io/) - Run `sdk list java` to see available distributions, then `sdk install java ` to install. * [jEnv](https://www.jenv.be/) - You can also use jEnv to manage Java versions. -* **Docker**: Required for integration tests and building container images. +* **Container Runtime**: A container runtime like [Docker] or [Podman] is required for integration tests, regression tests + and building container images. + +[Docker]: https://www.docker.com/ +[Podman]: https://podman.io/ ### Helm Chart Testing Requirements diff --git a/helm/polaris/README.md b/helm/polaris/README.md index 9a8750d99f..9d12fa6b57 100644 --- a/helm/polaris/README.md +++ b/helm/polaris/README.md @@ -28,7 +28,7 @@ weight: 675 Do not modify the README.md file directly, please modify README.md.gotmpl instead. To re-generate the README.md file, install helm-docs then run from the repo root: helm-docs --chart-search-root=helm - Alternatively, run ./gradlew helmDocs from the repo root. + Alternatively, run `./gradlew helmDocs` or `make helm-doc-generate` from the repo root. --> ![Version: 1.2.0-incubating-SNAPSHOT](https://img.shields.io/badge/Version-1.2.0--incubating--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0-incubating-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.2.0--incubating--SNAPSHOT-informational?style=flat-square) @@ -142,7 +142,13 @@ The following tools are required to run the tests: * [Helm Unit Test](https://github.com/helm-unittest/helm-unittest) * [Chart Testing](https://github.com/helm/chart-testing) -Quick installation of all required tools on macOS: +Quick installation instructions for these tools: +```bash +helm plugin install https://github.com/helm-unittest/helm-unittest.git +brew install chart-testing +``` + +Alternatively, on macOS, you can also run the following make targets: ```bash make install-dependencies-brew @@ -183,7 +189,7 @@ Integration tests are run with the Chart Testing tool: ct install --namespace polaris --charts ./helm/polaris ``` -### Running tets with Gradle +### Running tests with Gradle Both unit and integration tests can be run with Gradle. From the Polaris repo root: diff --git a/helm/polaris/README.md.gotmpl b/helm/polaris/README.md.gotmpl index af0ecf7aa9..cfae9b9b13 100644 --- a/helm/polaris/README.md.gotmpl +++ b/helm/polaris/README.md.gotmpl @@ -28,7 +28,7 @@ weight: 675 Do not modify the README.md file directly, please modify README.md.gotmpl instead. To re-generate the README.md file, install helm-docs then run from the repo root: helm-docs --chart-search-root=helm - Alternatively, run ./gradlew helmDocs from the repo root. + Alternatively, run `./gradlew helmDocs` or `make helm-doc-generate` from the repo root. --> {{ template "chart.deprecationWarning" . }} @@ -144,7 +144,13 @@ The following tools are required to run the tests: * [Helm Unit Test](https://github.com/helm-unittest/helm-unittest) * [Chart Testing](https://github.com/helm/chart-testing) -Quick installation of all required tools on macOS: +Quick installation instructions for these tools: +```bash +helm plugin install https://github.com/helm-unittest/helm-unittest.git +brew install chart-testing +``` + +Alternatively, on macOS, you can also run the following make targets: ```bash make install-dependencies-brew @@ -185,7 +191,7 @@ Integration tests are run with the Chart Testing tool: ct install --namespace polaris --charts ./helm/polaris ``` -### Running tets with Gradle +### Running tests with Gradle Both unit and integration tests can be run with Gradle. From the Polaris repo root: diff --git a/helm/polaris/build.gradle.kts b/helm/polaris/build.gradle.kts index 02716f592a..2fef86035c 100644 --- a/helm/polaris/build.gradle.kts +++ b/helm/polaris/build.gradle.kts @@ -24,6 +24,14 @@ plugins { description = "Apache Polaris (incubating) Helm Chart" +val runtimeServerDistribution by + configurations.creating { + isCanBeConsumed = false + isCanBeResolved = true + } + +dependencies { runtimeServerDistribution(project(":polaris-server", "distributionElements")) } + val helmTestReportsDir = layout.buildDirectory.dir("reports") val helmTemplateValidation by @@ -31,29 +39,36 @@ val helmTemplateValidation by group = "verification" description = "Run 'helm template' validation with all values files" - workingDir = projectDir + val outputFile = helmTestReportsDir.get().file("helm-template/validation.log").asFile - val outputFile = helmTestReportsDir.get().file("helm-template/validation.log") + doFirst { + outputFile.parentFile.mkdirs() + val outStream = outputFile.outputStream() + standardOutput = outStream + errorOutput = outStream + } commandLine = listOf( "sh", "-c", """ - mkdir -p ${outputFile.asFile.parent} - for f in values.yaml ci/*.yaml; do - echo "Validating helm template with ${'$'}f" - helm template --debug --namespace polaris-ns --values ${'$'}f . - done > ${outputFile.asFile} 2>&1 - """ + set -e + for f in values.yaml ci/*.yaml; do + echo "Validating helm template with ${'$'}f" + helm template --debug --namespace polaris-ns --values ${'$'}f . + done + """ .trimIndent(), ) + checkExitCode(outputFile) + inputs.files( fileTree(projectDir) { include("Chart.yaml", "values.yaml", "ci/*.yaml", "templates/**/*") } ) - outputs.file(outputFile) + outputs.cacheIf { true } } val helmUnitTest by @@ -61,28 +76,37 @@ val helmUnitTest by group = "verification" description = "Run Helm unit tests" - workingDir = projectDir + val outputFile = helmTestReportsDir.get().file("helm-unit/test.log").asFile - val outputFile = helmTestReportsDir.get().file("helm-unit/test.log") + doFirst { + outputFile.parentFile.mkdirs() + val outStream = outputFile.outputStream() + standardOutput = outStream + errorOutput = outStream + } - // Install the plugin if not already installed, then run tests commandLine = listOf( "sh", "-c", """ - mkdir -p ${outputFile.asFile.parent} - helm plugin install https://github.com/helm-unittest/helm-unittest.git 2>/dev/null || true - helm unittest . > ${outputFile.asFile} 2>&1 - """ + set -e + echo "====== Install helm-unittest plugin ======" + helm plugin install https://github.com/helm-unittest/helm-unittest.git || true + + echo "====== Run helm unit tests ======" + helm unittest . + """ .trimIndent(), ) + checkExitCode(outputFile) + inputs.files( fileTree(projectDir) { include("Chart.yaml", "values.yaml", "templates/**/*", "tests/**/*") } ) - outputs.file(outputFile) + outputs.cacheIf { true } } val chartTestingLint by @@ -90,26 +114,33 @@ val chartTestingLint by group = "verification" description = "Run chart-testing (lint)" - workingDir = rootProject.projectDir + val outputFile = helmTestReportsDir.get().file("ct/lint.log").asFile - val outputFile = helmTestReportsDir.get().file("ct/lint.log") + doFirst { + outputFile.parentFile.mkdirs() + val outStream = outputFile.outputStream() + standardOutput = outStream + errorOutput = outStream + } commandLine = listOf( "sh", "-c", """ - mkdir -p ${outputFile.asFile.parent} - ct lint --debug --charts ./helm/polaris > ${outputFile.asFile} 2>&1 - """ + set -e + ct lint --debug --charts . + """ .trimIndent(), ) + checkExitCode(outputFile) + inputs.files( fileTree(projectDir) { include("Chart.yaml", "values.yaml", "ci/*.yaml", "templates/**/*") } ) - outputs.file(outputFile) + outputs.cacheIf { true } } // Task to build Docker images in minikube's Docker environment @@ -118,45 +149,57 @@ val buildMinikubeImages by group = "build" description = "Build Polaris Docker images in minikube's Docker environment" + // Must be run from root project directory because Dockerfile.jvm must be + // contained within the build context workingDir = rootProject.projectDir - // Output: file containing SHA digest of built image - val outputFile = helmTestReportsDir.get().file("minikube-images.sha256") + val outputFile = helmTestReportsDir.get().file("minikube-images/build.log").asFile + val digestFile = helmTestReportsDir.get().file("minikube-images/build.sha256").asFile + val dockerFile = project(":polaris-server").projectDir.resolve("src/main/docker/Dockerfile.jvm") + + doFirst { + outputFile.parentFile.mkdirs() + digestFile.parentFile.mkdirs() + val outStream = outputFile.outputStream() + standardOutput = outStream + errorOutput = outStream + } commandLine = listOf( "sh", "-c", """ - # Check if minikube is running + set -e + echo "====== Check if minikube is running ======" if ! minikube status >/dev/null 2>&1; then echo "Minikube is not running. Starting minikube..." minikube start fi - # Set up docker environment and build images + echo "====== Set up docker environment and build images ======" eval $(minikube -p minikube docker-env) - # Build server image + echo "====== Build server image ======" docker build -t apache/polaris:latest \ - -f runtime/server/src/main/docker/Dockerfile.jvm \ - runtime/server + -f ${dockerFile.relativeTo(workingDir)} \ + ${project(":polaris-server").projectDir.relativeTo(workingDir)} - # Capture image digest - mkdir -p ${outputFile.asFile.parent} { docker inspect --format='{{.Id}}' apache/polaris:latest - } > ${outputFile.asFile} + } > ${digestFile.relativeTo(workingDir)} """ .trimIndent(), ) + checkExitCode(outputFile) + dependsOn(":polaris-server:quarkusBuild") - inputs.dir(rootProject.file("runtime/server/build/quarkus-app")) - inputs.file(rootProject.file("runtime/server/src/main/docker/Dockerfile.jvm")) + inputs.files(runtimeServerDistribution) + inputs.file(dockerFile) - outputs.file(outputFile) + outputs.cacheIf { true } } // Task to run chart-testing install on minikube @@ -165,62 +208,80 @@ val chartTestingInstall by group = "verification" description = "Run chart-testing (install) on minikube" - workingDir = rootProject.projectDir + val outputFile = helmTestReportsDir.get().file("ct/install.log").asFile - val outputFile = helmTestReportsDir.get().file("ct/install.log") + doFirst { + outputFile.parentFile.mkdirs() + val outStream = outputFile.outputStream() + standardOutput = outStream + errorOutput = outStream + } commandLine = listOf( "sh", "-c", """ - mkdir -p ${outputFile.asFile.parent} + set -eo pipefail + echo "====== Check if minikube is running ======" + if ! minikube status >/dev/null 2>&1; then + echo "Minikube is not running. Starting minikube..." + minikube start + fi - # Create namespace if it doesn't exist - kubectl create namespace polaris-ns --dry-run=client -o yaml | kubectl apply -f - + echo "====== Create namespace if it doesn't exist ======" + kubectl create namespace polaris-ns --dry-run=client -o yaml | kubectl apply -f - - # Install fixtures - kubectl apply --namespace polaris-ns -f helm/polaris/ci/fixtures + echo "===== Install fixtures ======" + kubectl apply --namespace polaris-ns -f helm/polaris/ci/fixtures - # Run chart-testing install - ct install \ - --namespace polaris-ns \ - --debug --charts ./helm/polaris > ${outputFile.asFile} 2>&1 - """ + echo "===== Run chart-testing install ======" + ct install --namespace polaris-ns --debug --charts . + """ .trimIndent(), ) + checkExitCode(outputFile) + + dependsOn(buildMinikubeImages) + inputs.files( fileTree(projectDir) { include("Chart.yaml", "values.yaml", "ci/**/*", "templates/**/*") } ) - outputs.file(outputFile) - - // Depend on the images being built in minikube - dependsOn(buildMinikubeImages) + outputs.cacheIf { true } } -// Task to generate helm documentation val helmDocs by tasks.registering(Exec::class) { group = "documentation" description = "Generate Helm chart documentation using helm-docs" - workingDir = rootProject.projectDir + val outputFile = helmTestReportsDir.get().file("helm-docs/build.log").asFile + + doFirst { + outputFile.parentFile.mkdirs() + val outStream = outputFile.outputStream() + standardOutput = outStream + errorOutput = outStream + } commandLine = listOf( "sh", "-c", """ - helm-docs --chart-search-root=helm + set -e + helm-docs --chart-search-root=. """ .trimIndent(), ) + checkExitCode(outputFile) + inputs.files(fileTree(projectDir) { include("Chart.yaml", "values.yaml", "README.md.gotmpl") }) - outputs.file(projectDir.resolve("README.md")) + outputs.cacheIf { true } } val test by @@ -235,11 +296,22 @@ val intTest by group = "verification" description = "Run Helm chart integration tests on minikube" dependsOn(chartTestingInstall) + mustRunAfter(test) } -tasks.named("check") { dependsOn(test) } +tasks.named("check") { dependsOn(test, intTest) } + +tasks.named("assemble") { dependsOn(helmDocs) } -tasks.named("build") { - dependsOn(intTest) - dependsOn(helmDocs) +fun Exec.checkExitCode(outputFile: File) { + isIgnoreExitValue = true + doLast { + val exitValue = executionResult.get().exitValue + if (exitValue != 0) { + logger.error("Shell script failed with exit code $exitValue.") + logger.error("To identify the cause of the failure, inspect the logs:") + logger.error(outputFile.absolutePath) + throw GradleException("Shell script failed with exit code $exitValue") + } + } } diff --git a/site/content/in-dev/unreleased/helm.md b/site/content/in-dev/unreleased/helm.md index 3d5a8f660b..17e57eefa1 100644 --- a/site/content/in-dev/unreleased/helm.md +++ b/site/content/in-dev/unreleased/helm.md @@ -28,7 +28,7 @@ weight: 675 Do not modify the README.md file directly, please modify README.md.gotmpl instead. To re-generate the README.md file, install helm-docs then run from the repo root: helm-docs --chart-search-root=helm - Alternatively, run ./gradlew helmDocs from the repo root. + Alternatively, run `./gradlew helmDocs` or `make helm-doc-generate` from the repo root. --> ![Version: 1.2.0-incubating-SNAPSHOT](https://img.shields.io/badge/Version-1.2.0--incubating--SNAPSHOT-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0-incubating-SNAPSHOT](https://img.shields.io/badge/AppVersion-1.2.0--incubating--SNAPSHOT-informational?style=flat-square) @@ -144,7 +144,13 @@ The following tools are required to run the tests: * [Helm Unit Test](https://github.com/helm-unittest/helm-unittest) * [Chart Testing](https://github.com/helm/chart-testing) -Quick installation of all required tools on macOS: +Quick installation instructions for these tools: +```bash +helm plugin install https://github.com/helm-unittest/helm-unittest.git +brew install chart-testing +``` + +Alternatively, on macOS, you can also run the following make targets: ```bash make install-dependencies-brew @@ -185,7 +191,7 @@ Integration tests are run with the Chart Testing tool: ct install --namespace polaris --charts ./helm/polaris ``` -### Running tets with Gradle +### Running tests with Gradle Both unit and integration tests can be run with Gradle. From the Polaris repo root: @@ -321,6 +327,11 @@ Both unit and integration tests can be run with Gradle. From the Polaris repo ro | persistence.relationalJdbc.secret.username | string | `"username"` | The secret key holding the database username for authentication | | persistence.type | string | `"in-memory"` | The type of persistence to use. Two built-in types are supported: in-memory and relational-jdbc. The eclipse-link type is also supported but is deprecated. | | podAnnotations | object | `{}` | Annotations to apply to polaris pods. | +| podDisruptionBudget | object | `{"annotations":{},"enabled":false,"maxUnavailable":null,"minAvailable":null}` | Pod disruption budget settings. | +| podDisruptionBudget.annotations | object | `{}` | Annotations to add to the pod disruption budget. | +| podDisruptionBudget.enabled | bool | `false` | Specifies whether a pod disruption budget should be created. | +| podDisruptionBudget.maxUnavailable | string | `nil` | The maximum number of pods that can be unavailable during disruptions. Can be an absolute number (ex: 5) or a percentage of desired pods (ex: 50%). IMPORTANT: Cannot be used simultaneously with minAvailable. | +| podDisruptionBudget.minAvailable | string | `nil` | The minimum number of pods that should remain available during disruptions. Can be an absolute number (ex: 5) or a percentage of desired pods (ex: 50%). IMPORTANT: Cannot be used simultaneously with maxUnavailable. | | podLabels | object | `{}` | Additional Labels to apply to polaris pods. | | podSecurityContext | object | `{"fsGroup":10001,"seccompProfile":{"type":"RuntimeDefault"}}` | Security context for the polaris pod. See https://kubernetes.io/docs/tasks/configure-pod-container/security-context/. | | podSecurityContext.fsGroup | int | `10001` | GID 10001 is compatible with Polaris OSS default images; change this if you are using a different image. | From bfbd4dd368c25efab8c7b07c7f3d00938f5cc6ac Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Wed, 5 Nov 2025 15:24:39 +0100 Subject: [PATCH 03/10] cleanup --- helm/polaris/build.gradle.kts | 152 ++++++++++------------------------ 1 file changed, 44 insertions(+), 108 deletions(-) diff --git a/helm/polaris/build.gradle.kts b/helm/polaris/build.gradle.kts index 2fef86035c..9097371ac7 100644 --- a/helm/polaris/build.gradle.kts +++ b/helm/polaris/build.gradle.kts @@ -41,28 +41,16 @@ val helmTemplateValidation by val outputFile = helmTestReportsDir.get().file("helm-template/validation.log").asFile - doFirst { - outputFile.parentFile.mkdirs() - val outStream = outputFile.outputStream() - standardOutput = outStream - errorOutput = outStream - } - - commandLine = - listOf( - "sh", - "-c", - """ + runShellScript( + """ set -e for f in values.yaml ci/*.yaml; do echo "Validating helm template with ${'$'}f" helm template --debug --namespace polaris-ns --values ${'$'}f . done - """ - .trimIndent(), - ) - - checkExitCode(outputFile) + """, + outputFile, + ) inputs.files( fileTree(projectDir) { include("Chart.yaml", "values.yaml", "ci/*.yaml", "templates/**/*") } @@ -78,29 +66,17 @@ val helmUnitTest by val outputFile = helmTestReportsDir.get().file("helm-unit/test.log").asFile - doFirst { - outputFile.parentFile.mkdirs() - val outStream = outputFile.outputStream() - standardOutput = outStream - errorOutput = outStream - } - - commandLine = - listOf( - "sh", - "-c", - """ + runShellScript( + """ set -e echo "====== Install helm-unittest plugin ======" helm plugin install https://github.com/helm-unittest/helm-unittest.git || true echo "====== Run helm unit tests ======" helm unittest . - """ - .trimIndent(), - ) - - checkExitCode(outputFile) + """, + outputFile, + ) inputs.files( fileTree(projectDir) { include("Chart.yaml", "values.yaml", "templates/**/*", "tests/**/*") } @@ -116,25 +92,13 @@ val chartTestingLint by val outputFile = helmTestReportsDir.get().file("ct/lint.log").asFile - doFirst { - outputFile.parentFile.mkdirs() - val outStream = outputFile.outputStream() - standardOutput = outStream - errorOutput = outStream - } - - commandLine = - listOf( - "sh", - "-c", - """ + runShellScript( + """ set -e ct lint --debug --charts . - """ - .trimIndent(), - ) - - checkExitCode(outputFile) + """, + outputFile, + ) inputs.files( fileTree(projectDir) { include("Chart.yaml", "values.yaml", "ci/*.yaml", "templates/**/*") } @@ -157,19 +121,10 @@ val buildMinikubeImages by val digestFile = helmTestReportsDir.get().file("minikube-images/build.sha256").asFile val dockerFile = project(":polaris-server").projectDir.resolve("src/main/docker/Dockerfile.jvm") - doFirst { - outputFile.parentFile.mkdirs() - digestFile.parentFile.mkdirs() - val outStream = outputFile.outputStream() - standardOutput = outStream - errorOutput = outStream - } + digestFile.parentFile.mkdirs() - commandLine = - listOf( - "sh", - "-c", - """ + runShellScript( + """ set -e echo "====== Check if minikube is running ======" if ! minikube status >/dev/null 2>&1; then @@ -188,11 +143,9 @@ val buildMinikubeImages by { docker inspect --format='{{.Id}}' apache/polaris:latest } > ${digestFile.relativeTo(workingDir)} - """ - .trimIndent(), - ) - - checkExitCode(outputFile) + """, + outputFile, + ) dependsOn(":polaris-server:quarkusBuild") @@ -210,18 +163,8 @@ val chartTestingInstall by val outputFile = helmTestReportsDir.get().file("ct/install.log").asFile - doFirst { - outputFile.parentFile.mkdirs() - val outStream = outputFile.outputStream() - standardOutput = outStream - errorOutput = outStream - } - - commandLine = - listOf( - "sh", - "-c", - """ + runShellScript( + """ set -eo pipefail echo "====== Check if minikube is running ======" if ! minikube status >/dev/null 2>&1; then @@ -233,15 +176,13 @@ val chartTestingInstall by kubectl create namespace polaris-ns --dry-run=client -o yaml | kubectl apply -f - echo "===== Install fixtures ======" - kubectl apply --namespace polaris-ns -f helm/polaris/ci/fixtures + kubectl apply --namespace polaris-ns -f ci/fixtures echo "===== Run chart-testing install ======" ct install --namespace polaris-ns --debug --charts . - """ - .trimIndent(), - ) - - checkExitCode(outputFile) + """, + outputFile, + ) dependsOn(buildMinikubeImages) @@ -259,25 +200,13 @@ val helmDocs by val outputFile = helmTestReportsDir.get().file("helm-docs/build.log").asFile - doFirst { - outputFile.parentFile.mkdirs() - val outStream = outputFile.outputStream() - standardOutput = outStream - errorOutput = outStream - } - - commandLine = - listOf( - "sh", - "-c", - """ + runShellScript( + """ set -e helm-docs --chart-search-root=. - """ - .trimIndent(), - ) - - checkExitCode(outputFile) + """, + outputFile, + ) inputs.files(fileTree(projectDir) { include("Chart.yaml", "values.yaml", "README.md.gotmpl") }) @@ -303,14 +232,21 @@ tasks.named("check") { dependsOn(test, intTest) } tasks.named("assemble") { dependsOn(helmDocs) } -fun Exec.checkExitCode(outputFile: File) { - isIgnoreExitValue = true +fun Exec.runShellScript(script: String, outputFile: File) { + doFirst { + outputFile.parentFile.mkdirs() + val outStream = outputFile.outputStream() + standardOutput = outStream + errorOutput = outStream + } + commandLine = listOf("sh", "-c", script.trimIndent()) + this.isIgnoreExitValue = true doLast { val exitValue = executionResult.get().exitValue if (exitValue != 0) { - logger.error("Shell script failed with exit code $exitValue.") - logger.error("To identify the cause of the failure, inspect the logs:") - logger.error(outputFile.absolutePath) + this.logger.error("Shell script failed with exit code $exitValue.") + this.logger.error("To identify the cause of the failure, inspect the logs:") + this.logger.error(outputFile.absolutePath) throw GradleException("Shell script failed with exit code $exitValue") } } From c7728992b6604f4272f4e648f023880bc3e9c8e2 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Wed, 5 Nov 2025 15:35:19 +0100 Subject: [PATCH 04/10] print logs if error --- helm/polaris/build.gradle.kts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/helm/polaris/build.gradle.kts b/helm/polaris/build.gradle.kts index 9097371ac7..a2d7ecdf90 100644 --- a/helm/polaris/build.gradle.kts +++ b/helm/polaris/build.gradle.kts @@ -244,9 +244,8 @@ fun Exec.runShellScript(script: String, outputFile: File) { doLast { val exitValue = executionResult.get().exitValue if (exitValue != 0) { - this.logger.error("Shell script failed with exit code $exitValue.") - this.logger.error("To identify the cause of the failure, inspect the logs:") - this.logger.error(outputFile.absolutePath) + this.logger.error("Shell script failed with exit code $exitValue:\n") + outputFile.readLines().forEach { this.logger.error(it) } throw GradleException("Shell script failed with exit code $exitValue") } } From cebb81a606e844cce4d1bd04cc8663ba0886dbd4 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Wed, 5 Nov 2025 15:50:03 +0100 Subject: [PATCH 05/10] check if helm-docs is installed --- helm/polaris/build.gradle.kts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/helm/polaris/build.gradle.kts b/helm/polaris/build.gradle.kts index a2d7ecdf90..09da086cc0 100644 --- a/helm/polaris/build.gradle.kts +++ b/helm/polaris/build.gradle.kts @@ -203,6 +203,30 @@ val helmDocs by runShellScript( """ set -e + + echo "====== Check if helm-docs is installed ======" + if ! command -v helm-docs >/dev/null 2>&1; then + echo "helm-docs is not installed." + + # Check if we're on macOS + if [[ "${'$'}(uname -s)" == "Darwin" ]]; then + # Check if brew is available + if command -v brew >/dev/null 2>&1; then + echo "Installing helm-docs using Homebrew..." + brew install norwoodj/tap/helm-docs + else + echo "WARNING: Homebrew is not installed. Cannot auto-install helm-docs." + echo "Please install Homebrew from https://brew.sh/ or install helm-docs manually." + exit 0 + fi + else + echo "WARNING: helm-docs is not installed. Skipping documentation generation." + echo "To install helm-docs on Linux, download from: https://github.com/norwoodj/helm-docs/releases" + exit 0 + fi + fi + + echo "====== Generate Helm documentation ======" helm-docs --chart-search-root=. """, outputFile, From 09e638ed63b7b13b2bea0ae40e472a2967d6ce75 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Wed, 5 Nov 2025 16:14:40 +0100 Subject: [PATCH 06/10] more checks --- helm/polaris/build.gradle.kts | 158 ++++++++++++++++++++++++++-------- 1 file changed, 123 insertions(+), 35 deletions(-) diff --git a/helm/polaris/build.gradle.kts b/helm/polaris/build.gradle.kts index 09da086cc0..06f3bc8750 100644 --- a/helm/polaris/build.gradle.kts +++ b/helm/polaris/build.gradle.kts @@ -34,6 +34,107 @@ dependencies { runtimeServerDistribution(project(":polaris-server", "distributio val helmTestReportsDir = layout.buildDirectory.dir("reports") +private val missingDependencyExitCode = -42 + +val helmChecks = + """ + echo "====== Check if helm is installed ======" + if ! command -v helm >/dev/null 2>&1; then + echo "helm is not installed." + # Check if we're on macOS + if [[ "${'$'}(uname -s)" == "Darwin" ]]; then + # Check if brew is available + if command -v brew >/dev/null 2>&1; then + echo "Installing helm using Homebrew..." + brew install helm + else + echo "WARNING: Homebrew is not installed. Cannot auto-install helm." + echo "Please install Homebrew from https://brew.sh/ or install helm manually." + exit $missingDependencyExitCode + fi + else + echo "WARNING: helm is not installed." + echo "To install helm, see https://helm.sh/docs/intro/install/" + exit $missingDependencyExitCode + fi + fi + """ + +val chartTestingChecks = + """ + echo "====== Check if chart-testing is installed ======" + if ! command -v ct >/dev/null 2>&1; then + echo "chart-testing is not installed." + # Check if we're on macOS + if [[ "${'$'}(uname -s)" == "Darwin" ]]; then + # Check if brew is available + if command -v brew >/dev/null 2>&1; then + echo "Installing chart-testing using Homebrew..." + brew install chart-testing + else + echo "WARNING: Homebrew is not installed. Cannot auto-install chart-testing." + echo "Please install Homebrew from https://brew.sh/ or install chart-testing manually." + exit $missingDependencyExitCode + fi + else + echo "WARNING: chart-testing is not installed." + echo "To install chart-testing, see https://github.com/helm/chart-testing." + exit $missingDependencyExitCode + fi + fi + """ + +val minikubeChecks = + """ + echo "====== Check if Minikube is installed ======" + if ! command -v minikube >/dev/null 2>&1; then + echo "Minikube is not installed." + echo "To install minikube, see https://minikube.sigs.k8s.io/docs/start/." + exit $missingDependencyExitCode + fi + + echo "====== Check if Minikube is running ======" + if ! minikube status >/dev/null 2>&1; then + echo "Minikube is not running. Starting minikube..." + minikube start + fi + """ + +val kubectlChecks = + """ + echo "====== Check if kubectl is installed ======" + if ! command -v kubectl >/dev/null 2>&1; then + echo "kubectl is not installed." + echo "To install kubectl, see https://kubernetes.io/docs/tasks/tools/" + exit $missingDependencyExitCode + fi + """ + +val helmDocsChecks = + """ + echo "====== Check if helm-docs is installed ======" + if ! command -v helm-docs >/dev/null 2>&1; then + echo "helm-docs is not installed." + + # Check if we're on macOS + if [[ "${'$'}(uname -s)" == "Darwin" ]]; then + # Check if brew is available + if command -v brew >/dev/null 2>&1; then + echo "Installing helm-docs using Homebrew..." + brew install norwoodj/tap/helm-docs + else + echo "WARNING: Homebrew is not installed. Cannot auto-install helm-docs." + echo "Please install Homebrew from https://brew.sh/ or install helm-docs manually." + exit $missingDependencyExitCode + fi + else + echo "WARNING: helm-docs is not installed. Skipping documentation generation." + echo "To install helm-docs on Linux, download from: https://github.com/norwoodj/helm-docs/releases" + exit $missingDependencyExitCode + fi + fi + """ + val helmTemplateValidation by tasks.registering(Exec::class) { group = "verification" @@ -44,6 +145,7 @@ val helmTemplateValidation by runShellScript( """ set -e + ${helmChecks.trimIndent()} for f in values.yaml ci/*.yaml; do echo "Validating helm template with ${'$'}f" helm template --debug --namespace polaris-ns --values ${'$'}f . @@ -69,6 +171,7 @@ val helmUnitTest by runShellScript( """ set -e + ${helmChecks.trimIndent()} echo "====== Install helm-unittest plugin ======" helm plugin install https://github.com/helm-unittest/helm-unittest.git || true @@ -95,6 +198,8 @@ val chartTestingLint by runShellScript( """ set -e + ${helmChecks.trimIndent()} + ${chartTestingChecks.trimIndent()} ct lint --debug --charts . """, outputFile, @@ -126,11 +231,8 @@ val buildMinikubeImages by runShellScript( """ set -e - echo "====== Check if minikube is running ======" - if ! minikube status >/dev/null 2>&1; then - echo "Minikube is not running. Starting minikube..." - minikube start - fi + + ${minikubeChecks.trimIndent()} echo "====== Set up docker environment and build images ======" eval $(minikube -p minikube docker-env) @@ -166,11 +268,11 @@ val chartTestingInstall by runShellScript( """ set -eo pipefail - echo "====== Check if minikube is running ======" - if ! minikube status >/dev/null 2>&1; then - echo "Minikube is not running. Starting minikube..." - minikube start - fi + + ${helmChecks.trimIndent()} + ${minikubeChecks.trimIndent()} + ${chartTestingChecks.trimIndent()} + ${kubectlChecks.trimIndent()} echo "====== Create namespace if it doesn't exist ======" kubectl create namespace polaris-ns --dry-run=client -o yaml | kubectl apply -f - @@ -204,27 +306,8 @@ val helmDocs by """ set -e - echo "====== Check if helm-docs is installed ======" - if ! command -v helm-docs >/dev/null 2>&1; then - echo "helm-docs is not installed." - - # Check if we're on macOS - if [[ "${'$'}(uname -s)" == "Darwin" ]]; then - # Check if brew is available - if command -v brew >/dev/null 2>&1; then - echo "Installing helm-docs using Homebrew..." - brew install norwoodj/tap/helm-docs - else - echo "WARNING: Homebrew is not installed. Cannot auto-install helm-docs." - echo "Please install Homebrew from https://brew.sh/ or install helm-docs manually." - exit 0 - fi - else - echo "WARNING: helm-docs is not installed. Skipping documentation generation." - echo "To install helm-docs on Linux, download from: https://github.com/norwoodj/helm-docs/releases" - exit 0 - fi - fi + ${helmChecks.trimIndent()} + ${helmDocsChecks.trimIndent()} echo "====== Generate Helm documentation ======" helm-docs --chart-search-root=. @@ -263,14 +346,19 @@ fun Exec.runShellScript(script: String, outputFile: File) { standardOutput = outStream errorOutput = outStream } - commandLine = listOf("sh", "-c", script.trimIndent()) + commandLine = listOf("bash", "-c", script.trimIndent()) this.isIgnoreExitValue = true doLast { val exitValue = executionResult.get().exitValue if (exitValue != 0) { - this.logger.error("Shell script failed with exit code $exitValue:\n") - outputFile.readLines().forEach { this.logger.error(it) } - throw GradleException("Shell script failed with exit code $exitValue") + if (exitValue == missingDependencyExitCode) { + this.logger.warn("Missing required executable, skipping task:") + outputFile.readLines().forEach { this.logger.warn(it) } + } else { + this.logger.error("Shell script failed with exit code $exitValue:\n") + outputFile.readLines().forEach { this.logger.error(it) } + throw GradleException("Shell script failed with exit code $exitValue") + } } } } From fffa90311d66bdc92dbfd0b96de9ef04c4b41515 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Wed, 5 Nov 2025 16:22:02 +0100 Subject: [PATCH 07/10] some fixes --- helm/polaris/build.gradle.kts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/helm/polaris/build.gradle.kts b/helm/polaris/build.gradle.kts index 06f3bc8750..ccd991f7ba 100644 --- a/helm/polaris/build.gradle.kts +++ b/helm/polaris/build.gradle.kts @@ -1,3 +1,5 @@ +import java.io.OutputStream + /* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -34,7 +36,7 @@ dependencies { runtimeServerDistribution(project(":polaris-server", "distributio val helmTestReportsDir = layout.buildDirectory.dir("reports") -private val missingDependencyExitCode = -42 +private val missingDependencyExitCode = 42 val helmChecks = """ @@ -340,15 +342,18 @@ tasks.named("check") { dependsOn(test, intTest) } tasks.named("assemble") { dependsOn(helmDocs) } fun Exec.runShellScript(script: String, outputFile: File) { + var outStream: OutputStream? = null doFirst { outputFile.parentFile.mkdirs() - val outStream = outputFile.outputStream() + outStream = outputFile.outputStream() standardOutput = outStream errorOutput = outStream } commandLine = listOf("bash", "-c", script.trimIndent()) this.isIgnoreExitValue = true doLast { + // Close the output stream before reading the file to avoid race conditions + outStream?.close() val exitValue = executionResult.get().exitValue if (exitValue != 0) { if (exitValue == missingDependencyExitCode) { From 024b3afc036ddf0e1ae35b353ef1b0b9d33e9f53 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Wed, 5 Nov 2025 16:29:17 +0100 Subject: [PATCH 08/10] remove checks (overkill) --- helm/polaris/build.gradle.kts | 127 +--------------------------------- 1 file changed, 3 insertions(+), 124 deletions(-) diff --git a/helm/polaris/build.gradle.kts b/helm/polaris/build.gradle.kts index ccd991f7ba..e7672f4d43 100644 --- a/helm/polaris/build.gradle.kts +++ b/helm/polaris/build.gradle.kts @@ -36,107 +36,6 @@ dependencies { runtimeServerDistribution(project(":polaris-server", "distributio val helmTestReportsDir = layout.buildDirectory.dir("reports") -private val missingDependencyExitCode = 42 - -val helmChecks = - """ - echo "====== Check if helm is installed ======" - if ! command -v helm >/dev/null 2>&1; then - echo "helm is not installed." - # Check if we're on macOS - if [[ "${'$'}(uname -s)" == "Darwin" ]]; then - # Check if brew is available - if command -v brew >/dev/null 2>&1; then - echo "Installing helm using Homebrew..." - brew install helm - else - echo "WARNING: Homebrew is not installed. Cannot auto-install helm." - echo "Please install Homebrew from https://brew.sh/ or install helm manually." - exit $missingDependencyExitCode - fi - else - echo "WARNING: helm is not installed." - echo "To install helm, see https://helm.sh/docs/intro/install/" - exit $missingDependencyExitCode - fi - fi - """ - -val chartTestingChecks = - """ - echo "====== Check if chart-testing is installed ======" - if ! command -v ct >/dev/null 2>&1; then - echo "chart-testing is not installed." - # Check if we're on macOS - if [[ "${'$'}(uname -s)" == "Darwin" ]]; then - # Check if brew is available - if command -v brew >/dev/null 2>&1; then - echo "Installing chart-testing using Homebrew..." - brew install chart-testing - else - echo "WARNING: Homebrew is not installed. Cannot auto-install chart-testing." - echo "Please install Homebrew from https://brew.sh/ or install chart-testing manually." - exit $missingDependencyExitCode - fi - else - echo "WARNING: chart-testing is not installed." - echo "To install chart-testing, see https://github.com/helm/chart-testing." - exit $missingDependencyExitCode - fi - fi - """ - -val minikubeChecks = - """ - echo "====== Check if Minikube is installed ======" - if ! command -v minikube >/dev/null 2>&1; then - echo "Minikube is not installed." - echo "To install minikube, see https://minikube.sigs.k8s.io/docs/start/." - exit $missingDependencyExitCode - fi - - echo "====== Check if Minikube is running ======" - if ! minikube status >/dev/null 2>&1; then - echo "Minikube is not running. Starting minikube..." - minikube start - fi - """ - -val kubectlChecks = - """ - echo "====== Check if kubectl is installed ======" - if ! command -v kubectl >/dev/null 2>&1; then - echo "kubectl is not installed." - echo "To install kubectl, see https://kubernetes.io/docs/tasks/tools/" - exit $missingDependencyExitCode - fi - """ - -val helmDocsChecks = - """ - echo "====== Check if helm-docs is installed ======" - if ! command -v helm-docs >/dev/null 2>&1; then - echo "helm-docs is not installed." - - # Check if we're on macOS - if [[ "${'$'}(uname -s)" == "Darwin" ]]; then - # Check if brew is available - if command -v brew >/dev/null 2>&1; then - echo "Installing helm-docs using Homebrew..." - brew install norwoodj/tap/helm-docs - else - echo "WARNING: Homebrew is not installed. Cannot auto-install helm-docs." - echo "Please install Homebrew from https://brew.sh/ or install helm-docs manually." - exit $missingDependencyExitCode - fi - else - echo "WARNING: helm-docs is not installed. Skipping documentation generation." - echo "To install helm-docs on Linux, download from: https://github.com/norwoodj/helm-docs/releases" - exit $missingDependencyExitCode - fi - fi - """ - val helmTemplateValidation by tasks.registering(Exec::class) { group = "verification" @@ -147,7 +46,6 @@ val helmTemplateValidation by runShellScript( """ set -e - ${helmChecks.trimIndent()} for f in values.yaml ci/*.yaml; do echo "Validating helm template with ${'$'}f" helm template --debug --namespace polaris-ns --values ${'$'}f . @@ -173,7 +71,6 @@ val helmUnitTest by runShellScript( """ set -e - ${helmChecks.trimIndent()} echo "====== Install helm-unittest plugin ======" helm plugin install https://github.com/helm-unittest/helm-unittest.git || true @@ -200,8 +97,6 @@ val chartTestingLint by runShellScript( """ set -e - ${helmChecks.trimIndent()} - ${chartTestingChecks.trimIndent()} ct lint --debug --charts . """, outputFile, @@ -234,8 +129,6 @@ val buildMinikubeImages by """ set -e - ${minikubeChecks.trimIndent()} - echo "====== Set up docker environment and build images ======" eval $(minikube -p minikube docker-env) @@ -271,11 +164,6 @@ val chartTestingInstall by """ set -eo pipefail - ${helmChecks.trimIndent()} - ${minikubeChecks.trimIndent()} - ${chartTestingChecks.trimIndent()} - ${kubectlChecks.trimIndent()} - echo "====== Create namespace if it doesn't exist ======" kubectl create namespace polaris-ns --dry-run=client -o yaml | kubectl apply -f - @@ -307,10 +195,6 @@ val helmDocs by runShellScript( """ set -e - - ${helmChecks.trimIndent()} - ${helmDocsChecks.trimIndent()} - echo "====== Generate Helm documentation ======" helm-docs --chart-search-root=. """, @@ -356,14 +240,9 @@ fun Exec.runShellScript(script: String, outputFile: File) { outStream?.close() val exitValue = executionResult.get().exitValue if (exitValue != 0) { - if (exitValue == missingDependencyExitCode) { - this.logger.warn("Missing required executable, skipping task:") - outputFile.readLines().forEach { this.logger.warn(it) } - } else { - this.logger.error("Shell script failed with exit code $exitValue:\n") - outputFile.readLines().forEach { this.logger.error(it) } - throw GradleException("Shell script failed with exit code $exitValue") - } + this.logger.error("Shell script failed with exit code $exitValue:\n") + outputFile.readLines().forEach { this.logger.error(it) } + throw GradleException("Shell script failed with exit code $exitValue") } } } From 50fae2f4bdff51ccf20551cc252418fd1546e20d Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Wed, 5 Nov 2025 16:29:26 +0100 Subject: [PATCH 09/10] fix CI --- .github/workflows/spark_client_regtests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spark_client_regtests.yml b/.github/workflows/spark_client_regtests.yml index 1ec0082c31..ae3f207b67 100644 --- a/.github/workflows/spark_client_regtests.yml +++ b/.github/workflows/spark_client_regtests.yml @@ -58,7 +58,7 @@ jobs: # publishToMavenLocal causes a GH API requests, use the token for those requests GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ./gradlew assemble publishToMavenLocal + run: ./gradlew assemble publishToMavenLocal -x :polaris-helm:assemble - name: Image build env: From b17a897ace81190c3fec743f0a20cb32fbb7b801 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Wed, 5 Nov 2025 16:31:22 +0100 Subject: [PATCH 10/10] fix CI --- .github/workflows/gradle.yml | 2 +- helm/polaris/build.gradle.kts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 7931b7e623..cb680f644c 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -215,7 +215,7 @@ jobs: ./gradlew \ intTest \ -x :polaris-runtime-service:intTest \ - -x :polaris-helm:check \ + -x :polaris-helm:intTest \ --continue env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} diff --git a/helm/polaris/build.gradle.kts b/helm/polaris/build.gradle.kts index e7672f4d43..06933f6a45 100644 --- a/helm/polaris/build.gradle.kts +++ b/helm/polaris/build.gradle.kts @@ -236,7 +236,6 @@ fun Exec.runShellScript(script: String, outputFile: File) { commandLine = listOf("bash", "-c", script.trimIndent()) this.isIgnoreExitValue = true doLast { - // Close the output stream before reading the file to avoid race conditions outStream?.close() val exitValue = executionResult.get().exitValue if (exitValue != 0) {