Merge remote-tracking branch 'apache/main' into pass-internal-properties-to-fileio

Author: XJDKC

.github/workflows/helm.yml

@@ -43,6 +43,12 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+
       - name: Set up Helm
         uses: azure/[email protected]
         with:
@@ -83,25 +89,30 @@ jobs:
         if: steps.list-changed.outputs.changed == 'true'
         uses: medyagh/[email protected]
 
-      - name: Docker build
+      - name: Print Docker info
+        if: steps.list-changed.outputs.changed == 'true'
+        run: |
+          docker -v
+          minikube docker-env
+
+      - name: Image build
         if: steps.list-changed.outputs.changed == 'true'
         run: |
-          eval $(minikube docker-env)
-          docker build -f ./Dockerfile \
-            --build-arg ECLIPSELINK=true \
-            --build-arg ECLIPSELINK_DEPS=com.h2database:h2:2.3.232 \
-            -t polaris:latest .
+          eval $(minikube -p minikube docker-env)
+          ./gradlew :polaris-quarkus-server:assemble \
+            -Dquarkus.container-image.build=true \
+            -PeclipseLinkDeps=com.h2database:h2:2.3.232
+          minikube image ls
 
       - name: Install fixtures
         if: steps.list-changed.outputs.changed == 'true'
         run: |
           kubectl create namespace polaris-ns
-          kubectl apply --namespace polaris-ns $(find helm/polaris/ci/fixtures -name "*.yaml" -exec echo -n "-f {} " \;) 
+          kubectl apply --namespace polaris-ns -f helm/polaris/ci/fixtures 
 
       - name: Run chart-testing (install)
         if: steps.list-changed.outputs.changed == 'true'
         run: |
           ct install --target-branch ${{ github.event.repository.default_branch }} \
             --namespace polaris-ns \
-            --helm-extra-set-args "--set=image.repository=polaris --set=image.tag=latest" \
             --debug --charts ./helm/polaris

.github/workflows/regtest.yml

@@ -33,10 +33,23 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - name: fix permissions
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+
+      - name: Fix permissions
         run: mkdir -p regtests/output && chmod 777 regtests/output && chmod 777 regtests/t_*/ref/*
+
+      - name: Image build
+        run: |
+          ./gradlew :polaris-quarkus-server:assemble -Dquarkus.container-image.build=true
+
       - name: Regression Test
         env:
           AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
           AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
-        run: docker compose up --build --exit-code-from regtest
+        run: |
+          docker compose -f regtests/docker-compose.yml up --build --exit-code-from regtest

.github/workflows/stale.yml

@@ -22,7 +22,7 @@ jobs:
   stale:
     runs-on: ubuntu-24.04
     steps:
-      - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639
+      - uses: actions/stale@ee7ef89499a3de6e4fe1fc1acb994e67c64e0a2a
         with:
           days-before-close: 5
           days-before-stale: 30

Dockerfile

@@ -39,15 +39,30 @@ for contribution guidelines.
 ## Building and Running 
 
 Apache Polaris is organized into the following modules:
+
 - `polaris-core` - The main Polaris entity definitions and core business logic
-- `polaris-server` - The Polaris REST API server
-- `polaris-eclipselink` - The Eclipselink implementation of the MetaStoreManager interface
+- API modules (implementing the Iceberg REST API and Polaris management API):
+  - `polaris-api-management-model` - The Polaris management model
+  - `polaris-api-management-service` - The Polaris management service
+  - `polaris-api-iceberg-service` - The Iceberg REST service
+- Service modules:
+  - `polaris-service-common` - The main components of the Polaris server
+- Quarkus runtime modules:
+  - `polaris-quarkus-service` - The Quarkus-specific components of the Polaris server
+  - `polaris-quarkus-defaults` - The Quarkus-specific configuration defaults
+  - `polaris-quarkus-server` - The Polaris server runtime
+  - `polaris-quarkus-admin-tool` - The Polaris admin & maintenance tool
+- Persistence modules:
+  - `polaris-jpa-model` - The JPA entity definitions
+  - `polaris-eclipselink` - The Eclipselink implementation of the MetaStoreManager interface
 
 Apache Polaris is built using Gradle with Java 21+ and Docker 27+.
+
 - `./gradlew build` - To build and run tests. Make sure Docker is running, as the integration tests depend on it.
 - `./gradlew assemble` - To skip tests.
 - `./gradlew test` - To run unit tests and integration tests.
-- `./gradlew runApp` - To run the Polaris server locally on localhost:8181. 
+- `./gradlew run` - To run the Polaris server locally; the server is reachable at 
+  localhost:8181. This is also suitable for running regression tests, or for connecting with Spark. 
 - `./regtests/run_spark_sql.sh` - To connect from Spark SQL. Here are some example commands to run in the Spark SQL shell:
 ```sql
 create database db1;
@@ -56,37 +71,33 @@ create table db1.table1 (id int, name string);
 insert into db1.table1 values (1, 'a');
 select * from db1.table1;
 ```
+- `env POLARIS_HOST=localhost ./regtests/run.sh` - To run regression tests locally, see more options [here](./regtests/README.md).
+### More build and run options
 
-Apache Polaris supports the following optional build options:
-- `-PeclipseLink=true` – Enables the EclipseLink extension.
-- `-PeclipseLinkDeps=[groupId]:[artifactId]:[version],...` – Specifies one or more additional dependencies for EclipseLink (e.g., JDBC drivers) separated by commas.
+#### Running in Docker
 
-### More build and run options
-Running in Docker
-- `docker build -t localhost:5001/polaris:latest .` - To build the image.
-  - Optional build options:
-    - `docker build -t localhost:5001/polaris:latest --build-arg ECLIPSELINK=true .` - Enables the EclipseLink extension.
-    - `docker build -t localhost:5001/polaris:latest --build-arg ECLIPSELINK=true --build-arg ECLIPSELINK_DEPS=[groupId]:[artifactId]:[version],... .` – Enables the EclipseLink extension with one or more additional dependencies for EclipseLink (e.g. JDBC drivers) separated by commas.
-- `docker run -p 8181:8181 localhost:5001/polaris:latest` - To run the image in standalone mode.
-
-Running in Kubernetes
-- `./run.sh` - To run Polaris as a mini-deployment locally. This will create one pod that bind itself to ports `8181` and `8182`.
-  - Optional run options:
-    - `./run.sh -b "ECLIPSELINK=true"` - Enables the EclipseLink extension.
-    - `./run.sh -b "ECLIPSELINK=true;ECLIPSELINK_DEPS=[groupId]:[artifactId]:[version],..."` – Enables the EclipseLink extension with one or more additional dependencies for EclipseLink (e.g. JDBC drivers) separated by commas.
-- `kubectl port-forward svc/polaris-service -n polaris 8181:8181 8182:8182` - To create secure connections between a local machine and a pod within the cluster for both service and metrics endpoints.
-  - Currently supported metrics endpoints:
-    - localhost:8182/metrics
-    - localhost:8182/healthcheck
+- `./gradlew clean :polaris-quarkus-server:assemble -Dquarkus.container-image.build=true` - To 
+  build the image locally.
+- `docker run -p 8181:8181 -p 8182:8182 apache/polaris:latest` - To run the image.
+
+The Polaris codebase contains some docker compose examples to quickly get started with Polaris,
+using different configurations. Check the `./getting-started` directory for more information.
+
+#### Running in Kubernetes
+
+- `./run.sh` - To run Polaris as a mini-deployment locally. This will create a Kind cluster, 
+  then deploy one pod and one service. The service is available on ports `8181` and `8182`.
+- `kubectl port-forward svc/polaris-service -n polaris 8181:8181 8182:8182` - To create secure 
+  connections between a local machine and a pod within the cluster for both service and 
+  health/metrics endpoints:
+  - http://localhost:8182/q/metrics
+  - http://localhost:8182/q/health
 - `kubectl get pods -n polaris` - To check the status of the pods.
 - `kubectl get deployment -n polaris` - To check the status of the deployment.
 - `kubectl describe deployment polaris-deployment -n polaris` - To troubleshoot if things aren't working as expected.
 
-Running regression tests
-- `./regtests/run.sh` - To run regression tests in another terminal.
-- `docker compose up --build --exit-code-from regtest` - To run regression tests in a Docker environment.
+#### Building docs
 
-Building docs
 - Docs are generated using [Hugo](https://gohugo.io/) using the [Docsy](https://www.docsy.dev/docs/) theme.
 - To view the site locally, run
   ```bash

README.md

@@ -90,7 +90,6 @@ tasks.named<RatTask>("rat").configure {
 
   excludes.add("logs/**")
   excludes.add("service/common/src/**/banner.txt")
-  excludes.add("quarkus/service/logs")
 
   excludes.add("site/node_modules/**")
   excludes.add("site/layouts/robots.txt")

build.gradle.kts

@@ -0,0 +1,44 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+ 
+   http://www.apache.org/licenses/LICENSE-2.0
+ 
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Getting Started with Apache Polaris
+
+You can quickly get started with Polaris by playing with the docker-compose examples provided in
+this directory. Each example has detailed instructions.
+
+## Prerequisites
+
+- [Docker](https://docs.docker.com/get-docker/)
+- [Docker Compose](https://docs.docker.com/compose/install/) 
+- [jq](https://stedolan.github.io/jq/download/) (for some examples)
+
+## Getting Started Examples
+
+- [Spark](spark): An example that uses an in-memory metastore, automatically bootstrapped, with
+  Apache Spark and a Jupyter notebook.
+
+- [Trino](trino): An example that uses Trino with Polaris.
+
+- [Telemetry](telemetry): An example that includes Prometheus and Jaeger to collect metrics and
+  traces from Apache Polaris. This example automatically creates a `polaris_demo` catalog.
+
+- [Eclipselink](elipselink): An example that uses an Eclipselink metastore and a Postgres
+  database. The realm is bootstrapped with the Polaris Admin tool. This example also creates a
+  `polaris_demo` catalog, and offers the ability to run Spark SQL queries. Finally, it shows how to
+  attach a debugger to the Polaris server.

getting-started/README.md

@@ -0,0 +1,43 @@
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one
+  ~ or more contributor license agreements.  See the NOTICE file
+  ~ distributed with this work for additional information
+  ~ regarding copyright ownership.  The ASF licenses this file
+  ~ to you under the Apache License, Version 2.0 (the
+  ~ "License"); you may not use this file except in compliance
+  ~ with the License.  You may obtain a copy of the License at
+  ~
+  ~   http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing,
+  ~ software distributed under the License is distributed on an
+  ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  ~ KIND, either express or implied.  See the License for the
+  ~ specific language governing permissions and limitations
+  ~ under the License.
+  -->
+
+<persistence version="2.0" xmlns="http://java.sun.com/xml/ns/persistence"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://java.sun.com/xml/ns/persistence http://java.sun.com/xml/ns/persistence/persistence_2_0.xsd">
+
+  <persistence-unit name="polaris" transaction-type="RESOURCE_LOCAL">
+    <provider>org.eclipse.persistence.jpa.PersistenceProvider</provider>
+    <class>org.apache.polaris.jpa.models.ModelEntity</class>
+    <class>org.apache.polaris.jpa.models.ModelEntityActive</class>
+    <class>org.apache.polaris.jpa.models.ModelEntityChangeTracking</class>
+    <class>org.apache.polaris.jpa.models.ModelEntityDropped</class>
+    <class>org.apache.polaris.jpa.models.ModelGrantRecord</class>
+    <class>org.apache.polaris.jpa.models.ModelPrincipalSecrets</class>
+    <class>org.apache.polaris.jpa.models.ModelSequenceId</class>
+    <shared-cache-mode>NONE</shared-cache-mode>
+    <properties>
+      <property name="jakarta.persistence.jdbc.url"
+        value="jdbc:postgresql://postgres:5432/{realm}"/>
+      <property name="jakarta.persistence.jdbc.user" value="postgres"/>
+      <property name="jakarta.persistence.jdbc.password" value="postgres"/>
+      <property name="jakarta.persistence.schema-generation.database.action" value="create"/>
+      <property name="eclipselink.persistence-context.flush-mode" value="auto"/>
+    </properties>
+  </persistence-unit>
+</persistence>