diff --git a/.github/workflows/go-integration.yml b/.github/workflows/go-integration.yml index 76941e45f..6a6696cf5 100644 --- a/.github/workflows/go-integration.yml +++ b/.github/workflows/go-integration.yml @@ -46,14 +46,8 @@ jobs: cache: true cache-dependency-path: go.sum - - name: Start docker - run: | - docker compose -f internal/recipe/docker-compose.yml up -d - sleep 10 - - name: Provision Tables - run: | - docker compose -f internal/recipe/docker-compose.yml exec -T spark-iceberg ipython ./provision.py - sleep 10 + - name: Integration setup + run: make integration-setup - name: Setup environment variables run: | @@ -62,22 +56,13 @@ jobs: echo "DOCKER_API_VERSION=$(docker version -f '{{.Server.APIVersion}}')" >> $GITHUB_ENV - name: Run integration tests - env: - AWS_S3_ENDPOINT: "${{ env.AWS_S3_ENDPOINT }}" - AWS_REGION: "us-east-1" - run: | - go test -tags integration -v -run="^TestScanner" ./table - go test -tags integration -v ./io - go test -tags integration -v -run="^TestRestIntegration$" ./catalog/rest - go test -tags=integration -v ./catalog/hive/... - - name: Run spark integration tests env: AWS_S3_ENDPOINT: "${{ env.AWS_S3_ENDPOINT }}" AWS_REGION: "us-east-1" SPARK_CONTAINER_ID: "${{ env.SPARK_CONTAINER_ID }}" DOCKER_API_VER: "${{ env.DOCKER_API_VER }}" - run: | - go test -tags integration -v -run="^TestSparkIntegration" ./table + run: | + make integration-test - name: Show debug logs if: ${{ failure() }} diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..6458b0415 --- /dev/null +++ b/Makefile @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the "License"); +# you may not use it except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# golangci-lint version (keep in sync with CI and README) +GOLANGCI_LINT_VERSION := v2.8.0 + +.PHONY: test lint lint-install integration-setup integration-test integration-scanner integration-io integration-rest integration-spark + +test: + go test -v ./... + +lint: + golangci-lint run --timeout=10m + +lint-install: + go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) + +integration-setup: + docker compose -f internal/recipe/docker-compose.yml up -d + sleep 10 + docker compose -f internal/recipe/docker-compose.yml exec -T spark-iceberg ipython ./provision.py + sleep 10 + +integration-test: integration-scanner integration-io integration-rest integration-spark integration-hive + +integration-scanner: + go test -tags=integration -v -run="^TestScanner" ./table + +integration-io: + go test -tags=integration -v ./io + +integration-rest: + go test -tags=integration -v -run="^TestRestIntegration$$" ./catalog/rest + +integration-spark: + go test -tags=integration -v -run="^TestSparkIntegration" ./table + +integration-hive: + go test -tags=integration -v ./catalog/hive/... diff --git a/README.md b/README.md index 0d3f2dea0..d4348f428 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,56 @@ $ git clone https://github.com/apache/iceberg-go.git $ cd iceberg-go/cmd/iceberg && go build . ``` +## Running Tests + +Use the [Makefile](Makefile) so commands stay in sync with CI (e.g. golangci-lint version). + +### Unit tests + +```shell +make test +``` + +### Linting + +```shell +make lint +``` + +Install the linter first + +```shell +make lint-install +# or: go install github.com/golangci/golangci-lint/cmd/golangci-lint@v2.8.0 +``` + +### Integration tests + +**Prerequisites:** Docker, Docker Compose + +1. Start the Docker containers using docker compose: + + ```shell + make integration-setup + ``` + +2. Export the required environment variables: + + ```shell + export AWS_S3_ENDPOINT=http://$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' minio):9000 + export AWS_REGION=us-east-1 + export SPARK_CONTAINER_ID=$(docker ps -qf 'name=spark-iceberg') + export DOCKER_API_VER=$(docker version -f '{{.Server.APIVersion}}') + ``` + +3. Run the integration tests: + + ```shell + make integration-test + ``` + + Or run a single suite: `make integration-scanner`, `make integration-io`, `make integration-rest`, `make integration-spark`. + ## Feature Support / Roadmap ### FileSystem Support