From 08cdb4b167ed419e3e16c3f23b90cfe382198bc5 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Sat, 18 Jan 2025 11:13:49 +0100 Subject: [PATCH] ci: Build the ubuntu-aarch64 binary with new ARM runners The new GitHub arm runners are available, so we can use them to build the ubuntu aarch64 binary instead of our custom machine. --- .github/actionlint.yml | 5 + .github/scripts/arm/build.sh | 72 --------- .github/scripts/arm/docker-publish.sh | 50 ------ .github/workflows/build.yaml | 9 ++ .github/workflows/ci.yaml | 142 ++++-------------- .../arm/docker-env/Dockerfile => Dockerfile | 4 +- 6 files changed, 44 insertions(+), 238 deletions(-) create mode 100644 .github/actionlint.yml delete mode 100644 .github/scripts/arm/build.sh delete mode 100644 .github/scripts/arm/docker-publish.sh rename .github/scripts/arm/docker-env/Dockerfile => Dockerfile (80%) diff --git a/.github/actionlint.yml b/.github/actionlint.yml new file mode 100644 index 0000000000..c0783796e7 --- /dev/null +++ b/.github/actionlint.yml @@ -0,0 +1,5 @@ +# TODO: Remove this once a new actionlint release has been cut +# and made its way to us through nixpkgs. +self-hosted-runner: + labels: + - ubuntu-24.04-arm diff --git a/.github/scripts/arm/build.sh b/.github/scripts/arm/build.sh deleted file mode 100644 index 59ec677e34..0000000000 --- a/.github/scripts/arm/build.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/bin/bash -set -Eeuo pipefail - -# This script builds PostgREST in a remote ARM server - -[ -z "$1" ] && { echo "Missing 1st argument: PostgREST github commit SHA"; exit 1; } -[ -z "$2" ] && { echo "Missing 2nd argument: Build environment directory name"; exit 1; } -[ -z "$3" ] && { echo "Missing 3rd argument: GHC version"; exit 1; } - -PGRST_GITHUB_COMMIT="$1" -SCRIPT_DIR="$2" - -DOCKER_BUILD_DIR="$SCRIPT_DIR/docker-env" -# latest is a shortcut documented on https://www.haskell.org/ghcup/guide/#tags-and-shortcuts -CABAL_VERSION="latest" -GHC_VERSION="$3" - -install_packages() { - sudo apt-get update -y - sudo apt-get upgrade -y - sudo apt-get install -y git build-essential curl libffi-dev libffi7 libgmp-dev libgmp10 libncurses-dev libncurses5 libtinfo5 llvm libnuma-dev zlib1g-dev libpq-dev jq gcc - sudo apt-get clean -} - -install_ghcup() { - export BOOTSTRAP_HASKELL_NONINTERACTIVE=1 - export BOOTSTRAP_HASKELL_MINIMAL=1 - curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | sh - source ~/.ghcup/env -} - -install_cabal() { - ghcup upgrade - ghcup install cabal $CABAL_VERSION - ghcup set cabal $CABAL_VERSION -} - -install_ghc() { - ghcup upgrade - ghcup install ghc $GHC_VERSION - ghcup set ghc $GHC_VERSION -} - -install_packages - -# Add ghcup to the PATH for this session -[ -f ~/.ghcup/env ] && source ~/.ghcup/env - -ghcup --version || install_ghcup -ghcup set cabal $CABAL_VERSION || install_cabal -ghcup set ghc $GHC_VERSION || install_ghc - -cd ~/$SCRIPT_DIR - -# Clone the repository and build the project -git clone https://github.com/PostgREST/postgrest.git -cd postgrest -git checkout $PGRST_GITHUB_COMMIT -cabal v2-update && cabal v2-build - -# Strip unused symbols from executable -PGRST_BIN=$(cabal exec which postgrest | tail -1) -strip $PGRST_BIN - -# Copy the built binary to the Dockerfile directory -cp $PGRST_BIN ~/$DOCKER_BUILD_DIR - -# Move and compress the built binary -mkdir -p ~/$SCRIPT_DIR/result -mv $PGRST_BIN ~/$SCRIPT_DIR/result -cd ~/$SCRIPT_DIR -tar -cJf result.tar.xz result diff --git a/.github/scripts/arm/docker-publish.sh b/.github/scripts/arm/docker-publish.sh deleted file mode 100644 index fb810b71e7..0000000000 --- a/.github/scripts/arm/docker-publish.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -set -Eeuo pipefail - -# This script publishes the Docker ARM images to Docker Hub. - -[ -z "$1" ] && { echo "Missing 1st argument: PostgREST github commit SHA"; exit 1; } -[ -z "$2" ] && { echo "Missing 2nd argument: Docker repo"; exit 1; } -[ -z "$3" ] && { echo "Missing 3rd argument: Docker username"; exit 1; } -[ -z "$4" ] && { echo "Missing 4th argument: Docker password"; exit 1; } -[ -z "$5" ] && { echo "Missing 5th argument: Build environment directory name"; exit 1; } -[ -z "$6" ] && { echo "Missing 6th argument: PostgREST version"; exit 1; } - -PGRST_GITHUB_COMMIT="$1" -DOCKER_REPO="$2" -DOCKER_USER="$3" -DOCKER_PASS="$4" -SCRIPT_DIR="$5" -PGRST_VERSION="$6" - -DOCKER_BUILD_DIR="$SCRIPT_DIR/docker-env" - -clean_env() -{ - sudo docker logout -} - -# Login to Docker -sudo docker logout -{ echo $DOCKER_PASS | sudo docker login -u $DOCKER_USER --password-stdin; } || { echo "Couldn't login to docker"; exit 1; } - -trap clean_env sigint sigterm exit - -# Move to the docker build environment -cd ~/$DOCKER_BUILD_DIR - -# Push final images to Docker hub -# NOTE: This command publishes a separate ARM image because the builds cannot -# be added to the manifest if they are not in the registry beforehand. -# This image must be manually deleted from Docker Hub at the end of the process. -sudo docker buildx build --build-arg PGRST_GITHUB_COMMIT=$PGRST_GITHUB_COMMIT \ - -t $DOCKER_REPO/postgrest:$PGRST_VERSION-arm \ - --push . - -# Add the arm images to the manifest -# NOTE: This assumes that there already is a `postgrest:` image -# for the amd64 architecture pushed to Docker Hub -sudo docker buildx imagetools create --append -t $DOCKER_REPO/postgrest:$PGRST_VERSION $DOCKER_REPO/postgrest:$PGRST_VERSION-arm -[ "$PGRST_VERSION" != "devel" ] && sudo docker buildx imagetools create --append -t $DOCKER_REPO/postgrest:latest $DOCKER_REPO/postgrest:$PGRST_VERSION-arm - -sudo docker logout diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 87764677cd..7bce087c64 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -95,6 +95,15 @@ jobs: ~/.stack/stack.sqlite3 # no artifact for Linux x86-64, because we use the static build + - name: Linux aarch64 + runs-on: ubuntu-24.04-arm + cache: | + ~/.stack/pantry + ~/.stack/snapshots + ~/.stack/stack.sqlite3 + artifact: postgrest-ubuntu-aarch64 + deps: sudo apt-get update && sudo apt-get install libpq-dev + - name: MacOS runs-on: macos-14 cache: | diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2f5a360cb8..0f55b74aed 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -49,63 +49,6 @@ jobs: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - arm: - name: Build / Cabal - aarch64 GHC 9.4.8 - if: vars.SSH_ARM_ENABLED - runs-on: ubuntu-24.04 - outputs: - remotepath: ${{ steps.Remote-Dir.outputs.remotepath }} - env: - GITHUB_COMMIT: ${{ github.sha }} - GHC_VERSION: '9.4.8' - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - id: Remote-Dir - name: Unique directory name for the remote build - run: echo "remotepath=postgrest-build-$(uuidgen)" >> "$GITHUB_OUTPUT" - - name: Copy script files to the remote server - uses: appleboy/scp-action@master - with: - host: ${{ secrets.SSH_ARM_HOST }} - username: ubuntu - key: ${{ secrets.SSH_ARM_PRIVATE_KEY }} - fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }} - source: ".github/scripts/arm/*" - target: ${{ steps.Remote-Dir.outputs.remotepath }} - strip_components: 3 - - name: Build ARM - uses: appleboy/ssh-action@master - env: - REMOTE_DIR: ${{ steps.Remote-Dir.outputs.remotepath }} - with: - host: ${{ secrets.SSH_ARM_HOST }} - username: ubuntu - key: ${{ secrets.SSH_ARM_PRIVATE_KEY }} - fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }} - command_timeout: 120m - script_stop: true - envs: GITHUB_COMMIT,REMOTE_DIR,GHC_VERSION - script: bash ~/$REMOTE_DIR/build.sh "$GITHUB_COMMIT" "$REMOTE_DIR" "$GHC_VERSION" - - name: Download binaries from remote server - uses: nicklasfrahm/scp-action@main - with: - direction: download - host: ${{ secrets.SSH_ARM_HOST }} - username: ubuntu - key: ${{ secrets.SSH_ARM_PRIVATE_KEY }} - fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }} - source: "${{ steps.Remote-Dir.outputs.remotepath }}/result.tar.xz" - target: "result.tar.xz" - - name: Extract downloaded binaries - run: tar -xvf result.tar.xz && rm result.tar.xz - - name: Save aarch64 executable as artifact - uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4.5.0 - with: - name: postgrest-ubuntu-aarch64 - path: result/postgrest - if-no-files-found: error - - tag: name: Release / Tag concurrency: @@ -118,8 +61,7 @@ jobs: startsWith(github.ref, 'refs/heads/') && needs.docs.result == 'success' && needs.test.result == 'success' && - needs.build.result == 'success' && - (needs.arm.result == 'skipped' || success()) + needs.build.result == 'success' permissions: contents: write runs-on: ubuntu-24.04 @@ -127,7 +69,6 @@ jobs: - docs - test - build - - arm steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -153,12 +94,10 @@ jobs: name: Release / Prepare if: | startsWith(github.ref, 'refs/tags/') && - needs.build.result == 'success' && - (needs.arm.result == 'skipped' || success()) + needs.build.result == 'success' runs-on: ubuntu-24.04 needs: - build - - arm steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Check the version to be released @@ -262,7 +201,7 @@ jobs: docker: name: Release / Docker Hub - runs-on: ubuntu-24.04 + runs-on: ubuntu-24.04-arm needs: - prepare if: | @@ -274,23 +213,45 @@ jobs: DOCKER_PASS: ${{ secrets.DOCKER_PASS }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Download Docker image + - name: Login on Dockerhub + run: docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" + - name: Download x86-64 Docker image uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: name: postgrest-docker-x86-64 + - name: Download aarch64 binary + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: postgrest-ubuntu-aarch64 + - name: Build aarch64 Docker image + run: | + # This only pushes the image via digest, not a tag. This will not appear + # in the image list on Docker Hub, yet. It will be later added to the main + # tag's manifest. + docker buildx build \ + -t "$DOCKER_REPO/postgrest:arm" \ + --platform linux/arm64 \ + --output push-by-digest=true,type=image,push=true \ + . + echo "SHA256_ARM=$(docker images --no-trunc --quiet "$DOCKER_REPO/postgrest:arm")" >> "$GITHUB_ENV" - name: Publish images on Docker Hub run: | - docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" docker load -i postgrest-docker.tar.gz docker tag postgrest:latest "$DOCKER_REPO/postgrest:${GITHUB_REF_NAME}" docker push "$DOCKER_REPO/postgrest:${GITHUB_REF_NAME}" + docker buildx imagetools create --append \ + -t "$DOCKER_REPO/postgrest:${GITHUB_REF_NAME}" \ + "$DOCKER_REPO/postgrest@$SHA256_ARM" # Only tag 'latest' for full releases if [ "${GITHUB_REF_NAME}" != "devel" ]; then echo "Pushing to 'latest' tag for full release of ${GITHUB_REF_NAME} ..." docker tag postgrest:latest "$DOCKER_REPO"/postgrest:latest docker push "$DOCKER_REPO"/postgrest:latest + docker buildx imagetools create --append \ + -t "$DOCKER_REPO/postgrest:latest" \ + "$DOCKER_REPO/postgrest@$SHA256_ARM" else echo "Skipping push to 'latest' tag for pre-release..." fi @@ -311,52 +272,3 @@ jobs: repository: ${{ vars.DOCKER_REPO }}/postgrest short-description: ${{ github.event.repository.description }} readme-filepath: ./docker-hub-readme.md - - - docker-arm: - name: Release / Docker Hub Arm - runs-on: ubuntu-24.04 - needs: - - arm - - docker - env: - GITHUB_COMMIT: ${{ github.sha }} - DOCKER_REPO: ${{ vars.DOCKER_REPO }} - DOCKER_USER: ${{ vars.DOCKER_USER }} - DOCKER_PASS: ${{ secrets.DOCKER_PASS }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Publish images for ARM builds on Docker Hub - uses: appleboy/ssh-action@master - env: - REMOTE_DIR: ${{ needs.arm.outputs.remotepath }} - with: - host: ${{ secrets.SSH_ARM_HOST }} - username: ubuntu - key: ${{ secrets.SSH_ARM_PRIVATE_KEY }} - fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }} - script_stop: true - envs: GITHUB_COMMIT,DOCKER_REPO,DOCKER_USER,DOCKER_PASS,REMOTE_DIR,GITHUB_REF_NAME - script: bash ~/$REMOTE_DIR/docker-publish.sh "$GITHUB_COMMIT" "$DOCKER_REPO" "$DOCKER_USER" "$DOCKER_PASS" "$REMOTE_DIR" "$GITHUB_REF_NAME" - - - clean-arm: - name: Build / Cleanup - needs: - - arm - - docker-arm - if: ${{ always() && vars.SSH_ARM_ENABLED }} - runs-on: ubuntu-24.04 - env: - REMOTE_DIR: ${{ needs.arm.outputs.remotepath }} - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Remove uploaded files from server - uses: appleboy/ssh-action@master - with: - host: ${{ secrets.SSH_ARM_HOST }} - username: ubuntu - key: ${{ secrets.SSH_ARM_PRIVATE_KEY }} - fingerprint: ${{ secrets.SSH_ARM_FINGERPRINT }} - envs: REMOTE_DIR - script: rm -rf $REMOTE_DIR diff --git a/.github/scripts/arm/docker-env/Dockerfile b/Dockerfile similarity index 80% rename from .github/scripts/arm/docker-env/Dockerfile rename to Dockerfile index 217767ee23..69459b9bf2 100644 --- a/.github/scripts/arm/docker-env/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -# PostgREST docker hub image +# PostgREST Docker Hub image for aarch64. +# The x86-64 is a single-static-binary image built via Nix, see: +# nix/tools/docker/README.md FROM ubuntu:noble@sha256:80dd3c3b9c6cecb9f1667e9290b3bc61b78c2678c02cbdae5f0fea92cc6734ab AS postgrest