Add a script for starting the Trino container

nineinchnick · Jan Waś · commit be9600867cec · 2024-09-09T11:17:57.000+02:00
diff --git a/.github/workflows/optimize.yml b/.github/workflows/optimize.yml
@@ -13,35 +13,14 @@ jobs:
   sync:
     runs-on: ubuntu-latest
     env:
+      GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       AWS_REGION: ${{ secrets.AWS_REGION }}
-      TRINO_VERSION: 445
     steps:
       - uses: actions/checkout@v4
       - name: Start Trino
-        run: |
-          CONTAINER_ID=$(docker run \
-            -v $(pwd)/catalog/trinocicd.properties:/etc/trino/catalog/trinocicd.properties \
-            -v $(pwd)/sql:/sql \
-            -e AWS_ACCESS_KEY_ID \
-            -e AWS_SECRET_ACCESS_KEY \
-            -e AWS_REGION \
-            -p 8080:8080 \
-            --name trino \
-            -d \
-            trinodb/trino:$TRINO_VERSION)
-          SERVER_IP=$(docker inspect --format '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CONTAINER_ID")
-          echo "CONTAINER_ID=$CONTAINER_ID" >> $GITHUB_ENV
-          echo "SERVER_IP=$SERVER_IP" >> $GITHUB_ENV
-          i=0
-          until docker inspect "${CONTAINER_ID}" --format "{{json .State.Health.Status }}" | grep -q '"healthy"'; do
-              if [[ $((i++)) -ge 10 ]]; then
-                  echo "🚨 Too many retries waiting for Trino to start"
-                  exit 1
-              fi
-              sleep 10
-          done
+        run: ./bin/run-trino.sh
       - name: Optimize tables
         run: |
           docker exec \
diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml
@@ -16,37 +16,18 @@ jobs:
   run:
     runs-on: ubuntu-latest
     env:
+      GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       AWS_REGION: ${{ secrets.AWS_REGION }}
-      TRINO_VERSION: 445
-      TRINO_GIT_VERSION: "0.74"
     steps:
       - uses: actions/checkout@v4
       - name: Install dependencies
         run: |
           sudo apt update
           sudo pip install ansi2html
-          curl -fLOsS https://github.com/nineinchnick/trino-git/releases/download/v$TRINO_GIT_VERSION/trino-git-$TRINO_GIT_VERSION.zip
-          unzip trino-git-$TRINO_GIT_VERSION.zip
       - name: Start Trino
-        run: |
-          CONTAINER_ID=$(docker run \
-            -v $(pwd)/trino-git-$TRINO_GIT_VERSION:/usr/lib/trino/plugin/git \
-            -v $(pwd)/catalog/git.properties:/etc/trino/catalog/git.properties \
-            -v $(pwd)/catalog/trinocicd.properties:/etc/trino/catalog/trinocicd.properties \
-            -v $(pwd)/hive-cache:/opt/hive-cache \
-            -e AWS_ACCESS_KEY_ID \
-            -e AWS_SECRET_ACCESS_KEY \
-            -e AWS_REGION \
-            -p 8080:8080 \
-            --name trino \
-            -d \
-            trinodb/trino:$TRINO_VERSION)
-          SERVER_IP=$(docker inspect --format '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CONTAINER_ID")
-          echo "CONTAINER_ID=$CONTAINER_ID" >> $GITHUB_ENV
-          echo "SERVER_IP=$SERVER_IP" >> $GITHUB_ENV
-          until docker logs trino 2>&1 | grep --quiet --fixed-strings --max-count=1 "SERVER STARTED"; do sleep 1 ; done
+        run: ./bin/run-trino.sh
       - name: Execute queries
         run: |
           ./bin/reports.sh reports/ci-cd/index.md "Trino CI/CD Reports" sql/ci-cd/{health,success-ratio-per-day,runs-queue-time-per-day,runs-duration-per-day,runs-job-cumulative-duration-per-day,jobs-duration}.sql
diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml
@@ -17,7 +17,6 @@ jobs:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       AWS_REGION: ${{ secrets.AWS_REGION }}
-      TRINO_VERSION: 445
       TRINO_REST_VERSION: "0.144"
       GITHUB_OWNER: trinodb
       GITHUB_REPO: trino
@@ -28,45 +27,11 @@ jobs:
       SYNC_TABLES: runs,jobs,steps,check_suites,check_runs,check_run_annotations,pulls,pull_commits,pull_stats,review_comments,reviews,issues,issue_comments,commits,teams,members
     steps:
       - uses: actions/checkout@v4
-      - name: Download trino-rest
-        run: |
-          curl -fLOsS https://github.com/nineinchnick/trino-rest/releases/download/v$TRINO_REST_VERSION/trino-rest-github-$TRINO_REST_VERSION.zip
-          unzip trino-rest-github-$TRINO_REST_VERSION.zip
       - name: Start Trino
         run: |
-          cat <<EOF >config.properties
-          coordinator=true
-          node-scheduler.include-coordinator=true
-          http-server.http.port=8080
-          discovery.uri=http://localhost:8080
-          query.max-memory-per-node=4086929818B
-          EOF
-          CONTAINER_ID=$(docker run \
-            -v $(pwd)/config.properties:/etc/trino/config.properties \
-            -v $(pwd)/trino-rest-github-$TRINO_REST_VERSION:/usr/lib/trino/plugin/github \
-            -v $(pwd)/catalog/github.properties:/etc/trino/catalog/github.properties \
-            -v $(pwd)/catalog/trinocicd.properties:/etc/trino/catalog/trinocicd.properties \
-            -v $(pwd)/hive-cache:/opt/hive-cache \
-            -v $(pwd)/sql:/sql \
-            -e AWS_ACCESS_KEY_ID \
-            -e AWS_SECRET_ACCESS_KEY \
-            -e AWS_REGION \
-            -e GITHUB_TOKEN \
-            -p 8080:8080 \
-            --name trino \
-            -d \
-            trinodb/trino:$TRINO_VERSION)
+          ./bin/run-trino.sh
           SERVER_IP=$(docker inspect --format '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CONTAINER_ID")
-          echo "CONTAINER_ID=$CONTAINER_ID" >> $GITHUB_ENV
           echo "SERVER_IP=$SERVER_IP" >> $GITHUB_ENV
-          i=0
-          until docker inspect "${CONTAINER_ID}" --format "{{json .State.Health.Status }}" | grep -q '"healthy"'; do
-              if [[ $((i++)) -ge 10 ]]; then
-                  echo "🚨 Too many retries waiting for Trino to start"
-                  exit 1
-              fi
-              sleep 10
-          done
       - name: Run Sync
         run: |
           docker run \
diff --git a/.gitignore b/.gitignore
@@ -107,8 +107,10 @@ dist
 .idea
 
 # custom Trino connectors
-/trino-rest-github-*
-/trino-git-*
+/bin/trino-rest-github-*
+/bin/trino-git-*
+/bin/config.properties
+/bin/hive-cache
 
 # Jekyll
 _site
diff --git a/bin/run-trino.sh b/bin/run-trino.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+#
+# Start a Trino container with additional connectors and catalogs
+# configured and wait for it to be ready.
+
+set -euo pipefail
+
+for cmd in curl unzip docker; do
+    if ! command -v "$cmd" >/dev/null; then
+        echo >&2 "Missing the $cmd command"
+        exit 1
+    fi
+done
+
+if [ -z "$AWS_REGION" ] || [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$GITHUB_TOKEN" ]; then
+    echo >&2 "Following environmental variables need to be set: AWS_REGION, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, GITHUB_TOKEN"
+    exit 1
+fi
+
+CONTAINER_NAME=trino
+TRINO_VERSION=455
+TRINO_GIT_VERSION=0.83
+TRINO_REST_VERSION=0.154
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+cd "$SCRIPT_DIR" || exit 1
+
+if [ ! -d trino-git-$TRINO_GIT_VERSION ]; then
+    curl -fLOsS https://github.com/nineinchnick/trino-git/releases/download/v$TRINO_GIT_VERSION/trino-git-$TRINO_GIT_VERSION.zip
+    unzip trino-git-$TRINO_GIT_VERSION.zip
+fi
+
+if [ ! -d trino-rest-github-$TRINO_REST_VERSION ]; then
+    curl -fLOsS https://github.com/nineinchnick/trino-rest/releases/download/v$TRINO_REST_VERSION/trino-rest-github-$TRINO_REST_VERSION.zip
+    unzip trino-rest-github-$TRINO_REST_VERSION.zip
+fi
+
+cat <<EOF >config.properties
+coordinator=true
+node-scheduler.include-coordinator=true
+http-server.http.port=8080
+discovery.uri=http://localhost:8080
+query.max-memory-per-node=4086929818B
+EOF
+
+mkdir -p "$SCRIPT_DIR"/hive-cache
+chmod 777 "$SCRIPT_DIR"/hive-cache
+docker run \
+    -v "$SCRIPT_DIR"/config.properties:/etc/trino/config.properties \
+    -v "$SCRIPT_DIR"/trino-git-$TRINO_GIT_VERSION:/usr/lib/trino/plugin/git \
+    -v "$SCRIPT_DIR"/trino-rest-github-$TRINO_REST_VERSION:/usr/lib/trino/plugin/github \
+    -v "$SCRIPT_DIR"/../catalog/git.properties:/etc/trino/catalog/git.properties \
+    -v "$SCRIPT_DIR"/../catalog/github.properties:/etc/trino/catalog/github.properties \
+    -v "$SCRIPT_DIR"/../catalog/trinocicd.properties:/etc/trino/catalog/trinocicd.properties \
+    -v "$SCRIPT_DIR"/hive-cache:/opt/hive-cache \
+    -v "$SCRIPT_DIR"/../sql:/sql \
+    -e AWS_ACCESS_KEY_ID \
+    -e AWS_SECRET_ACCESS_KEY \
+    -e AWS_REGION \
+    -e GITHUB_TOKEN \
+    -p 8080:8080 \
+    --name $CONTAINER_NAME \
+    -d \
+    trinodb/trino:$TRINO_VERSION
+
+i=0
+until status=$(docker inspect $CONTAINER_NAME --format "{{json .State.Health.Status }}") && echo "$status" | grep -q '"healthy"'; do
+    if [[ $((i++)) -ge 10 ]]; then
+        echo >&2 "🚨 Too many retries waiting for Trino to start"
+        exit 1
+    fi
+    echo >&2 "Status is: $status, sleeping 10 seconds"
+    sleep 10
+done
+
+echo >&2 "Status is: $status, connect to trino://localhost:8080/trinocicd/v2"
