diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 00000000..4be408e7 --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,10 @@ +.webassets-cache/ +*.sqlite3 +**/static/style.css +postgres-data/ +tmp-certs/ +.terraform/ +.git/ +.DS_Store +*.pyc +.vscode/ diff --git a/.github/workflows/cloudbuild.yml b/.github/workflows/cloudbuild.yml new file mode 100644 index 00000000..7dc3000d --- /dev/null +++ b/.github/workflows/cloudbuild.yml @@ -0,0 +1,44 @@ +name: Cloud Build + +on: + workflow_call: + inputs: + command: + description: "The justfile + Cloud build-related recipe to run" + required: true + type: string + outputs: + image: + description: "GCR image produced by the command where applicable" + value: ${{ jobs.run_cloudbuild_command.outputs.image }} + +jobs: + run_cloudbuild_command: + name: Submit Build + runs-on: ubuntu-latest + permissions: + contents: "read" + id-token: "write" + outputs: + image: ${{ steps.cloudbuild-command.outputs.image }} + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup just + uses: extractions/setup-just@v1 + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v0 + with: + workload_identity_provider: "projects/567739286055/locations/global/workloadIdentityPools/los-verdes-digital-membership/providers/los-verdes-digital-membership" + service_account: "github-deployer@lv-digital-membership.iam.gserviceaccount.com" + access_token_scopes: "https://www.googleapis.com/auth/cloud-platform,https://www.googleapis.com/auth/sqlservice.admin" + token_format: access_token + access_token_lifetime: 1800s + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0 + + - name: Build + run: just ${{ inputs.command }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 729469d5..e6aef938 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -16,18 +16,18 @@ jobs: permissions: contents: "read" id-token: "write" - uses: los-verdes/digital-membership/.github/workflows/flask_command.yml@main + uses: los-verdes/digital-membership/.github/workflows/cloudbuild.yml@main with: - command: build-worker-image + command: cloudbuild-image-worker build-website-image: name: Build Website Image permissions: contents: "read" id-token: "write" - uses: los-verdes/digital-membership/.github/workflows/flask_command.yml@main + uses: los-verdes/digital-membership/.github/workflows/cloudbuild.yml@main with: - command: build-website-image + command: cloudbuild-image-website deploy-infra-and-services: name: Deploy Infrastructure & Services @@ -102,10 +102,10 @@ jobs: permissions: contents: "read" id-token: "write" - uses: los-verdes/digital-membership/.github/workflows/flask_command.yml@main + uses: los-verdes/digital-membership/.github/workflows/cloudbuild.yml@main with: # TODO: add a CloudFlare cache purge somewhere 'round here.... - command: upload-statics + command: cloudbuild-upload-statics apply-database-config: name: Apply Database Configuration diff --git a/.gitignore b/.gitignore index 6c037780..451978d5 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ postgres-data/ .srl tmp-certs/ member_card/static/style.css +.terraform/ diff --git a/cloudbuild-statics.yaml b/cloudbuild-statics.yaml new file mode 100644 index 00000000..598ded81 --- /dev/null +++ b/cloudbuild-statics.yaml @@ -0,0 +1,18 @@ +substitutions: + _IMAGE_TAG: "n/a" + _BUCKET_ID: "statics-gcs-bucket" + +steps: + - name: "gcr.io/$PROJECT_ID/website:${_IMAGE_TAG}" + entrypoint: "flask" + args: + - assets + - build + + - name: "gcr.io/cloud-builders/gsutil" + args: + - "-m" + - "cp" + - "-r" + - "member_card/static/*" + - f"gs://${_BUCKET_ID}/static/" diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 00000000..a5960763 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,30 @@ +substitutions: + _IMAGE_TAG: "n/a" + _BUILD_TARGET: website + +steps: + # Download image to cache from + - name: "gcr.io/cloud-builders/docker" + entrypoint: "bash" + args: + - "-c" + - "docker pull gcr.io/$PROJECT_ID/worker:latest || exit 0" + + # Docker Build + - name: "gcr.io/cloud-builders/docker" + args: + - "build" + - "--cache-from=gcr.io/$PROJECT_ID/${_BUILD_TARGET}:latest" + - "--target=${_BUILD_TARGET}" + - "--tag=gcr.io/$PROJECT_ID/${_BUILD_TARGET}:${_IMAGE_TAG}" + - "." + + - name: "gcr.io/cloud-builders/docker" + args: + - "tag" + - "gcr.io/$PROJECT_ID/${_BUILD_TARGET}:${_IMAGE_TAG}" + - "gcr.io/$PROJECT_ID/${_BUILD_TARGET}:latest" + +images: + - "gcr.io/$PROJECT_ID/${_BUILD_TARGET}:${_IMAGE_TAG}" + - "gcr.io/$PROJECT_ID/${_BUILD_TARGET}:latest" diff --git a/justfile b/justfile index 206c98d8..49b777ab 100644 --- a/justfile +++ b/justfile @@ -120,17 +120,41 @@ build-worker: build: build-website build-worker -build-website-image: ci-install-python-reqs - just flask build-image website +cloudbuild CONFIG_FILE *SUBSTITUTIONS="": + #!/bin/bash + + logfile=$(mktemp /tmp/local_config_apply.XXXXXXXXXX) || { echo "Failed to create temp file"; exit 1; } + + gcloud builds submit {{ justfile_directory() }} \ + --format='json' \ + --config="{{ CONFIG_FILE }}" \ + --suppress-logs \ + --substitutions='_IMAGE_TAG={{ image_tag }}{{ if SUBSTITUTIONS != "" { "," + SUBSTITUTIONS } else { "" } }}' \ + | tee "$logfile" + + echo + echo "BUILD RESULTS:" + jq -r '.' "$logfile" + echo + + IMAGE="$(jq -r '.images[0]' "$logfile")" + if [[ -n "$IMAGE" ]] + then + echo "::set-output name=image::{build_result.images[0]}" + fi -build-worker-image: ci-install-python-reqs - just flask build-image worker +cloudbuild-image-website: + just cloudbuild 'cloudbuild.yaml' '_BUILD_TARGET=website' -build-base-image: ci-install-python-reqs - just flask build-image base +cloudbuild-image-worker: + just cloudbuild 'cloudbuild.yaml' '_BUILD_TARGET=worker' + +cloudbuild-upload-statics _BUCKET_ID='cstatic.losverd.es': + just cloudbuild 'cloudbuild-statics.yaml' '_BUCKET_ID={{ _BUCKET_ID }}' run-worker: build-worker docker run -it --rm '{{ worker_image_name }}:{{ image_tag }}' + shell-worker: build-worker docker run -it --rm --entrypoint='' '{{ worker_image_name }}:{{ image_tag }}' bash @@ -139,9 +163,6 @@ shell-website: build-website shell: shell-website -upload-statics: ci-install-python-reqs - just flask upload-statics - push: build echo "Pushing website image..." docker tag '{{ website_image_name }}:{{ image_tag }}' '{{ website_gcr_image_name }}:{{ image_tag }}' @@ -155,8 +176,6 @@ push: build docker push '{{ worker_gcr_image_name }}:{{ image_tag }}' docker push '{{ worker_gcr_image_name }}:latest' - echo "Uploading statics..." - just flask upload-statics set-tf-output-output output_name: output_value="$(just tf output -raw {{ output_name }})" @@ -191,7 +210,6 @@ apply-migrations: ci-install-python-reqs just flask db upgrade echo 'Any outstanding migrations have now been applied! :D' - sync-subscriptions: ci-install-python-reqs @echo "DIGITAL_MEMBERSHIP_DB_DATABASE_NAME: $DIGITAL_MEMBERSHIP_DB_DATABASE_NAME" @echo "DIGITAL_MEMBERSHIP_DB_USERNAME: $DIGITAL_MEMBERSHIP_DB_USERNAME" diff --git a/member_card/app.py b/member_card/app.py index cef0ff1d..7f630eea 100755 --- a/member_card/app.py +++ b/member_card/app.py @@ -676,23 +676,6 @@ def force_assets_bundle_build(): utils.force_assets_bundle_build(app) -@app.cli.command("upload-statics") -def upload_statics(): - from member_card.cloudbuild import create_upload_statics_build - - utils.force_assets_bundle_build(app) - create_upload_statics_build() - - -@app.cli.command("build-image") -@click.argument("image_name") -def build_image(image_name): - from member_card.cloudbuild import create_docker_image_build - - build_result = create_docker_image_build(image_name=image_name) - print(f"::set-output name=image::{build_result.images[0]}") - - @app.cli.command("insert-google-pass-class") def insert_google_pass_class(): from member_card import gpay diff --git a/member_card/cloudbuild.py b/member_card/cloudbuild.py deleted file mode 100644 index 1c417ea2..00000000 --- a/member_card/cloudbuild.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env python -import logging -from flask import current_app -from google.cloud.devtools import cloudbuild_v1 - -from member_card.utils import load_gcp_credentials - -logger = logging.getLogger(__name__) - - -def get_client(credentials=None): - if credentials is None: - credentials = load_gcp_credentials() - return cloudbuild_v1.CloudBuildClient(credentials=credentials) - - -def generate_upload_statics_build(project_id, repo_name, bucket_id, branch_name="main"): - build = cloudbuild_v1.Build() - build.source = { - "repo_source": { - "project_id": project_id, - "repo_name": repo_name, - "branch_name": branch_name, - } - } - build.steps = [ - { - "name": "gcr.io/cloud-builders/gsutil", - "args": [ - "-m", - "cp", - "-r", - "member_card/static/*", - f"gs://{bucket_id}/static/", - ], - }, - ] - return build - - -def create_upload_statics_build(client=None): - if client is None: - client = get_client() - - project_id = current_app.config["GCLOUD_PROJECT"] - repo_name = current_app.config["GCP_REPO_NAME"] - bucket_id = current_app.config["GCS_BUCKET_ID"] - - upload_statics_build = generate_upload_statics_build( - project_id=project_id, - repo_name=repo_name, - bucket_id=bucket_id, - ) - return create_build( - client=client, - project_id=project_id, - build=upload_statics_build, - ) - - -def generate_docker_image_build(project_id, repo_name, image_name, branch_name="main"): - base_image_gcr_name = "gcr.io/$PROJECT_ID/worker" - gcr_name = f"gcr.io/$PROJECT_ID/{image_name}" - - build_args = [ - "build", - "--cache-from", - f"{base_image_gcr_name}:latest", - "--target", - image_name, - "--tag", - f"{gcr_name}:$SHORT_SHA", - ".", - ] - if base_image_gcr_name != gcr_name: - build_args += [ - "--cache-from", - f"{gcr_name}:latest", - ] - build = cloudbuild_v1.Build() - - build.source = { - "repo_source": { - "project_id": project_id, - "repo_name": repo_name, - "branch_name": branch_name, - } - } - build.steps = [ - { - "name": "gcr.io/cloud-builders/docker", - "entrypoint": "bash", - "args": [ - "-c", - f"docker pull {base_image_gcr_name}:latest || exit 0", - ], - }, - { - "name": "gcr.io/cloud-builders/docker", - "args": build_args, - }, - { - "name": "gcr.io/cloud-builders/docker", - "args": [ - "tag", - f"{gcr_name}:$SHORT_SHA", - f"{gcr_name}:latest", - ], - }, - ] - build.images = [ - f"{gcr_name}:$SHORT_SHA", - f"{gcr_name}:latest", - ] - return build - - -def create_docker_image_build(image_name, client=None): - if client is None: - client = get_client() - - project_id = current_app.config["GCLOUD_PROJECT"] - repo_name = current_app.config["GCP_REPO_NAME"] - - docker_image_build = generate_docker_image_build( - project_id=project_id, - repo_name=repo_name, - image_name=image_name, - ) - return create_build( - client=client, - project_id=project_id, - build=docker_image_build, - ) - - -def create_build(client, project_id, build): - operation = client.create_build( - project_id=project_id, - build=build, - ) - logger.debug(f"Cloud build running: {operation.metadata}") - result = operation.result() - logger.info(f"Cloud build result: {result.status}") - return result diff --git a/requirements.in b/requirements.in index 8ed7b210..42070658 100644 --- a/requirements.in +++ b/requirements.in @@ -11,7 +11,6 @@ flask-cdn flask-login flask-security google-auth -google-cloud-build google-cloud-firestore google-cloud-logging google-cloud-pubsub diff --git a/requirements.txt b/requirements.txt index 8891549d..ab842e8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -103,7 +103,6 @@ google-api-core[grpc]==2.4.0 # via # google-api-python-client # google-cloud-appengine-logging - # google-cloud-build # google-cloud-core # google-cloud-firestore # google-cloud-logging @@ -127,8 +126,6 @@ google-cloud-appengine-logging==1.1.0 # via google-cloud-logging google-cloud-audit-log==0.2.0 # via google-cloud-logging -google-cloud-build==3.7.1 - # via -r requirements.in google-cloud-core==2.2.1 # via # google-cloud-firestore @@ -273,7 +270,6 @@ pillow==9.0.0 proto-plus==1.19.8 # via # google-cloud-appengine-logging - # google-cloud-build # google-cloud-firestore # google-cloud-logging # google-cloud-pubsub