From d54f106a49de68ae9182159e6a6d07e565fc183e Mon Sep 17 00:00:00 2001 From: Luca Osti Date: Tue, 10 Sep 2024 15:04:35 +0200 Subject: [PATCH] Split CI build, restore Cloudflare purge for datasheets (#2153) * Split CI build * Skip everything on cache hit * Fix output path * Move datasheets rendering to own action file * Set name * Add debug step for datasheets * Update production workflow, too * Fix APP_ENV * Checkout is inevitable * Fix purge * Explicit checkout * Typo * Remove restore-key * Use correct folder * 2.0.18-2-alpha.1 * 2.0.18-2 * Fail on purge fail, debug prefix and zone * Remove debug log --------- Co-authored-by: Matteo Marsala --- .github/actions/cleanup-disk/action.yml | 28 ++++++ .github/actions/cloudflare-upload/action.yml | 3 - .../actions/generate-datasheets/action.yml | 43 +++++++++ .github/actions/sync-s3/action.yml | 30 +++++++ .github/workflows/deploy-production.yml | 87 +++++++++++-------- .github/workflows/deploy-staging.yml | 87 +++++++++++-------- package-lock.json | 8 +- package.json | 2 +- 8 files changed, 210 insertions(+), 78 deletions(-) create mode 100644 .github/actions/cleanup-disk/action.yml create mode 100644 .github/actions/generate-datasheets/action.yml create mode 100644 .github/actions/sync-s3/action.yml diff --git a/.github/actions/cleanup-disk/action.yml b/.github/actions/cleanup-disk/action.yml new file mode 100644 index 0000000000..deb112f960 --- /dev/null +++ b/.github/actions/cleanup-disk/action.yml @@ -0,0 +1,28 @@ +name: Disk cleanup +description: "Cleanup disk space" +runs: + using: composite + steps: + - name: Free up disk space + shell: bash + run: | + echo "Disk space before cleanup..." + df -h / + echo "Removing unnecessary files to free up disk space..." + # https://github.com/actions/runner-images/issues/2840#issuecomment-2272410832 + sudo rm -rf \ + /opt/hostedtoolcache \ + /opt/google/chrome \ + /opt/microsoft/msedge \ + /opt/microsoft/powershell \ + /opt/pipx \ + /usr/lib/mono \ + /usr/local/julia* \ + /usr/local/lib/android \ + /usr/local/lib/node_modules \ + /usr/local/share/chromium \ + /usr/local/share/powershell \ + /usr/share/dotnet \ + /usr/share/swift + echo "Disk space after cleanup..." + df -h / diff --git a/.github/actions/cloudflare-upload/action.yml b/.github/actions/cloudflare-upload/action.yml index ef274e577b..ceac755bb3 100644 --- a/.github/actions/cloudflare-upload/action.yml +++ b/.github/actions/cloudflare-upload/action.yml @@ -20,7 +20,6 @@ inputs: runs: using: composite steps: - - name: Find PR Preview Comment if: github.event_name == 'pull_request' uses: peter-evans/find-comment@v1 @@ -67,7 +66,6 @@ runs: ๐Ÿš€ Preview this PR: ${{ steps.deploy-cloudflare.outputs.url }} ๐Ÿ“ Commit SHA: ${{ github.sha }} - - name: Update PR Preview Comment if: github.event_name == 'pull_request' && steps.deploy-preview-comment.outputs.comment-id != 0 uses: peter-evans/create-or-update-comment@v1.4.5 @@ -78,4 +76,3 @@ runs: ### ${{ inputs.project-name }} ๐Ÿš€ Preview this PR: ${{ steps.deploy-cloudflare.outputs.url }} ๐Ÿ“ Commit SHA: ${{ github.sha }} - diff --git a/.github/actions/generate-datasheets/action.yml b/.github/actions/generate-datasheets/action.yml new file mode 100644 index 0000000000..908c4f6d10 --- /dev/null +++ b/.github/actions/generate-datasheets/action.yml @@ -0,0 +1,43 @@ +name: "Generate Datasheets" +description: "Generate product datasheets from markdown files" +inputs: + datasheets-path: + description: "The datasheets path" + required: true + default: static/resources/datasheets + artifact-name: + description: "The name of the output artifact" + required: true + +runs: + using: composite + steps: + - uses: actions/cache@v4 + id: cache + with: + path: ${{ inputs.datasheets-path }} + key: ${{ runner.os }}-datasheets-${{ hashFiles('**/*datasheet.md') }} + + - uses: actions/setup-node@v4 + if: steps.cache.outputs.cache-hit != 'true' + with: + node-version: 18 + cache: "npm" + cache-dependency-path: "package-lock.json" + + - name: Render Datasheets + if: steps.cache.outputs.cache-hit != 'true' + run: | + cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering + ./render-datasheets.sh + cd $GITHUB_WORKSPACE + mkdir -p ${{ inputs.datasheets-path }} + find ./content/hardware -type f -name "*-datasheet.pdf" -exec cp {} ./${{ inputs.datasheets-path }}/ \; + shell: bash + + - name: Export artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name }} + path: ${{ inputs.datasheets-path }} + retention-days: 1 # Only needed to pass it to the next job diff --git a/.github/actions/sync-s3/action.yml b/.github/actions/sync-s3/action.yml new file mode 100644 index 0000000000..7f2a7524ac --- /dev/null +++ b/.github/actions/sync-s3/action.yml @@ -0,0 +1,30 @@ +name: Sync assets on S3 +description: "Sync Docs assets on S3" +inputs: + role-to-assume: + description: "The IAM role to assume" + required: true + bucket-name: + description: "The name of the S3 bucket to sync assets to" + required: true +runs: + using: composite + steps: + - name: Configure AWS credentials from Staging account + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ inputs.role-to-assume }} + aws-region: us-east-1 + + - name: Sync all cacheable assets + shell: bash + run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --exclude="*.pdf" --delete public/ s3://${{ inputs.bucket-name }}/ + + - name: Sync all non-cacheable assets + shell: bash + # Don't cache any HTML or JSON file: they should always be up-to-dates + run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.pdf" --delete public/ s3://${{ inputs.bucket-name }}/ + + - name: Sync PDF + shell: bash + run: aws s3 sync --cache-control "public, max-age=86400, must-revalidate" --include "*.pdf" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.css" --exclude="*.html" --exclude="*.json" --exclude="sw.json" --delete public/ s3://${{ inputs.bucket-name }}/ diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index cae7749e33..6c920f8340 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -10,13 +10,30 @@ concurrency: group: deploy-production cancel-in-progress: true +# Allow installation of dependencies permissions: id-token: write contents: read jobs: + #ย This job is used to render datasheets, but only if they have changed. + # It's a separate job so we don't have to cleanup the machine afterwards. + render-datasheets: + name: Render Datasheets + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - uses: ./.github/actions/generate-datasheets + with: + artifact-name: datasheets + datasheets-path: static/resources/datasheets + build: - if: "github.repository == 'arduino/docs-content'" + name: Build and Deploy + needs: render-datasheets runs-on: ubuntu-latest environment: production env: @@ -27,20 +44,28 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 1 + + - name: Cleanup runner disk + uses: ./.github/actions/cleanup-disk # Cleanup machine before starting the build + - uses: actions/setup-node@v4 with: node-version: 18 cache: "npm" - cache-dependency-path: "**/package-lock.json" + cache-dependency-path: "package-lock.json" + + - name: Retrieve Datasheets + uses: actions/download-artifact@v4 # Retrieve the datasheets generated in the previous job + with: + name: datasheets + path: static/resources/datasheets - - name: Render Datasheets - run: cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering;./render-datasheets.sh + - name: Debug datasheet list + run: ls -lah static/resources/datasheets - name: Copy Static Files run: | - mkdir -p static/resources/datasheets static/resources/schematics static/resources/pinouts static/resources/models - find ./content/hardware -type f -name "*-schematics.pdf" -exec cp {} ./static/resources/schematics/ \; - find ./content/hardware -type f -name "*-datasheet.pdf" -exec cp {} ./static/resources/datasheets/ \; + mkdir -p static/resources/schematics static/resources/pinouts static/resources/models find ./content/hardware -type f -name "*-full-pinout.pdf" -exec cp {} ./static/resources/pinouts/ \; find ./content/hardware -type f -name "*-pinout.png" -exec cp {} ./static/resources/pinouts/ \; find ./content/hardware -type f -name "*-step.zip" -exec cp {} ./static/resources/models/ \; @@ -54,7 +79,7 @@ jobs: restore-keys: | ${{ runner.os }}-cache-gatsby-main - - name: Gatsby Public Folder + - name: Gatsby Public Folder cache uses: actions/cache@v4 id: gatsby-public-folder with: @@ -64,37 +89,29 @@ jobs: ${{ runner.os }}-public-gatsby-main - run: npm install + - run: npm run build - - name: Clean up node_modules + - name: Clean up node_modules # Just to save space run: rm -rf node_modules - - name: Configure AWS credentials from Production account - uses: aws-actions/configure-aws-credentials@v4 + - name: Deploy to S3 + uses: ./.github/actions/sync-s3 with: role-to-assume: ${{ secrets.PRODUCTION_IAM_ROLE }} - aws-region: us-east-1 - - - name: Sync all cacheable assets - run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --exclude="*.pdf" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/ - - - name: Sync all non-cacheable assets - # Don't cache any HTML or JSON file: they should always be up-to-dates - run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.pdf" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/ - - # - name: Sync PDF - # run: aws s3 sync --cache-control "public, max-age=86400, must-revalidate" --include "*.pdf" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.css" --exclude="*.html" --exclude="*.json" --exclude="sw.json" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/ - - # - name: Purge cache on CloudFlare - # run: | - # curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.CLOUDFLARE_ZONE }}/purge_cache" \ - # -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ - # -H "Content-Type: application/json" \ - # --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}' + bucket-name: ${{ secrets.PRODUCTION_BUCKET_NAME }} - - name: Sync all cacheable assets - run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/ - - - name: Sync all non-cacheable assets - # Don't cache any HTML or JSON file: they should always be up-to-dates - run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --delete public/ s3://${{ secrets.PRODUCTION_BUCKET_NAME }}/ + purge-datasheets: + name: Purge Datasheets cache + needs: build + runs-on: ubuntu-latest + environment: production + steps: + - name: Purge Cloudflare Cache + shell: bash + run: | + echo "Purging Cloudflare cache for prefix: ${{ vars.DATASHEETS_BASE_URL }}, zone: ${{ vars.CLOUDFLARE_ZONE }}" + curl -f -X POST "https://api.cloudflare.com/client/v4/zones/${{ vars.CLOUDFLARE_ZONE }}/purge_cache" \ + -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ + -H "Content-Type: application/json" \ + --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}' diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 0c67404381..ba16e2bf08 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -10,12 +10,30 @@ concurrency: group: deploy-staging cancel-in-progress: true +# Allow installation of dependencies permissions: id-token: write contents: read jobs: + #ย This job is used to render datasheets, but only if they have changed. + # It's a separate job so we don't have to cleanup the machine afterwards. + render-datasheets: + name: Render Datasheets + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - uses: ./.github/actions/generate-datasheets + with: + artifact-name: datasheets + datasheets-path: static/resources/datasheets + build: + name: Build and Deploy + needs: render-datasheets runs-on: ubuntu-latest environment: staging env: @@ -26,20 +44,28 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 1 + + - name: Cleanup runner disk + uses: ./.github/actions/cleanup-disk # Cleanup machine before starting the build + - uses: actions/setup-node@v4 with: node-version: 18 cache: "npm" - cache-dependency-path: "**/package-lock.json" + cache-dependency-path: "package-lock.json" + + - name: Retrieve Datasheets + uses: actions/download-artifact@v4 # Retrieve the datasheets generated in the previous job + with: + name: datasheets + path: static/resources/datasheets - - name: Render Datasheets - run: cd ${GITHUB_WORKSPACE}/scripts/datasheet-rendering;./render-datasheets.sh + - name: Debug datasheet list + run: ls -lah static/resources/datasheets - name: Copy Static Files run: | - mkdir -p static/resources/datasheets static/resources/schematics static/resources/pinouts static/resources/models - find ./content/hardware -type f -name "*-schematics.pdf" -exec cp {} ./static/resources/schematics/ \; - find ./content/hardware -type f -name "*-datasheet.pdf" -exec cp {} ./static/resources/datasheets/ \; + mkdir -p static/resources/schematics static/resources/pinouts static/resources/models find ./content/hardware -type f -name "*-full-pinout.pdf" -exec cp {} ./static/resources/pinouts/ \; find ./content/hardware -type f -name "*-pinout.png" -exec cp {} ./static/resources/pinouts/ \; find ./content/hardware -type f -name "*-step.zip" -exec cp {} ./static/resources/models/ \; @@ -53,7 +79,7 @@ jobs: restore-keys: | ${{ runner.os }}-cache-gatsby-main - - name: Gatsby Public Folder + - name: Gatsby Public Folder cache uses: actions/cache@v4 id: gatsby-public-folder with: @@ -63,38 +89,29 @@ jobs: ${{ runner.os }}-public-gatsby-main - run: npm install + - run: npm run build - - name: Clean up node_modules + - name: Clean up node_modules # Just to save space run: rm -rf node_modules - - name: Configure AWS credentials from Staging account - uses: aws-actions/configure-aws-credentials@v4 + - name: Deploy to S3 + uses: ./.github/actions/sync-s3 with: role-to-assume: ${{ secrets.STAGING_IAM_ROLE }} - aws-region: us-east-1 - - # - name: Sync all cacheable assets - # run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --exclude="*.pdf" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/ - - # - name: Sync all non-cacheable assets - # # Don't cache any HTML or JSON file: they should always be up-to-dates - # run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.pdf" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/ - - # - name: Sync PDF - # run: aws s3 sync --cache-control "public, max-age=86400, must-revalidate" --include "*.pdf" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --exclude="*.css" --exclude="*.html" --exclude="*.json" --exclude="sw.json" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/ - - # - name: Purge cache on CloudFlare - # run: | - # curl -X POST "https://api.cloudflare.com/client/v4/zones/${{ secrets.CLOUDFLARE_ZONE }}/purge_cache" \ - # -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ - # -H "Content-Type: application/json" \ - # --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}' - - - name: Sync all cacheable assets - run: aws s3 sync --cache-control "public, max-age=31536000, immutable" --include "*.css" --include="*.js" --include="*.gif" --include="*.png" --include="*.svg" --exclude "*.html" --exclude="sw.js" --exclude="*.json" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/ - - - name: Sync all non-cacheable assets - # Don't cache any HTML or JSON file: they should always be up-to-dates - run: aws s3 sync --cache-control "public, max-age=0, must-revalidate" --include "*.html" --include="sw.js" --include="*.json" --include "*.css" --exclude="*.js" --exclude="*.gif" --exclude="*.png" --exclude="*.svg" --delete public/ s3://${{ secrets.STAGING_BUCKET_NAME }}/ + bucket-name: ${{ secrets.STAGING_BUCKET_NAME }} + purge-datasheets: + name: Purge Datasheets cache + needs: build + runs-on: ubuntu-latest + environment: staging + steps: + - name: Purge Cloudflare Cache + shell: bash + run: | + echo "Purging Cloudflare cache for prefix: ${{ vars.DATASHEETS_BASE_URL }}, zone: ${{ vars.CLOUDFLARE_ZONE }}" + curl -f -X POST "https://api.cloudflare.com/client/v4/zones/${{ vars.CLOUDFLARE_ZONE }}/purge_cache" \ + -H "Authorization: Bearer ${{ secrets.CLOUDFLARE_PURGE_API_TOKEN }}" \ + -H "Content-Type: application/json" \ + --data '{"prefixes":["${{ vars.DATASHEETS_BASE_URL }}"]}' diff --git a/package-lock.json b/package-lock.json index 34d271db0d..46007a2a0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "hasInstallScript": true, "license": "ISC", "dependencies": { - "@arduino/docs-arduino-cc": "^2.0.18-1", + "@arduino/docs-arduino-cc": "^2.0.18-2", "gatsby": "^5.11.0", "gatsby-background-image": "^1.6.0", "gatsby-image": "^3.11.0", @@ -339,9 +339,9 @@ } }, "node_modules/@arduino/docs-arduino-cc": { - "version": "2.0.18-1", - "resolved": "https://npm.pkg.github.com/download/@arduino/docs-arduino-cc/2.0.18-1/c0d48794d00fcc07359d7bec8a9b598457d8d6e4", - "integrity": "sha512-h78/M4NjezAHK9qHDjW1rwcDQGdGNK9mh+yMQVitVNFu9N2EvkR8Q/j0O5J1X3FthYsHjP9FhPyP6Qs8sseWPw==", + "version": "2.0.18-2-alpha.7", + "resolved": "https://npm.pkg.github.com/download/@arduino/docs-arduino-cc/2.0.18-2-alpha.7/6bd9abd7f466fd84529f1136ae43d19188a87382", + "integrity": "sha512-9Gz8XPTlDJidq45027/Vjh1Qn94aTxlxp4vTShe0g/dJDQ3Z8BT9LFz5PGSVBpcd4W0WA+tdMU/pGaYHCw88/Q==", "dependencies": { "@algolia/autocomplete-core": "^1.10.0", "@algolia/autocomplete-plugin-recent-searches": "^1.17.0", diff --git a/package.json b/package.json index 0119b99d23..2fa2064e9a 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ }, "homepage": "https://github.com/arduino/docs-content#readme", "dependencies": { - "@arduino/docs-arduino-cc": "^2.0.18-1", + "@arduino/docs-arduino-cc": "^2.0.18-2", "gatsby": "^5.11.0", "gatsby-background-image": "^1.6.0", "gatsby-image": "^3.11.0",