[CRE] mixed-env nightly: Slack-alert #cre-platform-operations on failure #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CRE Mixed-Env Nightly Sweep | ||
|
Check failure on line 1 in .github/workflows/cre-mixed-env-nightly.yaml
|
||
| # Runs the full CRE system-test matrix under the mixed-env topology (2 nodes on | ||
| # today's develop nightly image, 2 on yesterday's) to catch non-determinism / | ||
| # cross-version incompatibilities introduced into develop between nightlies — | ||
| # across the whole suite (EVM + Solana/Aptos/Stellar), not just the per-PR subset. | ||
| # | ||
| # It calls cre-system-tests.yaml directly with mixed_env_all: true, so no image | ||
| # build is needed: both images are the cached nightly develop builds. | ||
| on: | ||
| schedule: | ||
| # After the nightly image build (docker-build.yml runs at 03:00 UTC). | ||
| - cron: "0 5 * * *" | ||
| workflow_dispatch: | ||
| inputs: | ||
| baseline_date: | ||
| description: "Baseline nightly date (YYYYMMDD). Defaults to yesterday." | ||
| required: false | ||
| type: string | ||
| default: "" | ||
| permissions: | ||
| contents: read | ||
| id-token: write | ||
| jobs: | ||
| set-dates: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| today: ${{ steps.d.outputs.today }} | ||
| baseline: ${{ steps.d.outputs.baseline }} | ||
| steps: | ||
| - id: d | ||
| shell: bash | ||
| run: | | ||
| today="$(date -u +'%Y%m%d')" | ||
| baseline="${{ inputs.baseline_date }}" | ||
| if [[ -z "${baseline}" ]]; then | ||
| baseline="$(date -u -d 'yesterday' +'%Y%m%d')" | ||
| fi | ||
| echo "today=${today}" | tee -a "$GITHUB_OUTPUT" | ||
| echo "baseline=${baseline}" | tee -a "$GITHUB_OUTPUT" | ||
| mixed-env-sweep: | ||
| needs: set-dates | ||
| permissions: | ||
| contents: read | ||
| id-token: write | ||
| uses: ./.github/workflows/cre-mixed-env-tests.yaml | ||
| secrets: inherit | ||
| with: | ||
| ecr: "sdlc" | ||
| chainlink_image_repository_path: "chainlink" | ||
| # "PR" side = today's develop nightly. | ||
| chainlink_image_tag: "nightly-${{ needs.set-dates.outputs.today }}" | ||
| # Baseline side = a previous develop nightly, so the 2-2 split compares two | ||
| # develop builds and surfaces non-determinism introduced since then. | ||
| mixed_env_baseline_image: "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink:nightly-${{ needs.set-dates.outputs.baseline }}" | ||
| full_matrix: true | ||
| # There is no PR to block on a scheduled run, so alert the CRE team on Slack when | ||
| # the sweep fails (a failure usually means non-determinism was introduced into | ||
| # develop between the two nightlies, or a job/infra error). | ||
| notify-on-failure: | ||
| name: Notify Slack on failure | ||
| if: failure() | ||
| needs: [set-dates, mixed-env-sweep] | ||
| runs-on: ubuntu-latest | ||
| environment: | ||
| name: integration | ||
| deployment: false | ||
| steps: | ||
| - name: Send Slack notification (#cre-platform-operations) | ||
| uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 | ||
| with: | ||
| errors: "true" | ||
| method: chat.postMessage | ||
| token: ${{ secrets.QA_SLACK_API_KEY }} | ||
| # If delivery fails, invite the bot to the channel, or replace the name | ||
| # with the channel ID (Cxxxxxxxx) which chat.postMessage resolves more reliably. | ||
| payload: | | ||
| { | ||
| "channel": "#cre-platform-operations", | ||
| "text": "CRE Mixed-Env Nightly Sweep failed — possible non-determinism introduced into develop.", | ||
| "blocks": [ | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "*:rotating_light: CRE Mixed-Env Nightly Sweep failed :rotating_light:*" | ||
| } | ||
| }, | ||
| { | ||
| "type": "section", | ||
| "text": { | ||
| "type": "mrkdwn", | ||
| "text": "Today's develop nightly `nightly-${{ needs.set-dates.outputs.today }}` vs baseline `nightly-${{ needs.set-dates.outputs.baseline }}`. A failure usually means *non-determinism was introduced into develop* between these nightlies (or a job/infra error). Run: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.run_id }}>. Please investigate." | ||
| } | ||
| }, | ||
| { | ||
| "type": "divider" | ||
| } | ||
| ] | ||
| } | ||
