chore(ci): add Cloud Build failure reporter

.github/workflows/cloud_build_failure_reporter.yml

@@ -0,0 +1,180 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Cloud Build Failure Reporter
+
+on:
+  workflow_call:
+    inputs:
+      trigger_names:
+          required: true
+          type: string
+  workflow_dispatch:
+    inputs:
+      trigger_names:
+        description: 'Cloud Build trigger names separated by comma.'
+        required: true
+        default: ''
+
+jobs:
+  report:
+
+    permissions:
+      issues: 'write'
+      checks: 'read'
+
+    runs-on: 'ubuntu-latest'
+
+    steps:
+      - uses: 'actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea' # v7
+        with:
+          script: |-
+                  // parse test names
+                  const testNameSubstring = '${{ inputs.trigger_names }}';
+                  const testNameFound = new Map(); //keeps track of whether each test is found
+                  testNameSubstring.split(',').forEach(testName => {
+                    testNameFound.set(testName, false); 
+                  });
+
+                  // label for all issues opened by reporter
+                  const periodicLabel = 'periodic-failure';
+
+                  // check if any reporter opened any issues previously
+                  const prevIssues = await github.paginate(github.rest.issues.listForRepo, {
+                    ...context.repo,
+                    state: 'open',
+                    creator: 'github-actions[bot]',
+                    labels: [periodicLabel]
+                  });
+
+                  // createOrCommentIssue creates a new issue or comments on an existing issue.
+                  const createOrCommentIssue = async function (title, txt) {
+                    if (prevIssues.length < 1) {
+                      console.log('no previous issues found, creating one');
+                      await github.rest.issues.create({
+                        ...context.repo,
+                        title: title,
+                        body: txt,
+                        labels: [periodicLabel]
+                      });
+                      return;
+                    }
+                    // only comment on issue related to the current test
+                    for (const prevIssue of prevIssues) {
+                      if (prevIssue.title.includes(title)){
+                          console.log(
+                          `found previous issue ${prevIssue.html_url}, adding comment`
+                        );
+
+                        await github.rest.issues.createComment({
+                          ...context.repo,
+                          issue_number: prevIssue.number,
+                          body: txt
+                        });
+                        return;
+                      }
+                    }
+                  };
+
+                  // updateIssues comments on any existing issues. No-op if no issue exists.
+                  const updateIssues = async function (checkName, txt) {
+                    if (prevIssues.length < 1) {
+                      console.log('no previous issues found.');
+                      return;
+                    }
+                    // only comment on issue related to the current test
+                    for (const prevIssue of prevIssues) {
+                      if (prevIssue.title.includes(checkName)){
+                        console.log(`found previous issue ${prevIssue.html_url}, adding comment`);
+                        await github.rest.issues.createComment({
+                          ...context.repo,
+                          issue_number: prevIssue.number,
+                          body: txt
+                        });
+                      }
+                    }
+                  };
+
+                  // Find status of check runs.
+                  // We will find check runs for each commit and then filter for the periodic.
+                  // Checks API only allows for ref and if we use main there could be edge cases where
+                  // the check run happened on a SHA that is different from head.
+                  const commits = await github.paginate(github.rest.repos.listCommits, {
+                    ...context.repo
+                  });
+
+                  const relevantChecks = new Map();
+                  for (const commit of commits) {
+                    console.log(
+                      `checking runs at ${commit.html_url}: ${commit.commit.message}`
+                    );
+                    const checks = await github.rest.checks.listForRef({
+                      ...context.repo,
+                      ref: commit.sha
+                    });
+
+                    // Iterate through each check and find matching names
+                    for (const check of checks.data.check_runs) {
+                      console.log(`Handling test name ${check.name}`);
+                      for (const testName of testNameFound.keys()) {
+                        if (testNameFound.get(testName) === true){
+                          //skip if a check is already found for this name
+                          continue; 
+                        }
+                        if (check.name.includes(testName)) {
+                          relevantChecks.set(check, commit);
+                          testNameFound.set(testName, true);
+                        }
+                      }
+                    }
+                    // Break out of the loop early if all tests are found
+                    const allTestsFound = Array.from(testNameFound.values()).every(value => value === true);
+                    if (allTestsFound){
+                      break;
+                    }
+                  }
+
+                  // Handle each relevant check
+                  relevantChecks.forEach((commit, check) => {
+                    if (
+                        check.status === 'completed' &&
+                        check.conclusion === 'success'
+                    ) {
+                        updateIssues(
+                            check.name,
+                            `[Tests are passing](${check.html_url}) for commit [${commit.sha}](${commit.html_url}).`
+                        );
+                    } else if (check.status === 'in_progress') {
+                        console.log(
+                            `Check is pending ${check.html_url} for ${commit.html_url}. Retry again later.`
+                        );
+                    } else {
+                        createOrCommentIssue(
+                            `Cloud Build Failure Reporter: ${check.name} failed`,
+                            `Cloud Build Failure Reporter found test failure for [**${check.name}** ](${check.html_url}) at [${commit.sha}](${commit.html_url}). Please fix the error and then close the issue after the **${check.name}** test passes.`
+                        );
+                    }
+                  });
+
+                  // no periodic checks found across all commits, report it
+                  const noTestFound = Array.from(testNameFound.values()).every(value => value === false);
+                  if (noTestFound){
+                    createOrCommentIssue(
+                      'Missing periodic tests: ${{ inputs.trigger_names }}',
+                      `No periodic test is found for triggers: ${{ inputs.trigger_names }}. Last checked from ${
+                        commits[0].html_url
+                      } to ${commits[commits.length - 1].html_url}.`
+                    );
+                  }
+

.github/workflows/schedule_reporter.yml

@@ -0,0 +1,25 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Schedule Reporter
+
+on:
+  schedule:
+    - cron: '0 6 * * *'  # Runs at 6 AM every morning
+
+jobs:
+  run_reporter:
+    uses: ./.github/workflows/cloud_build_failure_reporter.yml
+    with:
+      trigger_names: "integration-test-nightly,continuous-test-on-merge"