Test: add result check.

Author: caozhanhao

.github/workflows/evaluate-artifacts-backend.yml

@@ -67,10 +67,16 @@ jobs:
 
       - name: Run tests and save results
         id: run-tests
+        # continue-on-error ensures the job doesn’t stop here even if tests fail
+        continue-on-error: true
         run: |
-          cat filtered_files.txt | main/build/test/ghaction -b -d ${{ env.TESTDATA_DIR }} \
-              --sha "${{ github.sha }}" \
-              --report "test_report.md"
+          set -o pipefail
+          cat filtered_files.txt \
+            | main/build/test/ghaction -b -d ${{ env.TESTDATA_DIR }} \
+                --sha "${{ github.sha }}" \
+                --report "test_report.md"
+          EXIT_CODE=$?
+          echo "exit_code=${EXIT_CODE}" >> $GITHUB_OUTPUT
 
       - name: Checkout test-result branch
         uses: actions/checkout@v4
@@ -104,6 +110,11 @@ jobs:
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           
           git add README.md
-          git commit -m "Add test results for artifacts commit ${{ github.sha }}"
-          
-          git push origin ${{ env.TEST_RESULT_BRANCH }}
+          git commit -m "Add test results for artifacts commit ${{ github.sha }}" || echo "Nothing to commit"
+          git push origin ${{ env.TEST_RESULT_BRANCH }}
+
+      - name: Fail if tests failed
+        if: steps.run-tests.outputs.exit_code != '0'
+        run: |
+          echo "🛑 ghaction exited with code ${{ steps.run-tests.outputs.exit_code }}. Failing job."
+          exit ${{ steps.run-tests.outputs.exit_code }}

test/ghaction.cpp

@@ -279,9 +279,14 @@ int main(int argc, char *argv[]) {
     };
     report << "- **Artifacts Commit:** " << (commit_sha.empty() ? "N/A" : build_md_url(commit_sha)) << "\n";
     report << "- **Total Tests:** " << entries.size() << "\n";
-    report << "- **Mode:** " << (only_frontend ? "Frontend only" : "With backend") << "\n\n";
+    report << "- **Mode:** " << (only_frontend ? "Frontend only" : "With backend") << "\n";
+
+    size_t passed = std::count_if(results.begin(), results.end(), [](const auto& tres) {
+        return tres.passed;
+    });
+
+    report << "- **Result: ** " << (passed == entries.size() ? "✅ PASSED" : "❌ FAILED") << "\n\n";
 
-    size_t passed = 0;
     std::vector<std::string> failed_tests;
     for (const auto& tres : results) {
         report << "#### Test: " << tres.id << "\n";
@@ -290,7 +295,6 @@ int main(int argc, char *argv[]) {
         report << "- **Status:** ";
 
         if (tres.passed) {
-            ++passed;
             report << "✅ PASSED\n";
         } else {
             failed_tests.push_back(tres.id);
@@ -318,5 +322,9 @@ int main(int argc, char *argv[]) {
     report << "| **Total** | **" << entries.size() << "** |\n";
 
     report.close();
+
+    if (!failed_tests.empty())
+        return -1;
+
     return 0;
 }

test/include/runner.hpp

@@ -137,8 +137,6 @@ static CheckResult check_ir_or_bin(const CheckIRBinData &data, bool only_run_fro
     // Not the one in sh or bash.
     exec_command += R"(;/bin/echo -e "\n"$? >> )" + output;
 
-    println("|  Running execute command: '{}'",  exec_command);
-
     if (std::system(exec_command.c_str()) != 0)
         return {"exec error", "", 0};