ATEX - Test and Upload Results #196
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ATEX - Test and Upload Results | |
| on: | |
| workflow_run: | |
| workflows: ["ATEX - Build Content"] | |
| types: | |
| - completed | |
| env: | |
| ATEX_REPO: RHSecurityCompliance/atex-results-testing-farm | |
| ATEX_HTML_REPO: RHSecurityCompliance/atex-html | |
| CONTEST_REPO: RHSecurityCompliance/contest | |
| ARTIFACT_RETENTION_DAYS: 1 | |
| TEST_TIMEOUT: 1440 # 24 hours | |
| # CentOS Stream versions to test (space-separated for shell loops) | |
| # NOTE: Keep in sync with matrix.centos_stream_major in the test job | |
| CS_VERSIONS: "8 9 10" | |
| permissions: | |
| contents: read | |
| actions: read | |
| pull-requests: write | |
| checks: write | |
| jobs: | |
| # Only run if the build workflow succeeded | |
| check_build: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event.workflow_run.conclusion == 'success' }} | |
| outputs: | |
| pr_number: ${{ steps.get_pr.outputs.pr_number }} | |
| pr_sha: ${{ steps.get_pr.outputs.pr_sha }} | |
| check_id: ${{ steps.create_check.outputs.check_id }} | |
| steps: | |
| - name: Download PR info | |
| uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| run-id: ${{ github.event.workflow_run.id }} | |
| name: pr-info | |
| path: pr-info/ | |
| - name: Get PR number and SHA | |
| id: get_pr | |
| run: | | |
| PR_NUMBER=$(cat pr-info/pr-number.txt) | |
| PR_SHA=$(cat pr-info/pr-sha.txt) | |
| echo "pr_number=${PR_NUMBER}" >> $GITHUB_OUTPUT | |
| echo "pr_sha=${PR_SHA}" >> $GITHUB_OUTPUT | |
| echo "PR Number: ${PR_NUMBER}" | |
| echo "PR SHA: ${PR_SHA}" | |
| if ! [[ "$PR_NUMBER" =~ ^[0-9]+$ ]]; then | |
| echo "Invalid PR number: $PR_NUMBER" | |
| exit 1 | |
| fi | |
| - name: Create GitHub check run | |
| uses: LouisBrunner/checks-action@6b626ffbad7cc56fd58627f774b9067e6118af23 # v2.0.0 | |
| id: create_check | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| name: ATEX - Test and Upload Results | |
| status: in_progress | |
| sha: ${{ steps.get_pr.outputs.pr_sha }} | |
| output: | | |
| {"summary":"Running ATEX tests: Job: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}","title":"ATEX Testing in Progress"} | |
| test: | |
| name: Test on CentOS Stream ${{ matrix.centos_stream_major }} | |
| runs-on: ubuntu-latest | |
| needs: check_build | |
| outputs: | |
| # Contest SHA from any matrix job (all use same ref, so same SHA) | |
| contest_sha: ${{ steps.get_contest.outputs.contest_sha }} | |
| contest_ref: ${{ steps.get_contest.outputs.contest_ref }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # NOTE: Keep in sync with env.CS_VERSIONS at the top of this file | |
| centos_stream_major: [8, 9, 10] | |
| container: | |
| image: fedora:latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Download build artifacts | |
| uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| run-id: ${{ github.event.workflow_run.id }} | |
| name: content-centos-stream${{ matrix.centos_stream_major }} | |
| path: content-centos-stream${{ matrix.centos_stream_major }}/ | |
| - name: Restore file permissions lost during artifact download | |
| run: | | |
| # GitHub Actions artifact download strips execute permissions | |
| # Restore permissions from the saved file created during build | |
| CONTENT_DIR="content-centos-stream${{ matrix.centos_stream_major }}" | |
| PERMS_FILE="${CONTENT_DIR}/file-permissions.txt" | |
| if [ -f "${PERMS_FILE}" ]; then | |
| echo "=== Restoring file permissions from ${PERMS_FILE} ===" | |
| cd "${CONTENT_DIR}" | |
| while IFS=' ' read -r mode filepath; do | |
| # Remove leading ./ from filepath if present | |
| filepath="${filepath#./}" | |
| if [ -f "${filepath}" ]; then | |
| chmod "${mode}" "${filepath}" | |
| fi | |
| done < file-permissions.txt | |
| echo "Restored permissions for $(wc -l < file-permissions.txt) files" | |
| # Show sample of restored executable files | |
| echo "=== Sample executable files after restore ===" | |
| find . -type f -executable -name "*.py" 2>/dev/null | head -5 || true | |
| find . -type f -executable -name "*.sh" 2>/dev/null | head -5 || true | |
| else | |
| echo "WARNING: ${PERMS_FILE} not found, permissions may be incorrect" | |
| exit 1 | |
| fi | |
| - name: Install git for checkout | |
| run: dnf -y install git | |
| - name: Checkout Contest Test Suite | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| repository: ${{ env.CONTEST_REPO }} | |
| ref: main | |
| path: contest | |
| fetch-depth: 1 | |
| - name: Get Contest SHA | |
| id: get_contest | |
| run: | | |
| CONTEST_SHA=$(cd contest && git rev-parse HEAD) | |
| CONTEST_REF="main" | |
| echo "contest_sha=${CONTEST_SHA}" >> $GITHUB_OUTPUT | |
| echo "contest_ref=${CONTEST_REF}" >> $GITHUB_OUTPUT | |
| echo "Contest: ${CONTEST_SHA:0:12} (${CONTEST_REF})" | |
| - name: Install test dependencies | |
| run: | | |
| dnf -y install python3-pip rsync | |
| pip install fmf atex==0.13 | |
| - name: Run tests on Testing Farm | |
| env: | |
| TESTING_FARM_API_TOKEN: ${{ secrets.TESTING_FARM_API_TOKEN }} | |
| CS_MAJOR: ${{ matrix.centos_stream_major }} | |
| REPO_URL: ${{ github.server_url }}/${{ github.repository }} | |
| ACTOR: ${{ github.actor }} | |
| run: | | |
| python3 tests/run_tests_testingfarm.py \ | |
| --contest-dir contest \ | |
| --content-dir content-centos-stream${CS_MAJOR} \ | |
| --plan "/plans/upstream" \ | |
| --compose "CentOS-Stream-${CS_MAJOR}" \ | |
| --arch x86_64 \ | |
| --os-major-version "${CS_MAJOR}" \ | |
| --timeout ${{ env.TEST_TIMEOUT }} \ | |
| --tag github_actor=$ACTOR \ | |
| --tag github_repo=$REPO_URL | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| with: | |
| name: test-results-centos-stream${{ matrix.centos_stream_major }} | |
| path: | | |
| results-centos-stream-${{ matrix.centos_stream_major }}-x86_64.json.xz | |
| files-centos-stream-${{ matrix.centos_stream_major }}-x86_64/ | |
| atex_debug.log.gz | |
| retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} | |
| upload: | |
| name: Upload and publish test results | |
| runs-on: ubuntu-latest | |
| needs: [check_build, test] | |
| if: always() # Run even if tests fail | |
| container: | |
| image: fedora:latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Install dependencies | |
| if: always() | |
| run: | | |
| dnf -y install python3-pip git rsync | |
| pip install fmf atex==0.13 | |
| - name: Checkout ATEX results repository | |
| if: always() | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| repository: ${{ env.ATEX_REPO }} | |
| ref: main | |
| path: atex-results-testing-farm | |
| token: ${{ secrets.ATEX_RESULTS_TF_REPO_TOKEN }} | |
| - name: Initialize FMF metadata | |
| if: always() | |
| working-directory: atex-results-testing-farm | |
| run: fmf init | |
| - name: Create TMT atex_results plan for artifact transport | |
| if: always() | |
| working-directory: atex-results-testing-farm | |
| run: | | |
| cat > main.fmf <<'EOF' | |
| /atex_results_plan: | |
| discover: | |
| how: shell | |
| tests: | |
| - name: /atex_results_test | |
| test: mv * "$TMT_TEST_DATA/." | |
| execute: | |
| how: tmt | |
| EOF | |
| # Download test results for all CentOS Stream versions | |
| - name: Download test results - CentOS Stream 8 | |
| if: always() | |
| uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| name: test-results-centos-stream8 | |
| path: test-results/cs8/ | |
| continue-on-error: true | |
| - name: Download test results - CentOS Stream 9 | |
| if: always() | |
| uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| name: test-results-centos-stream9 | |
| path: test-results/cs9/ | |
| continue-on-error: true | |
| - name: Download test results - CentOS Stream 10 | |
| if: always() | |
| uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| name: test-results-centos-stream10 | |
| path: test-results/cs10/ | |
| continue-on-error: true | |
| - name: Checkout ATEX HTML viewer | |
| if: always() | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| repository: ${{ env.ATEX_HTML_REPO }} | |
| ref: main | |
| path: atex-html | |
| - name: Update HTML title with PR number | |
| if: always() | |
| env: | |
| PR_NUMBER: ${{ needs.check_build.outputs.pr_number }} | |
| run: | | |
| sed "/<title>/s/>.*</>Test outputs from PR #${PR_NUMBER} HTML</" \ | |
| -i atex-html/index.html | |
| - name: Merge test results from all versions | |
| if: always() | |
| run: | | |
| mkdir -p atex-results-testing-farm/files_dir/ | |
| # Process and merge results for all CentOS Stream versions | |
| for version in ${{ env.CS_VERSIONS }}; do | |
| results_file="test-results/cs${version}/results-centos-stream-${version}-x86_64.json.xz" | |
| files_dir="test-results/cs${version}/files-centos-stream-${version}-x86_64" | |
| if [ -f "${results_file}" ]; then | |
| cat "${results_file}" | |
| rm -f "${results_file}" | |
| [ -d "${files_dir}" ] && cp -r "${files_dir}"/* atex-results-testing-farm/files_dir/ | |
| fi | |
| done > results.json.xz | |
| - name: Convert results to SQLite database | |
| if: always() | |
| run: | | |
| python atex-html/json2db.py results.json.xz atex-results-testing-farm/results.sqlite.gz | |
| - name: Prepare HTML results viewer | |
| if: always() | |
| run: | | |
| cp -rf atex-html/index.html atex-html/sqljs/ atex-results-testing-farm/ | |
| - name: Generate header.html for results page | |
| if: always() | |
| env: | |
| PR_NUMBER: ${{ needs.check_build.outputs.pr_number }} | |
| PR_SHA: ${{ needs.check_build.outputs.pr_sha }} | |
| CONTEST_SHA: ${{ needs.test.outputs.contest_sha }} | |
| CONTEST_REF: ${{ needs.test.outputs.contest_ref }} | |
| WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| REPO_URL: ${{ github.server_url }}/${{ github.repository }} | |
| ACTOR: ${{ github.actor }} | |
| RUN_STARTED: ${{ github.event.workflow_run.created_at }} | |
| CS_VERSIONS: ${{ env.CS_VERSIONS }} | |
| run: | | |
| cat > atex-results-testing-farm/header.html <<'HEADER_EOF' | |
| <style> | |
| #header table { | |
| display: inline-table; | |
| margin-right: 1em; | |
| vertical-align: top; | |
| border-collapse: collapse; | |
| } | |
| #header th, td { | |
| border: 1px solid black; | |
| padding: 0.3em; | |
| } | |
| #header h1 { | |
| color: #aaa; | |
| margin: 0; | |
| } | |
| #header h2 { | |
| color: #555; | |
| margin: 0; | |
| } | |
| </style> | |
| HEADER_EOF | |
| # Add dynamic content - header section | |
| cat >> atex-results-testing-farm/header.html <<EOF | |
| <h1>ATEX Upstream Testing</h1> | |
| <h2>PR <a href="${REPO_URL}/pull/${PR_NUMBER}">#${PR_NUMBER}</a> | |
| - Workflow <a href="${WORKFLOW_URL}">#${{ github.run_id }}</a> | |
| started on <span id="header-started-on"></span> | |
| by <a href="https://github.com/${ACTOR}"><code>${ACTOR}</code></a></h2> | |
| <script>document.getElementById('header-started-on').textContent = new Date('${RUN_STARTED}').toLocaleString()</script> | |
| <div style="margin-top: 1em; margin-bottom: 1em"> | |
| <table> | |
| <tr><th colspan="1">CentOS Stream</th></tr> | |
| EOF | |
| # List each CentOS Stream version that was tested | |
| for version in ${CS_VERSIONS}; do | |
| echo " <tr><td>${version}</td></tr>" >> atex-results-testing-farm/header.html | |
| done | |
| # Add commit info table | |
| cat >> atex-results-testing-farm/header.html <<EOF | |
| </table> | |
| <table> | |
| <tr><th>Repo</th><th>Commit used</th></tr> | |
| <tr><td>Content</td><td style="font-family: monospace, monospace;"><a href="${REPO_URL}/commit/${PR_SHA}">${PR_SHA:0:12}</a></td></tr> | |
| <tr><td>Contest (${CONTEST_REF})</td><td style="font-family: monospace, monospace;"><a href="https://github.com/${{ env.CONTEST_REPO }}/commit/${CONTEST_SHA}">${CONTEST_SHA:0:12}</a></td></tr> | |
| </table> | |
| </div> | |
| EOF | |
| echo "=== Generated header.html ===" | |
| cat atex-results-testing-farm/header.html | |
| - name: Commit and tag results in ATEX repository | |
| if: always() | |
| working-directory: atex-results-testing-farm | |
| env: | |
| GH_TOKEN: ${{ secrets.ATEX_RESULTS_TF_REPO_TOKEN }} | |
| PR_NUMBER: ${{ needs.check_build.outputs.pr_number }} | |
| run: | | |
| git config user.name "openscap-ci[bot]" | |
| git config user.email "[email protected]" | |
| git add . | |
| git commit -m "Test outputs from PR #${PR_NUMBER}" | |
| git tag "PR${PR_NUMBER}" | |
| git push origin "PR${PR_NUMBER}" | |
| - name: Submit results to Testing Farm | |
| if: always() | |
| id: testing_farm_request | |
| env: | |
| TESTING_FARM_API_TOKEN: ${{ secrets.TESTING_FARM_API_TOKEN }} | |
| PR_NUMBER: ${{ needs.check_build.outputs.pr_number }} | |
| run: | | |
| python3 tests/submit_results_to_testing_farm.py \ | |
| --repo-url "https://github.com/${{ env.ATEX_REPO }}" \ | |
| --pr-number "${PR_NUMBER}" 2>&1 | tee tf_output.log | |
| # Extract HTML link from output | |
| html_link=$(grep -oP 'HTML: \K.*' tf_output.log || echo 'No HTML link found') | |
| echo "HTML_LINK=${html_link}" >> "$GITHUB_OUTPUT" | |
| - name: Find existing PR comment | |
| if: always() | |
| uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e # v3 | |
| id: fc | |
| with: | |
| issue-number: ${{ needs.check_build.outputs.pr_number }} | |
| comment-author: 'github-actions[bot]' | |
| body-includes: ATEX Test Results | |
| - name: Create or update PR comment with results | |
| if: always() | |
| uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v4 | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| issue-number: ${{ needs.check_build.outputs.pr_number }} | |
| body: | | |
| ### ATEX Test Results | |
| Test artifacts have been submitted to Testing Farm. | |
| **Results:** [View Test Results](${{ steps.testing_farm_request.outputs.HTML_LINK }}) | |
| **Workflow Run:** [View Workflow Details](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| _This comment was automatically generated by the ATEX workflow._ | |
| edit-mode: replace | |
| - name: Cleanup temporary tag | |
| if: always() | |
| working-directory: atex-results-testing-farm | |
| env: | |
| GH_TOKEN: ${{ secrets.ATEX_RESULTS_TF_REPO_TOKEN }} | |
| PR_NUMBER: ${{ needs.check_build.outputs.pr_number }} | |
| run: | | |
| git push --delete origin "PR${PR_NUMBER}" | |
| - name: Update GitHub check run | |
| if: always() | |
| uses: LouisBrunner/checks-action@6b626ffbad7cc56fd58627f774b9067e6118af23 # v2.0.0 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| check_id: ${{ needs.check_build.outputs.check_id }} | |
| sha: ${{ needs.check_build.outputs.pr_sha }} | |
| status: completed | |
| # Use test job result to determine conclusion - needs.test.result will be 'failure' if any matrix job failed | |
| conclusion: ${{ needs.test.result }} | |
| output: | | |
| {"summary":"ATEX tests completed. Job: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}. View results: ${{ steps.testing_farm_request.outputs.HTML_LINK }}","title":"ATEX Testing Complete"} |