From 2813005c7370a5d6f5ca4c3a0863347efd653aa4 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 16 Apr 2025 08:25:41 -0700 Subject: [PATCH 01/27] X-Smart-Branch-Parent: master From 97d01587e9816355c0cfbef296cc7f60bfc3181a Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Fri, 18 Apr 2025 20:40:22 -0700 Subject: [PATCH 02/27] X-Smart-Squash: Squashed 18 commits: 1179d5740 Squash 835342eb1 rn 8430368d9 update f54326cca unit e9fc780d9 merge builder 951534628 split d7b2d89db workflow 19f9fc9bb update f2205847f always 1af882d83 unit 6a5d9a7e0 comment a22a493f9 unit 9424110d1 unit 83bb60f29 enable builder 4c01e09a3 arch 4f0df21e9 clean d07cefc3b arch 50aec13c9 typo --- .github/workflows/collector-builder.yml | 69 +++++----------- .github/workflows/collector.yml | 101 ++++++------------------ .github/workflows/init.yml | 46 +++++++++++ .github/workflows/integration-tests.yml | 19 +++-- .github/workflows/konflux.yml | 14 ++++ .github/workflows/main.yml | 13 ++- .github/workflows/unit-tests.yml | 40 ++++++++-- 7 files changed, 152 insertions(+), 150 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 710fee006e..3cdd27db97 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -8,6 +8,11 @@ on: required: true description: | The tag used to build the collector image + include-arch-json: + type: string + required: true + description: | + JSON for matrix.include to select architectures and runners to build outputs: collector-builder-tag: description: The builder tag used by the build @@ -40,7 +45,6 @@ jobs: build-builder-image: name: Build the builder image - runs-on: ubuntu-24.04 # Multiarch builds sometimes take for eeeeeeeeeever timeout-minutes: 480 needs: @@ -55,9 +59,9 @@ jobs: outputs: collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }} strategy: - fail-fast: false matrix: - arch: [amd64, ppc64le, s390x, arm64] + include: ${{ fromJSON(inputs.include-arch-json) }} + runs-on: ${{ matrix.runner }} env: PLATFORM: linux/${{ matrix.arch }} @@ -69,24 +73,30 @@ jobs: submodules: true - name: Set up QEMU + if: matrix.remote != true uses: docker/setup-qemu-action@v3 with: image: tonistiigi/binfmt:qemu-v8.1.5 - name: Set up Docker Buildx + if: matrix.remote != true uses: docker/setup-buildx-action@v3 - uses: actions/setup-python@v5 + if: matrix.remote with: python-version: "3.10" - uses: 'google-github-actions/auth@v2' + if: matrix.remote with: credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}' - uses: 'google-github-actions/setup-gcloud@v2' + if: matrix.remote - uses: ./.github/actions/setup-vm-creds + if: matrix.remote with: gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }} gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }} @@ -101,9 +111,7 @@ jobs: job-tag: builder - name: Create Build VMs - if: | - matrix.arch == 's390x' && - (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) + if: matrix.remote run: | make -C "${{ github.workspace }}/ansible" create-build-vms @@ -133,10 +141,7 @@ jobs: } > ${{ github.workspace }}/ansible/secrets.yml - name: Build images - if: | - (github.event_name != 'pull_request' && matrix.arch != 's390x') || - matrix.arch == 'amd64' || - (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch != 's390x') + if: matrix.remote != true timeout-minutes: 480 run: | ansible-galaxy install -r ansible/requirements.yml @@ -149,9 +154,7 @@ jobs: ansible/ci-build-builder.yml - name: Build s390x images - if: | - (github.event_name != 'pull_request' && matrix.arch == 's390x') || - (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x') + if: matrix.remote timeout-minutes: 480 run: | ansible-playbook \ @@ -162,7 +165,7 @@ jobs: ansible/ci-build-builder.yml - name: Destroy VMs - if: always() && matrix.arch == 's390x' + if: always() && matrix.remote run: | make -C ansible destroy-vms @@ -173,11 +176,10 @@ jobs: runs-on: ubuntu-24.04 if: | github.event_name != 'pull_request' || - (needs.build-builder-image.outputs.collector-builder-tag != 'cache' && - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) + needs.build-builder-image.outputs.collector-builder-tag != 'cache' env: COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }} - ARCHS: amd64 ppc64le s390x arm64 + ARCHS: ${{ join(fromJSON(inputs.include-arch-json).*.arch, ' ') }} steps: - uses: actions/checkout@v4 @@ -208,45 +210,12 @@ jobs: base-image: quay.io/rhacs-eng/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }} archs: ${{ env.ARCHS }} - retag-x86-image: - needs: - - build-builder-image - name: Retag x86 builder image - runs-on: ubuntu-24.04 - if: | - github.event_name == 'pull_request' && - needs.build-builder-image.outputs.collector-builder-tag != 'cache' && - !contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') - env: - COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }} - steps: - - name: Pull image to retag - run: | - docker pull "quay.io/stackrox-io/collector-builder:${COLLECTOR_BUILDER_TAG}-amd64" - - - name: Retag and push stackrox-io - uses: stackrox/actions/images/retag-and-push@v1 - with: - src-image: quay.io/stackrox-io/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }}-amd64 - dst-image: quay.io/stackrox-io/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }} - username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} - - - name: Retag and push rhacs-eng - uses: stackrox/actions/images/retag-and-push@v1 - with: - src-image: quay.io/stackrox-io/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }}-amd64 - dst-image: quay.io/rhacs-eng/collector-builder:${{ env.COLLECTOR_BUILDER_TAG }} - username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - notify: runs-on: ubuntu-24.04 if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request' needs: - build-builder-image - create-multiarch-manifest - - retag-x86-image steps: - name: Slack notification uses: rtCamp/action-slack-notify@v2 diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 0a401ca01b..0f52ec92cb 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -18,6 +18,11 @@ on: required: true description: | The builder tag to use in the build + include-arch-json: + type: string + required: true + description: | + JSON for matrix.include to select architectures and runners to build env: COLLECTOR_TAG: ${{ inputs.collector-tag }} @@ -30,11 +35,10 @@ env: jobs: build-collector-image: name: Build Collector - runs-on: ubuntu-24.04 strategy: - fail-fast: false matrix: - arch: [amd64, ppc64le, arm64] + include: ${{ fromJSON(inputs.include-arch-json) }} + runs-on: ${{ matrix.runner }} env: PLATFORM: linux/${{ matrix.arch }} @@ -46,11 +50,13 @@ jobs: submodules: true - name: Set up QEMU + if: matrix.remote != true uses: docker/setup-qemu-action@v3 with: image: tonistiigi/binfmt:qemu-v8.1.5 - name: Set up Docker Buildx + if: matrix.remote != true uses: docker/setup-buildx-action@v3 - name: Create ansible vars @@ -62,6 +68,7 @@ jobs: rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} collector_git_ref: ${{ github.ref }} + collector_git_sha: ${{ github.sha }} collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }} disable_profiling: ${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }} rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }} @@ -71,11 +78,8 @@ jobs: driver_version: ${DRIVER_VERSION} EOF - - name: Build images - if: | - github.event_name != 'pull_request' || - matrix.arch == 'amd64' || - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') + - name: Build ${{ matrix.arch }} image locally + if: matrix.remote != true timeout-minutes: 480 run: | ansible-playbook \ @@ -86,32 +90,21 @@ jobs: -e @'${{ github.workspace }}/ansible/secrets.yml' \ ansible/ci-build-collector.yml - build-collector-image-remote-vm: - name: Build Collector on a remote VM - runs-on: ubuntu-24.04 - if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') - strategy: - fail-fast: false - matrix: - arch: [s390x] - - env: - PLATFORM: linux/${{ matrix.arch }} - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + if: matrix.remote with: python-version: "3.10" - uses: 'google-github-actions/auth@v2' + if: matrix.remote with: credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}' - uses: 'google-github-actions/setup-gcloud@v2' + if: matrix.remote - uses: ./.github/actions/setup-vm-creds + if: matrix.remote with: gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }} gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }} @@ -126,28 +119,12 @@ jobs: job-tag: builder - name: Create Build VMs + if: matrix.remote run: | make -C "${{ github.workspace }}/ansible" create-build-vms - - name: Create ansible vars - run: | - cat << EOF > ${{ github.workspace }}/ansible/secrets.yml - --- - stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} - rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - collector_git_ref: ${{ github.ref }} - collector_git_sha: ${{ github.sha }} - collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }} - disable_profiling: ${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }} - rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }} - collector_image: ${{ inputs.collector-image }} - collector_tag: ${{ inputs.collector-tag }} - debug_mode: ${{ github.event_name == 'pull_request' }} - EOF - - - name: Build ${{ matrix.arch }} image + - name: Build ${{ matrix.arch }} image remotely + if: matrix.remote timeout-minutes: 480 run: | ansible-playbook \ @@ -158,21 +135,18 @@ jobs: ansible/ci-build-collector.yml - name: Destroy Build VMs - if: always() + if: always() && matrix.remote run: | make -C ansible destroy-vms create-multiarch-manifest: needs: - build-collector-image - - build-collector-image-remote-vm name: Create Multiarch manifest runs-on: ubuntu-24.04 - if: | - github.event_name != 'pull_request' || - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') + if: always() && !contains(join(needs.*.result, ','), 'failure') env: - ARCHS: amd64 ppc64le s390x arm64 + ARCHS: ${{ join(fromJSON(inputs.include-arch-json).*.arch, ' ') }} steps: - uses: actions/checkout@v4 @@ -203,43 +177,12 @@ jobs: base-image: quay.io/rhacs-eng/collector:${{ inputs.collector-tag }} archs: ${{ env.ARCHS }} - retag-x86-image: - needs: - - build-collector-image - name: Retag x86 image - runs-on: ubuntu-24.04 - if: | - github.event_name == 'pull_request' && - !contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') - steps: - - name: Pull image to retag - run: | - docker pull ${{ inputs.collector-image }}-amd64 - - - name: Retag and push stackrox-io - uses: stackrox/actions/images/retag-and-push@v1 - with: - src-image: ${{ inputs.collector-image }}-amd64 - dst-image: ${{ inputs.collector-image }} - username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} - - - name: Retag and push rhacs-eng - uses: stackrox/actions/images/retag-and-push@v1 - with: - src-image: ${{ inputs.collector-image }}-amd64 - dst-image: ${{ env.RHACS_ENG_IMAGE }} - username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - notify: runs-on: ubuntu-24.04 if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request' needs: - build-collector-image - - build-collector-image-remote-vm - create-multiarch-manifest - - retag-x86-image steps: - name: Slack notification uses: rtCamp/action-slack-notify@v2 diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml index be09716158..91abc27f17 100644 --- a/.github/workflows/init.yml +++ b/.github/workflows/init.yml @@ -58,6 +58,11 @@ on: description: | Trigger rebuild of QA containers value: ${{ jobs.common-variables.outputs.rebuild-qa-containers }} + include-arch-json: + description: | + JSON for matrix.include in later jobs + value: ${{ jobs.common-variables.outputs.include-arch-json }} + jobs: common-variables: @@ -75,6 +80,7 @@ jobs: cpaas-support-packages-bucket: ${{ steps.gcp-buckets.outputs.cpaas-support-packages-bucket }} support-packages-index-bucket: ${{ steps.gcp-buckets.outputs.support-packages-index-bucket }} rebuild-qa-containers: ${{ steps.filter.outputs.container }} + include-arch-json: ${{ steps.arch-json.outputs.include-arch-json }} steps: - uses: actions/checkout@v4 @@ -173,3 +179,43 @@ jobs: echo "support-packages-index-bucket=${SUPPORT_PACKAGES_BUCKET}" } >> "$GITHUB_OUTPUT" fi + + - name: Create settings for runner and remote builds per enabled arch + id: arch-json + uses: actions/github-script@v7 + with: + script: | + const all_archs = ['amd64', 'arm64', 'ppc64le', 's390x']; + const default_archs = ['amd64', 'arm64']; + + // the arch build configs for runner and remote used for + // collector-builder, collector image, and integration test jobs. + const arch_configs = { + amd64: { runner: 'ubuntu-24.04', remote: false }, + arm64: { runner: 'ubuntu-24.04-arm', remote: false }, + ppc64le: { runner: 'ubuntu-24.04', remote: false }, + s390x: { runner: 'ubuntu-24.04', remote: true }, + }; + + let selected_archs = all_archs; + if (context.payload.pull_request) { + const labels = context.payload.pull_request.labels.map(label => label.name); + const specific = all_archs.filter(a => labels.includes(`run-${a}-builds`)); + if (specific.length) { + selected_archs = specific + } else { + selected_archs = default_archs + if (labels.includes('run-multiarch-builds')) { + selected_archs.push('ppc64le', 's390x'); + } + } + } + // create output array of selected arch config objects + const arch_output = selected_archs.map(arch => ({ + arch: arch, + runner: arch_configs[arch].runner, + remote: arch_configs[arch].remote, + })); + core.setOutput('include-arch-json', JSON.stringify(arch_output)); + core.info(JSON.stringify(arch_output)); + diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 1290e8e7de..347a10460c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -42,10 +42,16 @@ on: number of CPU cores. type: boolean default: false + include-arch-json: + type: string + required: true + description: | + JSON for matrix.include in later jobs jobs: amd64-integration-tests: uses: ./.github/workflows/integration-tests-vm-type.yml + if: contains(inputs.include-arch-json, 'amd64') strategy: # ensure that if one part of the matrix fails, the # rest will continue @@ -91,9 +97,7 @@ jobs: arm64-integration-tests: uses: ./.github/workflows/integration-tests-vm-type.yml - if: | - github.event_name != 'pull_request' || inputs.is-konflux || - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') + if: contains(inputs.include-arch-json, 'arm64') strategy: # ensure that if one part of the matrix fails, the # rest will continue @@ -104,7 +108,6 @@ jobs: - ubuntu-arm - sles-arm64 - fcarm - with: vm_type: ${{ matrix.vm_type }} collector-tag: ${{ inputs.collector-tag }} @@ -116,9 +119,7 @@ jobs: s390x-integration-tests: uses: ./.github/workflows/integration-tests-vm-type.yml - if: | - github.event_name != 'pull_request' || inputs.is-konflux || - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') + if: contains(inputs.include-arch-json, 's390x') with: vm_type: rhel-s390x collector-tag: ${{ inputs.collector-tag }} @@ -130,9 +131,7 @@ jobs: ppc64le-integration-tests: uses: ./.github/workflows/integration-tests-vm-type.yml - if: | - github.event_name != 'pull_request' || inputs.is-konflux || - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') + if: contains(inputs.include-arch-json, 'ppc64le') with: vm_type: rhel-ppc64le collector-tag: ${{ inputs.collector-tag }} diff --git a/.github/workflows/konflux.yml b/.github/workflows/konflux.yml index 8a03295382..7093042e45 100644 --- a/.github/workflows/konflux.yml +++ b/.github/workflows/konflux.yml @@ -20,7 +20,20 @@ concurrency: cancel-in-progress: true jobs: + should-run: + outputs: + should-run: ${{ steps.should-run.outputs.should-run }} + runs-on: ubuntu-24.04 + steps: + - id: should-run + run: | + SHOULD_RUN="${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-konflux-steps') }}" + echo "should-run=${SHOULD_RUN}" >> "$GITHUB_OUTPUT" + init: + needs: + - should-run + if: needs.should-run.outputs.should-run == 'true' runs-on: ubuntu-24.04 outputs: collector-tag: ${{ steps.generate-tag.outputs.collector-tag }} @@ -89,6 +102,7 @@ jobs: collector-tests-tag: ${{ needs.integration-tests-containers.outputs.collector-tests-tag }} is-konflux: true job-tag: konf + include-arch-json: '[{"arch":"amd64"},{"arch","arm64"},{"arch":"ppc64le"},{"arch","s390x"}]' if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-integration-tests') }} secrets: inherit diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f002e6eb7b..1c2ce86a38 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,6 +40,7 @@ jobs: needs: init with: collector-tag: ${{ needs.init.outputs.collector-tag }} + include-arch-json: ${{ needs.init.outputs.include-arch-json }} secrets: inherit build-collector: @@ -51,6 +52,7 @@ jobs: collector-tag: ${{ needs.init.outputs.collector-tag }} collector-image: ${{ needs.init.outputs.collector-image }} collector-builder-tag: ${{ needs.build-builder-image.outputs.collector-builder-tag }} + include-arch-json: ${{ needs.init.outputs.include-arch-json }} secrets: inherit build-test-containers: @@ -66,9 +68,11 @@ jobs: unit-tests: uses: ./.github/workflows/unit-tests.yml needs: - - build-builder-image + - init + - build-builder-image with: collector-builder-tag: ${{ needs.build-builder-image.outputs.collector-builder-tag }} + include-arch-json: ${{ needs.init.outputs.include-arch-json }} secrets: inherit integration-tests: @@ -78,6 +82,7 @@ jobs: collector-qa-tag: ${{ needs.init.outputs.collector-qa-tag }} collector-tests-tag: ${{ needs.build-test-containers.outputs.collector-tests-tag }} large-box: ${{ github.event.schedule == '0 0 * * 0' }} + include-arch-json: ${{ needs.init.outputs.include-arch-json }} if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-integration-tests') }} needs: - init @@ -91,7 +96,9 @@ jobs: collector-tag: ${{ needs.init.outputs.collector-tag }} collector-qa-tag: ${{ needs.init.outputs.collector-qa-tag }} collector-tests-tag: ${{ needs.build-test-containers.outputs.collector-tests-tag }} - if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-integration-tests') }} + if: | + !contains(github.event.pull_request.labels.*.name, 'skip-integration-tests') && + contains(needs.init.outputs.include-arch-json, 'amd64') needs: - init - build-collector @@ -105,7 +112,7 @@ jobs: collector-qa-tag: ${{ needs.init.outputs.collector-qa-tag }} collector-tests-tag: ${{ needs.build-test-containers.outputs.collector-tests-tag }} if: | - always() && + always() && contains(needs.init.outputs.include-arch-json, 'amd64') && ((github.event_name != 'pull_request' && github.ref_name == 'master') || contains(github.event.pull_request.labels.*.name, 'run-benchmark')) needs: diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5ac589df89..4cdc5de2bd 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -8,30 +8,54 @@ on: required: true description: | The builder tag to use in the build + include-arch-json: + type: string + required: true + description: | + JSON list of {arch,runner,remote} build configurations + cmake-flags-list-json: + type: string + required: false + default: "['-DCMAKE_BUILD_TYPE=Release','-DADDRESS_SANITIZER=ON -DCMAKE_BUILD_TYPE=Debug','-DUSE_VALGRIND=ON -DCMAKE_BUILD_TYPE=Debug']" + description: | + JSON list of cmake flags for unit tests jobs: - unit-tests: + unit-tests-amd64: + if: contains(inputs.include-arch-json, 'amd64') runs-on: ubuntu-24.04 container: image: quay.io/stackrox-io/collector-builder:${{ inputs.collector-builder-tag }} strategy: - fail-fast: false matrix: - cmake-flags: - - -DCMAKE_BUILD_TYPE=Release - - -DADDRESS_SANITIZER=ON -DCMAKE_BUILD_TYPE=Debug - - -DUSE_VALGRIND=ON -DCMAKE_BUILD_TYPE=Debug + cmake-flags: ${{ fromJSON(inputs.cmake-flags-list-json) }} steps: - uses: actions/checkout@v4 with: submodules: true + - uses: ./.github/actions/run-unit-tests + with: + cmake-flags: ${{ matrix.cmake-flags }} + unit-tests-arm64: + if: contains(inputs.include-arch-json, 'arm64') + runs-on: ubuntu-24.04-arm + container: + image: quay.io/stackrox-io/collector-builder:${{ inputs.collector-builder-tag }} + strategy: + matrix: + cmake-flags: ${{ fromJSON(inputs.cmake-flags-list-json) }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true - uses: ./.github/actions/run-unit-tests with: cmake-flags: ${{ matrix.cmake-flags }} - coverage: - runs-on: ubuntu-24.04 + unit-tests-coverage: + if: contains(inputs.include-arch-json, 'arm64') + runs-on: ubuntu-24.04-arm container: image: quay.io/stackrox-io/collector-builder:${{ inputs.collector-builder-tag }} steps: From 9978e0cbfeb06d5d9826d081648d26272275920c Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 10:04:11 -0700 Subject: [PATCH 03/27] split --- .github/workflows/collector-builder.yml | 96 ++++++++++++++++++------- .github/workflows/collector.yml | 52 ++++++++++---- 2 files changed, 112 insertions(+), 36 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 3cdd27db97..3a4902a4d2 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -60,8 +60,8 @@ jobs: collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }} strategy: matrix: - include: ${{ fromJSON(inputs.include-arch-json) }} - runs-on: ${{ matrix.runner }} + arch: [amd64, arm64] + runs-on: ${{ (matrix.arch != 'arm64' && ubuntu-24.04) || 'ubuntu-24.04-arm' }} env: PLATFORM: linux/${{ matrix.arch }} @@ -73,30 +73,91 @@ jobs: submodules: true - name: Set up QEMU - if: matrix.remote != true uses: docker/setup-qemu-action@v3 with: image: tonistiigi/binfmt:qemu-v8.1.5 - name: Set up Docker Buildx - if: matrix.remote != true uses: docker/setup-buildx-action@v3 + - name: Define builder tag + id: builder-tag + run: | + COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}" + if [[ "${{ github.event_name }}" == 'pull_request' || \ + "${{ github.ref_type }}" == 'tag' || \ + "${{ github.ref_name }}" =~ ^release- ]]; then + COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}" + fi + + echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV" + echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT" + + - name: Create ansible vars + run: | + { + echo "---" + echo "stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}" + echo "stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}" + echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}" + echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}" + echo "collector_git_ref: ${{ github.ref }}" + echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}" + } > ${{ github.workspace }}/ansible/secrets.yml + + - name: Build images + timeout-minutes: 480 + run: | + ansible-galaxy install -r ansible/requirements.yml + ansible-playbook \ + --connection local \ + -i localhost, \ + --limit localhost \ + -e arch='${{ matrix.arch }}' \ + -e @'${{ github.workspace }}/ansible/secrets.yml' \ + ansible/ci-build-builder.yml + + build-builder-image-remote: + if: contains(inputs.include-arch-json, 's390x') + name: Build the builder image + # Multiarch builds sometimes take for eeeeeeeeeever + timeout-minutes: 480 + needs: + - builder-needs-rebuilding + if: | + needs.builder-needs-rebuilding.outputs.build-image == 'true' || + (github.event_name == 'push' && ( + github.ref_type == 'tag' || startsWith(github.ref_name, 'release-') + )) || + contains(github.event.pull_request.labels.*.name, 'build-builder-image') || + github.event_name == 'schedule' + outputs: + collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }} + strategy: + matrix: + arch: [s390x] + runs-on: ubuntu-24.04 + + env: + PLATFORM: linux/${{ matrix.arch }} + BUILD_TYPE: ci + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - uses: actions/setup-python@v5 - if: matrix.remote with: python-version: "3.10" - uses: 'google-github-actions/auth@v2' - if: matrix.remote with: credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}' - uses: 'google-github-actions/setup-gcloud@v2' - if: matrix.remote - uses: ./.github/actions/setup-vm-creds - if: matrix.remote with: gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }} gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }} @@ -111,7 +172,6 @@ jobs: job-tag: builder - name: Create Build VMs - if: matrix.remote run: | make -C "${{ github.workspace }}/ansible" create-build-vms @@ -140,21 +200,7 @@ jobs: echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}" } > ${{ github.workspace }}/ansible/secrets.yml - - name: Build images - if: matrix.remote != true - timeout-minutes: 480 - run: | - ansible-galaxy install -r ansible/requirements.yml - ansible-playbook \ - --connection local \ - -i localhost, \ - --limit localhost \ - -e arch='${{ matrix.arch }}' \ - -e @'${{ github.workspace }}/ansible/secrets.yml' \ - ansible/ci-build-builder.yml - - name: Build s390x images - if: matrix.remote timeout-minutes: 480 run: | ansible-playbook \ @@ -165,13 +211,14 @@ jobs: ansible/ci-build-builder.yml - name: Destroy VMs - if: always() && matrix.remote + if: always() run: | make -C ansible destroy-vms create-multiarch-manifest: needs: - build-builder-image + - build-builder-image-remote name: Create Multiarch manifest runs-on: ubuntu-24.04 if: | @@ -215,6 +262,7 @@ jobs: if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request' needs: - build-builder-image + - build-builder-image-remote - create-multiarch-manifest steps: - name: Slack notification diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 0f52ec92cb..e507b00f16 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -37,8 +37,8 @@ jobs: name: Build Collector strategy: matrix: - include: ${{ fromJSON(inputs.include-arch-json) }} - runs-on: ${{ matrix.runner }} + arch: [amd64, arm64] + runs-on: ${{ (matrix.arch != 'arm64' && ubuntu-24.04) || 'ubuntu-24.04-arm' }} env: PLATFORM: linux/${{ matrix.arch }} @@ -50,13 +50,11 @@ jobs: submodules: true - name: Set up QEMU - if: matrix.remote != true uses: docker/setup-qemu-action@v3 with: image: tonistiigi/binfmt:qemu-v8.1.5 - name: Set up Docker Buildx - if: matrix.remote != true uses: docker/setup-buildx-action@v3 - name: Create ansible vars @@ -79,7 +77,6 @@ jobs: EOF - name: Build ${{ matrix.arch }} image locally - if: matrix.remote != true timeout-minutes: 480 run: | ansible-playbook \ @@ -90,21 +87,53 @@ jobs: -e @'${{ github.workspace }}/ansible/secrets.yml' \ ansible/ci-build-collector.yml + build-collector-image-remote: + if: contains(inputs.include-arch-json, 's390x') + name: Build Collector + strategy: + matrix: + arch: [s390x] + runs-on: ubuntu-24.04 + + env: + PLATFORM: linux/${{ matrix.arch }} + CONTEXT_DRIVERS_DIR: ${{ github.workspace }}/collector/kernel-modules + + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Create ansible vars + run: | + cat << EOF > ${{ github.workspace }}/ansible/secrets.yml + --- + stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + collector_git_ref: ${{ github.ref }} + collector_git_sha: ${{ github.sha }} + collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }} + disable_profiling: ${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }} + rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }} + collector_image: ${{ inputs.collector-image }} + collector_tag: ${{ inputs.collector-tag }} + debug_mode: ${{ github.event_name == 'pull_request' }} + driver_version: ${DRIVER_VERSION} + EOF + - uses: actions/setup-python@v5 - if: matrix.remote with: python-version: "3.10" - uses: 'google-github-actions/auth@v2' - if: matrix.remote with: credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}' - uses: 'google-github-actions/setup-gcloud@v2' - if: matrix.remote - uses: ./.github/actions/setup-vm-creds - if: matrix.remote with: gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }} gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }} @@ -119,12 +148,10 @@ jobs: job-tag: builder - name: Create Build VMs - if: matrix.remote run: | make -C "${{ github.workspace }}/ansible" create-build-vms - name: Build ${{ matrix.arch }} image remotely - if: matrix.remote timeout-minutes: 480 run: | ansible-playbook \ @@ -135,7 +162,7 @@ jobs: ansible/ci-build-collector.yml - name: Destroy Build VMs - if: always() && matrix.remote + if: always() run: | make -C ansible destroy-vms @@ -182,6 +209,7 @@ jobs: if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request' needs: - build-collector-image + - build-collector-image-remote - create-multiarch-manifest steps: - name: Slack notification From 239509ca78a6b8c1b0aa8d01165f8ea51eb7ffb7 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 10:13:08 -0700 Subject: [PATCH 04/27] fix --- .github/workflows/collector-builder.yml | 8 ++++---- .github/workflows/collector.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 3a4902a4d2..24603ec4cf 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -61,7 +61,7 @@ jobs: strategy: matrix: arch: [amd64, arm64] - runs-on: ${{ (matrix.arch != 'arm64' && ubuntu-24.04) || 'ubuntu-24.04-arm' }} + runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} env: PLATFORM: linux/${{ matrix.arch }} @@ -118,19 +118,19 @@ jobs: ansible/ci-build-builder.yml build-builder-image-remote: - if: contains(inputs.include-arch-json, 's390x') name: Build the builder image # Multiarch builds sometimes take for eeeeeeeeeever timeout-minutes: 480 needs: - builder-needs-rebuilding if: | - needs.builder-needs-rebuilding.outputs.build-image == 'true' || + (needs.builder-needs-rebuilding.outputs.build-image == 'true' || (github.event_name == 'push' && ( github.ref_type == 'tag' || startsWith(github.ref_name, 'release-') )) || contains(github.event.pull_request.labels.*.name, 'build-builder-image') || - github.event_name == 'schedule' + github.event_name == 'schedule') && + contains(inputs.include-arch-json, 's390x') outputs: collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }} strategy: diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index e507b00f16..74134e3332 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -38,7 +38,7 @@ jobs: strategy: matrix: arch: [amd64, arm64] - runs-on: ${{ (matrix.arch != 'arm64' && ubuntu-24.04) || 'ubuntu-24.04-arm' }} + runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-22.04-arm') || 'ubuntu-22.04' }} env: PLATFORM: linux/${{ matrix.arch }} From 71d3704f513e59bb91b58e78f4ffa7c9182d37af Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 10:26:35 -0700 Subject: [PATCH 05/27] update --- .github/workflows/collector-builder.yml | 4 +-- .github/workflows/collector.yml | 41 ++++++++++++------------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 24603ec4cf..5b49f29ea7 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -44,7 +44,7 @@ jobs: - .github/workflows/collector-builder.yml build-builder-image: - name: Build the builder image + name: local builder image ${{ matrix.arch }} # Multiarch builds sometimes take for eeeeeeeeeever timeout-minutes: 480 needs: @@ -118,7 +118,7 @@ jobs: ansible/ci-build-builder.yml build-builder-image-remote: - name: Build the builder image + name: remote builder image ${{ matrix.arch }} # Multiarch builds sometimes take for eeeeeeeeeever timeout-minutes: 480 needs: diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 74134e3332..7791bc7752 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -97,31 +97,9 @@ jobs: env: PLATFORM: linux/${{ matrix.arch }} - CONTEXT_DRIVERS_DIR: ${{ github.workspace }}/collector/kernel-modules steps: - uses: actions/checkout@v4 - with: - submodules: true - - - name: Create ansible vars - run: | - cat << EOF > ${{ github.workspace }}/ansible/secrets.yml - --- - stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} - stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} - rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} - rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} - collector_git_ref: ${{ github.ref }} - collector_git_sha: ${{ github.sha }} - collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }} - disable_profiling: ${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }} - rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }} - collector_image: ${{ inputs.collector-image }} - collector_tag: ${{ inputs.collector-tag }} - debug_mode: ${{ github.event_name == 'pull_request' }} - driver_version: ${DRIVER_VERSION} - EOF - uses: actions/setup-python@v5 with: @@ -147,6 +125,24 @@ jobs: vm-type: rhel-${{ matrix.arch }} job-tag: builder + - name: Create ansible vars + run: | + cat << EOF > ${{ github.workspace }}/ansible/secrets.yml + --- + stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + collector_git_ref: ${{ github.ref }} + collector_git_sha: ${{ github.sha }} + collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }} + disable_profiling: ${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }} + rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }} + collector_image: ${{ inputs.collector-image }} + collector_tag: ${{ inputs.collector-tag }} + debug_mode: ${{ github.event_name == 'pull_request' }} + EOF + - name: Create Build VMs run: | make -C "${{ github.workspace }}/ansible" create-build-vms @@ -169,6 +165,7 @@ jobs: create-multiarch-manifest: needs: - build-collector-image + - build-collector-image-remote name: Create Multiarch manifest runs-on: ubuntu-24.04 if: always() && !contains(join(needs.*.result, ','), 'failure') From 9c524724565899d2911c801977d136c2d8ab7b6b Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 10:32:26 -0700 Subject: [PATCH 06/27] unit test --- .github/workflows/collector-builder.yml | 4 +-- .github/workflows/collector.yml | 4 +-- .github/workflows/unit-tests.yml | 39 +++++++------------------ 3 files changed, 15 insertions(+), 32 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 5b49f29ea7..3915fed4cf 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -44,7 +44,7 @@ jobs: - .github/workflows/collector-builder.yml build-builder-image: - name: local builder image ${{ matrix.arch }} + name: Local builder image ${{ matrix.arch }} # Multiarch builds sometimes take for eeeeeeeeeever timeout-minutes: 480 needs: @@ -118,7 +118,7 @@ jobs: ansible/ci-build-builder.yml build-builder-image-remote: - name: remote builder image ${{ matrix.arch }} + name: Remote builder image ${{ matrix.arch }} # Multiarch builds sometimes take for eeeeeeeeeever timeout-minutes: 480 needs: diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 7791bc7752..10de411d4f 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -34,7 +34,7 @@ env: jobs: build-collector-image: - name: Build Collector + name: Local collector image ${{ matrix.arch }} strategy: matrix: arch: [amd64, arm64] @@ -89,7 +89,7 @@ jobs: build-collector-image-remote: if: contains(inputs.include-arch-json, 's390x') - name: Build Collector + name: Remote collector image ${{ matrix.arch }} strategy: matrix: arch: [s390x] diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 4cdc5de2bd..ca1cffd527 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -13,49 +13,32 @@ on: required: true description: | JSON list of {arch,runner,remote} build configurations - cmake-flags-list-json: - type: string - required: false - default: "['-DCMAKE_BUILD_TYPE=Release','-DADDRESS_SANITIZER=ON -DCMAKE_BUILD_TYPE=Debug','-DUSE_VALGRIND=ON -DCMAKE_BUILD_TYPE=Debug']" - description: | - JSON list of cmake flags for unit tests jobs: - unit-tests-amd64: - if: contains(inputs.include-arch-json, 'amd64') - runs-on: ubuntu-24.04 + unit-tests: container: image: quay.io/stackrox-io/collector-builder:${{ inputs.collector-builder-tag }} strategy: + fail-fast: false matrix: - cmake-flags: ${{ fromJSON(inputs.cmake-flags-list-json) }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - uses: ./.github/actions/run-unit-tests - with: - cmake-flags: ${{ matrix.cmake-flags }} + arch: [amd64, arm64] + cmake-flags: + - -DCMAKE_BUILD_TYPE=Release + - -DADDRESS_SANITIZER=ON -DCMAKE_BUILD_TYPE=Debug + - -DUSE_VALGRIND=ON -DCMAKE_BUILD_TYPE=Debug + runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-22.04-arm') || 'ubuntu-22.04' }} - unit-tests-arm64: - if: contains(inputs.include-arch-json, 'arm64') - runs-on: ubuntu-24.04-arm - container: - image: quay.io/stackrox-io/collector-builder:${{ inputs.collector-builder-tag }} - strategy: - matrix: - cmake-flags: ${{ fromJSON(inputs.cmake-flags-list-json) }} steps: - uses: actions/checkout@v4 with: submodules: true + - uses: ./.github/actions/run-unit-tests with: cmake-flags: ${{ matrix.cmake-flags }} - unit-tests-coverage: - if: contains(inputs.include-arch-json, 'arm64') - runs-on: ubuntu-24.04-arm + coverage: + runs-on: ubuntu-24.04 container: image: quay.io/stackrox-io/collector-builder:${{ inputs.collector-builder-tag }} steps: From e83ff3950f3d89416316f700c14915358c0c23d0 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 11:09:18 -0700 Subject: [PATCH 07/27] ppc --- .github/workflows/collector-builder.yml | 52 ++++++++++--------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 3915fed4cf..627c50f808 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -29,6 +29,7 @@ jobs: runs-on: ubuntu-24.04 outputs: build-image: ${{ steps.changed.outputs.builder-changed }} + collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }} steps: - uses: actions/checkout@v4 @@ -43,6 +44,19 @@ jobs: - builder/Dockerfile - .github/workflows/collector-builder.yml + - name: Define builder tag + id: builder-tag + run: | + COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}" + if [[ "${{ github.event_name }}" == 'pull_request' || \ + "${{ github.ref_type }}" == 'tag' || \ + "${{ github.ref_name }}" =~ ^release- ]]; then + COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}" + fi + + echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV" + echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT" + build-builder-image: name: Local builder image ${{ matrix.arch }} # Multiarch builds sometimes take for eeeeeeeeeever @@ -57,10 +71,12 @@ jobs: contains(github.event.pull_request.labels.*.name, 'build-builder-image') || github.event_name == 'schedule' outputs: - collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }} + collector-builder-tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} strategy: matrix: - arch: [amd64, arm64] + arch: [amd64, arm64, ppc64le] + exclude: + - arch: {{ (!contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'ppc64le' || '' }} runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} env: @@ -80,19 +96,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Define builder tag - id: builder-tag - run: | - COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}" - if [[ "${{ github.event_name }}" == 'pull_request' || \ - "${{ github.ref_type }}" == 'tag' || \ - "${{ github.ref_name }}" =~ ^release- ]]; then - COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}" - fi - - echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV" - echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT" - - name: Create ansible vars run: | { @@ -102,7 +105,7 @@ jobs: echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}" echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}" echo "collector_git_ref: ${{ github.ref }}" - echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}" + echo "collector_builder_tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}" } > ${{ github.workspace }}/ansible/secrets.yml - name: Build images @@ -132,7 +135,7 @@ jobs: github.event_name == 'schedule') && contains(inputs.include-arch-json, 's390x') outputs: - collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }} + collector-builder-tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} strategy: matrix: arch: [s390x] @@ -175,19 +178,6 @@ jobs: run: | make -C "${{ github.workspace }}/ansible" create-build-vms - - name: Define builder tag - id: builder-tag - run: | - COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}" - if [[ "${{ github.event_name }}" == 'pull_request' || \ - "${{ github.ref_type }}" == 'tag' || \ - "${{ github.ref_name }}" =~ ^release- ]]; then - COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}" - fi - - echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV" - echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT" - - name: Create ansible vars run: | { @@ -197,7 +187,7 @@ jobs: echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}" echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}" echo "collector_git_ref: ${{ github.ref }}" - echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}" + echo "collector_builder_tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}" } > ${{ github.workspace }}/ansible/secrets.yml - name: Build s390x images From 0c88e3d3df28d2aa1f901943ca7d19f3424dfca8 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 11:18:21 -0700 Subject: [PATCH 08/27] fx --- .github/workflows/collector-builder.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 627c50f808..0b9ae5de9b 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -76,7 +76,7 @@ jobs: matrix: arch: [amd64, arm64, ppc64le] exclude: - - arch: {{ (!contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'ppc64le' || '' }} + - arch: ${{ (!contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'ppc64le' || '' }} runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} env: From 00ea55771dfc59a4db74bbcad0422ac4fd8dc71f Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 11:20:43 -0700 Subject: [PATCH 09/27] paren --- .github/workflows/collector-builder.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 0b9ae5de9b..8008ee22b3 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -76,7 +76,7 @@ jobs: matrix: arch: [amd64, arm64, ppc64le] exclude: - - arch: ${{ (!contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'ppc64le' || '' }} + - arch: ${{ (!contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'ppc64le') || '' }} runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} env: From 9cb683ebb58be35372d59e099731307c831b0b3f Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 11:50:48 -0700 Subject: [PATCH 10/27] name --- .github/workflows/collector-builder.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 8008ee22b3..e1cffb591b 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -82,6 +82,7 @@ jobs: env: PLATFORM: linux/${{ matrix.arch }} BUILD_TYPE: ci + COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} steps: - uses: actions/checkout@v4 @@ -121,7 +122,7 @@ jobs: ansible/ci-build-builder.yml build-builder-image-remote: - name: Remote builder image ${{ matrix.arch }} + name: Remote builder image s390x # Multiarch builds sometimes take for eeeeeeeeeever timeout-minutes: 480 needs: @@ -144,6 +145,7 @@ jobs: env: PLATFORM: linux/${{ matrix.arch }} BUILD_TYPE: ci + COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} steps: - uses: actions/checkout@v4 From b4acec9373f8c994ea3c8fa369bf08cea543ba93 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 12:31:23 -0700 Subject: [PATCH 11/27] simp --- .github/workflows/collector-builder.yml | 30 +++++++++++++------------ .github/workflows/collector.yml | 6 +++-- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index e1cffb591b..eaed5b7dfd 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -28,7 +28,7 @@ jobs: name: Determine if builder image needs to be built runs-on: ubuntu-24.04 outputs: - build-image: ${{ steps.changed.outputs.builder-changed }} + build-image: ${{ steps.check-builder-needed.outcome == 'success' }} collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }} steps: @@ -57,6 +57,18 @@ jobs: echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV" echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT" + - name: Check labels and branch status to check if builder is needed + id: check-builder-needed + if: | + steps.changed.outputs.builder-changed == 'true' || + (github.event_name == 'push' && ( + github.ref_type == 'tag' || startsWith(github.ref_name, 'release-') + )) || + contains(github.event.pull_request.labels.*.name, 'build-builder-image') || + github.event_name == 'schedule' + run: | + echo "Rebuild builder image with tag ${COLLECTOR_BUILDER_TAG}" + build-builder-image: name: Local builder image ${{ matrix.arch }} # Multiarch builds sometimes take for eeeeeeeeeever @@ -64,12 +76,7 @@ jobs: needs: - builder-needs-rebuilding if: | - needs.builder-needs-rebuilding.outputs.build-image == 'true' || - (github.event_name == 'push' && ( - github.ref_type == 'tag' || startsWith(github.ref_name, 'release-') - )) || - contains(github.event.pull_request.labels.*.name, 'build-builder-image') || - github.event_name == 'schedule' + needs.builder-needs-rebuilding.outputs.build-image == 'true' outputs: collector-builder-tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} strategy: @@ -128,13 +135,8 @@ jobs: needs: - builder-needs-rebuilding if: | - (needs.builder-needs-rebuilding.outputs.build-image == 'true' || - (github.event_name == 'push' && ( - github.ref_type == 'tag' || startsWith(github.ref_name, 'release-') - )) || - contains(github.event.pull_request.labels.*.name, 'build-builder-image') || - github.event_name == 'schedule') && - contains(inputs.include-arch-json, 's390x') + needs.builder-needs-rebuilding.outputs.build-image == 'true' || + contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') outputs: collector-builder-tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} strategy: diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 10de411d4f..60c0fa61f5 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -37,7 +37,9 @@ jobs: name: Local collector image ${{ matrix.arch }} strategy: matrix: - arch: [amd64, arm64] + arch: [amd64, arm64, ppc64le] + exclude: + - arch: ${{ (!contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'ppc64le') || '' }} runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-22.04-arm') || 'ubuntu-22.04' }} env: @@ -88,7 +90,7 @@ jobs: ansible/ci-build-collector.yml build-collector-image-remote: - if: contains(inputs.include-arch-json, 's390x') + if: contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') name: Remote collector image ${{ matrix.arch }} strategy: matrix: From ed2c95f229b1260893a2980c44f7b68e6f5ead0e Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 12:47:13 -0700 Subject: [PATCH 12/27] arch --- .github/workflows/collector-builder.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index eaed5b7dfd..2247106fbf 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -220,7 +220,7 @@ jobs: needs.build-builder-image.outputs.collector-builder-tag != 'cache' env: COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }} - ARCHS: ${{ join(fromJSON(inputs.include-arch-json).*.arch, ' ') }} + ARCHS: ${{ (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} steps: - uses: actions/checkout@v4 From 1deefaa45e4b10d39dbb48465de7dad2a43862c9 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 12:53:20 -0700 Subject: [PATCH 13/27] op --- .github/workflows/collector-builder.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 2247106fbf..df95b7e70c 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -135,7 +135,7 @@ jobs: needs: - builder-needs-rebuilding if: | - needs.builder-needs-rebuilding.outputs.build-image == 'true' || + needs.builder-needs-rebuilding.outputs.build-image == 'true' && contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') outputs: collector-builder-tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} From c560a708d6ccd0f7d40a192370af5aef9c17f1db Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 13:13:43 -0700 Subject: [PATCH 14/27] remove arch specific labels --- .github/workflows/collector-builder.yml | 5 --- .github/workflows/collector.yml | 7 +--- .github/workflows/init.yml | 46 ------------------------- .github/workflows/integration-tests.yml | 16 ++++----- .github/workflows/konflux.yml | 14 -------- .github/workflows/main.yml | 13 ++----- 6 files changed, 11 insertions(+), 90 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index df95b7e70c..966288d6d7 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -8,11 +8,6 @@ on: required: true description: | The tag used to build the collector image - include-arch-json: - type: string - required: true - description: | - JSON for matrix.include to select architectures and runners to build outputs: collector-builder-tag: description: The builder tag used by the build diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 60c0fa61f5..617428121f 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -18,11 +18,6 @@ on: required: true description: | The builder tag to use in the build - include-arch-json: - type: string - required: true - description: | - JSON for matrix.include to select architectures and runners to build env: COLLECTOR_TAG: ${{ inputs.collector-tag }} @@ -172,7 +167,7 @@ jobs: runs-on: ubuntu-24.04 if: always() && !contains(join(needs.*.result, ','), 'failure') env: - ARCHS: ${{ join(fromJSON(inputs.include-arch-json).*.arch, ' ') }} + ARCHS: ${{ (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml index 91abc27f17..be09716158 100644 --- a/.github/workflows/init.yml +++ b/.github/workflows/init.yml @@ -58,11 +58,6 @@ on: description: | Trigger rebuild of QA containers value: ${{ jobs.common-variables.outputs.rebuild-qa-containers }} - include-arch-json: - description: | - JSON for matrix.include in later jobs - value: ${{ jobs.common-variables.outputs.include-arch-json }} - jobs: common-variables: @@ -80,7 +75,6 @@ jobs: cpaas-support-packages-bucket: ${{ steps.gcp-buckets.outputs.cpaas-support-packages-bucket }} support-packages-index-bucket: ${{ steps.gcp-buckets.outputs.support-packages-index-bucket }} rebuild-qa-containers: ${{ steps.filter.outputs.container }} - include-arch-json: ${{ steps.arch-json.outputs.include-arch-json }} steps: - uses: actions/checkout@v4 @@ -179,43 +173,3 @@ jobs: echo "support-packages-index-bucket=${SUPPORT_PACKAGES_BUCKET}" } >> "$GITHUB_OUTPUT" fi - - - name: Create settings for runner and remote builds per enabled arch - id: arch-json - uses: actions/github-script@v7 - with: - script: | - const all_archs = ['amd64', 'arm64', 'ppc64le', 's390x']; - const default_archs = ['amd64', 'arm64']; - - // the arch build configs for runner and remote used for - // collector-builder, collector image, and integration test jobs. - const arch_configs = { - amd64: { runner: 'ubuntu-24.04', remote: false }, - arm64: { runner: 'ubuntu-24.04-arm', remote: false }, - ppc64le: { runner: 'ubuntu-24.04', remote: false }, - s390x: { runner: 'ubuntu-24.04', remote: true }, - }; - - let selected_archs = all_archs; - if (context.payload.pull_request) { - const labels = context.payload.pull_request.labels.map(label => label.name); - const specific = all_archs.filter(a => labels.includes(`run-${a}-builds`)); - if (specific.length) { - selected_archs = specific - } else { - selected_archs = default_archs - if (labels.includes('run-multiarch-builds')) { - selected_archs.push('ppc64le', 's390x'); - } - } - } - // create output array of selected arch config objects - const arch_output = selected_archs.map(arch => ({ - arch: arch, - runner: arch_configs[arch].runner, - remote: arch_configs[arch].remote, - })); - core.setOutput('include-arch-json', JSON.stringify(arch_output)); - core.info(JSON.stringify(arch_output)); - diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 347a10460c..a8f17c96b9 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -42,16 +42,10 @@ on: number of CPU cores. type: boolean default: false - include-arch-json: - type: string - required: true - description: | - JSON for matrix.include in later jobs jobs: amd64-integration-tests: uses: ./.github/workflows/integration-tests-vm-type.yml - if: contains(inputs.include-arch-json, 'amd64') strategy: # ensure that if one part of the matrix fails, the # rest will continue @@ -97,7 +91,6 @@ jobs: arm64-integration-tests: uses: ./.github/workflows/integration-tests-vm-type.yml - if: contains(inputs.include-arch-json, 'arm64') strategy: # ensure that if one part of the matrix fails, the # rest will continue @@ -108,6 +101,7 @@ jobs: - ubuntu-arm - sles-arm64 - fcarm + with: vm_type: ${{ matrix.vm_type }} collector-tag: ${{ inputs.collector-tag }} @@ -119,7 +113,9 @@ jobs: s390x-integration-tests: uses: ./.github/workflows/integration-tests-vm-type.yml - if: contains(inputs.include-arch-json, 's390x') + if: | + github.event_name != 'pull_request' || inputs.is-konflux || + contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') with: vm_type: rhel-s390x collector-tag: ${{ inputs.collector-tag }} @@ -131,7 +127,9 @@ jobs: ppc64le-integration-tests: uses: ./.github/workflows/integration-tests-vm-type.yml - if: contains(inputs.include-arch-json, 'ppc64le') + if: | + github.event_name != 'pull_request' || inputs.is-konflux || + contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') with: vm_type: rhel-ppc64le collector-tag: ${{ inputs.collector-tag }} diff --git a/.github/workflows/konflux.yml b/.github/workflows/konflux.yml index 7093042e45..8a03295382 100644 --- a/.github/workflows/konflux.yml +++ b/.github/workflows/konflux.yml @@ -20,20 +20,7 @@ concurrency: cancel-in-progress: true jobs: - should-run: - outputs: - should-run: ${{ steps.should-run.outputs.should-run }} - runs-on: ubuntu-24.04 - steps: - - id: should-run - run: | - SHOULD_RUN="${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-konflux-steps') }}" - echo "should-run=${SHOULD_RUN}" >> "$GITHUB_OUTPUT" - init: - needs: - - should-run - if: needs.should-run.outputs.should-run == 'true' runs-on: ubuntu-24.04 outputs: collector-tag: ${{ steps.generate-tag.outputs.collector-tag }} @@ -102,7 +89,6 @@ jobs: collector-tests-tag: ${{ needs.integration-tests-containers.outputs.collector-tests-tag }} is-konflux: true job-tag: konf - include-arch-json: '[{"arch":"amd64"},{"arch","arm64"},{"arch":"ppc64le"},{"arch","s390x"}]' if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-integration-tests') }} secrets: inherit diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1c2ce86a38..f002e6eb7b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,6 @@ jobs: needs: init with: collector-tag: ${{ needs.init.outputs.collector-tag }} - include-arch-json: ${{ needs.init.outputs.include-arch-json }} secrets: inherit build-collector: @@ -52,7 +51,6 @@ jobs: collector-tag: ${{ needs.init.outputs.collector-tag }} collector-image: ${{ needs.init.outputs.collector-image }} collector-builder-tag: ${{ needs.build-builder-image.outputs.collector-builder-tag }} - include-arch-json: ${{ needs.init.outputs.include-arch-json }} secrets: inherit build-test-containers: @@ -68,11 +66,9 @@ jobs: unit-tests: uses: ./.github/workflows/unit-tests.yml needs: - - init - - build-builder-image + - build-builder-image with: collector-builder-tag: ${{ needs.build-builder-image.outputs.collector-builder-tag }} - include-arch-json: ${{ needs.init.outputs.include-arch-json }} secrets: inherit integration-tests: @@ -82,7 +78,6 @@ jobs: collector-qa-tag: ${{ needs.init.outputs.collector-qa-tag }} collector-tests-tag: ${{ needs.build-test-containers.outputs.collector-tests-tag }} large-box: ${{ github.event.schedule == '0 0 * * 0' }} - include-arch-json: ${{ needs.init.outputs.include-arch-json }} if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-integration-tests') }} needs: - init @@ -96,9 +91,7 @@ jobs: collector-tag: ${{ needs.init.outputs.collector-tag }} collector-qa-tag: ${{ needs.init.outputs.collector-qa-tag }} collector-tests-tag: ${{ needs.build-test-containers.outputs.collector-tests-tag }} - if: | - !contains(github.event.pull_request.labels.*.name, 'skip-integration-tests') && - contains(needs.init.outputs.include-arch-json, 'amd64') + if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip-integration-tests') }} needs: - init - build-collector @@ -112,7 +105,7 @@ jobs: collector-qa-tag: ${{ needs.init.outputs.collector-qa-tag }} collector-tests-tag: ${{ needs.build-test-containers.outputs.collector-tests-tag }} if: | - always() && contains(needs.init.outputs.include-arch-json, 'amd64') && + always() && ((github.event_name != 'pull_request' && github.ref_name == 'master') || contains(github.event.pull_request.labels.*.name, 'run-benchmark')) needs: From 88cebddd1ad3e7553dfb205ce52a45eca8633ce6 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 13:44:17 -0700 Subject: [PATCH 15/27] manifest --- .github/workflows/collector-builder.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 966288d6d7..c98134e8b9 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -211,8 +211,9 @@ jobs: name: Create Multiarch manifest runs-on: ubuntu-24.04 if: | - github.event_name != 'pull_request' || - needs.build-builder-image.outputs.collector-builder-tag != 'cache' + always() && !contains(join(needs.*.result, ','), 'failure') + && (github.event_name != 'pull_request' || + needs.build-builder-image.outputs.collector-builder-tag != 'cache') env: COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }} ARCHS: ${{ (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} From e73d6d32571fb7cfd1101ac957d5ab4dcb7e742f Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 13:44:46 -0700 Subject: [PATCH 16/27] unit --- .github/workflows/unit-tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ca1cffd527..99d544cda3 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -8,11 +8,6 @@ on: required: true description: | The builder tag to use in the build - include-arch-json: - type: string - required: true - description: | - JSON list of {arch,runner,remote} build configurations jobs: unit-tests: From 3a04997695e2e28420ec615a276ce2141020b76c Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 14:28:21 -0700 Subject: [PATCH 17/27] clean --- .github/workflows/collector-builder.yml | 8 +++----- .github/workflows/collector.yml | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index c98134e8b9..e786e82a27 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -123,7 +123,7 @@ jobs: -e @'${{ github.workspace }}/ansible/secrets.yml' \ ansible/ci-build-builder.yml - build-builder-image-remote: + build-builder-image-remote-vm: name: Remote builder image s390x # Multiarch builds sometimes take for eeeeeeeeeever timeout-minutes: 480 @@ -132,8 +132,6 @@ jobs: if: | needs.builder-needs-rebuilding.outputs.build-image == 'true' && contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') - outputs: - collector-builder-tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} strategy: matrix: arch: [s390x] @@ -207,7 +205,7 @@ jobs: create-multiarch-manifest: needs: - build-builder-image - - build-builder-image-remote + - build-builder-image-remote-vm name: Create Multiarch manifest runs-on: ubuntu-24.04 if: | @@ -252,7 +250,7 @@ jobs: if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request' needs: - build-builder-image - - build-builder-image-remote + - build-builder-image-remote-vm - create-multiarch-manifest steps: - name: Slack notification diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 617428121f..62d69b7fea 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -84,13 +84,13 @@ jobs: -e @'${{ github.workspace }}/ansible/secrets.yml' \ ansible/ci-build-collector.yml - build-collector-image-remote: + build-collector-image-remote-vm: + name: Remote collector image s390x + runs-on: ubuntu-24.04 if: contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') - name: Remote collector image ${{ matrix.arch }} strategy: matrix: arch: [s390x] - runs-on: ubuntu-24.04 env: PLATFORM: linux/${{ matrix.arch }} @@ -122,6 +122,10 @@ jobs: vm-type: rhel-${{ matrix.arch }} job-tag: builder + - name: Create Build VMs + run: | + make -C "${{ github.workspace }}/ansible" create-build-vms + - name: Create ansible vars run: | cat << EOF > ${{ github.workspace }}/ansible/secrets.yml @@ -140,11 +144,7 @@ jobs: debug_mode: ${{ github.event_name == 'pull_request' }} EOF - - name: Create Build VMs - run: | - make -C "${{ github.workspace }}/ansible" create-build-vms - - - name: Build ${{ matrix.arch }} image remotely + - name: Build ${{ matrix.arch }} image timeout-minutes: 480 run: | ansible-playbook \ @@ -162,7 +162,7 @@ jobs: create-multiarch-manifest: needs: - build-collector-image - - build-collector-image-remote + - build-collector-image-remote-vm name: Create Multiarch manifest runs-on: ubuntu-24.04 if: always() && !contains(join(needs.*.result, ','), 'failure') @@ -203,7 +203,7 @@ jobs: if: always() && contains(join(needs.*.result, ','), 'failure') && github.event_name != 'pull_request' needs: - build-collector-image - - build-collector-image-remote + - build-collector-image-remote-vm - create-multiarch-manifest steps: - name: Slack notification From 9f1e69aef93304420e18c8e44c582f847deb65e0 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 15:50:07 -0700 Subject: [PATCH 18/27] fix builder tag --- .github/workflows/collector-builder.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index e786e82a27..cefbdee758 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -11,7 +11,7 @@ on: outputs: collector-builder-tag: description: The builder tag used by the build - value: ${{ jobs.build-builder-image.outputs.collector-builder-tag || 'master' }} + value: ${{ jobs.builder-needs-rebuilding.outputs.collector-builder-tag || 'master' }} env: COLLECTOR_TAG: ${{ inputs.collector-tag }} @@ -72,8 +72,6 @@ jobs: - builder-needs-rebuilding if: | needs.builder-needs-rebuilding.outputs.build-image == 'true' - outputs: - collector-builder-tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} strategy: matrix: arch: [amd64, arm64, ppc64le] @@ -204,6 +202,7 @@ jobs: create-multiarch-manifest: needs: + - builder-needs-rebuilding - build-builder-image - build-builder-image-remote-vm name: Create Multiarch manifest @@ -211,9 +210,9 @@ jobs: if: | always() && !contains(join(needs.*.result, ','), 'failure') && (github.event_name != 'pull_request' || - needs.build-builder-image.outputs.collector-builder-tag != 'cache') + needs.builder-needs-rebuilding.outputs.collector-builder-tag != 'cache') env: - COLLECTOR_BUILDER_TAG: ${{ needs.build-builder-image.outputs.collector-builder-tag }} + COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} ARCHS: ${{ (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} steps: From b4ae1d69e4dba3ff80fd4e862f903c5a91a451ec Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 15:57:30 -0700 Subject: [PATCH 19/27] fix multiarch --- .github/workflows/collector-builder.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index cefbdee758..38a35880ae 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -208,9 +208,7 @@ jobs: name: Create Multiarch manifest runs-on: ubuntu-24.04 if: | - always() && !contains(join(needs.*.result, ','), 'failure') - && (github.event_name != 'pull_request' || - needs.builder-needs-rebuilding.outputs.collector-builder-tag != 'cache') + needs.builder-needs-rebuilding.outputs.build-image == 'true' env: COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} ARCHS: ${{ (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} From cb463f26bc89ac7627710cdd0dec9810259ee751 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 16:04:23 -0700 Subject: [PATCH 20/27] default --- .github/workflows/collector-builder.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 38a35880ae..eeacd03fc0 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -26,6 +26,9 @@ jobs: build-image: ${{ steps.check-builder-needed.outcome == 'success' }} collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }} + env: + DEFAULT_BUILDER_TAG: master + steps: - uses: actions/checkout@v4 @@ -65,7 +68,7 @@ jobs: echo "Rebuild builder image with tag ${COLLECTOR_BUILDER_TAG}" build-builder-image: - name: Local builder image ${{ matrix.arch }} + name: Local builder image # Multiarch builds sometimes take for eeeeeeeeeever timeout-minutes: 480 needs: @@ -122,7 +125,7 @@ jobs: ansible/ci-build-builder.yml build-builder-image-remote-vm: - name: Remote builder image s390x + name: Remote builder image # Multiarch builds sometimes take for eeeeeeeeeever timeout-minutes: 480 needs: From 0b5207261764e3e97ff0df189f5eeb35377cd9ca Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 16:18:23 -0700 Subject: [PATCH 21/27] swap --- .github/workflows/collector-builder.yml | 32 +++++++++++++------------ 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index eeacd03fc0..5901b5416c 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -11,7 +11,7 @@ on: outputs: collector-builder-tag: description: The builder tag used by the build - value: ${{ jobs.builder-needs-rebuilding.outputs.collector-builder-tag || 'master' }} + value: ${{ jobs.builder-needs-rebuilding.outputs.collector-builder-tag }} env: COLLECTOR_TAG: ${{ inputs.collector-tag }} @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-24.04 outputs: build-image: ${{ steps.check-builder-needed.outcome == 'success' }} - collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag }} + collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag || 'master'}} env: DEFAULT_BUILDER_TAG: master @@ -42,19 +42,6 @@ jobs: - builder/Dockerfile - .github/workflows/collector-builder.yml - - name: Define builder tag - id: builder-tag - run: | - COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}" - if [[ "${{ github.event_name }}" == 'pull_request' || \ - "${{ github.ref_type }}" == 'tag' || \ - "${{ github.ref_name }}" =~ ^release- ]]; then - COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}" - fi - - echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV" - echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT" - - name: Check labels and branch status to check if builder is needed id: check-builder-needed if: | @@ -65,7 +52,22 @@ jobs: contains(github.event.pull_request.labels.*.name, 'build-builder-image') || github.event_name == 'schedule' run: | + echo "Rebuilding builder image" + + - name: Define builder tag + id: builder-tag + if: | + steps.check-builder-needed.outcome == 'success' + run: | + COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}" + if [[ "${{ github.event_name }}" == 'pull_request' || \ + "${{ github.ref_type }}" == 'tag' || \ + "${{ github.ref_name }}" =~ ^release- ]]; then + COLLECTOR_BUILDER_TAG="${{ inputs.collector-tag }}" + fi + echo "Rebuild builder image with tag ${COLLECTOR_BUILDER_TAG}" + echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT" build-builder-image: name: Local builder image From fdef81f0605eac70ddf52400a700c48b5863e7ea Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Mon, 21 Apr 2025 17:34:31 -0700 Subject: [PATCH 22/27] always --- .github/workflows/collector-builder.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 5901b5416c..95809b5455 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -213,6 +213,7 @@ jobs: name: Create Multiarch manifest runs-on: ubuntu-24.04 if: | + always() && !contains(join(needs.*.result, ','), 'failure') && needs.builder-needs-rebuilding.outputs.build-image == 'true' env: COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} From ffce335b84b0c825af688645eb5f09fddc2d5a05 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 22 Apr 2025 14:54:18 -0700 Subject: [PATCH 23/27] ensure all archs run on non pull-request triggers --- .github/workflows/collector-builder.yml | 18 ++++++++++++++---- .github/workflows/collector.yml | 17 +++++++++++++---- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 95809b5455..35fb044d3c 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -8,6 +8,16 @@ on: required: true description: | The tag used to build the collector image + architectures: + type: string + required: false + description: | + String seperated list of architectures + default: | + ${{ ((github.event.name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) && + 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} + outputs: collector-builder-tag: description: The builder tag used by the build @@ -81,7 +91,7 @@ jobs: matrix: arch: [amd64, arm64, ppc64le] exclude: - - arch: ${{ (!contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'ppc64le') || '' }} + - arch: ${{ contains(inputs.architectures, 'ppc64le') && '' || 'ppc64le' }} runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} env: @@ -134,7 +144,7 @@ jobs: - builder-needs-rebuilding if: | needs.builder-needs-rebuilding.outputs.build-image == 'true' && - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') + contains(inputs.architectures, 's390x') strategy: matrix: arch: [s390x] @@ -190,7 +200,7 @@ jobs: echo "collector_builder_tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}" } > ${{ github.workspace }}/ansible/secrets.yml - - name: Build s390x images + - name: Build images timeout-minutes: 480 run: | ansible-playbook \ @@ -217,7 +227,7 @@ jobs: needs.builder-needs-rebuilding.outputs.build-image == 'true' env: COLLECTOR_BUILDER_TAG: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} - ARCHS: ${{ (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} + ARCHS: ${{ inputs.architectures }} steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 62d69b7fea..cc0b303acd 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -18,6 +18,15 @@ on: required: true description: | The builder tag to use in the build + architectures: + type: string + required: false + description: | + String seperated list of architectures + default: | + ${{ ((github.event.name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) && + 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} env: COLLECTOR_TAG: ${{ inputs.collector-tag }} @@ -34,7 +43,7 @@ jobs: matrix: arch: [amd64, arm64, ppc64le] exclude: - - arch: ${{ (!contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'ppc64le') || '' }} + - arch: ${{ contains(inputs.architectures, 'ppc64le') && '' || 'ppc64le' }} runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-22.04-arm') || 'ubuntu-22.04' }} env: @@ -85,9 +94,9 @@ jobs: ansible/ci-build-collector.yml build-collector-image-remote-vm: - name: Remote collector image s390x + name: Remote collector image runs-on: ubuntu-24.04 - if: contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') + if: contains(inputs.architectures, 's390x') strategy: matrix: arch: [s390x] @@ -167,7 +176,7 @@ jobs: runs-on: ubuntu-24.04 if: always() && !contains(join(needs.*.result, ','), 'failure') env: - ARCHS: ${{ (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} + ARCHS: ${{ inputs.architectures }} steps: - uses: actions/checkout@v4 From ae7eac2a3c6f24f6461ab150b2e1833031b90709 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Tue, 22 Apr 2025 15:05:47 -0700 Subject: [PATCH 24/27] fix --- .github/workflows/collector-builder.yml | 6 +++--- .github/workflows/collector.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 35fb044d3c..f53df3d69d 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -12,9 +12,9 @@ on: type: string required: false description: | - String seperated list of architectures + Space-seperated list of architectures default: | - ${{ ((github.event.name != 'pull_request' || + ${{ ((github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) && 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} @@ -121,7 +121,7 @@ jobs: echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}" echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}" echo "collector_git_ref: ${{ github.ref }}" - echo "collector_builder_tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}" + echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}" } > ${{ github.workspace }}/ansible/secrets.yml - name: Build images diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index cc0b303acd..cb7f850a96 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -22,9 +22,9 @@ on: type: string required: false description: | - String seperated list of architectures + Space-seperated list of architectures default: | - ${{ ((github.event.name != 'pull_request' || + ${{ ((github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) && 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} From 37ab4e6c81b9e92572fa57e9d25a73bd51889bbc Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 23 Apr 2025 10:50:21 -0700 Subject: [PATCH 25/27] pr feedback --- .github/workflows/collector-builder.yml | 58 ++++++++++--------------- .github/workflows/collector.yml | 8 +--- .github/workflows/init.yml | 8 ++++ .github/workflows/main.yml | 2 + 4 files changed, 36 insertions(+), 40 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index f53df3d69d..36ba27b6cb 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -10,13 +10,9 @@ on: The tag used to build the collector image architectures: type: string - required: false + required: true description: | - Space-seperated list of architectures - default: | - ${{ ((github.event_name != 'pull_request' || - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) && - 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} + Space-seperated list of architectures to build outputs: collector-builder-tag: @@ -33,7 +29,7 @@ jobs: name: Determine if builder image needs to be built runs-on: ubuntu-24.04 outputs: - build-image: ${{ steps.check-builder-needed.outcome == 'success' }} + build-image: ${{ steps.builder-tag.build-image || false }} collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag || 'master'}} env: @@ -52,8 +48,8 @@ jobs: - builder/Dockerfile - .github/workflows/collector-builder.yml - - name: Check labels and branch status to check if builder is needed - id: check-builder-needed + - name: Check labels and define builder tag + id: builder-tag if: | steps.changed.outputs.builder-changed == 'true' || (github.event_name == 'push' && ( @@ -61,13 +57,6 @@ jobs: )) || contains(github.event.pull_request.labels.*.name, 'build-builder-image') || github.event_name == 'schedule' - run: | - echo "Rebuilding builder image" - - - name: Define builder tag - id: builder-tag - if: | - steps.check-builder-needed.outcome == 'success' run: | COLLECTOR_BUILDER_TAG="${DEFAULT_BUILDER_TAG}" if [[ "${{ github.event_name }}" == 'pull_request' || \ @@ -78,6 +67,7 @@ jobs: echo "Rebuild builder image with tag ${COLLECTOR_BUILDER_TAG}" echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT" + echo "build-image=true" >> "$GITHUB_OUTPUT" build-builder-image: name: Local builder image @@ -114,15 +104,15 @@ jobs: - name: Create ansible vars run: | - { - echo "---" - echo "stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}" - echo "stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}" - echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}" - echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}" - echo "collector_git_ref: ${{ github.ref }}" - echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}" - } > ${{ github.workspace }}/ansible/secrets.yml + cat << EOF > ${{ github.workspace }}/ansible/secrets.yml + --- + stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + collector_git_ref: ${{ github.ref }} + collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }} + EOF - name: Build images timeout-minutes: 480 @@ -190,15 +180,15 @@ jobs: - name: Create ansible vars run: | - { - echo "---" - echo "stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}" - echo "stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}" - echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}" - echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}" - echo "collector_git_ref: ${{ github.ref }}" - echo "collector_builder_tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }}" - } > ${{ github.workspace }}/ansible/secrets.yml + cat << EOF > ${{ github.workspace }}/ansible/secrets.yml + --- + stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }} + stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }} + rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }} + rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }} + collector_git_ref: ${{ github.ref }} + collector_builder_tag: ${{ needs.builder-needs-rebuilding.outputs.collector-builder-tag }} + EOF - name: Build images timeout-minutes: 480 diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index cb7f850a96..5d0963b43e 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -20,13 +20,9 @@ on: The builder tag to use in the build architectures: type: string - required: false + required: true description: | - Space-seperated list of architectures - default: | - ${{ ((github.event_name != 'pull_request' || - contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) && - 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} + Space-seperated list of architectures to build env: COLLECTOR_TAG: ${{ inputs.collector-tag }} diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml index be09716158..7cfb043f2a 100644 --- a/.github/workflows/init.yml +++ b/.github/workflows/init.yml @@ -58,6 +58,14 @@ on: description: | Trigger rebuild of QA containers value: ${{ jobs.common-variables.outputs.rebuild-qa-containers }} + architectures: + type: string + description: | + Space-seperated list of architectures to build + value: | + ${{ ((github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')) && + 'amd64 arm64 ppc64le s390x') || 'amd64 arm64' }} jobs: common-variables: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f002e6eb7b..9a7d8df27d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,6 +40,7 @@ jobs: needs: init with: collector-tag: ${{ needs.init.outputs.collector-tag }} + architectures: ${{ needs.init.architectures }} secrets: inherit build-collector: @@ -51,6 +52,7 @@ jobs: collector-tag: ${{ needs.init.outputs.collector-tag }} collector-image: ${{ needs.init.outputs.collector-image }} collector-builder-tag: ${{ needs.build-builder-image.outputs.collector-builder-tag }} + architectures: ${{ needs.init.architectures }} secrets: inherit build-test-containers: From a1291312a0926f61aa3d7fb09d345a960921e993 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Wed, 23 Apr 2025 13:05:32 -0700 Subject: [PATCH 26/27] outputs --- .github/workflows/collector-builder.yml | 2 +- .github/workflows/init.yml | 1 - .github/workflows/main.yml | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/collector-builder.yml b/.github/workflows/collector-builder.yml index 36ba27b6cb..9ff9184782 100644 --- a/.github/workflows/collector-builder.yml +++ b/.github/workflows/collector-builder.yml @@ -29,7 +29,7 @@ jobs: name: Determine if builder image needs to be built runs-on: ubuntu-24.04 outputs: - build-image: ${{ steps.builder-tag.build-image || false }} + build-image: ${{ steps.builder-tag.outputs.build-image || false }} collector-builder-tag: ${{ steps.builder-tag.outputs.collector-builder-tag || 'master'}} env: diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml index 7cfb043f2a..b9bea60ec6 100644 --- a/.github/workflows/init.yml +++ b/.github/workflows/init.yml @@ -59,7 +59,6 @@ on: Trigger rebuild of QA containers value: ${{ jobs.common-variables.outputs.rebuild-qa-containers }} architectures: - type: string description: | Space-seperated list of architectures to build value: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9a7d8df27d..c9d8e3384d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,7 @@ jobs: needs: init with: collector-tag: ${{ needs.init.outputs.collector-tag }} - architectures: ${{ needs.init.architectures }} + architectures: ${{ needs.init.outputs.architectures }} secrets: inherit build-collector: @@ -52,7 +52,7 @@ jobs: collector-tag: ${{ needs.init.outputs.collector-tag }} collector-image: ${{ needs.init.outputs.collector-image }} collector-builder-tag: ${{ needs.build-builder-image.outputs.collector-builder-tag }} - architectures: ${{ needs.init.architectures }} + architectures: ${{ needs.init.outputs.architectures }} secrets: inherit build-test-containers: From 216401d9a36767659780b6e133e94f0f77b7fe22 Mon Sep 17 00:00:00 2001 From: Robby Cochran Date: Thu, 24 Apr 2025 14:45:08 -0700 Subject: [PATCH 27/27] ubuntu-24.04 --- .github/workflows/collector.yml | 2 +- .github/workflows/unit-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/collector.yml b/.github/workflows/collector.yml index 5d0963b43e..736f68d1cb 100644 --- a/.github/workflows/collector.yml +++ b/.github/workflows/collector.yml @@ -40,7 +40,7 @@ jobs: arch: [amd64, arm64, ppc64le] exclude: - arch: ${{ contains(inputs.architectures, 'ppc64le') && '' || 'ppc64le' }} - runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-22.04-arm') || 'ubuntu-22.04' }} + runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} env: PLATFORM: linux/${{ matrix.arch }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 99d544cda3..d13562ff21 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -21,7 +21,7 @@ jobs: - -DCMAKE_BUILD_TYPE=Release - -DADDRESS_SANITIZER=ON -DCMAKE_BUILD_TYPE=Debug - -DUSE_VALGRIND=ON -DCMAKE_BUILD_TYPE=Debug - runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-22.04-arm') || 'ubuntu-22.04' }} + runs-on: ${{ (matrix.arch == 'arm64' && 'ubuntu-24.04-arm') || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v4