From aacac48032ae4365c1f4588b71721d3cf0dc3f00 Mon Sep 17 00:00:00 2001 From: NikhilMM89 Date: Fri, 2 Oct 2026 10:36:20 -0700 Subject: [PATCH 1/2] fix(ci): deploy the merged commit's own image, not whatever :latest points at (CCP-5824) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit diff_branch defaulted to main, so a push to main diffed main against main — always empty, so the build never triggered and the action retagged :latest onto :dev instead. :latest is whichever image was tagged most recently by any build, so the deployed image depended on timing. After #229 merged, DEV and TEST ran the #210 frontend and the run still went green. - merge.yml: diff_branch HEAD^ so only packages the merge touched rebuild, from the merge commit; tag_fallback dev so an untouched package keeps the image it had - pr-open.yml: same tag_fallback change — unchanged packages in a PR env were also being filled from :latest - Log per package whether the image was built or retagged, plus the digest Verified the HEAD^ diff against the last five merge commits: #245 frontend only, #242 backend+frontend, #244 all three. --- .github/workflows/merge.yml | 18 +++++++++++++++++- .github/workflows/pr-open.yml | 3 ++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml index 308db6aeb..8093e3cf7 100644 --- a/.github/workflows/merge.yml +++ b/.github/workflows/merge.yml @@ -38,12 +38,28 @@ jobs: timeout-minutes: 10 steps: - uses: bcgov/action-builder-ghcr@cb2629351c87dd1c2130073e4ebb7233a9653a63 # v4.4.1 + id: build with: package: ${{ matrix.package }} tags: dev - tag_fallback: latest + # Diff the merge commit against the previous main; diff_branch defaults to + # main, which on a push to main is always empty so nothing ever built. + diff_branch: HEAD^ + # :latest is whichever image was tagged most recently by any build. + tag_fallback: dev triggers: ('${{ matrix.package }}/') + - name: Report deployed image + shell: bash + env: + PKG: ${{ matrix.package }} + TRIGGERED: ${{ steps.build.outputs.triggered }} + DIGEST: ${{ steps.build.outputs.digest }} + run: | + [ "${TRIGGERED}" = "true" ] && SRC="fresh build of ${GITHUB_SHA}" || SRC="retagged existing :dev (unchanged)" + echo "${PKG}: ${SRC} (digest ${DIGEST:-none})" + echo "- **${PKG}**: ${SRC} — \`${DIGEST:-none}\`" >> "${GITHUB_STEP_SUMMARY}" + import: name: Import to ImageStream needs: [builds] diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml index 8e84117c6..4dd3de453 100644 --- a/.github/workflows/pr-open.yml +++ b/.github/workflows/pr-open.yml @@ -34,7 +34,8 @@ jobs: with: package: ${{ matrix.package }} tags: ${{ github.event.number }} - tag_fallback: latest + # :latest is whichever image was tagged most recently by any build. + tag_fallback: dev triggers: ('${{ matrix.package }}/') import: From 0ea3fb43efe6e653daf23b71c7c15a5f745672cd Mon Sep 17 00:00:00 2001 From: NikhilMM89 Date: Tue, 6 Oct 2026 09:59:53 -0700 Subject: [PATCH 2/2] fix(charts): raise the promtail sidecar memory limit from 128Mi to 256Mi ContainerMemoryHigh fired on two prod frontend pods. The frontend container itself is nowhere near its limit (39-57Mi against 8Gi); it is the promtail sidecar enabled alongside it, sitting at 109Mi of a 128Mi limit - 85.2%, which is exactly the alert threshold. It is not log volume. access.log is 664KB after four days and promtail's positions file shows it fully consumed, so there is no backlog. 60-110Mi is simply promtail's resting Go footprint, and the chart's 128Mi left no headroom. Test shows the same spread, 55Mi and 103Mi. Nothing has OOMKilled yet. The reason to fix it ahead of that: the app container has its own limit, so an OOMKilled promtail costs no user-visible service - it just stops frontend logs reaching Loki and quietly blinds the frontend log alerts again. Requests go to 128Mi and limits to 256Mi, roughly 2.5x the observed peak. Costs 320Mi of requests across five prod replicas, against 7.4Gi of headroom. --- charts/app/values.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/charts/app/values.yaml b/charts/app/values.yaml index bac120e4c..b68719e61 100644 --- a/charts/app/values.yaml +++ b/charts/app/values.yaml @@ -231,11 +231,14 @@ frontend: environment: ~ #-- Job name for logs (auto-generated if not set) jobName: ~ - #-- Resource requests and limits for Promtail sidecar + #-- Resource requests and limits for Promtail sidecar. + #-- 128Mi was too tight: promtail idles at 60-110Mi in prod, so pods sat at + #-- 85% of the limit and ContainerMemoryHigh fired. An OOMKill here is silent + #-- (the app container is unaffected) and stops frontend logs reaching Loki. resources: requests: cpu: 50m - memory: 64Mi + memory: 128Mi limits: cpu: 200m - memory: 128Mi + memory: 256Mi