MongoDbPodRecovering alert firing once per reporting pod instead of once per recovering member

VERSION

@@ -1,4 +1,4 @@
-VERSION="2.13.10"
+VERSION="2.13.11"
 
 VERSION_HOTFIX=
 

monitoring/mongodb/alerts.test.yaml

@@ -369,31 +369,108 @@ tests:
               summary: MongoDb has low disk space
 
 
+  # mongodb_rs_members_state has one series per (reporting pod, member):
+  # every pod's exporter reports the state of every replica-set member. Here
+  # member data-1 is RECOVERING (3) while data-0 is PRIMARY (1) and data-2 is
+  # SECONDARY (2); all three pods report it, and exactly one alert per
+  # severity must fire, naming the recovering member (ZENKO-5332).
   - name: MongoDbPodRecovering
     interval: 1m
     input_series:
-      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-mongos-0", rs_state="3"}
+      # Reported by data-0 (itself PRIMARY, rs_state=1).
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-0", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-0.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="PRIMARY", rs_state="1"}
+        values: 1x1440
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-0", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-1.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="RECOVERING", rs_state="1"}
         values: 3x1440
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-0", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-2.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="SECONDARY", rs_state="1"}
+        values: 2x1440
+      # Reported by data-1 (itself RECOVERING, rs_state=3).
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-1", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-0.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="PRIMARY", rs_state="3"}
+        values: 1x1440
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-1", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-1.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="RECOVERING", rs_state="3"}
+        values: 3x1440
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-1", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-2.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="SECONDARY", rs_state="3"}
+        values: 2x1440
+      # Reported by data-2 (itself SECONDARY, rs_state=2).
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-2", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-0.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="PRIMARY", rs_state="2"}
+        values: 1x1440
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-2", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-1.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="RECOVERING", rs_state="2"}
+        values: 3x1440
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-2", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-2.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="SECONDARY", rs_state="2"}
+        values: 2x1440
 
     alert_rule_test:
+      # Before the 1h "for" elapses, nothing fires.
+      - { alertname: MongoDbPodRecovering, eval_time: 30m, exp_alerts: [] }
+
+      # After 1h, exactly one warning, naming the recovering member.
+      - alertname: MongoDbPodRecovering
+        eval_time: 2h
+        exp_alerts:
+          - exp_labels:
+              namespace: zenko
+              severity: warning
+              rs_nm: data-db-mongodb-sharded-shard-0
+              member_idx: data-db-mongodb-sharded-shard0-data-1.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017
+              pod: data-db-mongodb-sharded-shard0-data-1
+            exp_annotations:
+              description: "MongoDB pod `data-db-mongodb-sharded-shard0-data-1` has been in 'RECOVERING' state for more than 1 hour. This may be expected if the 'Resync a Data Services MongoDB Member' procedure has recently been executed."
+              summary: MongoDB is recovering
+
+      # After 1d, exactly one critical (plus the warning still firing).
       - alertname: MongoDbPodRecovering
         eval_time: 1d
         exp_alerts:
           - exp_labels:
               namespace: zenko
               severity: critical
-              pod: data-db-mongodb-sharded-mongos-0
-              rs_state: 3
+              rs_nm: data-db-mongodb-sharded-shard-0
+              member_idx: data-db-mongodb-sharded-shard0-data-1.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017
+              pod: data-db-mongodb-sharded-shard0-data-1
             exp_annotations:
-              description: "MongoDB pod `data-db-mongodb-sharded-mongos-0` has been in the 'RECOVERING' state more than 24 hours. The instance may be failing to catch up and recover."
+              description: "MongoDB pod `data-db-mongodb-sharded-shard0-data-1` has been in the 'RECOVERING' state more than 24 hours. The instance may be failing to catch up and recover."
               summary: MongoDB is recovering
           - exp_labels:
               namespace: zenko
-              pod: data-db-mongodb-sharded-mongos-0
-              rs_state: 3
               severity: warning
+              rs_nm: data-db-mongodb-sharded-shard-0
+              member_idx: data-db-mongodb-sharded-shard0-data-1.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017
+              pod: data-db-mongodb-sharded-shard0-data-1
+            exp_annotations:
+              description: "MongoDB pod `data-db-mongodb-sharded-shard0-data-1` has been in 'RECOVERING' state for more than 1 hour. This may be expected if the 'Resync a Data Services MongoDB Member' procedure has recently been executed."
+              summary: MongoDB is recovering
+
+  # Same scenario, but the recovering member's own exporter is not scrapeable
+  # (its pod reports nothing): the peers' view must still fire exactly one
+  # alert per severity.
+  - name: MongoDbPodRecoveringUnscrapeableMember
+    interval: 1m
+    input_series:
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-0", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-0.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="PRIMARY", rs_state="1"}
+        values: 1x1440
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-0", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-1.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="RECOVERING", rs_state="1"}
+        values: 3x1440
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-0", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-2.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="SECONDARY", rs_state="1"}
+        values: 2x1440
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-2", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-0.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="PRIMARY", rs_state="2"}
+        values: 1x1440
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-2", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-1.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="RECOVERING", rs_state="2"}
+        values: 3x1440
+      - series: mongodb_rs_members_state{namespace="zenko", pod="data-db-mongodb-sharded-shard0-data-2", rs_nm="data-db-mongodb-sharded-shard-0", member_idx="data-db-mongodb-sharded-shard0-data-2.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017", member_state="SECONDARY", rs_state="2"}
+        values: 2x1440
+
+    alert_rule_test:
+      - alertname: MongoDbPodRecovering
+        eval_time: 2h
+        exp_alerts:
+          - exp_labels:
+              namespace: zenko
+              severity: warning
+              rs_nm: data-db-mongodb-sharded-shard-0
+              member_idx: data-db-mongodb-sharded-shard0-data-1.data-db-mongodb-sharded-headless.zenko.svc.cluster.local:27017
+              pod: data-db-mongodb-sharded-shard0-data-1
             exp_annotations:
-              description: "MongoDB pod `data-db-mongodb-sharded-mongos-0` has been in 'RECOVERING' state for more than 1 hour. This may be expected if the 'Resync a Data Services MongoDB Member' procedure has recently been executed."
+              description: "MongoDB pod `data-db-mongodb-sharded-shard0-data-1` has been in 'RECOVERING' state for more than 1 hour. This may be expected if the 'Resync a Data Services MongoDB Member' procedure has recently been executed."
               summary: MongoDB is recovering
 
   - name: MongoDbInvalidState

monitoring/mongodb/alerts.yaml

@@ -229,9 +229,21 @@ groups:
       description: 'MongoDb has low disk space'
       summary: 'MongoDb has low disk space'
 
+  # mongodb_rs_members_state has one series per (reporting pod, member): every
+  # pod's exporter reports the state of every replica-set member, so a single
+  # recovering member matches once per reporting pod. Aggregate by member_idx
+  # to fire exactly once per recovering member — this also keeps the alert
+  # firing when the recovering member's own exporter cannot be scraped, since
+  # its peers still report it. The pod label is rebuilt from the first DNS
+  # segment of member_idx (the member's pod name).
   - alert: MongoDbPodRecovering
     expr: |
-      mongodb_rs_members_state{namespace="${namespace}", pod=~"${service}.*"} == 3
+      label_replace(
+        min by (namespace, rs_nm, member_idx) (
+          mongodb_rs_members_state{namespace="${namespace}", pod=~"${service}.*"}
+        ),
+        "pod", "$1", "member_idx", "([^.:]+)[.:].*"
+      ) == 3
     for: 1h
     labels:
       severity: warning
@@ -241,7 +253,12 @@ groups:
 
   - alert: MongoDbPodRecovering
     expr: |
-      mongodb_rs_members_state{namespace="${namespace}", pod=~"${service}.*"} == 3
+      label_replace(
+        min by (namespace, rs_nm, member_idx) (
+          mongodb_rs_members_state{namespace="${namespace}", pod=~"${service}.*"}
+        ),
+        "pod", "$1", "member_idx", "([^.:]+)[.:].*"
+      ) == 3
     for: 1d
     labels:
       severity: critical

tests/workflows/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "description": "Tests for GHA workflows",
   "scripts": {
-    "postinstall": "yarn --cwd node_modules/@kie/act-js node scripts/postinstall.js 0.2.75",
+    "postinstall": "yarn --cwd node_modules/@kie/act-js node scripts/postinstall.js 0.2.84",
     "test": "jest"
   },
   "license": "Apache-2.0",

tests/workflows/release.spec.ts

@@ -209,20 +209,10 @@ test.each([
                 .listReleases()
                 // First call from release notes generation, to get the previous release
                 .reply({ status: 200, data: [{ tag_name: '2.3.6', id: 122 }] })
-                // Second call made by action-gh-release@v2.5.2+ (after release creation) to handle
-                // race condition when release is created by multiple jobs in parallel...
-                .reply({
-                    status: 200, data: [{ tag_name: '2.3.6', id: 122 }, {
-                        id: 123,
-                        draft: true,
-                        name: `Release ${tag}`,
-                        prerelease: tag === '2.3.7-rc.1',
-                        tag_name: tag,
-                        target_commitish: await getCommitHash(),
-                        upload_url: 'http://uploads.github.com/repos/scality/Zenko/releases/456/assets{?name,label}',
-                        html_url: 'http://github.com/repos/scality/Zenko/releases/456',
-                    }],
-                }),
+                // Second call made by action-gh-release@v2.5.0's findTagFromReleases, which scans
+                // all releases before creating one: it must not contain the target tag, or the
+                // action takes the update-existing-release path instead of creating the release.
+                .reply({ status: 200, data: [{ tag_name: '2.3.6', id: 122 }] }),
             moctokit.rest.repos
                 .generateReleaseNotes({
                     owner: 'scality',
@@ -309,19 +299,25 @@ test.each([
         }
     });
 
-    var lastResult = result[result.length - 1];
-    var postSteps = [];
+    // act >=0.2.81 appends a timing suffix to success/failure lines ("Main foo [40ms]"), which
+    // act-js's OutputParser splits into a named "Run" entry followed by an unnamed status entry.
+    // So the real status of result[i] lives on result[i + 1]; unnamed entries have status set.
+    var lastResult = result.length - 2;
+    var postSteps: number[] = [];
 
     // action-artifacts keep executing Post step, need to skip it...
     for (let i = result.length - 1; i >= 0; i--) {
-        if (result[i].name.startsWith('Main ')) {
-            lastResult = result[i];
+        if (!result[i].name) {
+            postSteps.push(result[i].status);
+        } else if (result[i].name.startsWith('Main ')) {
+            lastResult = i;
             break;
         }
-        postSteps.push(result[i]);
     }
 
-    expect(lastResult.name).toStrictEqual('Main ' + stepName);
-    expect(lastResult.status).toStrictEqual(status.value());
-    postSteps.forEach(r => expect(r.status).toStrictEqual(Pass.value()));
+    postSteps.pop(); // last pushed entry is the matched step's own status, not a post-step
+
+    expect(result[lastResult].name.startsWith('Main ' + stepName)).toBe(true);
+    expect(result[lastResult + 1].status).toStrictEqual(status.value());
+    postSteps.forEach(s => expect(s).toStrictEqual(Pass.value()));
 })