Skip to content

Commit 3e570ae

Browse files
authored
[obs] Make sure to not alert for 0 backup failures (#20458)
1 parent 6b4fae6 commit 3e570ae

File tree

1 file changed

+4
-2
lines changed

1 file changed

+4
-2
lines changed

operations/observability/mixins/workspace/rules/satellite/workspaces.yaml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,9 @@ spec:
5454
summary: Workspace backups failed recently in cluster {{ $labels.cluster }}
5555
description: This can happen when a single node has failed in the cloud provider
5656
expr: |
57-
sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) <= 16
57+
sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) > 0
58+
AND
59+
sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) < 16
5860
- alert: GitpodWsManagerMk2BackupFailureCritical
5961
labels:
6062
severity: critical
@@ -64,4 +66,4 @@ spec:
6466
summary: Workspace backups failed recently in cluster {{ $labels.cluster }}
6567
description: This can be an indicator of two or more nodes failing in a cloud provider
6668
expr: |
67-
sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) > 16
69+
sum by (cluster) (increase(gitpod_ws_manager_mk2_workspace_backups_failure_total{cluster!~"ephemeral.*"}[1h])) >= 16

0 commit comments

Comments
 (0)