Add VMNonRecoverableOSPanic alert

Signed-off-by: avlitman <alitman@redhat.com>

Author: avlitman

hack/prom-rule-ci/observability-prom-rules-tests.yaml

@@ -446,6 +446,44 @@ tests:
               kubernetes_operator_part_of: "kubevirt"
               kubernetes_operator_component: "cnv-observability"
 
+  # VM non-recoverable OS panic
+  - interval: 1m
+    input_series:
+      - series: 'kubevirt_vmi_guest_os_panic_total{namespace="test", name="vm-01", type="pvpanic", bugcheck_code="unknown"}'
+        values: '0 0 1 1 1 1 1'
+
+    alert_rule_test:
+      # No alert at 1m (no panics yet)
+      - eval_time: 1m
+        alertname: VMNonRecoverableOSPanic
+        exp_alerts: []
+      # Alert fires at 4m (1 panic detected, within 1-5 range)
+      - eval_time: 4m
+        alertname: VMNonRecoverableOSPanic
+        exp_alerts:
+          - exp_annotations:
+              summary: "Non-recoverable guest OS panic detected on VM vm-01 in namespace test"
+              description: "At least one non-recoverable guest OS panic has been detected on the VM in the last 24 hours (1 total)."
+              runbook_url: "https://kubevirt.io/monitoring/runbooks/VMNonRecoverableOSPanic"
+            exp_labels:
+              severity: "warning"
+              operator_health_impact: "none"
+              kubernetes_operator_part_of: "kubevirt"
+              kubernetes_operator_component: "kubevirt"
+              namespace: "test"
+              name: "vm-01"
+
+  # VM non-recoverable OS panic - should NOT fire above 5
+  - interval: 1m
+    input_series:
+      - series: 'kubevirt_vmi_guest_os_panic_total{namespace="test", name="vm-02", type="pvpanic", bugcheck_code="unknown"}'
+        values: '0 0 6 6 6 6 6'
+
+    alert_rule_test:
+      - eval_time: 4m
+        alertname: VMNonRecoverableOSPanic
+        exp_alerts: []
+
   - interval: 1m
     input_series:
       - series: 'kubevirt_hco_memory_overcommit_percentage'

pkg/monitoring/observability/rules/alerts/alerts.go

@@ -28,7 +28,9 @@ func Register(operatorRegistry *operatorrules.Registry) error {
 	for _, alertGroup := range alerts {
 		for _, alert := range alertGroup {
 			alert.Labels["kubernetes_operator_part_of"] = "kubevirt"
-			alert.Labels["kubernetes_operator_component"] = "cnv-observability"
+			if _, ok := alert.Labels["kubernetes_operator_component"]; !ok {
+				alert.Labels["kubernetes_operator_component"] = "cnv-observability"
+			}
 			if _, ok := alert.Annotations["runbook_url"]; !ok {
 				alert.Annotations["runbook_url"] = fmt.Sprintf(runbookURLTemplate, alert.Alert)
 			}

pkg/monitoring/observability/rules/alerts/cluster_alerts.go

@@ -118,6 +118,20 @@ func clusterAlerts() []promv1.Rule {
 				"operator_health_impact": "none",
 			},
 		},
+		{
+			Alert: "VMNonRecoverableOSPanic",
+			Expr:  intstr.FromString(`sum by (namespace, name) (increase(kubevirt_vmi_guest_os_panic_total[24h])) > 0 and sum by (namespace, name) (increase(kubevirt_vmi_guest_os_panic_total[24h])) <= 5`),
+			For:   ptr.To[promv1.Duration]("1m"),
+			Annotations: map[string]string{
+				"summary":     "Non-recoverable guest OS panic detected on VM {{ $labels.name }} in namespace {{ $labels.namespace }}",
+				"description": "At least one non-recoverable guest OS panic has been detected on the VM in the last 24 hours ({{ $value }} total).",
+			},
+			Labels: map[string]string{
+				"severity":                        "warning",
+				"operator_health_impact":          "none",
+				"kubernetes_operator_component":   "kubevirt",
+			},
+		},
 		{
 			Alert: "DuplicateWaspAgentDSDetected",
 			Expr: intstr.FromString(