notify: add grafana_alerting_alertmanager_templates metric (#557)

Author: yuri-tceretian

notify/grafana_alertmanager.go

@@ -869,6 +869,7 @@ func (am *GrafanaAlertmanager) ApplyConfig(cfg NotificationsConfiguration) (err
 
 	am.setReceiverMetrics(cfg.Receivers, len(activeReceivers))
 	am.setInhibitionRulesMetrics(cfg.InhibitRules)
+	am.setTemplateMetrics(cfg.Templates)
 
 	am.receivers = receivers
 
@@ -894,6 +895,21 @@ func (am *GrafanaAlertmanager) setInhibitionRulesMetrics(r []InhibitRule) {
 	am.opts.Metrics.configuredInhibitionRules.WithLabelValues(am.tenantString()).Set(float64(len(r)))
 }
 
+func (am *GrafanaAlertmanager) setTemplateMetrics(cfgTemplates []templates.TemplateDefinition) {
+	// Start all known kinds at 0 so a kind that no longer has any templates
+	// gets its gauge reset instead of keeping a stale value from a previous config.
+	byKind := map[templates.Kind]int{
+		templates.GrafanaKind: 0,
+		templates.MimirKind:   0,
+	}
+	for _, t := range cfgTemplates {
+		byKind[t.Kind]++
+	}
+	for kind, count := range byKind {
+		am.opts.Metrics.configuredTemplates.WithLabelValues(am.tenantString(), kind.String()).Set(float64(count))
+	}
+}
+
 func (am *GrafanaAlertmanager) setReceiverMetrics(cfgReceivers []models.ReceiverConfig, countActiveReceivers int) {
 	am.opts.Metrics.configuredReceivers.WithLabelValues(am.tenantString(), ActiveStateLabelValue).Set(float64(countActiveReceivers))
 	am.opts.Metrics.configuredReceivers.WithLabelValues(am.tenantString(), InactiveStateLabelValue).Set(float64(len(cfgReceivers) - countActiveReceivers))

notify/grafana_alertmanager_metrics.go

@@ -18,6 +18,7 @@ type GrafanaAlertmanagerMetrics struct {
 	configuredReceivers       *prometheus.GaugeVec
 	configuredIntegrations    *prometheus.GaugeVec
 	configuredInhibitionRules *prometheus.GaugeVec
+	configuredTemplates       *prometheus.GaugeVec
 }
 
 // NewGrafanaAlertmanagerMetrics creates a set of metrics for the Alertmanager.
@@ -43,5 +44,11 @@ func NewGrafanaAlertmanagerMetrics(r prometheus.Registerer, l log.Logger) *Grafa
 			Name:      "alertmanager_inhibition_rules",
 			Help:      "Number of configured inhibition rules.",
 		}, []string{"org"}),
+		configuredTemplates: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "alertmanager_templates",
+			Help:      "Number of configured templates by kind.",
+		}, []string{"org", "kind"}),
 	}
 }

notify/grafana_alertmanager_test.go

@@ -604,6 +604,37 @@ func TestGrafanaAlertmanager_setInhibitionRulesMetrics(t *testing.T) {
 `), "grafana_alerting_alertmanager_inhibition_rules"))
 }
 
+func TestGrafanaAlertmanager_setTemplateMetrics(t *testing.T) {
+	am, reg := setupAMTest(t)
+
+	cfgTemplates := []templates.TemplateDefinition{
+		{Name: "grafana-1", Kind: templates.GrafanaKind},
+		{Name: "grafana-2", Kind: templates.GrafanaKind},
+		{Name: "mimir-1", Kind: templates.MimirKind},
+	}
+	am.setTemplateMetrics(cfgTemplates)
+
+	require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(`
+        	            	# HELP grafana_alerting_alertmanager_templates Number of configured templates by kind.
+        	            	# TYPE grafana_alerting_alertmanager_templates gauge
+        	            	grafana_alerting_alertmanager_templates{kind="Grafana",org="1"} 2
+        	            	grafana_alerting_alertmanager_templates{kind="Mimir",org="1"} 1
+`), "grafana_alerting_alertmanager_templates"))
+
+	// Applying a config that no longer has Mimir templates must reset that
+	// kind's gauge to 0 instead of leaving the stale previous value.
+	am.setTemplateMetrics([]templates.TemplateDefinition{
+		{Name: "grafana-1", Kind: templates.GrafanaKind},
+	})
+
+	require.NoError(t, testutil.GatherAndCompare(reg, bytes.NewBufferString(`
+        	            	# HELP grafana_alerting_alertmanager_templates Number of configured templates by kind.
+        	            	# TYPE grafana_alerting_alertmanager_templates gauge
+        	            	grafana_alerting_alertmanager_templates{kind="Grafana",org="1"} 1
+        	            	grafana_alerting_alertmanager_templates{kind="Mimir",org="1"} 0
+`), "grafana_alerting_alertmanager_templates"))
+}
+
 func TestGrafanaAlertmanager_setReceiverMetrics(t *testing.T) {
 	am, reg := setupAMTest(t)