don't fail during a scrape

Either report all metrics, or none at all.

Author: alicebob

latency.go

@@ -57,34 +57,41 @@ func (lc latencyCollector) describe(ch chan<- *prometheus.Desc) {
 	}
 }
 
-func (lc latencyCollector) collect(conn *as.Connection, ch chan<- prometheus.Metric) error {
+func (lc latencyCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) {
 	stats, err := as.RequestInfo(conn, "latency:")
 	if err != nil {
-		return err
+		return nil, err
 	}
 	lat, err := parseLatency(stats["latency:"])
 	if err != nil {
-		return err
+		return nil, err
 	}
-	for key, metrics := range lat {
+	var metrics []prometheus.Metric
+	for key, ms := range lat {
 		if key == "batch-index" {
 			continue // TODO: would be nice to do something with this key
 		}
 		ns, op, err := readNS(key)
 		if err != nil {
-			return fmt.Errorf("weird latency key %q: %s", key, err)
+			return nil, fmt.Errorf("weird latency key %q: %s", key, err)
 		}
-		for threshold, data := range metrics {
+		for threshold, data := range ms {
 			if threshold == "ops/sec" {
 				m := lc.ops[op]
-				ch <- prometheus.MustNewConstMetric(m.desc, m.typ, data, ns)
+				metrics = append(
+					metrics,
+					prometheus.MustNewConstMetric(m.desc, m.typ, data, ns),
+				)
 				continue
 			}
 			m := lc.latency[op]
-			ch <- prometheus.MustNewConstMetric(m.desc, m.typ, data, ns, threshold)
+			metrics = append(
+				metrics,
+				prometheus.MustNewConstMetric(m.desc, m.typ, data, ns, threshold),
+			)
 		}
 	}
-	return nil
+	return metrics, nil
 }
 
 // parseLatency returns map with: "[{namespace}]-[op]" -> map[threshold]measurement

main.go

@@ -83,7 +83,7 @@ func main() {
 }
 
 type collector interface {
-	collect(*as.Connection, chan<- prometheus.Metric) error
+	collect(*as.Connection) ([]prometheus.Metric, error)
 	describe(ch chan<- *prometheus.Desc)
 }
 
@@ -131,34 +131,44 @@ func (asc *asCollector) Collect(ch chan<- prometheus.Metric) {
 	asc.totalScrapes.Inc()
 	ch <- asc.totalScrapes
 
-	conn, err := as.NewConnection(asc.nodeAddr, 3*time.Second)
+	ms, err := asc.collect()
 	if err != nil {
+		log.Print(err)
 		ch <- prometheus.MustNewConstMetric(upDesc, prometheus.GaugeValue, 0.0)
 		return
 	}
+	ch <- prometheus.MustNewConstMetric(upDesc, prometheus.GaugeValue, 1.0)
+	for _, m := range ms {
+		ch <- m
+	}
+}
+
+func (asc *asCollector) collect() ([]prometheus.Metric, error) {
+	conn, err := as.NewConnection(asc.nodeAddr, 3*time.Second)
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
 	if asc.username != "" {
 		hp, err := hashPassword(asc.password)
 		if err != nil {
-			log.Printf("hashPassword: %s", err)
-			ch <- prometheus.MustNewConstMetric(upDesc, prometheus.GaugeValue, 0.0)
-			return
+			return nil, fmt.Errorf("hashPassword: %s", err)
 		}
 		if err := conn.Authenticate(asc.username, hp); err != nil {
-			log.Printf("auth error: %s", err)
-			ch <- prometheus.MustNewConstMetric(upDesc, prometheus.GaugeValue, 0.0)
-			return
+			return nil, fmt.Errorf("auth error: %s", err)
 		}
 	}
-	ch <- prometheus.MustNewConstMetric(upDesc, prometheus.GaugeValue, 1.0)
-
-	defer conn.Close()
 
+	var metrics []prometheus.Metric
 	for _, c := range asc.collectors {
-		if err := c.collect(conn, ch); err != nil {
-			log.Print(err)
-			return
+		ms, err := c.collect(conn)
+		if err != nil {
+			return nil, err
 		}
+		metrics = append(metrics, ms...)
 	}
+	return metrics, nil
 }
 
 // take from github.com/aerospike/aerospike-client-go/admin_command.go

metric.go

@@ -39,11 +39,11 @@ func parseFloatOrBool(v string) (float64, error) {
 
 // infoCollect parses RequestInfo() results and handles the metrics
 func infoCollect(
-	ch chan<- prometheus.Metric,
 	metrics cmetrics,
 	info string,
 	labelValues ...string,
-) {
+) []prometheus.Metric {
+	var res []prometheus.Metric
 	stats := parseInfo(info)
 	for key, m := range metrics {
 		v, ok := stats[key]
@@ -56,8 +56,12 @@ func infoCollect(
 			log.Printf("%q invalid value %q: %s", key, v, err)
 			continue
 		}
-		ch <- prometheus.MustNewConstMetric(m.desc, m.typ, f, labelValues...)
+		res = append(
+			res,
+			prometheus.MustNewConstMetric(m.desc, m.typ, f, labelValues...),
+		)
 	}
+	return res
 }
 
 func parseInfo(s string) map[string]string {

metric_test.go

@@ -3,7 +3,6 @@ package main
 import (
 	"reflect"
 	"testing"
-	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
@@ -105,17 +104,13 @@ func TestInfoCollect(t *testing.T) {
 			want: `gauge:<value:1 > `,
 		},
 	} {
-		ch := make(chan prometheus.Metric)
 		metrics := cmetrics{c.field: c.metric}
-		go infoCollect(ch, metrics, c.payload)
+		ms := infoCollect(metrics, c.payload)
 
-		var metric prometheus.Metric
-		select {
-		case metric = <-ch:
-		case <-time.After(time.Second):
-			t.Fatalf("timeout for case %d (%q)", n, c.want)
+		if have, want := len(ms), 1; have != want {
+			t.Fatalf("have %d, want %d", have, want)
 		}
-
+		metric := ms[0]
 		m := &dto.Metric{}
 		metric.Write(m)
 		if have, want := m.String(), c.want; have != want {

namespaces.go

@@ -222,17 +222,21 @@ func (nc nsCollector) describe(ch chan<- *prometheus.Desc) {
 	}
 }
 
-func (nc nsCollector) collect(conn *as.Connection, ch chan<- prometheus.Metric) error {
+func (nc nsCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) {
 	info, err := as.RequestInfo(conn, "namespaces")
 	if err != nil {
-		return err
+		return nil, err
 	}
+	var metrics []prometheus.Metric
 	for _, ns := range strings.Split(info["namespaces"], ";") {
 		nsinfo, err := as.RequestInfo(conn, "namespace/"+ns)
 		if err != nil {
-			return err
+			return nil, err
 		}
-		infoCollect(ch, cmetrics(nc), nsinfo["namespace/"+ns], ns)
+		metrics = append(
+			metrics,
+			infoCollect(cmetrics(nc), nsinfo["namespace/"+ns], ns)...,
+		)
 	}
-	return nil
+	return metrics, nil
 }

sets.go

@@ -42,17 +42,21 @@ func (setc setCollector) describe(ch chan<- *prometheus.Desc) {
 	}
 }
 
-func (setc setCollector) collect(conn *as.Connection, ch chan<- prometheus.Metric) error {
+func (setc setCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) {
+	var metrics []prometheus.Metric
 	info, err := as.RequestInfo(conn, "sets")
 	if err != nil {
-		return err
+		return nil, err
 	}
 	for _, setInfo := range strings.Split(info["sets"], ";") {
 		if setInfo == "" {
 			continue
 		}
 		setStats := parseInfo(setInfo)
-		infoCollect(ch, cmetrics(setc), setInfo, setStats["ns"], setStats["set"])
+		metrics = append(
+			metrics,
+			infoCollect(cmetrics(setc), setInfo, setStats["ns"], setStats["set"])...,
+		)
 	}
-	return nil
+	return metrics, nil
 }

stats.go

@@ -137,11 +137,10 @@ func (sc statsCollector) describe(ch chan<- *prometheus.Desc) {
 	}
 }
 
-func (sc statsCollector) collect(conn *as.Connection, ch chan<- prometheus.Metric) error {
+func (sc statsCollector) collect(conn *as.Connection) ([]prometheus.Metric, error) {
 	res, err := as.RequestInfo(conn, "statistics")
 	if err != nil {
-		return err
+		return nil, err
 	}
-	infoCollect(ch, cmetrics(sc), res["statistics"])
-	return nil
+	return infoCollect(cmetrics(sc), res["statistics"]), nil
 }