Add metrics for tracking live servers

This should allow us to correlate the servers that drone thinks it knows
about with those that GCP has

Author: iainlane

cmd/drone-autoscaler/main.go

@@ -62,9 +62,11 @@ func main() {
 			Fatalln("Invalid or missing hosting provider")
 	}
 
+	collector := metrics.New()
+
 	// instruments the provider with prometheus metrics.
 	provider = metrics.ServerCreate(provider)
-	provider = metrics.ServerDelete(provider)
+	provider = metrics.ServerDelete(provider, collector)
 
 	db, err := store.Connect(conf.Database.Driver, conf.Database.Datasource)
 	if err != nil {
@@ -90,7 +92,7 @@ func main() {
 		conf,
 		servers,
 		provider,
-		metrics.New(),
+		collector,
 	)
 
 	//

engine/alloc.go

@@ -123,5 +123,7 @@ func (a *allocator) allocate(ctx context.Context, server *autoscaler.Server) err
 		return err
 	}
 
+	a.metrics.RegisterKnownInstance(instance)
+
 	return nil
 }

metrics/metrics.go

@@ -9,6 +9,8 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/drone/autoscaler"
 )
 
 var noContext = context.Background()
@@ -40,6 +42,10 @@ type Collector interface {
 	// IncrServerSetupError keeps a count of errors encountered
 	// when installing software on servers.
 	IncrServerSetupError()
+
+	RegisterKnownInstance(instance *autoscaler.Instance)
+
+	UnregisterKnownInstance(instance *autoscaler.Instance)
 }
 
 // Prometheus is a Prometheus metrics collector.
@@ -50,6 +56,7 @@ type Prometheus struct {
 	countServerCreateErr  prometheus.Counter
 	countServerInitErr    prometheus.Counter
 	countServerSetupErr   prometheus.Counter
+	knownInstance         *prometheus.GaugeVec
 }
 
 // New returns a new Prometheus metrics provider.
@@ -82,12 +89,23 @@ func New() *Prometheus {
 		Name: "drone_server_install_errors_total",
 		Help: "Total number of errors installing software on a server.",
 	})
+	p.knownInstance = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "drone_server_known_instance",
+		Help: "Known server instances.",
+	},
+		[]string{
+			"name",
+			"provider",
+			"region",
+			"size",
+		})
 	prometheus.MustRegister(p.trackServerCreateTime)
 	prometheus.MustRegister(p.trackServerInitTime)
 	prometheus.MustRegister(p.trackServerSetupTime)
 	prometheus.MustRegister(p.countServerCreateErr)
 	prometheus.MustRegister(p.countServerInitErr)
 	prometheus.MustRegister(p.countServerSetupErr)
+	prometheus.MustRegister(p.knownInstance)
 	return p
 }
 
@@ -135,6 +153,26 @@ func (m *Prometheus) IncrServerSetupError() {
 	m.countServerSetupErr.Inc()
 }
 
+// RegisterKnownInstance registers that we know about a server.
+func (m *Prometheus) RegisterKnownInstance(instance *autoscaler.Instance) {
+	m.knownInstance.With(prometheus.Labels{
+		"name":     instance.Name,
+		"provider": string(instance.Provider),
+		"region":   instance.Region,
+		"size":     instance.Size,
+	}).Set(1)
+}
+
+// UnregisterKnownInstance forgets a server we once knew.
+func (m *Prometheus) UnregisterKnownInstance(instance *autoscaler.Instance) {
+	m.knownInstance.Delete(prometheus.Labels{
+		"name":     instance.Name,
+		"provider": string(instance.Provider),
+		"region":   instance.Region,
+		"size":     instance.Size,
+	})
+}
+
 // NopCollector provides a no-op metrics collector.
 type NopCollector struct{}
 
@@ -163,3 +201,9 @@ func (*NopCollector) IncrServerInitError() {}
 // IncrServerSetupError keeps a count of errors encountered
 // when installing software on servers.
 func (*NopCollector) IncrServerSetupError() {}
+
+// RegisterKnownInstance registers that we know about a server.
+func (*NopCollector) RegisterKnownInstance(instance *autoscaler.Instance) {}
+
+// UnregisterKnownInstance forgets a server we once knew.
+func (*NopCollector) UnregisterKnownInstance(instance *autoscaler.Instance) {}

metrics/server_delete.go

@@ -12,7 +12,7 @@ import (
 )
 
 // ServerDelete provides metrics for servers deleted.
-func ServerDelete(provider autoscaler.Provider) autoscaler.Provider {
+func ServerDelete(provider autoscaler.Provider, collector Collector) autoscaler.Provider {
 	created := prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "drone_servers_deleted",
 		Help: "Total number of servers deleted.",
@@ -24,17 +24,19 @@ func ServerDelete(provider autoscaler.Provider) autoscaler.Provider {
 	prometheus.MustRegister(created)
 	prometheus.MustRegister(errors)
 	return &providerWrapDestroy{
-		Provider: provider,
-		created:  created,
-		errors:   errors,
+		Provider:  provider,
+		collector: collector,
+		created:   created,
+		errors:    errors,
 	}
 }
 
 // instruments the Provider to count server destroy events.
 type providerWrapDestroy struct {
 	autoscaler.Provider
-	created prometheus.Counter
-	errors  prometheus.Counter
+	collector Collector
+	created   prometheus.Counter
+	errors    prometheus.Counter
 }
 
 func (p *providerWrapDestroy) Destroy(ctx context.Context, instance *autoscaler.Instance) error {
@@ -44,5 +46,6 @@ func (p *providerWrapDestroy) Destroy(ctx context.Context, instance *autoscaler.
 	} else {
 		p.errors.Add(1)
 	}
+	p.collector.UnregisterKnownInstance(instance)
 	return err
 }