Migrate metrics from Opencensus to Opentelemetry (#245)

* migrate metrics to opentelemetry
* move loadtest count metric from controller to proxy

cmd/controller.go

@@ -5,6 +5,14 @@ import (
 	"strings"
 	"time"
 
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
+
+	"go.opentelemetry.io/otel/exporters/prometheus"
+
+	"go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/aggregation"
+
 	"github.com/kelseyhightower/envconfig"
 	"github.com/spf13/cobra"
 	kubeInformers "k8s.io/client-go/informers"
@@ -17,6 +25,10 @@ import (
 	informers "github.com/hellofresh/kangal/pkg/kubernetes/generated/informers/externalversions"
 )
 
+// reconcileDistribution defines the bucket boundaries for the histogram of reconcile latency metric
+// Bucket boundaries are 10ms, 100ms, 1s, 10s, 30s and 60s.
+var reconcileDistribution = []float64{10, 100, 1000, 10000, 30000, 60000}
+
 type controllerCmdOptions struct {
 	kubeConfig           string
 	masterURL            string
@@ -51,9 +63,9 @@ func NewControllerCmd() *cobra.Command {
 				return fmt.Errorf("could not build logger instance: %w", err)
 			}
 
-			pe, err := observability.NewPrometheusExporter("kangal-controller", observability.ControllerViews)
+			pe, err := prometheus.New()
 			if err != nil {
-				return err
+				return fmt.Errorf("could not build prometheus exporter: %w", err)
 			}
 
 			kubeCfg, err := kubernetes.BuildClientConfig(cfg.MasterURL, cfg.KubeConfig, cfg.KubeClientTimeout)
@@ -71,7 +83,17 @@ func NewControllerCmd() *cobra.Command {
 				return fmt.Errorf("error building kangal clientSet: %w", err)
 			}
 
-			statsClient, err := observability.NewStatsReporter("kangal")
+			provider := metric.NewMeterProvider(
+				metric.WithReader(pe),
+				metric.WithResource(
+					resource.NewSchemaless(semconv.ServiceNameKey.String("kangal-controller"))),
+				metric.WithView(metric.NewView(
+					metric.Instrument{Name: "kangal_reconcile_latency"},
+					metric.Stream{Aggregation: aggregation.ExplicitBucketHistogram{
+						Boundaries: reconcileDistribution,
+					}},
+				)))
+			statsReporter, err := controller.NewMetricsReporter(provider.Meter("controller"))
 			if err != nil {
 				return fmt.Errorf("error getting stats client:  %w", err)
 			}
@@ -84,7 +106,7 @@ func NewControllerCmd() *cobra.Command {
 				Exporter:       pe,
 				KubeClient:     kubeClient,
 				KangalClient:   kangalClient,
-				StatsReporter:  statsClient,
+				StatsReporter:  statsReporter,
 				KubeInformer:   kubeInformerFactory,
 				KangalInformer: kangalInformerFactory,
 			})

cmd/proxy.go

@@ -4,6 +4,13 @@ import (
 	"flag"
 	"fmt"
 
+	"go.opentelemetry.io/otel/metric/global"
+
+	"go.opentelemetry.io/otel/exporters/prometheus"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.12.0"
+
 	"github.com/kelseyhightower/envconfig"
 	"github.com/spf13/cobra"
 	kubernetesClient "k8s.io/client-go/kubernetes"
@@ -42,9 +49,9 @@ func NewProxyCmd() *cobra.Command {
 				return fmt.Errorf("could not build logger instance: %w", err)
 			}
 
-			pe, err := observability.NewPrometheusExporter("kangal-proxy", nil)
+			pe, err := prometheus.New()
 			if err != nil {
-				return fmt.Errorf("could not initialise Prometheus exporter: %w", err)
+				return fmt.Errorf("could not build prometheus exporter: %w", err)
 			}
 
 			k8sConfig, err := kubernetes.BuildClientConfig(opts.masterURL, opts.kubeConfig, cfg.KubeClientTimeout)
@@ -65,6 +72,16 @@ func NewProxyCmd() *cobra.Command {
 			loadTestClient := kangalClientSet.LoadTests()
 			kubeClient := kubernetes.NewClient(loadTestClient, kubeClientSet, logger)
 
+			provider := metric.NewMeterProvider(metric.WithReader(pe), metric.WithResource(
+				resource.NewSchemaless(semconv.ServiceNameKey.String("kangal-proxy"))))
+
+			global.SetMeterProvider(provider)
+
+			statsReporter, err := proxy.NewMetricsReporter(provider.Meter("proxy"), kubeClient)
+			if err != nil {
+				return fmt.Errorf("error getting stats client:  %w", err)
+			}
+
 			err = report.InitObjectStorageClient(cfg.Report)
 			if err != nil {
 				return fmt.Errorf("building reportingClient client: %w", err)
@@ -74,9 +91,10 @@ func NewProxyCmd() *cobra.Command {
 			cfg.MasterURL = opts.masterURL
 
 			return proxy.RunServer(cfg, proxy.Runner{
-				Exporter:   pe,
-				KubeClient: kubeClient,
-				Logger:     logger,
+				Exporter:      pe,
+				KubeClient:    kubeClient,
+				Logger:        logger,
+				StatsReporter: statsReporter,
 			})
 		},
 	}

go.mod

@@ -3,20 +3,25 @@ module github.com/hellofresh/kangal
 go 1.19
 
 require (
-	contrib.go.opencensus.io/exporter/prometheus v0.4.2
 	github.com/felixge/httpsnoop v1.0.3
 	github.com/go-chi/chi/v5 v5.0.7
 	github.com/go-chi/render v1.0.2
 	github.com/golang/mock v1.6.0
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/minio/minio-go/v6 v6.0.57
+	github.com/prometheus/client_golang v1.13.0
 	github.com/rs/cors v1.8.2
 	github.com/spf13/afero v1.9.2
 	github.com/spf13/cobra v1.5.0
-	github.com/stretchr/testify v1.8.0
+	github.com/stretchr/testify v1.8.1
 	github.com/technosophos/moniker v0.0.0-20210218184952-3ea787d3943b
 	github.com/tidwall/sjson v1.2.5
-	go.opencensus.io v0.23.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0
+	go.opentelemetry.io/otel v1.11.2
+	go.opentelemetry.io/otel/exporters/prometheus v0.33.0
+	go.opentelemetry.io/otel/metric v0.34.0
+	go.opentelemetry.io/otel/sdk v1.11.2
+	go.opentelemetry.io/otel/sdk/metric v0.34.0
 	go.uber.org/zap v1.23.0
 	k8s.io/api v0.25.1
 	k8s.io/apimachinery v0.25.1
@@ -32,9 +37,8 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
-	github.com/go-kit/log v0.2.1 // indirect
-	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
@@ -60,15 +64,14 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.13.0 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
-	github.com/prometheus/statsd_exporter v0.22.8 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/tidwall/gjson v1.14.3 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
+	go.opentelemetry.io/otel/trace v1.11.2 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 // indirect