diff --git a/cmd/controller.go b/cmd/controller.go index 5821963c..00358a8d 100644 --- a/cmd/controller.go +++ b/cmd/controller.go @@ -5,6 +5,14 @@ import ( "strings" "time" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.12.0" + + "go.opentelemetry.io/otel/exporters/prometheus" + + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/aggregation" + "github.com/kelseyhightower/envconfig" "github.com/spf13/cobra" kubeInformers "k8s.io/client-go/informers" @@ -17,6 +25,10 @@ import ( informers "github.com/hellofresh/kangal/pkg/kubernetes/generated/informers/externalversions" ) +// reconcileDistribution defines the bucket boundaries for the histogram of reconcile latency metric +// Bucket boundaries are 10ms, 100ms, 1s, 10s, 30s and 60s. +var reconcileDistribution = []float64{10, 100, 1000, 10000, 30000, 60000} + type controllerCmdOptions struct { kubeConfig string masterURL string @@ -51,9 +63,9 @@ func NewControllerCmd() *cobra.Command { return fmt.Errorf("could not build logger instance: %w", err) } - pe, err := observability.NewPrometheusExporter("kangal-controller", observability.ControllerViews) + pe, err := prometheus.New() if err != nil { - return err + return fmt.Errorf("could not build prometheus exporter: %w", err) } kubeCfg, err := kubernetes.BuildClientConfig(cfg.MasterURL, cfg.KubeConfig, cfg.KubeClientTimeout) @@ -71,7 +83,17 @@ func NewControllerCmd() *cobra.Command { return fmt.Errorf("error building kangal clientSet: %w", err) } - statsClient, err := observability.NewStatsReporter("kangal") + provider := metric.NewMeterProvider( + metric.WithReader(pe), + metric.WithResource( + resource.NewSchemaless(semconv.ServiceNameKey.String("kangal-controller"))), + metric.WithView(metric.NewView( + metric.Instrument{Name: "kangal_reconcile_latency"}, + metric.Stream{Aggregation: aggregation.ExplicitBucketHistogram{ + Boundaries: reconcileDistribution, + }}, + ))) + statsReporter, err := controller.NewMetricsReporter(provider.Meter("controller")) if err != nil { return fmt.Errorf("error getting stats client: %w", err) } @@ -84,7 +106,7 @@ func NewControllerCmd() *cobra.Command { Exporter: pe, KubeClient: kubeClient, KangalClient: kangalClient, - StatsReporter: statsClient, + StatsReporter: statsReporter, KubeInformer: kubeInformerFactory, KangalInformer: kangalInformerFactory, }) diff --git a/cmd/proxy.go b/cmd/proxy.go index a971699a..8a5439f6 100644 --- a/cmd/proxy.go +++ b/cmd/proxy.go @@ -4,6 +4,13 @@ import ( "flag" "fmt" + "go.opentelemetry.io/otel/metric/global" + + "go.opentelemetry.io/otel/exporters/prometheus" + "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/resource" + semconv "go.opentelemetry.io/otel/semconv/v1.12.0" + "github.com/kelseyhightower/envconfig" "github.com/spf13/cobra" kubernetesClient "k8s.io/client-go/kubernetes" @@ -42,9 +49,9 @@ func NewProxyCmd() *cobra.Command { return fmt.Errorf("could not build logger instance: %w", err) } - pe, err := observability.NewPrometheusExporter("kangal-proxy", nil) + pe, err := prometheus.New() if err != nil { - return fmt.Errorf("could not initialise Prometheus exporter: %w", err) + return fmt.Errorf("could not build prometheus exporter: %w", err) } k8sConfig, err := kubernetes.BuildClientConfig(opts.masterURL, opts.kubeConfig, cfg.KubeClientTimeout) @@ -65,6 +72,16 @@ func NewProxyCmd() *cobra.Command { loadTestClient := kangalClientSet.LoadTests() kubeClient := kubernetes.NewClient(loadTestClient, kubeClientSet, logger) + provider := metric.NewMeterProvider(metric.WithReader(pe), metric.WithResource( + resource.NewSchemaless(semconv.ServiceNameKey.String("kangal-proxy")))) + + global.SetMeterProvider(provider) + + statsReporter, err := proxy.NewMetricsReporter(provider.Meter("proxy"), kubeClient) + if err != nil { + return fmt.Errorf("error getting stats client: %w", err) + } + err = report.InitObjectStorageClient(cfg.Report) if err != nil { return fmt.Errorf("building reportingClient client: %w", err) @@ -74,9 +91,10 @@ func NewProxyCmd() *cobra.Command { cfg.MasterURL = opts.masterURL return proxy.RunServer(cfg, proxy.Runner{ - Exporter: pe, - KubeClient: kubeClient, - Logger: logger, + Exporter: pe, + KubeClient: kubeClient, + Logger: logger, + StatsReporter: statsReporter, }) }, } diff --git a/go.mod b/go.mod index f2667ae5..63ad7fd3 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,25 @@ module github.com/hellofresh/kangal go 1.19 require ( - contrib.go.opencensus.io/exporter/prometheus v0.4.2 github.com/felixge/httpsnoop v1.0.3 github.com/go-chi/chi/v5 v5.0.7 github.com/go-chi/render v1.0.2 github.com/golang/mock v1.6.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/minio/minio-go/v6 v6.0.57 + github.com/prometheus/client_golang v1.13.0 github.com/rs/cors v1.8.2 github.com/spf13/afero v1.9.2 github.com/spf13/cobra v1.5.0 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 github.com/technosophos/moniker v0.0.0-20210218184952-3ea787d3943b github.com/tidwall/sjson v1.2.5 - go.opencensus.io v0.23.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0 + go.opentelemetry.io/otel v1.11.2 + go.opentelemetry.io/otel/exporters/prometheus v0.33.0 + go.opentelemetry.io/otel/metric v0.34.0 + go.opentelemetry.io/otel/sdk v1.11.2 + go.opentelemetry.io/otel/sdk/metric v0.34.0 go.uber.org/zap v1.23.0 k8s.io/api v0.25.1 k8s.io/apimachinery v0.25.1 @@ -32,9 +37,8 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -60,15 +64,14 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.13.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - github.com/prometheus/statsd_exporter v0.22.8 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/gjson v1.14.3 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect + go.opentelemetry.io/otel/trace v1.11.2 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 // indirect diff --git a/go.sum b/go.sum index b0263e11..2ddf1451 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= -contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= -contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -48,7 +46,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -102,17 +99,17 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= -github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -169,10 +166,8 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -285,7 +280,6 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -297,7 +291,6 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -307,9 +300,6 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= -github.com/prometheus/statsd_exporter v0.22.8 h1:Qo2D9ZzaQG+id9i5NYNGmbf1aa/KxKbB9aKfMS+Yib0= -github.com/prometheus/statsd_exporter v0.22.8/go.mod h1:/DzwbTEaFTE0Ojz5PqcSk6+PFHOPWGxdXVr6yC8eFOM= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= @@ -332,16 +322,16 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/technosophos/moniker v0.0.0-20210218184952-3ea787d3943b h1:fo0GUa0B+vxSZ8bgnL3fpCPHReM/QPlALdak9T/Zw5Y= github.com/technosophos/moniker v0.0.0-20210218184952-3ea787d3943b/go.mod h1:O1c8HleITsZqzNZDjSNzirUGsMT0oGu9LhHKoJrqO+A= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -368,8 +358,20 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0 h1:yt2NKzK7Vyo6h0+X8BA4FpreZQTlVEIarnsBP/H5mzs= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0/go.mod h1:+ARmXlUlc51J7sZeCBkBJNdHGySrdOzgzxp6VWRWM1U= +go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0= +go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI= +go.opentelemetry.io/otel/exporters/prometheus v0.33.0 h1:xXhPj7SLKWU5/Zd4Hxmd+X1C4jdmvc0Xy+kvjFx2z60= +go.opentelemetry.io/otel/exporters/prometheus v0.33.0/go.mod h1:ZSmYfKdYWEdSDBB4njLBIwTf4AU2JNsH3n2quVQDebI= +go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8= +go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8= +go.opentelemetry.io/otel/sdk v1.11.2 h1:GF4JoaEx7iihdMFu30sOyRx52HDHOkl9xQ8SMqNXUiU= +go.opentelemetry.io/otel/sdk v1.11.2/go.mod h1:wZ1WxImwpq+lVRo4vsmSOxdd+xwoUJ6rqyLc3SyX9aU= +go.opentelemetry.io/otel/sdk/metric v0.34.0 h1:7ElxfQpXCFZlRTvVRTkcUvK8Gt5DC8QzmzsLsO2gdzo= +go.opentelemetry.io/otel/sdk/metric v0.34.0/go.mod h1:l4r16BIqiqPy5rd14kkxllPy/fOI4tWo1jkpD9Z3ffQ= +go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0= +go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= @@ -457,7 +459,6 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -492,7 +493,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -540,9 +540,7 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -719,7 +717,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index ed1bfa0a..57e25dfd 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -3,26 +3,25 @@ package controller import ( "fmt" - "contrib.go.opencensus.io/exporter/prometheus" "go.uber.org/zap" kubeInformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "github.com/hellofresh/kangal/pkg/backends" - "github.com/hellofresh/kangal/pkg/core/observability" clientSetV "github.com/hellofresh/kangal/pkg/kubernetes/generated/clientset/versioned" "github.com/hellofresh/kangal/pkg/kubernetes/generated/informers/externalversions" + otelPrometheus "go.opentelemetry.io/otel/exporters/prometheus" ) // Runner encapsulates all Kangal Controller dependencies type Runner struct { - Exporter *prometheus.Exporter + Exporter *otelPrometheus.Exporter Logger *zap.Logger KubeClient kubernetes.Interface KangalClient *clientSetV.Clientset KubeInformer kubeInformers.SharedInformerFactory KangalInformer externalversions.SharedInformerFactory - StatsReporter observability.StatsReporter + StatsReporter *MetricsReporter } // Run runs an instance of kubernetes kubeController @@ -39,7 +38,7 @@ func Run(cfg Config, rr Runner) error { backends.WithTolerations(cfg.Tolerations.KubeToleration()), ) - c := NewController(cfg, rr.KubeClient, rr.KangalClient, rr.KubeInformer, rr.KangalInformer, rr.StatsReporter, registry, rr.Logger) + c := NewController(cfg, rr.KubeClient, rr.KangalClient, rr.KubeInformer, rr.KangalInformer, *rr.StatsReporter, registry, rr.Logger) // notice that there is no need to run Start methods in a separate goroutine. (i.e. go kubeInformerFactory.Start(stopCh) // Start method is non-blocking and runs all registered informers in a dedicated goroutine. diff --git a/pkg/controller/loadtest.go b/pkg/controller/loadtest.go index 277e9b1e..f808ec21 100644 --- a/pkg/controller/loadtest.go +++ b/pkg/controller/loadtest.go @@ -5,7 +5,12 @@ import ( "fmt" "time" - "go.opencensus.io/stats" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/metric/instrument" + "go.opentelemetry.io/otel/metric/instrument/syncint64" + "go.opentelemetry.io/otel/metric/unit" + "go.uber.org/zap" batchV1 "k8s.io/api/batch/v1" coreV1 "k8s.io/api/core/v1" @@ -23,7 +28,6 @@ import ( "k8s.io/client-go/util/workqueue" "github.com/hellofresh/kangal/pkg/backends" - "github.com/hellofresh/kangal/pkg/core/observability" loadTestV1 "github.com/hellofresh/kangal/pkg/kubernetes/apis/loadtest/v1" clientSetV "github.com/hellofresh/kangal/pkg/kubernetes/generated/clientset/versioned" sampleScheme "github.com/hellofresh/kangal/pkg/kubernetes/generated/clientset/versioned/scheme" @@ -37,6 +41,52 @@ const ( trueString = "true" ) +// MetricsReporter used to interface with the metrics configurations +type MetricsReporter struct { + workQueueDepthStat syncint64.UpDownCounter + reconcileCountStat syncint64.UpDownCounter + reconcileLatencyStat syncint64.Histogram +} + +// NewMetricsReporter contains loadtest metrics definition +func NewMetricsReporter(meter metric.Meter) (*MetricsReporter, error) { + workQueueDepthStat, err := meter.SyncInt64().UpDownCounter( + "kangal_work_queue_depth", + instrument.WithDescription("Depth of the work queue"), + instrument.WithUnit(unit.Dimensionless), + ) + if err != nil { + fmt.Errorf("could not register workQueueDepthStat metric: %w", err) + return nil, err + } + + reconcileCountStat, err := meter.SyncInt64().UpDownCounter( + "kangal_reconcile_count", + instrument.WithDescription("Number of reconcile operations"), + instrument.WithUnit(unit.Dimensionless), + ) + if err != nil { + fmt.Errorf("could not register reconcileCountStat metric: %w", err) + return nil, err + } + + reconcileLatencyStat, err := meter.SyncInt64().Histogram( + "kangal_reconcile_latency", + instrument.WithDescription("Latency of reconcile operations"), + instrument.WithUnit(unit.Milliseconds), + ) + if err != nil { + fmt.Errorf("could not register reconcileLatencyStat metric: %w", err) + return nil, err + } + + return &MetricsReporter{ + workQueueDepthStat: workQueueDepthStat, + reconcileCountStat: reconcileCountStat, + reconcileLatencyStat: reconcileLatencyStat, + }, nil +} + // Controller is the controller implementation for LoadTest resources type Controller struct { cfg Config @@ -62,7 +112,7 @@ type Controller struct { // Kubernetes API. recorder record.EventRecorder - statsClient observability.StatsReporter + statsClient MetricsReporter registry backends.Registry logger *zap.Logger @@ -75,7 +125,7 @@ func NewController( kangalClientSet clientSetV.Interface, kubeInformerFactory informers.SharedInformerFactory, kangalInformerFactory externalversions.SharedInformerFactory, - statsClient observability.StatsReporter, + statsClient MetricsReporter, registry backends.Registry, logger *zap.Logger) *Controller { @@ -209,9 +259,9 @@ func (c *Controller) processNextWorkItem() bool { } // Send the metrics for the current queue depth - c.statsClient.ReportQueueDepth(int64(c.workQueue.Len())) + c.statsClient.workQueueDepthStat.Add(context.Background(), int64(c.workQueue.Len())) - // We wrap this block in a func so we can defer c.workQueue.Done. + // We wrap this block in a func, so we can defer c.workQueue.Done. err := func(obj interface{}) error { startTime := time.Now() @@ -231,7 +281,9 @@ func (c *Controller) processNextWorkItem() bool { if err != nil { status = falseString } - c.statsClient.ReportReconcile(time.Since(startTime), key, status) + + c.statsClient.reconcileCountStat.Add(context.Background(), 1, attribute.String("key", key), attribute.String("success", status)) + c.statsClient.reconcileLatencyStat.Record(context.Background(), int64(time.Since(startTime)/time.Millisecond), attribute.String("key", key), attribute.String("success", status)) }() // We expect strings to come off the workQueue. These are of the @@ -277,7 +329,6 @@ func (c *Controller) syncHandler(key string) error { ctx, cancel := context.WithTimeout(context.Background(), c.cfg.SyncHandlerTimeout) defer cancel() - stats.Record(ctx, observability.MRunningLoadtestCountStat.M(c.countRunningLoadtests())) logger := c.logger.With( zap.String("loadtest", key), ) @@ -384,7 +435,6 @@ func (c *Controller) handleObject(obj interface{}) { } c.logger.Debug("Processing object", zap.String("object-name", object.GetName())) - stats.Record(context.Background(), observability.MRunningLoadtestCountStat.M(c.countRunningLoadtests())) foo, err := c.loadtestsLister.Get(ownerRef.Name) if err != nil { @@ -507,24 +557,6 @@ func checkLoadTestLifeTimeExceeded(loadTest *loadTestV1.LoadTest, deleteThreshol return false } -func (c *Controller) countRunningLoadtests() int64 { - tt, err := c.kangalClientSet.KangalV1().LoadTests().List(context.Background(), metaV1.ListOptions{}) - if err != nil { - c.logger.Error("Couldn't list existing loadtests", zap.Error(err)) - return 0 - } - - var rt = 0 - for _, loadTest := range tt.Items { - if loadTest.Status.Phase == loadTestV1.LoadTestRunning { - rt++ - } - } - - return int64(rt) - -} - func (c *Controller) deleteLoadTest(ctx context.Context, key string, loadTest *loadTestV1.LoadTest) { err := c.kangalClientSet.KangalV1().LoadTests().Delete(ctx, loadTest.Name, metaV1.DeleteOptions{}) if err == nil { diff --git a/pkg/controller/metrics.go b/pkg/controller/metrics.go index f042c723..c5057561 100644 --- a/pkg/controller/metrics.go +++ b/pkg/controller/metrics.go @@ -4,10 +4,11 @@ import ( "fmt" "net/http" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/render" - "go.opencensus.io/plugin/ochttp" "go.uber.org/zap" cHttp "github.com/hellofresh/kangal/pkg/core/http" @@ -26,7 +27,7 @@ func RunMetricsServer(cfg Config, rr Runner, stopChan chan struct{}) error { // Register Routes r.Get("/", cHttp.LivenessHandler("Kangal Controller")) r.Get("/status", cHttp.LivenessHandler("Kangal Controller")) - r.Handle("/metrics", rr.Exporter) + r.Handle("/metrics", promhttp.Handler()) // Run HTTP Server address := fmt.Sprintf(":%d", cfg.HTTPPort) @@ -34,7 +35,7 @@ func RunMetricsServer(cfg Config, rr Runner, stopChan chan struct{}) error { go func() { // Try and run http server, fail on error - if err := http.ListenAndServe(address, &ochttp.Handler{Handler: r}); err != nil { + if err := http.ListenAndServe(address, promhttp.Handler()); err != nil { rr.Logger.Error("Failed to run HTTP server", zap.Error(err)) close(stopChan) } diff --git a/pkg/core/observability/controller.go b/pkg/core/observability/controller.go deleted file mode 100644 index 24cff01b..00000000 --- a/pkg/core/observability/controller.go +++ /dev/null @@ -1,118 +0,0 @@ -package observability - -import ( - "context" - "errors" - "time" - - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" - "go.opencensus.io/tag" -) - -var ( - workQueueDepthStat = stats.Int64("work_queue_depth", "Depth of the work queue", stats.UnitDimensionless) - reconcileCountStat = stats.Int64("reconcile_count", "Number of reconcile operations", stats.UnitDimensionless) - reconcileLatencyStat = stats.Int64("reconcile_latency", "Latency of reconcile operations", stats.UnitMilliseconds) - - // MRunningLoadtestCountStat counts the number of running loadtests - MRunningLoadtestCountStat = stats.Int64("running_loadtests_count", "Number of running loadtests", stats.UnitDimensionless) - - // reconcileDistribution defines the bucket boundaries for the histogram of reconcile latency metric. - // Bucket boundaries are 10ms, 100ms, 1s, 10s, 30s and 60s. - reconcileDistribution = view.Distribution(10, 100, 1000, 10000, 30000, 60000) - - // Create the tag keys that will be used to add tags to our measurements. - reconcilerTagKey = mustNewTagKey("reconciler") - keyTagKey = mustNewTagKey("key") - successTagKey = mustNewTagKey("success") -) - -// ControllerViews are the views needed to be registered for getting metrics -// from the kangal controller -var ControllerViews = []*view.View{ - { - Description: "Depth of the work queue", - Measure: workQueueDepthStat, - Aggregation: view.LastValue(), - TagKeys: []tag.Key{reconcilerTagKey}, - }, - { - Description: "Number of reconcile operations", - Measure: reconcileCountStat, - Aggregation: view.Count(), - TagKeys: []tag.Key{reconcilerTagKey, keyTagKey, successTagKey}, - }, - { - Description: "Latency of reconcile operations", - Measure: reconcileLatencyStat, - Aggregation: reconcileDistribution, - TagKeys: []tag.Key{reconcilerTagKey, keyTagKey, successTagKey}, - }, - { - Description: MRunningLoadtestCountStat.Description(), - Measure: MRunningLoadtestCountStat, - Aggregation: view.LastValue(), - }, -} - -// StatsReporter defines the interface for sending metrics -type StatsReporter interface { - // ReportQueueDepth reports the queue depth metric - ReportQueueDepth(v int64) error - - // ReportReconcile reports the count and latency metrics for a reconcile operation - ReportReconcile(duration time.Duration, key, success string) error -} - -// Reporter holds cached metric objects to report metrics -type reporter struct { - reconciler string - globalCtx context.Context -} - -// NewStatsReporter creates a reporter that collects and reports metrics -func NewStatsReporter(reconciler string) (StatsReporter, error) { - // Reconciler tag is static. Create a context containing that and cache it. - ctx, err := tag.New( - context.Background(), - tag.Insert(reconcilerTagKey, reconciler)) - if err != nil { - return nil, err - } - - return &reporter{reconciler: reconciler, globalCtx: ctx}, nil -} - -// ReportQueueDepth reports the queue depth metric -func (r *reporter) ReportQueueDepth(v int64) error { - if r.globalCtx == nil { - return errors.New("reporter is not initialized correctly") - } - stats.Record(r.globalCtx, workQueueDepthStat.M(v)) - return nil -} - -// ReportReconcile reports the count and latency metrics for a reconcile operation -func (r *reporter) ReportReconcile(duration time.Duration, key, success string) error { - ctx, err := tag.New( - context.Background(), - tag.Insert(reconcilerTagKey, r.reconciler), - tag.Insert(keyTagKey, key), - tag.Insert(successTagKey, success)) - if err != nil { - return err - } - - stats.Record(ctx, reconcileCountStat.M(1)) - stats.Record(ctx, reconcileLatencyStat.M(int64(duration/time.Millisecond))) - return nil -} - -func mustNewTagKey(s string) tag.Key { - tagKey, err := tag.NewKey(s) - if err != nil { - panic(err) - } - return tagKey -} diff --git a/pkg/core/observability/metrics.go b/pkg/core/observability/metrics.go deleted file mode 100644 index 7b311ada..00000000 --- a/pkg/core/observability/metrics.go +++ /dev/null @@ -1,49 +0,0 @@ -package observability - -import ( - "fmt" - "strings" - "time" - - "contrib.go.opencensus.io/exporter/prometheus" - "go.opencensus.io/plugin/ochttp" - "go.opencensus.io/stats/view" - "go.opencensus.io/tag" -) - -// NewPrometheusExporter builds and configures Prometheus exporter -func NewPrometheusExporter(serviceName string, serviceViews []*view.View) (*prometheus.Exporter, error) { - prometheusNS := strings.ReplaceAll(serviceName, "-", "_") - - prometheusExporter, err := prometheus.NewExporter(prometheus.Options{Namespace: prometheusNS}) - if err != nil { - return nil, err - } - - view.RegisterExporter(prometheusExporter) - view.SetReportingPeriod(time.Second) - - ocHTTPServerViews := []*view.View{ - ochttp.ServerRequestCountView, - ochttp.ServerRequestBytesView, - ochttp.ServerResponseBytesView, - ochttp.ServerLatencyView, - ochttp.ServerRequestCountByMethod, - ochttp.ServerResponseCountByStatusCode, - { - Name: "opencensus.io/http/server/latency_by_path", - Description: "Latency distribution of HTTP requests by route", - TagKeys: []tag.Key{ochttp.KeyServerRoute}, - Measure: ochttp.ServerLatency, - Aggregation: ochttp.DefaultLatencyDistribution, - }, - } - - vv := append(ocHTTPServerViews, serviceViews...) - - if err := view.Register(vv...); err != nil { - return nil, fmt.Errorf("could not register prometheus server views: %w", err) - } - - return prometheusExporter, nil -} diff --git a/pkg/kubernetes/client.go b/pkg/kubernetes/client.go index c6ba42e5..612d1c53 100644 --- a/pkg/kubernetes/client.go +++ b/pkg/kubernetes/client.go @@ -247,3 +247,21 @@ func BuildClientConfig(masterURL string, kubeConfigPath string, timeout time.Dur kubeCfg.Timeout = timeout return kubeCfg, nil } + +// CountRunningLoadtests used in metrics to report running loadtests +func (c *Client) CountRunningLoadtests() int64 { + tt, err := c.ltClient.List(context.Background(), metaV1.ListOptions{}) + if err != nil { + c.logger.Error("Couldn't list existing loadtests", zap.Error(err)) + return 0 + } + + var rt = 0 + for _, loadTest := range tt.Items { + if loadTest.Status.Phase == apisLoadTestV1.LoadTestRunning { + rt++ + } + } + + return int64(rt) +} diff --git a/pkg/proxy/proxy.go b/pkg/proxy/proxy.go index f9928377..bdde00e8 100644 --- a/pkg/proxy/proxy.go +++ b/pkg/proxy/proxy.go @@ -7,6 +7,12 @@ import ( "io" "net/http" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/metric/instrument" + "go.opentelemetry.io/otel/metric/instrument/asyncint64" + "go.opentelemetry.io/otel/metric/unit" + "github.com/go-chi/chi/v5" "github.com/go-chi/render" "go.uber.org/zap" @@ -33,6 +39,34 @@ type Proxy struct { allowedCustomImages bool } +// MetricsReporter used to interface with the metrics configurations +type MetricsReporter struct { + countRunningLoadtests asyncint64.UpDownCounter +} + +// NewMetricsReporter contains loadtest metrics definition +func NewMetricsReporter(meter metric.Meter, kubeClient *kube.Client) (*MetricsReporter, error) { + countRunningLoadtests, err := meter.AsyncInt64().UpDownCounter( + "kangal_running_loadtests_count", + instrument.WithDescription("The number of currently running loadtests"), + instrument.WithUnit(unit.Dimensionless), + ) + if err != nil { + return nil, fmt.Errorf("could not register countRunningLoadtests metric: %w", err) + } + + if err := meter.RegisterCallback([]instrument.Asynchronous{countRunningLoadtests}, func(ctx context.Context) { + lt := kubeClient.CountRunningLoadtests() + countRunningLoadtests.Observe(ctx, lt, attribute.String("loadtest", "running")) + }, + ); err != nil { + return nil, err + } + return &MetricsReporter{ + countRunningLoadtests: countRunningLoadtests}, + nil +} + // NewProxy returns new Proxy handlers func NewProxy(maxLoadTestsRun int, registry backends.Registry, kubeClient *kube.Client, maxListLimit int64, allowedCustomImages bool) *Proxy { return &Proxy{ diff --git a/pkg/proxy/server.go b/pkg/proxy/server.go index 76ad894a..5e503846 100644 --- a/pkg/proxy/server.go +++ b/pkg/proxy/server.go @@ -4,11 +4,13 @@ import ( "fmt" "net/http" - "contrib.go.opencensus.io/exporter/prometheus" + "go.opentelemetry.io/otel/metric/global" + "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/render" - "go.opencensus.io/plugin/ochttp" + "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.uber.org/zap" "github.com/hellofresh/kangal/pkg/backends" @@ -16,13 +18,15 @@ import ( mPkg "github.com/hellofresh/kangal/pkg/core/middleware" kube "github.com/hellofresh/kangal/pkg/kubernetes" "github.com/hellofresh/kangal/pkg/report" + otelPrometheus "go.opentelemetry.io/otel/exporters/prometheus" ) // Runner encapsulates all Kangal Proxy API server dependencies type Runner struct { - Exporter *prometheus.Exporter - KubeClient *kube.Client - Logger *zap.Logger + Exporter *otelPrometheus.Exporter + KubeClient *kube.Client + Logger *zap.Logger + StatsReporter *MetricsReporter } // RunServer runs Kangal proxy API @@ -45,7 +49,7 @@ func RunServer(cfg Config, rr Runner) error { r.Use(OpenAPISpecCORSMiddleware(cfg.OpenAPI)) r.Get("/status", cHttp.LivenessHandler("Kangal Proxy")) - r.Handle("/metrics", rr.Exporter) + r.Handle("/metrics", promhttp.Handler()) // ---------------------------------------------------------------------- // // LoadTest Proxy CRUD @@ -55,22 +59,22 @@ func RunServer(cfg Config, rr Runner) error { r.Method(http.MethodGet, loadtestRoute, - ochttp.WithRouteTag(http.HandlerFunc(proxyHandler.List), loadtestRoute), + otelhttp.NewHandler(http.HandlerFunc(proxyHandler.List), loadtestRoute), ) r.Method(http.MethodPost, loadtestRoute, - ochttp.WithRouteTag(http.HandlerFunc(proxyHandler.Create), loadtestRoute), + otelhttp.NewHandler(http.HandlerFunc(proxyHandler.Create), loadtestRoute), ) r.Method(http.MethodGet, loadtestRouteWithID, - ochttp.WithRouteTag(http.HandlerFunc(proxyHandler.Get), loadtestRouteWithID), + otelhttp.NewHandler(http.HandlerFunc(proxyHandler.Get), loadtestRouteWithID), ) r.Method(http.MethodDelete, loadtestRouteWithID, - ochttp.WithRouteTag(http.HandlerFunc(proxyHandler.Delete), loadtestRouteWithID), + otelhttp.NewHandler(http.HandlerFunc(proxyHandler.Delete), loadtestRouteWithID), ) // ---------------------------------------------------------------------- // @@ -96,7 +100,7 @@ func RunServer(cfg Config, rr Runner) error { rr.Logger.Info("Running HTTP server...", zap.String("address", address)) // Try and run http server, fail on error - err := http.ListenAndServe(address, &ochttp.Handler{Handler: r}) + err := http.ListenAndServe(address, otelhttp.NewHandler(r, "kangal", otelhttp.WithMeterProvider(global.MeterProvider()), otelhttp.WithMessageEvents(otelhttp.ReadEvents, otelhttp.WriteEvents))) if err != nil { return fmt.Errorf("failed to run HTTP server: %w", err) }