Ingest metrics and traces via otel collector

* add firehose enabled property
* otel configuration helper script
* bump minor version

Add needed go dependencies

fix tests

changes to hopefully eliminate data gaps

Author: jbooherl

go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/prometheus/client_model v0.5.0
 	github.com/prometheus/common v0.45.0
 	github.com/prometheus/prometheus v0.47.0
+	go.opentelemetry.io/proto/otlp v1.0.0
 	go.uber.org/atomic v1.11.0
 	go.uber.org/zap v1.26.0
 	golang.org/x/net v0.18.0
@@ -110,6 +111,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/gophercloud/gophercloud v1.5.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/hashicorp/consul/api v1.22.0 // indirect
 	github.com/hashicorp/cronexpr v1.1.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect

go.sum

@@ -828,6 +828,7 @@ github.com/golang/geo v0.0.0-20190916061304-5b978397cfec h1:lJwO/92dFXWeXOZdoGXg
 github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
+github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -970,6 +971,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.1/go.mod h1:G+WkljZi4mflcqVxYSgvt8MNctRQHjEH8ubKtt1Ka3w=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
 github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M=
 github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0=
 github.com/hashicorp/consul/api v1.18.0/go.mod h1:owRRGJ9M5xReDC5nfT8FTJrNAPbT4NM6p/k+d03q2v4=
@@ -1556,6 +1559,8 @@ go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
+go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
+go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
 go.step.sm/crypto v0.16.2 h1:Pr9aazTwWBBZNogUsOqhOrPSdwAa9pPs+lMB602lnDA=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=

jobs/metric-store-nozzle/spec

@@ -7,6 +7,9 @@ templates:
   logs_provider.crt.erb: config/certs/logs_provider.crt
   logs_provider.key.erb: config/certs/logs_provider.key
   logs_provider_ca.crt.erb: config/certs/logs_provider_ca.crt
+  otel_provider.crt.erb: config/certs/otel_provider.crt
+  otel_provider.key.erb: config/certs/otel_provider.key
+  otel_provider_ca.crt.erb: config/certs/otel_provider_ca.crt
   metric_store_client_ca.crt.erb: config/certs/metric_store_client_ca.crt
   metric_store_client.crt.erb: config/certs/metric_store_client.crt
   metric_store_client.key.erb: config/certs/metric_store_client.key
@@ -44,6 +47,16 @@ properties:
   logs_provider.tls.key:
     description: "TLS key for the logs-provider connection"
 
+  otel_provider.tls.ca_cert:
+    description: "CA root required for key/cert verification to connect to the otel-provider"
+  otel_provider.tls.cert:
+    description: "TLS certificate for the otel-provider connection"
+  otel_provider.tls.key:
+    description: "TLS key for the otel-provider connection"
+  otel_port:
+    description: "The port for the Nozzle to bind an otel endpoint"
+    default: 6062
+
   metrics_port:
     description: "The port for the Nozzle to bind a prometheus metrics endpoint"
     default: 6061
@@ -63,3 +76,6 @@ properties:
   enable_envelope_selector:
     description: Enable envelopes with specified tags. If enabled, only envelopes with specific tags will keep. By default envelopes with app_id and applicationGuid tags are enabled.
     default: false
+  firehose_enabled:
+    description: If enabled, connect to the firehose.
+    default: false

jobs/metric-store-nozzle/templates/bpm.yml.erb

@@ -30,6 +30,12 @@ processes:
     LOGS_PROVIDER_CERT_PATH: "<%= "#{cert_dir}/logs_provider.crt" %>"
     LOGS_PROVIDER_KEY_PATH:  "<%= "#{cert_dir}/logs_provider.key" %>"
 
+    # Otel Provider
+    OTEL_ADDR:               "<%= "0.0.0.0:#{p('otel_port')}" %>"
+    OTEL_PROVIDER_CA_PATH:   "<%= "#{cert_dir}/otel_provider_ca.crt" %>"
+    OTEL_PROVIDER_CERT_PATH: "<%= "#{cert_dir}/otel_provider.crt" %>"
+    OTEL_PROVIDER_KEY_PATH:  "<%= "#{cert_dir}/otel_provider.key" %>"
+
     # Metric Store Client Mutual TLS
     METRIC_STORE_CLIENT_CA_PATH:   "<%= "#{cert_dir}/metric_store_client_ca.crt" %>"
     METRIC_STORE_CLIENT_CERT_PATH: "<%= "#{cert_dir}/metric_store_client.crt" %>"
@@ -40,9 +46,9 @@ processes:
     METRIC_STORE_METRICS_CERT_PATH: "<%= "#{cert_dir}/metric_store_metrics.crt" %>"
     METRIC_STORE_METRICS_KEY_PATH:  "<%= "#{cert_dir}/metric_store_metrics.key" %>"
 
-    GODEBUG: "x509ignoreCN=0" # temporary (15Sep2020) while we wait for upstream (firehose) certs to be updated
     GRPC_GO_LOG_SEVERITY_LEVEL: "warning"
     GRPC_GO_LOG_VERBOSITY_LEVEL: "1"
     ENABLE_ENVELOPE_SELECTOR: "<%= p('enable_envelope_selector') %>"
+    FIREHOSE_ENABLED: "<%= p('firehose_enabled') %>"
   limits:
     open_files: 8192

jobs/metric-store-nozzle/templates/otel_provider.crt.erb

@@ -0,0 +1 @@
+<%= p("otel_provider.tls.cert") %>

jobs/metric-store-nozzle/templates/otel_provider.key.erb

@@ -0,0 +1 @@
+<%= p("otel_provider.tls.key") %>

jobs/metric-store-nozzle/templates/otel_provider_ca.crt.erb

@@ -0,0 +1 @@
+<%= p("otel_provider.tls.ca_cert") %>