diff --git a/.chloggen/fix_red-metrics.yaml b/.chloggen/fix_red-metrics.yaml new file mode 100755 index 000000000..4421bb9fc --- /dev/null +++ b/.chloggen/fix_red-metrics.yaml @@ -0,0 +1,41 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Use new default metrics namespace/prefix for span RED metrics in Jaeger query. + + +# One or more tracking issues related to the change +issues: [1072] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Use the new RED metrics default namespace `traces.span.metrics` for retrieval from Prometheus. + Since OpenTelemetry Collector version 0.109.0 the default namespace is set to traces.span.metrics. + The namespace taken into account by jaeger-query can be configured via a TempoStack CR entry. + To achieve this the Operator will set the jaeger-query `--prometheus.query.namespace=` flag. + Since Jaeger version 1.62, jaeger-query uses `traces.span.metrics` as default too. + + Example how to restore the default namespace used prior to version `0.109.0`, by configuring an empty value for `redMetricsNamespace` in the TempoStack CR: + ``` + apiVersion: tempo.grafana.com/v1alpha1 + kind: TempoStack + ... + spec: + template: + queryFrontend: + jaegerQuery: + enabled: true + monitorTab: + enabled: true + prometheusEndpoint: "http://myPromInstance:9090" + redMetricsNamespace: "" + ``` + More details can be found here: + - https://github.com/jaegertracing/jaeger/pull/6007 + - https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/34485 diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go index f71123172..8a9103e12 100644 --- a/apis/tempo/v1alpha1/tempostack_types.go +++ b/apis/tempo/v1alpha1/tempostack_types.go @@ -689,6 +689,13 @@ type JaegerQueryMonitor struct { // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Prometheus endpoint" PrometheusEndpoint string `json:"prometheusEndpoint"` + // REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. + // By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0. + // +optional + // +kubebuilder:validation:Optional + // +kubebuilder:default:=traces.span.metrics + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="RED Metric Namespace" + REDMetricsNamespace string `json:"redMetricsNamespace"` } // IngressSpec defines Jaeger Query Ingress options. diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index 103fd3d01..89d1450ae 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -1070,6 +1070,12 @@ spec: on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint + - description: REDMetricsNamespace defines the a prefix used retrieve span rate, + error, and duration (RED) metrics. By default it is set to `traces.span.metrics` + following the default namespace of the OpenTelemetry Collector since Version + 0.109.0. + displayName: RED Metric Namespace + path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override the calculated resources derived from total displayName: Resources diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml index ad90ab922..68eb5046f 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml @@ -2470,6 +2470,12 @@ spec: PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 type: string + redMetricsNamespace: + default: traces.span.metrics + description: |- + REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. + By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0. + type: string type: object resources: description: Resources defines resources for this component, diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index 7a2ead8e5..f58a33ce7 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -1070,6 +1070,12 @@ spec: on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint + - description: REDMetricsNamespace defines the a prefix used retrieve span rate, + error, and duration (RED) metrics. By default it is set to `traces.span.metrics` + following the default namespace of the OpenTelemetry Collector since Version + 0.109.0. + displayName: RED Metric Namespace + path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override the calculated resources derived from total displayName: Resources diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml index ad90ab922..68eb5046f 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml @@ -2470,6 +2470,12 @@ spec: PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 type: string + redMetricsNamespace: + default: traces.span.metrics + description: |- + REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. + By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0. + type: string type: object resources: description: Resources defines resources for this component, diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml index da0c38c5e..61e06807a 100644 --- a/config/crd/bases/tempo.grafana.com_tempostacks.yaml +++ b/config/crd/bases/tempo.grafana.com_tempostacks.yaml @@ -2466,6 +2466,12 @@ spec: PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 type: string + redMetricsNamespace: + default: traces.span.metrics + description: |- + REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. + By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0. + type: string type: object resources: description: Resources defines resources for this component, diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml index 6500bcdf8..c0f246a74 100644 --- a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml @@ -999,6 +999,12 @@ spec: on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint + - description: REDMetricsNamespace defines the a prefix used retrieve span rate, + error, and duration (RED) metrics. By default it is set to `traces.span.metrics` + following the default namespace of the OpenTelemetry Collector since Version + 0.109.0. + displayName: RED Metric Namespace + path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override the calculated resources derived from total displayName: Resources diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml index 4f3f7f264..3215680de 100644 --- a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml @@ -999,6 +999,12 @@ spec: on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint + - description: REDMetricsNamespace defines the a prefix used retrieve span rate, + error, and duration (RED) metrics. By default it is set to `traces.span.metrics` + following the default namespace of the OpenTelemetry Collector since Version + 0.109.0. + displayName: RED Metric Namespace + path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override the calculated resources derived from total displayName: Resources diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml index a7641ae03..b946cb156 100644 --- a/docs/spec/tempo.grafana.com_tempostacks.yaml +++ b/docs/spec/tempo.grafana.com_tempostacks.yaml @@ -342,6 +342,7 @@ spec: # TempoStackSpec defines the desired st monitorTab: # MonitorTab defines the monitor tab configuration. enabled: false # Enabled enables the monitor tab in the Jaeger console. The PrometheusEndpoint must be configured to enable this feature. prometheusEndpoint: "" # PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 + redMetricsNamespace: "traces.span.metrics" # REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0. servicesQueryDuration: "" # ServicesQueryDuration defines how long the services will be available in the services list tempoQuery: # TempoQuery defines options specific to the Tempoo Query component. resources: # Resources defines resources for this component, this will override the calculated resources derived from total diff --git a/internal/manifests/queryfrontend/query_frontend.go b/internal/manifests/queryfrontend/query_frontend.go index d79b53ee7..55d1afe70 100644 --- a/internal/manifests/queryfrontend/query_frontend.go +++ b/internal/manifests/queryfrontend/query_frontend.go @@ -398,6 +398,11 @@ func enableMonitoringTab(tempo v1alpha1.TempoStack, jaegerQueryContainer corev1. // However, we do not intend to support them. // --prometheus.query.normalize-calls // --prometheus.query.normalize-duration + // + // NOTE: Jaeger 1.62 default namespace changed to "traces_span_metrics". + // We fallback to no namespace. + // See https://github.com/jaegertracing/jaeger/pull/6007. + fmt.Sprintf("--prometheus.query.namespace=%s", tempo.Spec.Template.QueryFrontend.JaegerQuery.MonitorTab.REDMetricsNamespace), }, } // If the endpoint matches Prometheus on OpenShift, configure TLS and token based auth diff --git a/internal/manifests/queryfrontend/query_frontend_test.go b/internal/manifests/queryfrontend/query_frontend_test.go index 7b1316df4..9ffb26b6a 100644 --- a/internal/manifests/queryfrontend/query_frontend_test.go +++ b/internal/manifests/queryfrontend/query_frontend_test.go @@ -621,15 +621,16 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) { JaegerQuery: v1alpha1.JaegerQuerySpec{ Enabled: true, MonitorTab: v1alpha1.JaegerQueryMonitor{ - Enabled: true, - PrometheusEndpoint: "http://prometheus:9091", + Enabled: true, + PrometheusEndpoint: "http://prometheus:9091", + REDMetricsNamespace: "test", }, }, }, }, }, }, - args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true"}, + args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", "--prometheus.query.namespace=test"}, env: []corev1.EnvVar{{Name: "METRICS_STORAGE_TYPE", Value: "prometheus"}, {Name: "PROMETHEUS_SERVER_URL", Value: "http://prometheus:9091"}}, }, { @@ -657,6 +658,7 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) { "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", + "--prometheus.query.namespace=", "--prometheus.tls.enabled=true", "--prometheus.token-file=/var/run/secrets/kubernetes.io/serviceaccount/token", "--prometheus.token-override-from-context=false", diff --git a/tests/e2e-openshift/red-metrics/03-install-tempo.yaml b/tests/e2e-openshift/red-metrics/03-install-tempo.yaml index dd8121223..6d8653f9f 100644 --- a/tests/e2e-openshift/red-metrics/03-install-tempo.yaml +++ b/tests/e2e-openshift/red-metrics/03-install-tempo.yaml @@ -28,5 +28,6 @@ spec: monitorTab: enabled: true prometheusEndpoint: https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 + redMetricsNamespace: "" ingress: type: route