diff --git a/components/shared/K8sNextSteps.tsx b/components/shared/K8sNextSteps.tsx index 3d742666e1..d95b97fdef 100644 --- a/components/shared/K8sNextSteps.tsx +++ b/components/shared/K8sNextSteps.tsx @@ -9,7 +9,7 @@ export default function K8sNextSteps() {
  • - + Use OpenTelemetry Operator for automatic instrumentation
  • diff --git a/constants/listicles/dashboard-templates.json b/constants/listicles/dashboard-templates.json index 748b765b64..a40c5b3ce6 100644 --- a/constants/listicles/dashboard-templates.json +++ b/constants/listicles/dashboard-templates.json @@ -445,6 +445,12 @@ "clickName": "OpenRouter Dashboard Template", "icon": "/svgs/icons/LLMMonitoring/openrouter-logo.webp" }, + { + "name": "OpenTelemetry Collector", + "href": "/docs/dashboards/dashboard-templates/opentelemetry-collector-dashboard", + "clickName": "OpenTelemetry Collector Dashboard Template", + "icon": "/img/icons/listicle/si-opentelemetry.svg" + }, { "name": "PgBouncer", "href": "/docs/dashboards/dashboard-templates/pgbouncer-dashboard", diff --git a/constants/listicles/metrics-quick-start.json b/constants/listicles/metrics-quick-start.json index 39fbc5e91a..7d4f8df2a8 100644 --- a/constants/listicles/metrics-quick-start.json +++ b/constants/listicles/metrics-quick-start.json @@ -186,6 +186,12 @@ "href": "/docs/metrics-management/keda-metrics", "clickName": "KEDA Metrics Link", "icon": "/img/icons/listicle/lucide-zap-purple.svg" + }, + { + "name": "OpenTelemetry Collector", + "href": "/docs/metrics-management/opentelemetry-collector-metrics", + "clickName": "OpenTelemetry Collector Metrics Link", + "icon": "/img/icons/listicle/si-opentelemetry.svg" } ] }, diff --git a/data-assets/img/docs/dashboards/dashboard-templates/opentelemetry-collector-dashboard.webp b/data-assets/img/docs/dashboards/dashboard-templates/opentelemetry-collector-dashboard.webp new file mode 100644 index 0000000000..99d765c30a Binary files /dev/null and b/data-assets/img/docs/dashboards/dashboard-templates/opentelemetry-collector-dashboard.webp differ diff --git a/data-assets/img/docs/metrics/otel-collector/sample-metrics.webp b/data-assets/img/docs/metrics/otel-collector/sample-metrics.webp new file mode 100644 index 0000000000..985fbf1d2e Binary files /dev/null and b/data-assets/img/docs/metrics/otel-collector/sample-metrics.webp differ diff --git a/data/docs-side-nav/main.json b/data/docs-side-nav/main.json index e7a4614755..6db1364140 100644 --- a/data/docs-side-nav/main.json +++ b/data/docs-side-nav/main.json @@ -1936,6 +1936,11 @@ "type": "doc", "route": "/docs/metrics-management/keda-metrics", "label": "KEDA Metrics" + }, + { + "type": "doc", + "route": "/docs/metrics-management/opentelemetry-collector-metrics", + "label": "OpenTelemetry Collector Metrics" } ] }, @@ -2641,6 +2646,11 @@ "route": "/docs/dashboards/dashboard-templates/openrouter-dashboard", "label": "OpenRouter" }, + { + "type": "doc", + "route": "/docs/dashboards/dashboard-templates/opentelemetry-collector-dashboard", + "label": "OpenTelemetry Collector" + }, { "type": "doc", "route": "/docs/dashboards/dashboard-templates/pgbouncer-dashboard", diff --git a/data/docs/dashboards/dashboard-templates/opentelemetry-collector-dashboard.mdx b/data/docs/dashboards/dashboard-templates/opentelemetry-collector-dashboard.mdx new file mode 100644 index 0000000000..7d43570f67 --- /dev/null +++ b/data/docs/dashboards/dashboard-templates/opentelemetry-collector-dashboard.mdx @@ -0,0 +1,90 @@ +--- +date: 2026-06-23 +title: OpenTelemetry Collector Pipeline Health Dashboard Template +description: Monitor OTel Collector receiver throughput, processor drops, exporter delivery, queue depth, and process resource usage. +doc_type: explanation +--- + +Use this dashboard to monitor your OpenTelemetry Collector instances across receiver throughput, processor drop rates, exporter delivery, queue depth, and process resource usage. + + + Before importing this dashboard, send OTel Collector internal metrics to SigNoz. Follow the [Send OTel Collector Metrics](https://signoz.io/docs/metrics-management/opentelemetry-collector-metrics/) guide to configure the built-in OTLP telemetry reader. + + +
    + +
    + +
    + +## What This Dashboard Monitors + +- **Overview**: Spans, metric points, and log records received and sent per second at a glance +- **Receivers**: Accepted and refused/failed signals per second, broken down by receiver +- **Processors**: Items entering and leaving each processor, batch send size percentiles, and timeout-triggered flushes +- **Exporters**: Sent and failed signals by exporter, queue size vs. capacity, queue utilization percentage, and in-flight requests +- **Process Resources**: Heap memory, RSS memory, CPU usage, and memory allocation rate per Collector instance + +## Metrics Included + +### Overview + +- **Spans Received /s**: Spans entering the pipeline per second across all receivers +- **Metric Points Received /s**: Metric data points entering the pipeline per second +- **Log Records Received /s**: Log records entering the pipeline per second +- **Spans Sent /s**: Spans delivered to the backend per second across all exporters +- **Metric Points Sent /s**: Metric data points delivered to the backend per second +- **Log Records Sent /s**: Log records delivered to the backend per second + +### Receivers + +- **Accepted Spans /s by Receiver**: Spans entering the pipeline per second, by receiver +- **Refused & Failed Spans /s by Receiver**: Refused spans from pipeline back-pressure and failed spans from receiver errors, grouped by receiver. Investigate any non-zero value. +- **Accepted Metric Points /s by Receiver**: Metric points entering the pipeline per second, by receiver +- **Refused & Failed Metric Points /s by Receiver**: Refused and failed metric points by receiver +- **Accepted Log Records /s by Receiver**: Log records entering the pipeline per second, by receiver +- **Refused & Failed Log Records /s by Receiver**: Refused and failed log records by receiver + +### Processors + +- **Items Incoming /s by Processor**: Signals entering each processor per second +- **Items Outgoing /s by Processor**: Signals leaving each processor per second. A rate below incoming means the processor is dropping or filtering data. +- **Batch Send Size (p50/p95/p99)**: Items per batch at three percentiles. A large gap between p50 and p99 points to bursty traffic. +- **Batch Timeout Trigger Sends /s by Processor**: Batches flushed by timeout per second by processor. A high rate with small batch sizes means the configured batch size is too large for current traffic. + +### Exporters + +- **Spans Sent /s by Exporter**: Spans delivered to the backend per second, by exporter +- **Span Send Failures /s by Exporter**: Spans the exporter failed to deliver per second. Investigate any non-zero value. +- **Metric Points Sent /s by Exporter**: Metric points delivered per second, by exporter +- **Metric Point Send Failures /s by Exporter**: Metric points the exporter failed to deliver per second +- **Log Records Sent /s by Exporter**: Log records delivered per second, by exporter +- **Log Record Send Failures /s by Exporter**: Log records the exporter failed to deliver per second +- **Exporter Queue Size vs Capacity**: Queue depth vs. capacity per exporter. Size approaching capacity means the exporter cannot keep up with incoming data. +- **Exporter Queue Utilization %**: Queue fill percentage per exporter. Above 80% the exporter risks dropping data under sustained load. +- **Exporter In-Flight Requests by Exporter**: Active export requests including retries. High values alongside a slow-draining queue point to backend latency or connectivity issues. + +### Process Resources + +- **Heap Memory Allocated**: Heap bytes held by live objects per Collector instance. Sustained growth between GC cycles points to a memory leak. +- **Process RSS Memory**: Physical memory per Collector instance, including Go runtime overhead +- **CPU Usage (user + system)**: CPU seconds consumed per second per instance. Values near the available core count indicate CPU saturation. +- **Memory Allocation Rate**: Heap allocation throughput in bytes per second per instance. High rates increase GC pressure and CPU overhead. + +## Dashboard Variables + +- **service_name**: Filter by Collector service name + +## Related Dashboards + +- [Host Metrics (VM)](https://signoz.io/docs/dashboards/dashboard-templates/hostmetrics-vm/) +- [Kubernetes Node Metrics](https://signoz.io/docs/dashboards/dashboard-templates/kubernetes-node-metrics-overall/) +- [FluxCD](https://signoz.io/docs/dashboards/dashboard-templates/fluxcd-dashboard/) +- [KEDA](https://signoz.io/docs/dashboards/dashboard-templates/keda/) diff --git a/data/docs/metrics-management/opentelemetry-collector-metrics.mdx b/data/docs/metrics-management/opentelemetry-collector-metrics.mdx new file mode 100644 index 0000000000..98eba0b807 --- /dev/null +++ b/data/docs/metrics-management/opentelemetry-collector-metrics.mdx @@ -0,0 +1,175 @@ +--- +date: 2026-06-23 +title: Monitor OpenTelemetry Collector Pipeline Health with SigNoz +description: Push OpenTelemetry Collector internal metrics to SigNoz using the built-in OTLP telemetry reader to monitor pipeline health. +doc_type: howto +--- + +The OpenTelemetry Collector emits its own pipeline health metrics under the `otelcol_*` prefix. Configure the built-in OTLP telemetry reader to push those metrics to SigNoz. + + + Steps are the same. Update the exporter endpoint and remove the ingestion key header or `signozApiKey` value as shown in [Cloud to Self-Hosted](https://signoz.io/docs/ingestion/cloud-vs-self-hosted/#cloud-to-self-hosted). + + +## Prerequisites + +- OpenTelemetry Collector v0.92 or later +- An instance of SigNoz ([Cloud](https://signoz.io/teams/) or [Self-Hosted](https://signoz.io/docs/install/self-host/)) + +## How it works + +The Collector has a built-in telemetry pipeline that reports internal counters and gauges, including received spans, processor drops, exporter queue depth, and process memory. Add an OTLP periodic reader under **service.telemetry.metrics** to send those metrics to SigNoz at a configured interval. + +`level: normal` (the default) covers receiver throughput, exporter queues, batch processor stats, and process resources. `level: detailed` adds per-method HTTP/RPC dimensions on exporter calls. + +## Steps + +### Step 1: Add the telemetry reader + +Add a `service.telemetry` block to your Collector config with a periodic OTLP exporter pointing at SigNoz: + +```yaml:config.yaml +service: + telemetry: + resource: + service.name: "" + metrics: + level: normal + readers: + - periodic: + interval: 60000 + exporter: + otlp: + protocol: "http/protobuf" + endpoint: "https://ingest..signoz.cloud:443" + headers: + signoz-ingestion-key: "" +``` + +Verify these values: + +- ``: Your [SigNoz Cloud region](https://signoz.io/docs/ingestion/signoz-cloud/overview/#endpoint) +- ``: Your SigNoz [ingestion key](https://signoz.io/docs/ingestion/signoz-cloud/keys/) +- ``: A unique name for this Collector instance. If you run multiple instances, use different names to distinguish them in the dashboard. + + + This config block is independent of your pipeline. You do not need to add a receiver, processor, or pipeline. + + +### Step 2: Apply and restart + + + + +Validate the config and restart the Collector service: + +```bash +sudo /usr/bin/otelcol-contrib validate --config /etc/otelcol-contrib/config.yaml +sudo systemctl restart otelcol-contrib +``` + +Tail the logs to confirm the telemetry reader started with no errors: + +```bash +sudo journalctl -u otelcol-contrib -f +``` + + + + +Restart the Collector container: + +```bash +docker compose up -d +docker logs -f signoz-collection-agent +``` + +For `docker run`, stop and rerun the container with the updated config mount. + + + + +Enable self-telemetry metrics in your `override-values.yaml` for the [SigNoz k8s-infra chart](https://signoz.io/docs/opentelemetry-collection-agents/k8s/k8s-infra/install-k8s-infra/): + +```yaml:override-values.yaml +presets: + selfTelemetry: + endpoint: https://ingest..signoz.cloud:443 + insecure: false + insecureSkipVerify: false + signozApiKey: + metrics: + enabled: true +``` + +Use this preset for both the DaemonSet collector (`otelAgent`) and the deployment collector (`otelDeployment`). + +If you use the OpenTelemetry Helm chart instead of k8s-infra, add the `service.telemetry` block from Step 1 under `config.service.telemetry` in your chart values. + +Deploy the chart update: + +```bash +helm upgrade --install signoz/k8s-infra \ + -n signoz \ + -f override-values.yaml +``` + +Check the Collector pod logs: + +```bash +kubectl get pods -n signoz +kubectl logs -n signoz -f +``` + + + + +## Validate + +Open [Metrics Explorer](https://signoz.io/docs/metrics-management/metrics-explorer/) in SigNoz and search for `otelcol_`. You should see metrics such as `otelcol_exporter_send_failed_spans`. + +
    + +
    + + +## Troubleshooting + + +### No `otelcol_` metrics in SigNoz + +Check the Collector logs for export errors: + +```bash +sudo journalctl -u otelcol-contrib -f +``` + +For Docker, use `docker logs -f signoz-collection-agent`. For Kubernetes, use the Collector pod logs. + +If the logs show authentication errors, confirm the ingestion key and endpoint region are correct. If they show connection errors, confirm the Collector can reach `ingest..signoz.cloud:443`. + +Wait at least one full `interval` (60 seconds with the config above) after restarting before checking. + +### Some panels are empty + +Check `service.telemetry.metrics.level`. `basic` omits batch processor metrics. `none` disables telemetry. Use `normal` or `detailed`. + +### Duplicate metric points + +You can run multiple Collector instances with the same `service.name`. The dashboard groups them by `service.instance.id`, which the Collector assigns per process. All instances appear under the shared service name and can be filtered by instance. + +
    + +## Next Steps + +- Import the prebuilt [OpenTelemetry Collector dashboard](https://signoz.io/docs/dashboards/dashboard-templates/opentelemetry-collector-dashboard/). +- Set up [alerts](https://signoz.io/docs/userguide/alerts-management/). +- Learn more about the [OpenTelemetry Collector configuration](https://signoz.io/docs/opentelemetry-collection-agents/opentelemetry-collector/configuration/). + +## Get Help + +