From 4fe2afd65de83b83823a48efb39b0e54e61d2041 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Fri, 17 Jul 2026 15:40:00 +0530 Subject: [PATCH] docs: generate the metrics reference and bring the config reference to v1.6.5 --- .../vocabularies/TraceMachina/accept.txt | 3 + .../docs/content/docs/reference/metrics.mdx | 781 ++++++++++++++++++ .../reference/nativelink-config/meta.json | 1 - web/apps/docs/package.json | 1 + web/apps/docs/scripts/gen-changelog.mjs | 21 +- .../docs/scripts/gen-config-reference.mjs | 12 +- .../docs/scripts/gen-metrics-reference.mjs | 616 ++++++++++++++ web/apps/docs/scripts/lib/metrics-to-mdx.mjs | 413 +++++++++ 8 files changed, 1842 insertions(+), 6 deletions(-) create mode 100644 web/apps/docs/content/docs/reference/metrics.mdx create mode 100644 web/apps/docs/scripts/gen-metrics-reference.mjs create mode 100644 web/apps/docs/scripts/lib/metrics-to-mdx.mjs diff --git a/.github/styles/config/vocabularies/TraceMachina/accept.txt b/.github/styles/config/vocabularies/TraceMachina/accept.txt index a5ad239b4..9c4817d12 100644 --- a/.github/styles/config/vocabularies/TraceMachina/accept.txt +++ b/.github/styles/config/vocabularies/TraceMachina/accept.txt @@ -326,3 +326,6 @@ inlined protobufs arg rustfmt +unreferenced +quantile +OTel diff --git a/web/apps/docs/content/docs/reference/metrics.mdx b/web/apps/docs/content/docs/reference/metrics.mdx new file mode 100644 index 000000000..5d8c3b820 --- /dev/null +++ b/web/apps/docs/content/docs/reference/metrics.mdx @@ -0,0 +1,781 @@ +--- +title: Metrics reference +description: Every OpenTelemetry instrument NativeLink declares, with its type, unit, attributes, Prometheus series, and whether the binary actually emits it. Autogenerated from the Rust source and its call sites. +full: true +--- + +{/* AUTOGENERATED. Do not edit by hand. + Source: nativelink-util/src/metrics.rs @ v1.6.5 (c479a38f) + Regenerate from web/: bun --filter @nativelink/docs gen:metrics-reference */} + +NativeLink **v1.6.5** declares 30 OpenTelemetry instruments. This page is +generated from the declarations in +[`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs) +and from every `.add()` and `.record()` call site across the workspace, so the +**Emitted** column is not a claim, it is the result of looking. + +That column is the reason this page is generated rather than written. Declaring +an instrument costs nothing and emits nothing; the instrument only produces a +series when some code path calls it. A catalogue written by hand cannot tell +those two states apart, and the gap between them is where dashboards go quietly +wrong. + + + + They are constructed at startup and stored on the metrics struct, but no + code path calls them. See [Declared but never + emitted](#declared-but-never-emitted) for which ones, and for the shipped + Prometheus rules that query them anyway. + + + +## Every instrument at a glance + +| Instrument | Prometheus series | Kind | Emitted | Under what conditions | +| --- | --- | --- | --- | --- | +| [`cache.operation.duration`](#cache-operation-duration) | `nativelink_cache_operation_duration` | Histogram | Conditionally | only for stores explicitly wrapped in a `cache_metrics` spec | +| [`cache.operations`](#cache-operations) | `nativelink_cache_operations_total` | Counter | Conditionally | only for stores explicitly wrapped in a `cache_metrics` spec | +| [`cache.io`](#cache-io) | `nativelink_cache_io_total` | Counter | Conditionally | only for stores explicitly wrapped in a `cache_metrics` spec | +| [`cache.size`](#cache-size) | `nativelink_cache_size` | UpDownCounter | **Never** | no call site anywhere in the workspace | +| [`cache.entries`](#cache-entries) | `nativelink_cache_entries` | UpDownCounter | **Never** | no call site anywhere in the workspace | +| [`cache.item.size`](#cache-item-size) | `nativelink_cache_item_size` | Histogram | Conditionally | only for stores explicitly wrapped in a `cache_metrics` spec | +| [`execution.stage.duration`](#execution-stage-duration) | `nativelink_execution_stage_duration` | Histogram | Conditionally | scheduler process only | +| [`execution.total.duration`](#execution-total-duration) | `nativelink_execution_total_duration` | Histogram | Conditionally | scheduler process only | +| [`execution.queue.time`](#execution-queue-time) | `nativelink_execution_queue_time` | Histogram | Conditionally | scheduler process only | +| [`execution.active.count`](#execution-active-count) | `nativelink_execution_active_count` (unreferenced) | UpDownCounter | Conditionally | scheduler process only | +| [`execution.completed.count`](#execution-completed-count) | `nativelink_execution_completed_count_total` | Counter | Conditionally | scheduler process only | +| [`execution.stage.transitions`](#execution-stage-transitions) | `nativelink_execution_stage_transitions_total` | Counter | Conditionally | scheduler process only | +| [`execution.cpu.time`](#execution-cpu-time) | `nativelink_execution_cpu_time` (unreferenced) | Histogram | Conditionally | scheduler process only | +| [`execution.peak.memory`](#execution-peak-memory) | `nativelink_execution_peak_memory` (unreferenced) | Histogram | Conditionally | scheduler process only | +| [`execution.output.size`](#execution-output-size) | `nativelink_execution_output_size` | Histogram | Conditionally | scheduler process only | +| [`execution.retry.count`](#execution-retry-count) | `nativelink_execution_retry_count_total` | Counter | Conditionally | scheduler process only | +| [`worker.connected.count`](#worker-connected-count) | `nativelink_worker_connected_count` (unreferenced) | UpDownCounter | Conditionally | scheduler process only | +| [`worker.connections`](#worker-connections) | `nativelink_worker_connections` (unreferenced) | Counter | Conditionally | scheduler process only | +| [`worker.disconnections`](#worker-disconnections) | `nativelink_worker_disconnections` (unreferenced) | Counter | Conditionally | scheduler process only | +| [`worker.keepalives`](#worker-keepalives) | `nativelink_worker_keepalives` (unreferenced) | Counter | Conditionally | scheduler process only | +| [`worker.state.count`](#worker-state-count) | `nativelink_worker_state_count` (unreferenced) | UpDownCounter | Conditionally | scheduler process only | +| [`rpc.server.duration`](#rpc-server-duration) | `nativelink_rpc_server_duration` (unreferenced) | Histogram | Yes | whenever the code path runs | +| [`scheduler.matching.duration`](#scheduler-matching-duration) | `nativelink_scheduler_matching_duration` (unreferenced) | Histogram | Conditionally | scheduler process only | +| [`scheduler.matching.passes`](#scheduler-matching-passes) | `nativelink_scheduler_matching_passes` (unreferenced) | Counter | Conditionally | scheduler process only | +| [`store.tier.operations`](#store-tier-operations) | `nativelink_store_tier_operations` (unreferenced) | Counter | Conditionally | only through a `fast_slow` store | +| [`store.tier.io`](#store-tier-io) | `nativelink_store_tier_io` (unreferenced) | Counter | Conditionally | only through a `fast_slow` store | +| [`health.checks`](#health-checks) | `nativelink_health_checks` (unreferenced) | Counter | Yes | whenever the code path runs | +| [`connection.pool.available`](#connection-pool-available) | `nativelink_connection_pool_available` (unreferenced) | Histogram | Yes | whenever the code path runs | +| [`connection.pool.acquisitions`](#connection-pool-acquisitions) | `nativelink_connection_pool_acquisitions` (unreferenced) | Counter | Yes | whenever the code path runs | +| [`connection.reconnects`](#connection-reconnects) | `nativelink_connection_reconnects` (unreferenced) | Counter | Yes | whenever the code path runs | + +## How an instrument becomes a Prometheus series + +NativeLink speaks OTLP only; there is no Prometheus scrape endpoint in the +binary. The names in the middle column above are what the shipped OpenTelemetry +collector produces after two transformations: + +1. The Prometheus exporter in + [`otel-collector-config.yaml`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/deployment-examples/metrics/otel-collector-config.yaml) + is configured with `namespace: nativelink`, which prefixes every series with + `nativelink_`. +2. The exporter rewrites the OTLP name into Prometheus form: dots become + underscores, and monotonic counters gain a `_total` suffix. Histograms fan out + into `_bucket`, `_sum` and `_count` series. + +`cache.operations` (a counter) is therefore scraped as `nativelink_cache_operations_total`, +and `cache.operation.duration` (a histogram) as `nativelink_cache_operation_duration_bucket` +and friends. Change the collector's `namespace` and every name on this page changes +with it. + +Where a name in the **Prometheus series** column is unmarked, it is not a +prediction: it is the name that actually appears in the shipped rules, +dashboards and alerts under `deployment-examples/metrics/` and `kubernetes/`. +A name marked _(unreferenced)_ is derived from the two rules above, because no +shipped artifact mentions that metric at all: nothing has ever queried it, so +nothing has confirmed the exporter's spelling of it. + +## Declared but never emitted + +2 instruments are built by the +meter and stored on the metrics struct, but no code anywhere in the workspace +ever calls `.add()` or `.record()` on them. They will never appear in your +Prometheus output, at any configuration, on any release this page was +generated from. + +| Instrument | Series it would produce | Referenced by shipped config | +| --- | --- | --- | +| `cache.size` | `nativelink_cache_size` | [`deployment-examples/metrics/README.md:112`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/deployment-examples/metrics/README.md#L112), [`deployment-examples/metrics/prometheus-config.yaml:135`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/deployment-examples/metrics/prometheus-config.yaml#L135), [`deployment-examples/metrics/prometheus-recording-rules.yml:164`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/deployment-examples/metrics/prometheus-recording-rules.yml#L164) | +| `cache.entries` | `nativelink_cache_entries` | [`deployment-examples/metrics/README.md:113`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/deployment-examples/metrics/README.md#L113), [`deployment-examples/metrics/prometheus-recording-rules.yml:168`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/deployment-examples/metrics/prometheus-recording-rules.yml#L168) | + + + + `nativelink_cache_size` and `nativelink_cache_entries` appear in the recording rules and the metrics + README under `deployment-examples/metrics/`, but nothing emits them. Any + recording rule, dashboard panel or alert built on them evaluates to an + empty vector forever, which reads on a dashboard as a healthy zero, not + as a missing signal. Delete those panels or treat them as known-empty. + + + +## Cache metrics + +Global cache metrics instruments. + +Declared as `CACHE_METRICS` in [`nativelink-util/src/metrics.rs:419`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L419), backed by the `CacheMetrics` struct. + +Every cache instrument is emitted from one place: the `cache_metrics` store wrapper. A store that is not wrapped emits nothing, no matter how the collector is configured. + +### `cache.operation.duration` + +Histogram of cache operation durations in milliseconds + +| Property | Value | +| --- | --- | +| Instrument type | `Histogram` | +| Field | `cache_operation_duration` | +| Unit | `ms` | +| Prometheus series | `nativelink_cache_operation_duration` | +| Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | +| Declared at | [`nativelink-util/src/metrics.rs:423`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L423) | +| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:67`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L67) | + +**Histogram buckets:** 19 explicit boundaries, from +`0.001` to `5000.0` `ms`. Anything past the last boundary lands in the +`+Inf` bucket, so a quantile computed above it is a lower bound, not a +measurement. + +### `cache.operations` + +Counter of cache operations by type and result + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `cache_operations` | +| Unit | — | +| Prometheus series | `nativelink_cache_operations_total` | +| Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | +| Declared at | [`nativelink-util/src/metrics.rs:457`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L457) | +| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:121`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L121), [`nativelink-store/src/cache_metrics_store.rs:126`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L126), [`nativelink-store/src/cache_metrics_store.rs:137`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L137), [`nativelink-store/src/cache_metrics_store.rs:162`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L162), [`nativelink-store/src/cache_metrics_store.rs:168`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L168), [`nativelink-store/src/cache_metrics_store.rs:194`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L194), [`nativelink-store/src/cache_metrics_store.rs:200`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L200), [`nativelink-store/src/cache_metrics_store.rs:213`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L213), [`nativelink-store/src/cache_metrics_store.rs:219`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L219), [`nativelink-store/src/cache_metrics_store.rs:240`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L240), [`nativelink-store/src/cache_metrics_store.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L248), [`nativelink-store/src/cache_metrics_store.rs:254`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L254) | + +### `cache.io` + +Counter of bytes read/written during cache operations + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `cache_io` | +| Unit | `By` | +| Prometheus series | `nativelink_cache_io_total` | +| Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | +| Declared at | [`nativelink-util/src/metrics.rs:462`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L462) | +| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:74`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L74), [`nativelink-store/src/cache_metrics_store.rs:242`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L242) | + +### `cache.size` + +Current total size of all cached data in bytes + +| Property | Value | +| --- | --- | +| Instrument type | `UpDownCounter` | +| Field | `cache_size` | +| Unit | `By` | +| Prometheus series | `nativelink_cache_size` | +| Emitted | **Never**. no call site anywhere in the workspace | +| Declared at | [`nativelink-util/src/metrics.rs:468`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L468) | +| Emitted from | — | + +### `cache.entries` + +Current number of entries in cache + +| Property | Value | +| --- | --- | +| Instrument type | `UpDownCounter` | +| Field | `cache_entries` | +| Unit | `{entry}` | +| Prometheus series | `nativelink_cache_entries` | +| Emitted | **Never**. no call site anywhere in the workspace | +| Declared at | [`nativelink-util/src/metrics.rs:474`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L474) | +| Emitted from | — | + +### `cache.item.size` + +Histogram of individual cache entry sizes in bytes + +| Property | Value | +| --- | --- | +| Instrument type | `Histogram` | +| Field | `cache_entry_size` | +| Unit | `By` | +| Prometheus series | `nativelink_cache_item_size` | +| Emitted | Conditionally. only for stores explicitly wrapped in a `cache_metrics` spec | +| Declared at | [`nativelink-util/src/metrics.rs:480`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L480) | +| Emitted from | [`nativelink-store/src/cache_metrics_store.rs:77`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs#L77) | + +## Execution metrics + +Global remote execution metrics instruments. + +Declared as `EXECUTION_METRICS` in [`nativelink-util/src/metrics.rs:506`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L506), backed by the `ExecutionMetrics` struct. + +Execution instruments are emitted from the scheduler process as actions change state, plus the per-action CPU time and peak memory a worker reports back when it finishes. Worker processes emit no execution metrics of their own. + +### `execution.stage.duration` + +Histogram of stage durations in seconds + +| Property | Value | +| --- | --- | +| Instrument type | `Histogram` | +| Field | `execution_stage_duration` | +| Unit | `s` | +| Prometheus series | `nativelink_execution_stage_duration` | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:510`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L510) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L919) | + +**Histogram buckets:** 15 explicit boundaries, from +`0.001` to `3600.0` `s`. Anything past the last boundary lands in the +`+Inf` bucket, so a quantile computed above it is a lower bound, not a +measurement. + +### `execution.total.duration` + +Histogram of total execution durations in seconds + +| Property | Value | +| --- | --- | +| Instrument type | `Histogram` | +| Field | `execution_total_duration` | +| Unit | `s` | +| Prometheus series | `nativelink_execution_total_duration` | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:535`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L535) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L919) | + +**Histogram buckets:** 13 explicit boundaries, from +`0.01` to `7200.0` `s`. Anything past the last boundary lands in the +`+Inf` bucket, so a quantile computed above it is a lower bound, not a +measurement. + +### `execution.queue.time` + +Histogram of queue wait times in seconds + +| Property | Value | +| --- | --- | +| Instrument type | `Histogram` | +| Field | `execution_queue_time` | +| Unit | `s` | +| Prometheus series | `nativelink_execution_queue_time` | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:560`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L560) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L919) | + +**Histogram buckets:** 12 explicit boundaries, from +`0.001` to `600.0` `s`. Anything past the last boundary lands in the +`+Inf` bucket, so a quantile computed above it is a lower bound, not a +measurement. + +### `execution.active.count` + +Current number of actions in each stage + +| Property | Value | +| --- | --- | +| Instrument type | `UpDownCounter` | +| Field | `execution_active_count` | +| Unit | `{action}` | +| Prometheus series | `nativelink_execution_active_count` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:580`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L580) | +| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:662`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/memory_awaited_action_db.rs#L662), [`nativelink-scheduler/src/memory_awaited_action_db.rs:669`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/memory_awaited_action_db.rs#L669), [`nativelink-scheduler/src/memory_awaited_action_db.rs:814`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/memory_awaited_action_db.rs#L814) | + +### `execution.completed.count` + +Total number of completed executions + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `execution_completed_count` | +| Unit | `{action}` | +| Prometheus series | `nativelink_execution_completed_count_total` | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:586`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L586) | +| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:688`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/memory_awaited_action_db.rs#L688), [`nativelink-scheduler/src/memory_awaited_action_db.rs:700`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/memory_awaited_action_db.rs#L700), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:918`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L918), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:928`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L928) | + +### `execution.stage.transitions` + +Number of stage transitions + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `execution_stage_transitions` | +| Unit | `{transition}` | +| Prometheus series | `nativelink_execution_stage_transitions_total` | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:592`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L592) | +| Emitted from | [`nativelink-scheduler/src/memory_awaited_action_db.rs:655`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/memory_awaited_action_db.rs#L655), [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:907`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L907) | + +### `execution.cpu.time` + +CPU time an action used, as sampled by the worker. + +| Property | Value | +| --- | --- | +| Instrument type | `Histogram` | +| Field | `execution_cpu_time` | +| Unit | `s` | +| Prometheus series | `nativelink_execution_cpu_time` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:598`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L598) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:709`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/api_worker_scheduler.rs#L709) | + +**Histogram buckets:** 12 explicit boundaries, from +`0.1` to `7200.0` `s`. Anything past the last boundary lands in the +`+Inf` bucket, so a quantile computed above it is a lower bound, not a +measurement. + +### `execution.peak.memory` + +Peak memory an action used, as sampled by the worker. + +| Property | Value | +| --- | --- | +| Instrument type | `Histogram` | +| Field | `execution_peak_memory` | +| Unit | `By` | +| Prometheus series | `nativelink_execution_peak_memory` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:607`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L607) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:706`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/api_worker_scheduler.rs#L706) | + +**Histogram buckets:** 8 explicit boundaries, from +`1_048_576.0` to `68_719_476_736.0` `By`. Anything past the last boundary lands in the +`+Inf` bucket, so a quantile computed above it is a lower bound, not a +measurement. + +### `execution.output.size` + +Histogram of output sizes in bytes + +| Property | Value | +| --- | --- | +| Instrument type | `Histogram` | +| Field | `execution_output_size` | +| Unit | `By` | +| Prometheus series | `nativelink_execution_output_size` | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:623`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L623) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:919`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L919) | + +**Histogram buckets:** 8 explicit boundaries, from +`1_024.0` to `10_737_418_240.0` `By`. Anything past the last boundary lands in the +`+Inf` bucket, so a quantile computed above it is a lower bound, not a +measurement. + +### `execution.retry.count` + +Counter for execution retries + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `execution_retry_count` | +| Unit | `{retry}` | +| Prometheus series | `nativelink_execution_retry_count_total` | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:639`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L639) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler_state_manager.rs:940`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/simple_scheduler_state_manager.rs#L940) | + +## Worker fleet metrics + +Global worker fleet metrics instruments. + +Declared as `WORKER_METRICS` in [`nativelink-util/src/metrics.rs:817`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L817), backed by the `WorkerMetrics` struct. + +Worker fleet instruments are emitted from the scheduler process as workers connect, heartbeat, pause, drain and leave. They describe the pool as this scheduler instance sees it. + +### `worker.connected.count` + +Workers currently connected. + +| Property | Value | +| --- | --- | +| Instrument type | `UpDownCounter` | +| Field | `worker_connected_count` | +| Unit | `{worker}` | +| Prometheus series | `nativelink_worker_connected_count` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:821`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L821) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:227`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/api_worker_scheduler.rs#L227), [`nativelink-scheduler/src/api_worker_scheduler.rs:504`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/api_worker_scheduler.rs#L504) | + +### `worker.connections` + +Workers that have joined, cumulative. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `worker_connections` | +| Unit | `{worker}` | +| Prometheus series | `nativelink_worker_connections` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:827`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L827) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:227`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/api_worker_scheduler.rs#L227) | + +### `worker.disconnections` + +Workers that have left, cumulative, by reason. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `worker_disconnections` | +| Unit | `{worker}` | +| Prometheus series | `nativelink_worker_disconnections` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:833`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L833) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:504`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/api_worker_scheduler.rs#L504) | + +### `worker.keepalives` + +Keepalives received, cumulative. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `worker_keepalives` | +| Unit | `{keepalive}` | +| Prometheus series | `nativelink_worker_keepalives` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:839`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L839) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:187`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/api_worker_scheduler.rs#L187) | + +### `worker.state.count` + +Connected workers currently paused or draining. + +| Property | Value | +| --- | --- | +| Instrument type | `UpDownCounter` | +| Field | `worker_state_count` | +| Unit | `{worker}` | +| Prometheus series | `nativelink_worker_state_count` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:845`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L845) | +| Emitted from | [`nativelink-scheduler/src/api_worker_scheduler.rs:221`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/api_worker_scheduler.rs#L221), [`nativelink-scheduler/src/api_worker_scheduler.rs:224`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/api_worker_scheduler.rs#L224), [`nativelink-scheduler/src/api_worker_scheduler.rs:256`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/api_worker_scheduler.rs#L256), [`nativelink-scheduler/src/api_worker_scheduler.rs:413`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/api_worker_scheduler.rs#L413) | + +## gRPC server metrics + +Global gRPC serving metrics. + +One duration histogram covers rate, errors and latency: the count gives +rate, the status attribute separates errors, and the buckets give latency. + +Declared as `RPC_METRICS` in [`nativelink-util/src/metrics.rs:921`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L921), backed by the `RpcMetrics` struct. + +Emitted by the `OtlpLayer` tower middleware on every gRPC server NativeLink starts, so a CAS, a scheduler and a worker API endpoint each report their own. + +### `rpc.server.duration` + +Duration of inbound gRPC calls, by service, method and status. + +| Property | Value | +| --- | --- | +| Instrument type | `Histogram` | +| Field | `rpc_server_duration` | +| Unit | `s` | +| Prometheus series | `nativelink_rpc_server_duration` (not referenced by any shipped rule or dashboard) | +| Emitted | Yes. whenever the code path runs | +| Declared at | [`nativelink-util/src/metrics.rs:925`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L925) | +| Emitted from | [`nativelink-util/src/telemetry.rs:375`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/telemetry.rs#L375) | + +**Histogram buckets:** 15 explicit boundaries, from +`0.001` to `300.0` `s`. Anything past the last boundary lands in the +`+Inf` bucket, so a quantile computed above it is a lower bound, not a +measurement. + +## Scheduler metrics + +Global scheduler metrics. + +Declared as `SCHEDULER_METRICS` in [`nativelink-util/src/metrics.rs:974`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L974), backed by the `SchedulerOtlpMetrics` struct. + +Emitted from the scheduler process once per matching pass, whether or not the pass assigned anything. + +### `scheduler.matching.duration` + +How long a matching pass takes. The saturation signal: this climbing +while the queue is non-empty means matching is the bottleneck. + +| Property | Value | +| --- | --- | +| Instrument type | `Histogram` | +| Field | `matching_duration` | +| Unit | `s` | +| Prometheus series | `nativelink_scheduler_matching_duration` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:978`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L978) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:281`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/simple_scheduler.rs#L281) | + +**Histogram buckets:** 13 explicit boundaries, from +`0.0001` to `10.0` `s`. Anything past the last boundary lands in the +`+Inf` bucket, so a quantile computed above it is a lower bound, not a +measurement. + +### `scheduler.matching.passes` + +Matching passes, by result. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `matching_passes` | +| Unit | `{pass}` | +| Prometheus series | `nativelink_scheduler_matching_passes` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. scheduler process only | +| Declared at | [`nativelink-util/src/metrics.rs:987`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L987) | +| Emitted from | [`nativelink-scheduler/src/simple_scheduler.rs:281`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/simple_scheduler.rs#L281) | + +## Store tier metrics + +Global tiered-store metrics. + +Declared as `STORE_TIER_METRICS` in [`nativelink-util/src/metrics.rs:1023`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L1023), backed by the `StoreTierMetrics` struct. + +Emitted from the `fast_slow` store wrapper only. A deployment without a `fast_slow` store emits none of these. + +### `store.tier.operations` + +Reads served per tier, by result. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `tier_operations` | +| Unit | `{operation}` | +| Prometheus series | `nativelink_store_tier_operations` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. only through a `fast_slow` store | +| Declared at | [`nativelink-util/src/metrics.rs:1027`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L1027) | +| Emitted from | [`nativelink-store/src/fast_slow_store.rs:340`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/fast_slow_store.rs#L340), [`nativelink-store/src/fast_slow_store.rs:807`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/fast_slow_store.rs#L807), [`nativelink-store/src/fast_slow_store.rs:823`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/fast_slow_store.rs#L823), [`nativelink-store/src/fast_slow_store.rs:833`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/fast_slow_store.rs#L833), [`nativelink-store/src/fast_slow_store.rs:858`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/fast_slow_store.rs#L858), [`nativelink-store/src/fast_slow_store.rs:880`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/fast_slow_store.rs#L880) | + +### `store.tier.io` + +Bytes moved per tier and direction. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `tier_io` | +| Unit | `By` | +| Prometheus series | `nativelink_store_tier_io` (not referenced by any shipped rule or dashboard) | +| Emitted | Conditionally. only through a `fast_slow` store | +| Declared at | [`nativelink-util/src/metrics.rs:1033`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L1033) | +| Emitted from | [`nativelink-store/src/fast_slow_store.rs:349`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/fast_slow_store.rs#L349), [`nativelink-store/src/fast_slow_store.rs:824`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/fast_slow_store.rs#L824), [`nativelink-store/src/fast_slow_store.rs:859`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/fast_slow_store.rs#L859), [`nativelink-store/src/fast_slow_store.rs:881`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/fast_slow_store.rs#L881) | + +## Health check metrics + +Global health-check metrics. + +Declared as `HEALTH_METRICS` in [`nativelink-util/src/metrics.rs:1074`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L1074), backed by the `HealthMetrics` struct. + +Emitted each time the health check endpoint runs, one observation per registered health indicator. + +### `health.checks` + +Health check results, by namespace and status. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `health_checks` | +| Unit | `{check}` | +| Prometheus series | `nativelink_health_checks` (not referenced by any shipped rule or dashboard) | +| Emitted | Yes. whenever the code path runs | +| Declared at | [`nativelink-util/src/metrics.rs:1078`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L1078) | +| Emitted from | [`nativelink-util/src/health_utils.rs:248`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/health_utils.rs#L248) | + +## Connection pool metrics + +Global connection-pool metrics. + +Declared as `CONNECTION_METRICS` in [`nativelink-util/src/metrics.rs:1105`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L1105), backed by the `ConnectionMetrics` struct. + +Emitted by the gRPC connection manager and the Redis store as they hand out and re-establish connections. A deployment that uses neither emits none of these. + +### `connection.pool.available` + +Free slots at the moment a connection was taken. + +| Property | Value | +| --- | --- | +| Instrument type | `Histogram` | +| Field | `pool_available` | +| Unit | `{connection}` | +| Prometheus series | `nativelink_connection_pool_available` (not referenced by any shipped rule or dashboard) | +| Emitted | Yes. whenever the code path runs | +| Declared at | [`nativelink-util/src/metrics.rs:1109`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L1109) | +| Emitted from | [`nativelink-store/src/redis_store.rs:558`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/redis_store.rs#L558), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/connection_manager.rs#L351) | + +**Histogram buckets:** 11 explicit boundaries, from +`0.0` to `512.0` `{connection}`. Anything past the last boundary lands in the +`+Inf` bucket, so a quantile computed above it is a lower bound, not a +measurement. + +### `connection.pool.acquisitions` + +Acquisitions, by pool and result. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `pool_acquisitions` | +| Unit | `{acquisition}` | +| Prometheus series | `nativelink_connection_pool_acquisitions` (not referenced by any shipped rule or dashboard) | +| Emitted | Yes. whenever the code path runs | +| Declared at | [`nativelink-util/src/metrics.rs:1118`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L1118) | +| Emitted from | [`nativelink-store/src/redis_store.rs:558`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/redis_store.rs#L558), [`nativelink-util/src/connection_manager.rs:341`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/connection_manager.rs#L341), [`nativelink-util/src/connection_manager.rs:351`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/connection_manager.rs#L351) | + +### `connection.reconnects` + +Reconnects, by pool. + +| Property | Value | +| --- | --- | +| Instrument type | `Counter` | +| Field | `reconnects` | +| Unit | `{reconnect}` | +| Prometheus series | `nativelink_connection_reconnects` (not referenced by any shipped rule or dashboard) | +| Emitted | Yes. whenever the code path runs | +| Declared at | [`nativelink-util/src/metrics.rs:1124`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs#L1124) | +| Emitted from | [`nativelink-store/src/redis_store.rs:318`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/redis_store.rs#L318) | + +## Attributes + +Every attribute key NativeLink attaches to a metric, and the closed set of +values it can take. The Prometheus label is the key with dots replaced by +underscores. + +### Cache attributes + +| Attribute | Prometheus label | Rust constant | Values | +| --- | --- | --- | --- | +| `cache.type` | `cache_type` | `CACHE_TYPE` | open (free-form string) | +| `cache.operation.name` | `cache_operation_name` | `CACHE_OPERATION` | `read`, `write`, `delete`, `evict` | +| `cache.operation.result` | `cache_operation_result` | `CACHE_RESULT` | `hit`, `miss`, `expired`, `success`, `error` | + +### Execution attributes + +| Attribute | Prometheus label | Rust constant | Values | +| --- | --- | --- | --- | +| `execution.stage` | `execution_stage` | `EXECUTION_STAGE` | `unknown`, `cache_check`, `queued`, `executing`, `completed` | +| `execution.result` | `execution_result` | `EXECUTION_RESULT` | `success`, `failure`, `cancelled`, `timeout`, `cache_hit` | +| `execution.instance` | `execution_instance` | `EXECUTION_INSTANCE` | open (free-form string) | +| `execution.priority` | `execution_priority` | `EXECUTION_PRIORITY` | open (free-form string) | +| `execution.worker_id` | `execution_worker_id` | `EXECUTION_WORKER_ID` | open (free-form string) | +| `execution.exit_code` | `execution_exit_code` | `EXECUTION_EXIT_CODE` | open (free-form string) | +| `execution.action_digest` | `execution_action_digest` | `EXECUTION_ACTION_DIGEST` | open (free-form string) | + +### Worker attributes + +| Attribute | Prometheus label | Rust constant | Values | +| --- | --- | --- | --- | +| `worker.state` | `worker_state` | `WORKER_STATE` | open (free-form string) | +| `worker.disconnect.reason` | `worker_disconnect_reason` | `WORKER_DISCONNECT_REASON` | open (free-form string) | + +### gRPC attributes + +| Attribute | Prometheus label | Rust constant | Values | +| --- | --- | --- | --- | +| `rpc.service` | `rpc_service` | `RPC_SERVICE` | open (free-form string) | +| `rpc.method` | `rpc_method` | `RPC_METHOD` | open (free-form string) | +| `rpc.grpc.status_code` | `rpc_grpc_status_code` | `RPC_STATUS_CODE` | open (free-form string) | + +### Scheduler attributes + +| Attribute | Prometheus label | Rust constant | Values | +| --- | --- | --- | --- | +| `scheduler.match.result` | `scheduler_match_result` | `SCHEDULER_MATCH_RESULT` | open (free-form string) | + +### Store tier attributes + +| Attribute | Prometheus label | Rust constant | Values | +| --- | --- | --- | --- | +| `store.tier` | `store_tier` | `STORE_TIER` | open (free-form string) | +| `store.result` | `store_result` | `STORE_RESULT` | open (free-form string) | +| `store.direction` | `store_direction` | `STORE_DIRECTION` | open (free-form string) | + +### Health attributes + +| Attribute | Prometheus label | Rust constant | Values | +| --- | --- | --- | --- | +| `health.namespace` | `health_namespace` | `HEALTH_NAMESPACE` | open (free-form string) | +| `health.status` | `health_status` | `HEALTH_STATUS` | open (free-form string) | + +### Connection attributes + +| Attribute | Prometheus label | Rust constant | Values | +| --- | --- | --- | --- | +| `connection.pool` | `connection_pool` | `CONNECTION_POOL` | open (free-form string) | +| `connection.result` | `connection_result` | `CONNECTION_RESULT` | open (free-form string) | + +### `cache.operation.name` values + +| Value | Meaning | +| --- | --- | +| `read` | Data retrieval operations (get, peek, contains, etc.) | +| `write` | Data storage operations (insert, update, replace, etc.) | +| `delete` | Explicit data removal operations | +| `evict` | Automatic cache maintenance (evictions, TTL cleanup, etc.) | + +### `cache.operation.result` values + +| Value | Meaning | +| --- | --- | +| `hit` | Data found and valid (Read operations) | +| `miss` | Data not found (Read operations) | +| `expired` | Data found but invalid/expired (Read operations) | +| `success` | Operation completed successfully (Write/Delete/Evict operations) | +| `error` | Operation failed (any operation type) | + +### `execution.stage` values + +| Value | Meaning | +| --- | --- | +| `unknown` | Unknown stage | +| `cache_check` | Checking cache for existing results | +| `queued` | Action is queued waiting for execution | +| `executing` | Action is being executed by a worker | +| `completed` | Action execution completed | + +### `execution.result` values + +| Value | Meaning | +| --- | --- | +| `success` | Execution completed successfully | +| `failure` | Execution failed | +| `cancelled` | Execution was cancelled | +| `timeout` | Execution timed out | +| `cache_hit` | Result was found in cache | + +## Reading the source + +If anything here disagrees with the binary, the source wins: + +- [`nativelink-util/src/metrics.rs`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/metrics.rs): every declaration +- [`nativelink-util/src/telemetry.rs`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-util/src/telemetry.rs): the OTLP exporter and its interval +- [`nativelink-store/src/cache_metrics_store.rs`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-store/src/cache_metrics_store.rs): every cache call site +- [`nativelink-scheduler/src/simple_scheduler_state_manager.rs`](https://github.com/TraceMachina/nativelink/blob/v1.6.5/nativelink-scheduler/src/simple_scheduler_state_manager.rs): every execution call site + +## Where to go next + + + The collector, the exporter interval and the queries worth alerting on: the + operational half of this reference. + + + + Every configuration field, generated the same way from the same source of + truth. + diff --git a/web/apps/docs/content/docs/reference/nativelink-config/meta.json b/web/apps/docs/content/docs/reference/nativelink-config/meta.json index 22e3bb1a6..5cb533eca 100644 --- a/web/apps/docs/content/docs/reference/nativelink-config/meta.json +++ b/web/apps/docs/content/docs/reference/nativelink-config/meta.json @@ -1,6 +1,5 @@ { "pages": [ - "index", "store-overview" ], "title": "NativeLink config" diff --git a/web/apps/docs/package.json b/web/apps/docs/package.json index 37cdaf96d..244070405 100644 --- a/web/apps/docs/package.json +++ b/web/apps/docs/package.json @@ -32,6 +32,7 @@ "gen:changelog": "node scripts/gen-changelog.mjs", "gen:config-reference": "node scripts/gen-config-reference.mjs", "gen:llms": "node scripts/gen-llms.mjs", + "gen:metrics-reference": "node scripts/gen-metrics-reference.mjs", "lint": "biome check .", "lint:anchors": "node scripts/lint-anchors.mjs", "lint:boundaries": "node scripts/lint-boundaries.mjs", diff --git a/web/apps/docs/scripts/gen-changelog.mjs b/web/apps/docs/scripts/gen-changelog.mjs index ee373e68b..3fea34cd4 100644 --- a/web/apps/docs/scripts/gen-changelog.mjs +++ b/web/apps/docs/scripts/gen-changelog.mjs @@ -134,6 +134,22 @@ const GIT_CLIFF_MARKER_LINES = new Set([ "", ]); +/** + * git-cliff links each release heading to its compare view: + * `## [1.6.3](https://.../compare/v1.6.2..v1.6.3) - 2026-07-24`. Fumadocs + * wraps every heading in its own anchor link, and an `` inside an `` + * is invalid HTML that breaks hydration. Keep the version and date in the + * heading and move the compare link to a line of its own underneath. + */ +const unlinkReleaseHeading = (line) => { + const match = /^## \[([^\]]+)\]\((https?:\/\/[^)\s]+)\)(.*)$/.exec(line); + if (!match) { + return line; + } + const [, version, url, rest] = match; + return `## ${version}${rest}\n\n[Compare with the previous release](${url})`; +}; + const { text: raw, origin } = await readChangelog(); // Keep everything from the first release heading on; the git-cliff header @@ -156,7 +172,10 @@ const body = raw if (inFence) { return line; } - return GIT_CLIFF_MARKER_LINES.has(line.trim()) ? null : escapeLine(line); + if (GIT_CLIFF_MARKER_LINES.has(line.trim())) { + return null; + } + return escapeLine(unlinkReleaseHeading(line)); }) .filter((line) => line !== null) .join("\n") diff --git a/web/apps/docs/scripts/gen-config-reference.mjs b/web/apps/docs/scripts/gen-config-reference.mjs index 610fb256a..b19c4821b 100644 --- a/web/apps/docs/scripts/gen-config-reference.mjs +++ b/web/apps/docs/scripts/gen-config-reference.mjs @@ -237,7 +237,10 @@ function generateOne(v, sharedTargetDir) { } function writeManifest(versions) { - const entries = versions.map((v) => ({ + // A tag that could not be built (pre-1.0.2 releases have no build-schema + // binary) has no page, and a manifest entry for it would be a dead link in + // the version switcher. + const entries = versions.filter((v) => existsSync(outFileFor(v))).map((v) => ({ version: v.version, label: v.label, href: hrefFor(v), @@ -276,11 +279,12 @@ export const CONFIG_VERSIONS: ConfigVersion[] = ${JSON.stringify(entries, null, const HAND_WRITTEN_PAGES = ["store-overview"]; function writeMeta() { - // The canonical page plus any hand-written companion pages appear in the - // sidebar; other versions are reached through the in-page version switcher. + // The canonical page is the folder's own link (Fumadocs uses index.mdx for + // that when meta.json does not list it); the hand-written companion pages + // appear under it. Other versions are reached through the in-page switcher. writeFileSync( join(contentDir, "meta.json"), - `${JSON.stringify({ pages: ["index", ...HAND_WRITTEN_PAGES], title: "NativeLink config" }, null, 2)}\n`, + `${JSON.stringify({ pages: [...HAND_WRITTEN_PAGES], title: "NativeLink config" }, null, 2)}\n`, ); } diff --git a/web/apps/docs/scripts/gen-metrics-reference.mjs b/web/apps/docs/scripts/gen-metrics-reference.mjs new file mode 100644 index 000000000..cfead1fbc --- /dev/null +++ b/web/apps/docs/scripts/gen-metrics-reference.mjs @@ -0,0 +1,616 @@ +#!/usr/bin/env node +// Generate the NativeLink metrics reference from the Rust source. +// +// Unlike `gen-config-reference.mjs`, this generator needs no Rust toolchain: the +// metric declarations live in one file (`nativelink-util/src/metrics.rs`) as +// plain `meter.("name")` builder chains, and the question this page exists +// to answer (*is this metric actually emitted?*) is answered by finding call +// sites, which is a source scan either way. Running it costs a few hundred +// milliseconds and no build. +// +// Usage, from web/: +// bun --filter @nativelink/docs gen:metrics-reference +// or directly: +// node scripts/gen-metrics-reference.mjs +// +// What it does: +// 1. Parses every instrument out of the `LazyLock` metric groups in metrics.rs: +// field name, OTLP name, Rust instrument type, unit, description, and +// histogram bucket boundaries. +// 2. Parses the attribute-key constants and the `impl From for Value` +// blocks that define the closed set of values each attribute can take. +// 3. Scans every crate for `.field.add(..)` / `.field.record(..)` call sites, +// and for indirect references (a field handed to a helper), resolving the +// helper back to its own call sites. Test files do not count. +// 4. Scans the shipped observability artifacts for `nativelink_*` series names +// and matches them back to instruments, so the page can report both +// directions of drift: instruments nothing emits, and series nothing +// declares. +// 5. Feeds the inventory to the pure `metricsToMdx` transform. + +import { execFileSync } from "node:child_process"; +import { existsSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; +import { dirname, join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { metricsToMdx } from "./lib/metrics-to-mdx.mjs"; + +const here = dirname(fileURLToPath(import.meta.url)); +const docsRoot = join(here, ".."); +// The checkout to scan. Defaults to the repository this docs app lives in; +// override to document a different checkout, for example a newer main than the +// docs branch is based on. +const repoRoot = process.env.NATIVELINK_REPO_ROOT ?? join(docsRoot, "..", "..", ".."); + +const METRICS_RS = "nativelink-util/src/metrics.rs"; +const GITHUB_BASE = "https://github.com/TraceMachina/nativelink"; +const OUT_FILE = join(docsRoot, "content/docs/reference/metrics.mdx"); + +/** Crates scanned for metric call sites. */ +const SOURCE_DIRS = [ + "nativelink-store", + "nativelink-scheduler", + "nativelink-service", + "nativelink-worker", + "nativelink-util", + "nativelink-config", + "src", +]; + +/** Shipped observability artifacts scanned for `nativelink_*` series names. */ +const ARTIFACT_DIRS = ["deployment-examples/metrics", "kubernetes"]; +const ARTIFACT_EXTS = [".yaml", ".yml", ".json", ".md"]; + +/** Which crate owns a call site, and what that implies about when it runs. */ +const GATES = [ + { + file: "nativelink-store/src/cache_metrics_store.rs", + verdict: "gated", + gate: "only for stores explicitly wrapped in a `cache_metrics` spec", + }, + { + prefix: "nativelink-scheduler/", + verdict: "gated", + gate: "scheduler process only", + }, + { + file: "nativelink-store/src/fast_slow_store.rs", + verdict: "gated", + gate: "only through a `fast_slow` store", + }, +]; + +// -------------------------------------------------------------------------- +// Small helpers. +// -------------------------------------------------------------------------- + +function git(args) { + return execFileSync("git", args, { cwd: repoRoot, encoding: "utf8" }).trim(); +} + +function lineOf(text, index) { + let line = 1; + for (let i = 0; i < index; i++) if (text.charCodeAt(i) === 10) line++; + return line; +} + +/** Index just past the `{` … `}` block that starts at `open`. */ +function matchBrace(text, open) { + let depth = 0; + for (let i = open; i < text.length; i++) { + const c = text[i]; + if (c === "{") depth++; + else if (c === "}") { + depth--; + if (depth === 0) return i; + } + } + return text.length; +} + +function walk(dir, exts, out = []) { + let entries; + try { + entries = readdirSync(dir); + } catch { + return out; + } + for (const name of entries) { + if (name === "target" || name === "node_modules" || name === ".git") continue; + const full = join(dir, name); + let st; + try { + st = statSync(full); + } catch { + continue; + } + if (st.isDirectory()) walk(full, exts, out); + else if (exts.some((e) => name.endsWith(e))) out.push(full); + } + return out; +} + +/** Strip Rust `///` and `//!` doc-comment markers from a captured run. */ +function docText(raw) { + return raw + .split("\n") + .map((l) => l.trim().replace(/^\/{3}!?/, "").replace(/^\/{2}!/, "").trim()) + .join("\n") + .trim(); +} + +/** The `///` block immediately preceding `index`, if any. */ +function precedingDoc(text, index) { + const before = text.slice(0, index); + const m = before.match(/((?:[ \t]*\/\/\/[^\n]*\n)+)[ \t]*(?:#\[[^\]]*\]\s*)*$/); + return m ? docText(m[1]) : ""; +} + +// -------------------------------------------------------------------------- +// 1. Instrument declarations. +// -------------------------------------------------------------------------- + +const RUST_KIND = { + f64_histogram: { kind: "Histogram", rust: "Histogram" }, + u64_histogram: { kind: "Histogram", rust: "Histogram" }, + u64_counter: { kind: "Counter", rust: "Counter" }, + f64_counter: { kind: "Counter", rust: "Counter" }, + i64_up_down_counter: { kind: "UpDownCounter", rust: "UpDownCounter" }, + f64_up_down_counter: { kind: "UpDownCounter", rust: "UpDownCounter" }, + u64_gauge: { kind: "Gauge", rust: "Gauge" }, + i64_gauge: { kind: "Gauge", rust: "Gauge" }, + f64_gauge: { kind: "Gauge", rust: "Gauge" }, +}; + +/** Concatenated Rust string literal(s) inside a call, e.g. `.with_description("a" "b")`. */ +function stringArg(chunk) { + const parts = [...chunk.matchAll(/"((?:[^"\\]|\\.)*)"/g)].map((m) => + m[1].replace(/\\"/g, '"').replace(/\\n/g, "\n").replace(/\\\\/g, "\\"), + ); + return parts.join(""); +} + +function parseGroups(src) { + const groups = []; + const groupRe = /pub static (\w+): LazyLock<(\w+)> = LazyLock::new\(\|\| \{/g; + let g = groupRe.exec(src); + while (g !== null) { + const [, name, struct] = g; + const open = src.indexOf("{", g.index + g[0].length - 1); + const end = matchBrace(src, open); + const block = src.slice(open, end); + const blockOffset = open; + + // Field docs live on the struct, not the builder. + const structRe = new RegExp(`pub struct ${struct} \\{`); + const sm = structRe.exec(src); + const fieldDocs = new Map(); + let structRange = null; + if (sm) { + const sOpen = src.indexOf("{", sm.index); + const sEnd = matchBrace(src, sOpen); + structRange = [sOpen, sEnd]; + const sBlock = src.slice(sOpen, sEnd); + const fRe = /((?:[ \t]*\/\/\/[^\n]*\n)+)[ \t]*pub (\w+):/g; + let f = fRe.exec(sBlock); + while (f !== null) { + fieldDocs.set(f[2], docText(f[1])); + f = fRe.exec(sBlock); + } + } + + const metrics = []; + const instRe = /(\w+): meter\s*\.(\w+)\("([^"]+)"\)/g; + let i = instRe.exec(block); + while (i !== null) { + const [, field, builder, otelName] = i; + const kindInfo = RUST_KIND[builder]; + if (!kindInfo) { + throw new Error( + `Unknown OpenTelemetry builder \`${builder}\` for \`${field}\`; teach RUST_KIND about it.`, + ); + } + // The chain runs to the next `.build()`. + const chainEnd = block.indexOf(".build()", i.index); + const chain = block.slice(i.index, chainEnd === -1 ? block.length : chainEnd); + + const descM = /\.with_description\(([\s\S]*?)\)\s*\n/.exec(chain); + const unitM = /\.with_unit\("([^"]*)"\)/.exec(chain); + const boundsM = /\.with_boundaries\(vec!\[([\s\S]*?)\]\)/.exec(chain); + const boundaries = boundsM + ? boundsM[1] + .replace(/\/\/[^\n]*/g, "") + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + : []; + + metrics.push({ + field, + otelName, + kind: kindInfo.kind, + rustType: kindInfo.rust, + unit: unitM ? unitM[1] : "", + description: fieldDocs.get(field) || (descM ? stringArg(descM[1]) : ""), + boundaries, + declaredAt: { file: METRICS_RS, line: lineOf(src, blockOffset + i.index) }, + }); + i = instRe.exec(block); + } + + groups.push({ + name, + struct, + doc: precedingDoc(src, g.index), + declaredAt: { file: METRICS_RS, line: lineOf(src, g.index) }, + excluded: [[open, end], structRange].filter(Boolean), + metrics, + }); + g = groupRe.exec(src); + } + return groups; +} + +// -------------------------------------------------------------------------- +// 2. Attribute keys and their closed value sets. +// -------------------------------------------------------------------------- + +function parseAttributes(src) { + const keys = []; + const re = /((?:[ \t]*\/\/[^\n/][^\n]*\n)*)pub const (\w+): &str = "([^"]+)";/g; + let m = re.exec(src); + while (m !== null) { + keys.push({ + const: m[2], + key: m[3], + comment: docText(m[1]), + line: lineOf(src, m.index), + }); + m = re.exec(src); + } + + const valueEnums = []; + const implRe = /impl From<(\w+)> for Value \{([\s\S]*?)\n\}/g; + let im = implRe.exec(src); + while (im !== null) { + const enumName = im[1]; + const arms = [...im[2].matchAll(/(\w+)::(\w+) => Self::from\("([^"]+)"\)/g)].map((a) => ({ + variant: a[2], + value: a[3], + })); + if (arms.length) { + // Pull each variant's `///` doc off the enum declaration. + const decl = new RegExp(`pub enum ${enumName} \\{`).exec(src); + const docs = new Map(); + if (decl) { + const open = src.indexOf("{", decl.index); + const body = src.slice(open, matchBrace(src, open)); + const vRe = /((?:[ \t]*\/\/\/[^\n]*\n)+)[ \t]*(\w+),/g; + let v = vRe.exec(body); + while (v !== null) { + docs.set(v[2], docText(v[1])); + v = vRe.exec(body); + } + } + valueEnums.push({ + name: enumName, + variants: arms.map((a) => ({ ...a, doc: docs.get(a.variant) || "" })), + }); + } + im = implRe.exec(src); + } + return { keys, valueEnums }; +} + +/** Map an attribute-key constant to the enum whose values fill it, by name. */ +const ATTR_ENUM = { + CACHE_OPERATION: "CacheOperationName", + CACHE_RESULT: "CacheOperationResult", + EXECUTION_STAGE: "ExecutionStage", + EXECUTION_RESULT: "ExecutionResult", +}; + +// -------------------------------------------------------------------------- +// 3. Call sites. +// -------------------------------------------------------------------------- + +function isTestPath(rel) { + return /(^|\/)tests?\//.test(rel) || rel.endsWith("_test.rs") || rel.endsWith("_tests.rs"); +} + +function sourceFiles() { + const files = []; + for (const d of SOURCE_DIRS) files.push(...walk(join(repoRoot, d), [".rs"])); + return files + .map((f) => ({ abs: f, rel: relative(repoRoot, f).split("\\").join("/") })) + .filter((f) => !isTestPath(f.rel)); +} + +/** Nearest enclosing `fn NAME(` before `index`. */ +function enclosingFn(text, index) { + const before = text.slice(0, index); + const matches = [...before.matchAll(/\bfn\s+(\w+)\s*\(/g)]; + return matches.length ? matches[matches.length - 1][1] : null; +} + +function findCallSites(files, metricsSrc, groups) { + /** field -> { direct: [site], indirect: [site], viaFn: Set } */ + const byField = new Map(); + const fields = []; + for (const g of groups) for (const m of g.metrics) fields.push(m.field); + for (const f of fields) byField.set(f, { direct: [], indirect: [], viaFn: new Set() }); + + const excludedInMetricsRs = groups.flatMap((g) => g.excluded); + const inExcluded = (idx) => excludedInMetricsRs.some(([a, b]) => idx >= a && idx <= b); + + for (const { abs, rel } of files) { + const text = readFileSync(abs, "utf8"); + for (const field of fields) { + const anyRe = new RegExp(`\\.\\s*${field}\\b`, "g"); + let m = anyRe.exec(text); + while (m !== null) { + const isMetricsRs = rel === METRICS_RS; + if (isMetricsRs && inExcluded(m.index)) { + m = anyRe.exec(text); + continue; + } + const after = text.slice(m.index + m[0].length, m.index + m[0].length + 40); + const direct = /^\s*\.\s*(add|record)\s*\(/.test(after); + const site = { file: rel, line: lineOf(text, m.index) }; + const entry = byField.get(field); + if (direct) entry.direct.push(site); + else entry.indirect.push(site); + // Any reference inside metrics.rs itself sits in a helper body; credit + // it to whoever calls that helper, not to metrics.rs. + if (!direct || isMetricsRs) { + const fn = enclosingFn(text, m.index); + if (fn) entry.viaFn.add(`${rel}::${fn}`); + } + m = anyRe.exec(text); + } + } + } + + // An indirect reference inside metrics.rs only emits if the helper that holds + // it is itself called from somewhere that is not metrics.rs and not a test. + const helperCallers = new Map(); + const helpers = new Set(); + for (const entry of byField.values()) + for (const v of entry.viaFn) if (v.startsWith(`${METRICS_RS}::`)) helpers.add(v.split("::")[1]); + + for (const helper of helpers) { + const callers = []; + const re = new RegExp(`\\b${helper}\\s*\\(`, "g"); + for (const { abs, rel } of files) { + if (rel === METRICS_RS) continue; + const text = readFileSync(abs, "utf8"); + let m = re.exec(text); + while (m !== null) { + callers.push({ file: rel, line: lineOf(text, m.index) }); + m = re.exec(text); + } + re.lastIndex = 0; + } + helperCallers.set(helper, callers); + } + + return { byField, helperCallers }; +} + +function verdictFor(field, { byField, helperCallers }) { + const entry = byField.get(field); + const sites = []; + for (const s of entry.direct) if (s.file !== METRICS_RS) sites.push(s); + + // Direct calls inside metrics.rs (outside the declaration blocks) are helper + // bodies; credit them to the helper's own callers. + const helperSites = []; + for (const via of entry.viaFn) { + const [file, fn] = via.split("::"); + if (file !== METRICS_RS) continue; + for (const c of helperCallers.get(fn) ?? []) helperSites.push({ ...c, via: fn }); + } + // Deduplicate: two instruments recorded by the same helper share its callers. + const all = [...sites, ...helperSites].filter( + (s, i, a) => a.findIndex((o) => o.file === s.file && o.line === s.line) === i, + ); + + if (all.length === 0) { + return { verdict: "never", gate: "no call site anywhere in the workspace", sites: [] }; + } + + const gate = GATES.find((g) => + all.every((s) => (g.file ? s.file === g.file : s.file.startsWith(g.prefix))), + ); + if (gate) return { verdict: gate.verdict, gate: gate.gate, sites: all }; + return { verdict: "emitted", gate: "whenever the code path runs", sites: all }; +} + +// -------------------------------------------------------------------------- +// 4. Shipped observability artifacts. +// -------------------------------------------------------------------------- + +function scanArtifacts() { + const found = new Map(); // series name -> [{file, line}] + for (const d of ARTIFACT_DIRS) { + for (const abs of walk(join(repoRoot, d), ARTIFACT_EXTS)) { + const rel = relative(repoRoot, abs).split("\\").join("/"); + const text = readFileSync(abs, "utf8"); + const re = /\bnativelink_[a-z0-9_]+/g; + let m = re.exec(text); + while (m !== null) { + const name = m[0]; + // `- name: nativelink_performance` is a Prometheus *rule group*, not a + // series. Same for an alert-rule group. Skip those or every group name + // reads as an orphan series. + const lineStart = text.lastIndexOf("\n", m.index) + 1; + if (/^\s*-?\s*name:\s*$/.test(text.slice(lineStart, m.index))) { + m = re.exec(text); + continue; + } + if (!found.has(name)) found.set(name, []); + const sites = found.get(name); + if (sites.length < 4) sites.push({ file: rel, line: lineOf(text, m.index) }); + m = re.exec(text); + } + } + } + return found; +} + +const SUFFIXES = ["_bucket", "_sum", "_count", "_total"]; + +/** Base series name an instrument would produce, before exporter suffixes. */ +function promBase(otelName) { + return `nativelink_${otelName.replace(/\./g, "_")}`; +} + +function stripSuffix(name) { + for (const s of SUFFIXES) if (name.endsWith(s)) return name.slice(0, -s.length); + return name; +} + +// -------------------------------------------------------------------------- +// Assemble. +// -------------------------------------------------------------------------- + +const GROUP_TITLES = { + CACHE_METRICS: "Cache metrics", + EXECUTION_METRICS: "Execution metrics", + WORKER_METRICS: "Worker fleet metrics", + RPC_METRICS: "gRPC server metrics", + SCHEDULER_METRICS: "Scheduler metrics", + STORE_TIER_METRICS: "Store tier metrics", + HEALTH_METRICS: "Health check metrics", + CONNECTION_METRICS: "Connection pool metrics", +}; + +const GROUP_GATES = { + CACHE_METRICS: + "Every cache instrument is emitted from one place: the `cache_metrics` store wrapper. A store that is not wrapped emits nothing, no matter how the collector is configured.", + EXECUTION_METRICS: + "Execution instruments are emitted from the scheduler process as actions change state, plus the per-action CPU time and peak memory a worker reports back when it finishes. Worker processes emit no execution metrics of their own.", + WORKER_METRICS: + "Worker fleet instruments are emitted from the scheduler process as workers connect, heartbeat, pause, drain and leave. They describe the pool as this scheduler instance sees it.", + RPC_METRICS: + "Emitted by the `OtlpLayer` tower middleware on every gRPC server NativeLink starts, so a CAS, a scheduler and a worker API endpoint each report their own.", + SCHEDULER_METRICS: + "Emitted from the scheduler process once per matching pass, whether or not the pass assigned anything.", + STORE_TIER_METRICS: + "Emitted from the `fast_slow` store wrapper only. A deployment without a `fast_slow` store emits none of these.", + HEALTH_METRICS: + "Emitted each time the health check endpoint runs, one observation per registered health indicator.", + CONNECTION_METRICS: + "Emitted by the gRPC connection manager and the Redis store as they hand out and re-establish connections. A deployment that uses neither emits none of these.", +}; + +function main() { + const metricsPath = join(repoRoot, METRICS_RS); + if (!existsSync(metricsPath)) { + throw new Error(`Cannot find ${METRICS_RS}. Is this running inside the NativeLink repo?`); + } + const src = readFileSync(metricsPath, "utf8"); + + let ref = "main"; + let commit = ""; + try { + commit = git(["rev-parse", "--short", "HEAD"]); + ref = git(["describe", "--tags", "--abbrev=0"]); + } catch { + // Detached or shallow checkout; fall back to `main`. + } + + const groups = parseGroups(src); + if (groups.length === 0) throw new Error("Parsed zero metric groups; the parser has drifted."); + + const files = sourceFiles(); + const calls = findCallSites(files, src, groups); + const artifacts = scanArtifacts(); + + const matchedArtifactNames = new Set(); + for (const g of groups) { + g.title = GROUP_TITLES[g.name] ?? g.name; + g.gate = GROUP_GATES[g.name] ?? ""; + for (const m of g.metrics) { + m.emit = verdictFor(m.field, calls); + m.promNameGuess = promBase(m.otelName); + const base = m.promNameGuess; + const hits = [...artifacts.entries()].filter(([name]) => stripSuffix(name) === base); + if (hits.length) { + // Prefer the un-suffixed form when both appear. + hits.sort((a, b) => a[0].length - b[0].length); + m.promName = hits[0][0]; + m.artifacts = hits.flatMap(([, sites]) => sites).slice(0, 4); + for (const [name] of hits) matchedArtifactNames.add(name); + } else { + m.promName = null; + m.artifacts = []; + } + } + } + + const { keys, valueEnums } = parseAttributes(src); + for (const e of valueEnums) { + const attr = Object.entries(ATTR_ENUM).find(([, v]) => v === e.name); + if (attr) e.attribute = keys.find((k) => k.const === attr[0])?.key ?? e.name; + } + for (const k of keys) { + const enumName = ATTR_ENUM[k.const]; + k.values = enumName ? (valueEnums.find((e) => e.name === enumName)?.variants ?? []) : []; + } + + const attributeGroups = [ + { title: "Cache attributes", keys: keys.filter((k) => k.key.startsWith("cache.")) }, + { title: "Execution attributes", keys: keys.filter((k) => k.key.startsWith("execution.")) }, + { title: "Worker attributes", keys: keys.filter((k) => k.key.startsWith("worker.")) }, + { title: "gRPC attributes", keys: keys.filter((k) => k.key.startsWith("rpc.")) }, + { title: "Scheduler attributes", keys: keys.filter((k) => k.key.startsWith("scheduler.")) }, + { title: "Store tier attributes", keys: keys.filter((k) => k.key.startsWith("store.")) }, + { title: "Health attributes", keys: keys.filter((k) => k.key.startsWith("health.")) }, + { title: "Connection attributes", keys: keys.filter((k) => k.key.startsWith("connection.")) }, + ].filter((g) => g.keys.length); + + // Series the shipped config mentions that no instrument produces. Names that + // are recording-rule outputs (`nativelink:`-prefixed in Prometheus, but written + // with an underscore in a few places) and the rule-group names themselves are + // not series, so drop anything that matches no instrument *and* is a prefix of + // no instrument base. + const allBases = new Set(groups.flatMap((g) => g.metrics.map((m) => m.promNameGuess))); + const orphanArtifactNames = [...artifacts.entries()] + .filter(([name]) => !matchedArtifactNames.has(name)) + .filter(([name]) => { + const base = stripSuffix(name); + if (allBases.has(base)) return false; + // Rule-group names (`nativelink_execution`, `nativelink_cache`, …) and + // truncated fragments are prefixes of a real base; they are not series. + for (const b of allBases) if (b.startsWith(name) || b.startsWith(base)) return false; + return true; + }) + .map(([name, sites]) => ({ name, sites })); + + const inventory = { groups, attributeGroups, valueEnums, orphanArtifactNames }; + const mdx = metricsToMdx(inventory, { + version: ref, + ref, + commit, + githubBase: GITHUB_BASE, + }); + writeFileSync(OUT_FILE, mdx); + + const total = groups.reduce((n, g) => n + g.metrics.length, 0); + const never = groups.reduce( + (n, g) => n + g.metrics.filter((m) => m.emit.verdict === "never").length, + 0, + ); + console.log(`Wrote ${relative(repoRoot, OUT_FILE)}`); + console.log(` ${total} instruments in ${groups.length} groups (${ref}${commit ? ` ${commit}` : ""})`); + console.log(` ${never} never emitted, ${orphanArtifactNames.length} orphan series in shipped config`); + for (const g of groups) { + for (const m of g.metrics) { + const flag = m.emit.verdict === "never" ? "NEVER " : m.emit.verdict === "gated" ? "gated " : "emitted"; + console.log(` ${flag} ${m.otelName}`); + } + } +} + +main(); diff --git a/web/apps/docs/scripts/lib/metrics-to-mdx.mjs b/web/apps/docs/scripts/lib/metrics-to-mdx.mjs new file mode 100644 index 000000000..2fec877e2 --- /dev/null +++ b/web/apps/docs/scripts/lib/metrics-to-mdx.mjs @@ -0,0 +1,413 @@ +// Transform a scanned NativeLink metric inventory into an MDX reference page. +// +// This is the metrics-side sibling of `schema-to-mdx.mjs`. Where that module +// turns a schemars JSON Schema into the configuration reference, this one turns +// the inventory produced by `scripts/gen-metrics-reference.mjs`, read straight +// out of `nativelink-util/src/metrics.rs` and the call sites across the +// workspace, into the metrics reference. +// +// The reason this exists rather than a hand-written catalogue: a metric that is +// *declared* is not a metric that is *emitted*. NativeLink declares fourteen +// OpenTelemetry instruments; several of them have no call site anywhere in the +// binary, and the shipped Prometheus rules reference two of those by name. A +// hand-written catalogue cannot notice that. A generated one notices it every +// time it runs. +// +// The module has no dependencies and no side effects: `metricsToMdx(inventory, +// opts)` is a pure function from inventory JSON to an MDX string. + +/** @typedef {Record} Inventory */ + +// -------------------------------------------------------------------------- +// MDX-safe text. Rust doc-comments are CommonMark, but MDX also treats `<` as +// the start of JSX and `{` as the start of a JS expression. +// -------------------------------------------------------------------------- + +function escapeMdxText(text) { + return text.replace(/ + segment.startsWith("`") && segment.endsWith("`") ? segment : escapeMdxText(segment), + ) + .join(""); +} + +function slug(name) { + return name + .toLowerCase() + .replace(/[^a-z0-9 -]/g, "-") + .trim() + .replace(/\s+/g, "-") + .replace(/-+/g, "-"); +} + +// -------------------------------------------------------------------------- +// Human labels for the mechanical verdicts the scanner produces. +// -------------------------------------------------------------------------- + +const VERDICT_LABEL = { + emitted: "Yes", + gated: "Conditionally", + never: "**Never**", +}; + +function sourceLink(githubBase, ref, file, line) { + const anchor = line ? `#L${line}` : ""; + const label = line ? `${file}:${line}` : file; + return `[\`${label}\`](${githubBase}/blob/${ref}/${file}${anchor})`; +} + +function siteList(sites, githubBase, ref) { + if (!sites.length) return "—"; + return sites.map((s) => sourceLink(githubBase, ref, s.file, s.line)).join(", "); +} + +// -------------------------------------------------------------------------- +// Sections. +// -------------------------------------------------------------------------- + +function renderSummaryTable(inventory, githubBase, ref) { + const rows = []; + for (const group of inventory.groups) { + for (const m of group.metrics) { + const prom = m.promName + ? `\`${m.promName}\`` + : `\`${m.promNameGuess}\` (unreferenced)`; + rows.push( + `| [\`${m.otelName}\`](#${slug(m.otelName)}) | ${prom} | ${m.kind} | ${ + VERDICT_LABEL[m.emit.verdict] + } | ${escapeTableCell(m.emit.gate)} |`, + ); + } + } + return [ + "| Instrument | Prometheus series | Kind | Emitted | Under what conditions |", + "| --- | --- | --- | --- | --- |", + ...rows, + "", + ].join("\n"); +} + +function renderNeverEmitted(inventory, githubBase, ref) { + const dead = []; + for (const group of inventory.groups) { + for (const m of group.metrics) { + if (m.emit.verdict === "never") dead.push({ group, m }); + } + } + if (dead.length === 0) { + return [ + "## Declared but never emitted", + "", + "Every instrument declared in `metrics.rs` has at least one call site.", + "", + ].join("\n"); + } + + const out = [ + "## Declared but never emitted", + "", + `${dead.length === 1 ? "One instrument is" : `${dead.length} instruments are`} built by the`, + "meter and stored on the metrics struct, but no code anywhere in the workspace", + "ever calls `.add()` or `.record()` on them. They will never appear in your", + "Prometheus output, at any configuration, on any release this page was", + "generated from.", + "", + ]; + + const rows = dead.map(({ m }) => { + const refs = m.artifacts.length + ? m.artifacts.map((a) => sourceLink(githubBase, ref, a.file, a.line)).join(", ") + : "—"; + return `| \`${m.otelName}\` | \`${m.promName ?? m.promNameGuess}\` | ${refs} |`; + }); + out.push( + "| Instrument | Series it would produce | Referenced by shipped config |", + "| --- | --- | --- |", + ...rows, + "", + ); + + const withRefs = dead.filter((d) => d.m.artifacts.length); + if (withRefs.length) { + out.push( + ``, + "", + ` ${withRefs + .map((d) => `\`${d.m.promName ?? d.m.promNameGuess}\``) + .join(" and ")} appear in the recording rules and the metrics`, + " README under `deployment-examples/metrics/`, but nothing emits them. Any", + " recording rule, dashboard panel or alert built on them evaluates to an", + " empty vector forever, which reads on a dashboard as a healthy zero, not", + " as a missing signal. Delete those panels or treat them as known-empty.", + "", + "", + "", + ); + } + return out.join("\n"); +} + +function renderOrphans(inventory, githubBase, ref) { + const orphans = inventory.orphanArtifactNames ?? []; + if (orphans.length === 0) return ""; + const rows = orphans.map( + (o) => `| \`${o.name}\` | ${siteList(o.sites, githubBase, ref)} |`, + ); + return [ + "## Series referenced by shipped config with no matching instrument", + "", + "These `nativelink_`-prefixed series appear in the shipped Prometheus rules,", + "dashboards or alerts, but do not correspond to any instrument declared in", + "`metrics.rs`. Either they are stale names from an older release, or they are", + "typos. Either way they resolve to nothing.", + "", + "| Series | Referenced from |", + "| --- | --- |", + ...rows, + "", + ].join("\n"); +} + +function renderMetric(m, githubBase, ref) { + const out = [`### \`${m.otelName}\``, ""]; + if (m.description) out.push(prose(m.description), ""); + + const facts = [ + `| Instrument type | \`${m.rustType}\` |`, + `| Field | \`${m.field}\` |`, + `| Unit | ${m.unit ? `\`${m.unit}\`` : "—"} |`, + `| Prometheus series | ${m.promName ? `\`${m.promName}\`` : `\`${m.promNameGuess}\` (not referenced by any shipped rule or dashboard)`} |`, + `| Emitted | ${VERDICT_LABEL[m.emit.verdict]}. ${escapeTableCell(m.emit.gate)} |`, + `| Declared at | ${sourceLink(githubBase, ref, m.declaredAt.file, m.declaredAt.line)} |`, + `| Emitted from | ${siteList(m.emit.sites, githubBase, ref)} |`, + ]; + out.push("| Property | Value |", "| --- | --- |", ...facts, ""); + + if (m.boundaries?.length) { + out.push( + `**Histogram buckets:** ${m.boundaries.length} explicit boundaries, from`, + `\`${m.boundaries[0]}\` to \`${m.boundaries[m.boundaries.length - 1]}\`${ + m.unit ? ` \`${m.unit}\`` : "" + }. Anything past the last boundary lands in the`, + "`+Inf` bucket, so a quantile computed above it is a lower bound, not a", + "measurement.", + "", + ); + } + return out.join("\n"); +} + +function renderGroup(group, githubBase, ref) { + const out = [`## ${group.title}`, ""]; + if (group.doc) out.push(prose(group.doc), ""); + out.push( + `Declared as \`${group.name}\` in ${sourceLink( + githubBase, + ref, + group.declaredAt.file, + group.declaredAt.line, + )}, backed by the \`${group.struct}\` struct.`, + "", + ); + if (group.gate) out.push(group.gate, ""); + for (const m of group.metrics) out.push(renderMetric(m, githubBase, ref)); + return out.join("\n"); +} + +function renderAttributes(inventory, githubBase, ref) { + const out = ["## Attributes", ""]; + out.push( + "Every attribute key NativeLink attaches to a metric, and the closed set of", + "values it can take. The Prometheus label is the key with dots replaced by", + "underscores.", + "", + ); + for (const group of inventory.attributeGroups) { + out.push(`### ${group.title}`, ""); + const rows = group.keys.map((k) => { + const values = k.values.length + ? k.values.map((v) => `\`${v.value}\``).join(", ") + : "open (free-form string)"; + return `| \`${k.key}\` | \`${k.key.replace(/\./g, "_")}\` | \`${k.const}\` | ${escapeTableCell(values)} |`; + }); + out.push( + "| Attribute | Prometheus label | Rust constant | Values |", + "| --- | --- | --- | --- |", + ...rows, + "", + ); + } + const enums = inventory.valueEnums.filter((e) => e.variants.some((v) => v.doc)); + for (const e of enums) { + out.push(`### \`${e.attribute ?? e.name}\` values`, ""); + out.push( + "| Value | Meaning |", + "| --- | --- |", + ...e.variants.map((v) => `| \`${v.value}\` | ${cell(v.doc)} |`), + "", + ); + } + return out.join("\n"); +} + +// -------------------------------------------------------------------------- +// Public entry point. +// -------------------------------------------------------------------------- + +/** + * @param {Inventory} inventory Produced by scripts/gen-metrics-reference.mjs. + * @param {object} opts + * @param {string} opts.version Display version, e.g. "v1.6.3". + * @param {string} opts.ref Git ref the sources were read from. + * @param {string} [opts.commit] Short commit the sources were read from. + * @param {string} [opts.githubBase] e.g. "https://github.com/TraceMachina/nativelink". + * @returns {string} MDX document. + */ +export function metricsToMdx(inventory, opts) { + const { + version, + ref, + commit, + githubBase = "https://github.com/TraceMachina/nativelink", + } = opts; + + const total = inventory.groups.reduce((n, g) => n + g.metrics.length, 0); + const never = inventory.groups.reduce( + (n, g) => n + g.metrics.filter((m) => m.emit.verdict === "never").length, + 0, + ); + + const frontmatter = [ + "---", + "title: Metrics reference", + "description: Every OpenTelemetry instrument NativeLink declares, with its type, unit, attributes, Prometheus series, and whether the binary actually emits it. Autogenerated from the Rust source and its call sites.", + "full: true", + "---", + "", + ].join("\n"); + + const provenance = [ + "{/* AUTOGENERATED. Do not edit by hand.", + ` Source: nativelink-util/src/metrics.rs @ ${ref}${commit ? ` (${commit})` : ""}`, + " Regenerate from web/: bun --filter @nativelink/docs gen:metrics-reference */}", + "", + ].join("\n"); + + const intro = [ + `NativeLink **${version}** declares ${total} OpenTelemetry instruments. This page is`, + "generated from the declarations in", + `[\`nativelink-util/src/metrics.rs\`](${githubBase}/blob/${ref}/nativelink-util/src/metrics.rs)`, + "and from every `.add()` and `.record()` call site across the workspace, so the", + "**Emitted** column is not a claim, it is the result of looking.", + "", + "That column is the reason this page is generated rather than written. Declaring", + "an instrument costs nothing and emits nothing; the instrument only produces a", + "series when some code path calls it. A catalogue written by hand cannot tell", + "those two states apart, and the gap between them is where dashboards go quietly", + "wrong.", + "", + ].join("\n"); + + const headline = never + ? [ + ``, + "", + " They are constructed at startup and stored on the metrics struct, but no", + " code path calls them. See [Declared but never", + " emitted](#declared-but-never-emitted) for which ones, and for the shipped", + " Prometheus rules that query them anyway.", + "", + "", + "", + ].join("\n") + : ""; + + const naming = [ + "## How an instrument becomes a Prometheus series", + "", + "NativeLink speaks OTLP only; there is no Prometheus scrape endpoint in the", + "binary. The names in the middle column above are what the shipped OpenTelemetry", + "collector produces after two transformations:", + "", + "1. The Prometheus exporter in", + ` [\`otel-collector-config.yaml\`](${githubBase}/blob/${ref}/deployment-examples/metrics/otel-collector-config.yaml)`, + " is configured with `namespace: nativelink`, which prefixes every series with", + " `nativelink_`.", + "2. The exporter rewrites the OTLP name into Prometheus form: dots become", + " underscores, and monotonic counters gain a `_total` suffix. Histograms fan out", + " into `_bucket`, `_sum` and `_count` series.", + "", + "`cache.operations` (a counter) is therefore scraped as `nativelink_cache_operations_total`,", + "and `cache.operation.duration` (a histogram) as `nativelink_cache_operation_duration_bucket`", + "and friends. Change the collector's `namespace` and every name on this page changes", + "with it.", + "", + "Where a name in the **Prometheus series** column is unmarked, it is not a", + "prediction: it is the name that actually appears in the shipped rules,", + "dashboards and alerts under `deployment-examples/metrics/` and `kubernetes/`.", + "A name marked _(unreferenced)_ is derived from the two rules above, because no", + "shipped artifact mentions that metric at all: nothing has ever queried it, so", + "nothing has confirmed the exporter's spelling of it.", + "", + ].join("\n"); + + const body = [ + frontmatter, + provenance, + intro, + headline, + "## Every instrument at a glance", + "", + renderSummaryTable(inventory, githubBase, ref), + naming, + renderNeverEmitted(inventory, githubBase, ref), + renderOrphans(inventory, githubBase, ref), + ...inventory.groups.map((g) => renderGroup(g, githubBase, ref)), + renderAttributes(inventory, githubBase, ref), + "## Reading the source", + "", + "If anything here disagrees with the binary, the source wins:", + "", + `- [\`nativelink-util/src/metrics.rs\`](${githubBase}/blob/${ref}/nativelink-util/src/metrics.rs): every declaration`, + `- [\`nativelink-util/src/telemetry.rs\`](${githubBase}/blob/${ref}/nativelink-util/src/telemetry.rs): the OTLP exporter and its interval`, + `- [\`nativelink-store/src/cache_metrics_store.rs\`](${githubBase}/blob/${ref}/nativelink-store/src/cache_metrics_store.rs): every cache call site`, + `- [\`nativelink-scheduler/src/simple_scheduler_state_manager.rs\`](${githubBase}/blob/${ref}/nativelink-scheduler/src/simple_scheduler_state_manager.rs): every execution call site`, + "", + "## Where to go next", + "", + "", + " The collector, the exporter interval and the queries worth alerting on: the", + " operational half of this reference.", + "", + "", + "", + " Every configuration field, generated the same way from the same source of", + " truth.", + "", + "", + ].join("\n"); + + return `${body.replace(/\n{3,}/g, "\n\n").trimEnd()}\n`; +}