refactor: standardize Prometheus metric naming conventions (part 1)

Updated metric names to use consistent _total suffix for gauge metrics:
- inflight_request_total -> inflight_requests_total
- Added client_disconnects_total to prometheus_names.rs
- Updated documentation and test comments to reflect new naming

This is one of many PRs to address the naming convention. Only the Prometheus external names are updated for this PR. Not addressed for this PR: NATS naming (in the next PR), and variable name changes (another PR).

Signed-off-by: Keiven Chang <[email protected]>

Author: keivenchang

deploy/metrics/README.md

@@ -40,7 +40,7 @@ As of Q2 2025, Dynamo HTTP Frontend metrics are exposed when you build container
 
 The core Dynamo backend system automatically exposes metrics with the `dynamo_component_*` prefix for all components that use the `DistributedRuntime` framework:
 
-- `dynamo_component_inflight_requests`: Requests currently being processed (gauge)
+- `dynamo_component_inflight_requests_total`: Requests currently being processed (gauge)
 - `dynamo_component_request_bytes_total`: Total bytes received in requests (counter)
 - `dynamo_component_request_duration_seconds`: Request processing time (histogram)
 - `dynamo_component_requests_total`: Total requests processed (counter)

lib/llm/src/http/service/metrics.rs

@@ -117,7 +117,8 @@ impl Metrics {
     ///
     /// The following metrics will be created with the configured prefix:
     /// - `{prefix}_requests_total` - IntCounterVec for the total number of requests processed
-    /// - `{prefix}_inflight_requests` - IntGaugeVec for the number of inflight requests
+    /// - `{prefix}_inflight_requests_total` - IntGaugeVec for the number of inflight requests
+    /// - `{prefix}_client_disconnects_total` - IntGauge for the number of connections dropped by clients
     /// - `{prefix}_request_duration_seconds` - HistogramVec for the duration of requests
     /// - `{prefix}_input_sequence_tokens` - HistogramVec for input sequence length in tokens
     /// - `{prefix}_output_sequence_tokens` - HistogramVec for output sequence length in tokens
@@ -156,7 +157,7 @@ impl Metrics {
         .unwrap();
 
         let client_disconnect_gauge = prometheus::IntGauge::new(
-            frontend_metric_name("client_disconnects"),
+            frontend_metric_name(frontend_service::CLIENT_DISCONNECTS_TOTAL),
             "Number of connections dropped by clients",
         )
         .unwrap();

lib/runtime/examples/system_metrics/README.md

@@ -62,7 +62,7 @@ The `dynamo_component_errors_total` metric includes the following error types:
 - `dynamo_component_request_duration_seconds` - Request processing time
 
 ### Gauges
-- `dynamo_component_inflight_requests` - Number of requests currently being processed
+- `dynamo_component_inflight_requests_total` - Number of requests currently being processed
 
 ### Custom Metrics (Optional)
 - `dynamo_component_bytes_processed_total` - Total data bytes processed by system handler (example)
@@ -80,9 +80,9 @@ These labels are prefixed with "dynamo_" to avoid collisions with Kubernetes and
 When the system is running, you'll see metrics from http://<ip>:<port>/metrics like this:
 
 ```prometheus
-# HELP dynamo_component_inflight_requests Number of requests currently being processed by component endpoint
-# TYPE dynamo_component_inflight_requests gauge
-dynamo_component_inflight_requests{dynamo_component="example_component",dynamo_endpoint="example_endpoint9881",dynamo_namespace="example_namespace"} 0
+# HELP dynamo_component_inflight_requests_total Number of requests currently being processed by component endpoint
+# TYPE dynamo_component_inflight_requests_total gauge
+dynamo_component_inflight_requests_total{dynamo_component="example_component",dynamo_endpoint="example_endpoint9881",dynamo_namespace="example_namespace"} 0
 
 # HELP dynamo_component_bytes_processed_total Example of a custom metric. Total number of data bytes processed by system handler
 # TYPE dynamo_component_bytes_processed_total counter

lib/runtime/examples/system_metrics/tests/integration_test.rs

@@ -106,7 +106,7 @@ async fn test_backend_with_metrics() -> Result<()> {
                 println!("{}", metrics_content);
                 println!("=== END METRICS CONTENT ===");
 
-                // Parse and verify ingress metrics are greater than 0 (except inflight_requests)
+                // Parse and verify ingress metrics are greater than 0 (except inflight_requests_total)
                 verify_ingress_metrics_greater_than_0(&metrics_content);
 
                 println!("Successfully retrieved and verified metrics!");
@@ -131,7 +131,7 @@ async fn test_backend_with_metrics() -> Result<()> {
 }
 
 fn verify_ingress_metrics_greater_than_0(metrics_content: &str) {
-    // Define the work handler metrics we want to verify (excluding inflight_requests which can be 0)
+    // Define the work handler metrics we want to verify (excluding inflight_requests_total which can be 0)
     let metrics_to_verify = [
         "my_custom_bytes_processed_total",
         "requests_total",

lib/runtime/src/metrics.rs

@@ -1619,7 +1619,7 @@ mod test_metricsregistry_nats {
                 23000.0,
             ), // ~75-125% of 20660
             (
-                build_component_metric_name(work_handler::INFLIGHT_REQUESTS),
+                build_component_metric_name(work_handler::INFLIGHT_REQUESTS_TOTAL),
                 0.0,
                 1.0,
             ), // 0 or very low

lib/runtime/src/metrics/prometheus_names.rs

@@ -79,6 +79,9 @@ pub mod frontend_service {
     /// Number of inflight requests going to the engine (vLLM, SGLang, ...)
     pub const INFLIGHT_REQUESTS_TOTAL: &str = "inflight_requests_total";
 
+    /// Number of connections dropped by clients
+    pub const CLIENT_DISCONNECTS_TOTAL: &str = "client_disconnects_total";
+
     /// Duration of LLM requests
     pub const REQUEST_DURATION_SECONDS: &str = "request_duration_seconds";
 
@@ -125,7 +128,7 @@ pub mod work_handler {
     pub const RESPONSE_BYTES_TOTAL: &str = "response_bytes_total";
 
     /// Number of requests currently being processed by work handler
-    pub const INFLIGHT_REQUESTS: &str = "inflight_requests";
+    pub const INFLIGHT_REQUESTS_TOTAL: &str = "inflight_requests_total";
 
     /// Time spent processing requests by work handler (histogram)
     pub const REQUEST_DURATION_SECONDS: &str = "request_duration_seconds";

lib/runtime/src/pipeline/network/ingress/push_handler.rs

@@ -61,7 +61,7 @@ impl WorkHandlerMetrics {
         )?;
 
         let inflight_requests = endpoint.create_intgauge(
-            work_handler::INFLIGHT_REQUESTS,
+            work_handler::INFLIGHT_REQUESTS_TOTAL,
             "Number of requests currently being processed by work handler",
             metrics_labels,
         )?;