feat: add Rust-to-Python const code generator for prometheus_names.py

Cargo.toml

@@ -11,6 +11,7 @@ members = [
     "lib/async-openai",
     "lib/parsers",
     "lib/bindings/c",
+    "lib/bindings/python/codegen",
     "lib/engines/*",
 ]
 # Exclude certain packages that are slow to build and we don't ship as flagship

components/src/dynamo/planner/utils/prometheus.py

@@ -19,7 +19,7 @@
 from prometheus_api_client import PrometheusConnect
 from pydantic import BaseModel, ValidationError
 
-from dynamo._core import prometheus_names
+from dynamo import prometheus_names
 from dynamo.runtime.logging import configure_dynamo_logging
 
 configure_dynamo_logging()
@@ -94,23 +94,23 @@ def _get_average_metric(
 
     def get_avg_inter_token_latency(self, interval: str, model_name: str):
         return self._get_average_metric(
-            prometheus_names.frontend.inter_token_latency_seconds,
+            prometheus_names.frontend_service.INTER_TOKEN_LATENCY_SECONDS,
             interval,
             "avg inter token latency",
             model_name,
         )
 
     def get_avg_time_to_first_token(self, interval: str, model_name: str):
         return self._get_average_metric(
-            prometheus_names.frontend.time_to_first_token_seconds,
+            prometheus_names.frontend_service.TIME_TO_FIRST_TOKEN_SECONDS,
             interval,
             "avg time to first token",
             model_name,
         )
 
     def get_avg_request_duration(self, interval: str, model_name: str):
         return self._get_average_metric(
-            prometheus_names.frontend.request_duration_seconds,
+            prometheus_names.frontend_service.REQUEST_DURATION_SECONDS,
             interval,
             "avg request duration",
             model_name,
@@ -119,7 +119,7 @@ def get_avg_request_duration(self, interval: str, model_name: str):
     def get_avg_request_count(self, interval: str, model_name: str):
         # This function follows a different query pattern than the other metrics
         try:
-            requests_total_metric = prometheus_names.frontend.requests_total
+            requests_total_metric = prometheus_names.frontend_service.REQUESTS_TOTAL
             raw_res = self.prom.custom_query(
                 query=f"increase({requests_total_metric}[{interval}])"
             )
@@ -138,15 +138,15 @@ def get_avg_request_count(self, interval: str, model_name: str):
 
     def get_avg_input_sequence_tokens(self, interval: str, model_name: str):
         return self._get_average_metric(
-            prometheus_names.frontend.input_sequence_tokens,
+            prometheus_names.frontend_service.INPUT_SEQUENCE_TOKENS,
             interval,
             "avg input sequence tokens",
             model_name,
         )
 
     def get_avg_output_sequence_tokens(self, interval: str, model_name: str):
         return self._get_average_metric(
-            prometheus_names.frontend.output_sequence_tokens,
+            prometheus_names.frontend_service.OUTPUT_SEQUENCE_TOKENS,
             interval,
             "avg output sequence tokens",
             model_name,

lib/bindings/python/codegen/Cargo.toml

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "dynamo-codegen"
+version = "0.1.0"
+edition = "2021"
+license = "Apache-2.0"
+
+[dependencies]
+syn = { version = "2.0", features = ["full", "extra-traits"] }
+quote = "1.0"
+proc-macro2 = "1.0"
+anyhow = "1.0"
+
+[[bin]]
+name = "gen-python-prometheus-names"
+path = "src/gen_python_prometheus_names.rs"

lib/bindings/python/codegen/README.md

@@ -0,0 +1,38 @@
+# Dynamo Codegen
+
+Python code generator for Dynamo Python bindings.
+
+## gen-python-prometheus-names
+
+Generates `prometheus_names.py` from Rust source `lib/runtime/src/metrics/prometheus_names.rs`.
+
+### Usage
+
+```bash
+cargo run -p dynamo-codegen --bin gen-python-prometheus-names
+```
+
+### What it does
+
+- Parses Rust AST from `lib/runtime/src/metrics/prometheus_names.rs`
+- Generates Python classes with constants at `lib/bindings/python/src/dynamo/prometheus_names.py`
+- Handles macro-generated constants (e.g., `kvstats_name!("active_blocks")` → `"kvstats_active_blocks"`)
+
+### Example
+
+**Rust input:**
+```rust
+pub mod kvstats {
+    pub const ACTIVE_BLOCKS: &str = kvstats_name!("active_blocks");
+}
+```
+
+**Python output:**
+```python
+class kvstats:
+    ACTIVE_BLOCKS = "kvstats_active_blocks"
+```
+
+### When to run
+
+Run after modifying `lib/runtime/src/metrics/prometheus_names.rs` to regenerate the Python file.

lib/bindings/python/codegen/src/gen_python_prometheus_names.rs

@@ -0,0 +1,218 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Binary to generate Python prometheus_names from Rust source
+
+use anyhow::{Context, Result};
+use dynamo_codegen::prometheus_parser::{ModuleDef, PrometheusParser};
+use std::collections::HashMap;
+use std::path::PathBuf;
+
+/// Generates Python module code from parsed Rust prometheus_names modules.
+/// Converts Rust const declarations into Python class attributes with deterministic ordering.
+struct PythonGenerator<'a> {
+    modules: &'a HashMap<String, ModuleDef>,
+}
+
+impl<'a> PythonGenerator<'a> {
+    fn new(parser: &'a PrometheusParser) -> Self {
+        Self {
+            modules: &parser.modules,
+        }
+    }
+
+    fn load_template(template_name: &str) -> String {
+        let template_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .join("templates")
+            .join(template_name);
+
+        std::fs::read_to_string(&template_path)
+            .unwrap_or_else(|_| panic!("Failed to read template: {}", template_path.display()))
+    }
+
+    fn generate_python_file(&self) -> String {
+        let mut output = Self::load_template("prometheus_names.py.template");
+
+        // Append generated classes
+        output.push_str(&self.generate_classes());
+
+        output
+    }
+
+    fn generate_classes(&self) -> String {
+        let mut lines = Vec::new();
+
+        // Sort module names to ensure deterministic output
+        let mut module_names: Vec<&String> = self.modules.keys().collect();
+        module_names.sort();
+
+        // Generate simple classes with constants as class attributes
+        for module_name in module_names {
+            let module = &self.modules[module_name];
+            lines.push(format!("class {}:", module_name));
+
+            // Use doc comment from module if available
+            if !module.doc_comment.is_empty() {
+                let first_line = module.doc_comment.lines().next().unwrap_or("").trim();
+                if !first_line.is_empty() {
+                    lines.push(format!("    \"\"\"{}\"\"\"", first_line));
+                }
+            }
+            lines.push("".to_string());
+
+            for constant in &module.constants {
+                if !constant.doc_comment.is_empty() {
+                    for comment_line in constant.doc_comment.lines() {
+                        lines.push(format!("    # {}", comment_line));
+                    }
+                }
+                lines.push(format!("    {} = \"{}\"", constant.name, constant.value));
+            }
+
+            lines.push("".to_string());
+        }
+
+        lines.join("\n")
+    }
+}
+
+fn main() -> Result<()> {
+    let args: Vec<String> = std::env::args().collect();
+
+    let mut source_path: Option<PathBuf> = None;
+    let mut output_path: Option<PathBuf> = None;
+
+    let mut i = 1;
+    while i < args.len() {
+        match args[i].as_str() {
+            "--source" => {
+                i += 1;
+                if i < args.len() {
+                    source_path = Some(PathBuf::from(&args[i]));
+                }
+            }
+            "--output" => {
+                i += 1;
+                if i < args.len() {
+                    output_path = Some(PathBuf::from(&args[i]));
+                }
+            }
+            "--help" | "-h" => {
+                print_usage();
+                return Ok(());
+            }
+            _ => {
+                eprintln!("Unknown argument: {}", args[i]);
+                print_usage();
+                std::process::exit(1);
+            }
+        }
+        i += 1;
+    }
+
+    // Determine paths relative to codegen directory
+    let codegen_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+
+    let source = source_path.unwrap_or_else(|| {
+        // From: lib/bindings/python/codegen
+        // To:   lib/runtime/src/metrics/prometheus_names.rs
+        codegen_dir
+            .join("../../../runtime/src/metrics/prometheus_names.rs")
+            .canonicalize()
+            .expect("Failed to resolve source path")
+    });
+
+    let output = output_path.unwrap_or_else(|| {
+        // From: lib/bindings/python/codegen
+        // To:   lib/bindings/python/src/dynamo/prometheus_names.py
+        codegen_dir
+            .join("../src/dynamo/prometheus_names.py")
+            .canonicalize()
+            .unwrap_or_else(|_| {
+                // If file doesn't exist yet, resolve the parent directory
+                let dir = codegen_dir
+                    .join("../src/dynamo")
+                    .canonicalize()
+                    .expect("Failed to resolve output directory");
+                dir.join("prometheus_names.py")
+            })
+    });
+
+    println!("Generating Python prometheus_names from Rust source");
+    println!("Source: {}", source.display());
+    println!("Output: {}", output.display());
+    println!();
+
+    let content = std::fs::read_to_string(&source)
+        .with_context(|| format!("Failed to read source file: {}", source.display()))?;
+
+    println!("Parsing Rust AST...");
+    let parser = PrometheusParser::parse_file(&content)?;
+
+    println!("Found {} modules:", parser.modules.len());
+    let mut module_names: Vec<&String> = parser.modules.keys().collect();
+    module_names.sort();
+    for name in module_names.iter() {
+        let module = &parser.modules[name.as_str()];
+        println!(
+            "  - {}: {} constants{}",
+            name,
+            module.constants.len(),
+            if module.is_macro_generated {
+                " (macro-generated)"
+            } else {
+                ""
+            }
+        );
+    }
+
+    println!("\nGenerating Python prometheus_names module...");
+    let generator = PythonGenerator::new(&parser);
+    let python_code = generator.generate_python_file();
+
+    // Ensure output directory exists
+    if let Some(parent) = output.parent() {
+        std::fs::create_dir_all(parent)
+            .with_context(|| format!("Failed to create output directory: {}", parent.display()))?;
+    }
+
+    std::fs::write(&output, python_code)
+        .with_context(|| format!("Failed to write output file: {}", output.display()))?;
+
+    println!("✓ Generated Python prometheus_names: {}", output.display());
+    println!("\nSuccess! Python module ready for import.");
+
+    Ok(())
+}
+
+fn print_usage() {
+    println!(
+        r#"
+gen-python-prometheus-names - Generate Python prometheus_names from Rust source
+
+Usage: gen-python-prometheus-names [OPTIONS]
+
+Parses lib/runtime/src/metrics/prometheus_names.rs and generates a pure Python
+module with 1:1 constant mappings at lib/bindings/python/src/dynamo/prometheus_names.py
+
+This allows Python code to import Prometheus metric constants without Rust bindings:
+    from dynamo.prometheus_names import frontend_service, kvstats
+
+OPTIONS:
+    --source PATH    Path to Rust source file
+                     (default: lib/runtime/src/metrics/prometheus_names.rs)
+
+    --output PATH    Path to Python output file
+                     (default: lib/bindings/python/src/dynamo/prometheus_names.py)
+
+    --help, -h       Print this help message
+
+EXAMPLES:
+    # Generate with default paths
+    cargo run -p dynamo-codegen --bin gen-python-prometheus-names
+
+    # Generate with custom output
+    cargo run -p dynamo-codegen --bin gen-python-prometheus-names -- --output /tmp/test.py
+"#
+    );
+}

lib/bindings/python/codegen/src/lib.rs

@@ -0,0 +1,8 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Code generation utilities for Dynamo project
+//!
+//! This crate provides tools to generate code from Rust sources to other languages.
+
+pub mod prometheus_parser;