feat(observability): expose dashboard API metrics

ods/extensions/services/dashboard-api/README.md

@@ -14,6 +14,7 @@ It runs at `http://localhost:3002` and is the single backend used by the React d
 - **Service health**: Health checks for all ODS services via Docker network
 - **LLM metrics**: Tokens/second, lifetime tokens, loaded model, context size
 - **System metrics**: CPU usage, RAM usage, uptime, disk space
+- **API observability**: Prometheus request counts, in-flight gauge, and latency histograms with bounded route labels
 - **Workflow management**: n8n workflow catalog — install, enable, disable, track executions
 - **Feature discovery**: Hardware-aware feature recommendations with VRAM tier detection
 - **Setup wizard**: First-run setup, persona selection, diagnostic tests
@@ -44,6 +45,7 @@ Environment variables (set in `.env`):
 | Method | Path | Auth | Description |
 |--------|------|------|-------------|
 | `GET` | `/health` | No | Health check |
+| `GET` | `/metrics` | No | Prometheus API request metrics (route templates, status, and latency) |
 | `GET` | `/gpu` | Yes | GPU metrics (VRAM, temp, utilization) |
 | `GET` | `/services` | Yes | All service health statuses |
 | `GET` | `/disk` | Yes | Disk usage |

ods/extensions/services/dashboard-api/api_metrics.py

@@ -0,0 +1,146 @@
+"""Low-cardinality Prometheus metrics for the dashboard API."""
+
+import threading
+import time
+from collections import defaultdict
+
+
+_DURATION_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0)
+
+
+def _label_value(value: str) -> str:
+    return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+
+class ApiMetrics:
+    """Thread-safe process-local request metrics registry."""
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._started_at = time.time()
+        self._in_flight = 0
+        self._requests: dict[tuple[str, str, int], int] = defaultdict(int)
+        self._duration_count: dict[tuple[str, str], int] = defaultdict(int)
+        self._duration_sum: dict[tuple[str, str], float] = defaultdict(float)
+        self._duration_buckets: dict[tuple[str, str, float], int] = defaultdict(int)
+
+    def request_started(self) -> None:
+        with self._lock:
+            self._in_flight += 1
+
+    def request_finished(
+        self,
+        method: str,
+        route: str,
+        status_code: int,
+        duration_seconds: float,
+    ) -> None:
+        method = method.upper()
+        route = route or "unmatched"
+        duration_seconds = max(0.0, duration_seconds)
+        key = (method, route)
+        with self._lock:
+            self._in_flight -= 1
+            self._requests[(method, route, status_code)] += 1
+            self._duration_count[key] += 1
+            self._duration_sum[key] += duration_seconds
+            for bucket in _DURATION_BUCKETS:
+                if duration_seconds <= bucket:
+                    self._duration_buckets[(method, route, bucket)] += 1
+
+    def reset(self) -> None:
+        """Clear observations; intended for isolated tests."""
+        with self._lock:
+            self._started_at = time.time()
+            self._in_flight = 0
+            self._requests.clear()
+            self._duration_count.clear()
+            self._duration_sum.clear()
+            self._duration_buckets.clear()
+
+    def render(self) -> str:
+        with self._lock:
+            started_at = self._started_at
+            in_flight = self._in_flight
+            requests = dict(self._requests)
+            duration_count = dict(self._duration_count)
+            duration_sum = dict(self._duration_sum)
+            duration_buckets = dict(self._duration_buckets)
+
+        lines = [
+            "# HELP ods_dashboard_api_process_start_time_seconds Start time of this API process.",
+            "# TYPE ods_dashboard_api_process_start_time_seconds gauge",
+            f"ods_dashboard_api_process_start_time_seconds {started_at:.3f}",
+            "# HELP ods_dashboard_api_http_requests_in_flight Requests currently being served.",
+            "# TYPE ods_dashboard_api_http_requests_in_flight gauge",
+            f"ods_dashboard_api_http_requests_in_flight {in_flight}",
+            "# HELP ods_dashboard_api_http_requests_total Completed HTTP requests.",
+            "# TYPE ods_dashboard_api_http_requests_total counter",
+        ]
+        for (method, route, status), count in sorted(requests.items()):
+            labels = f'method="{_label_value(method)}",route="{_label_value(route)}",status="{status}"'
+            lines.append(f"ods_dashboard_api_http_requests_total{{{labels}}} {count}")
+
+        lines.extend([
+            "# HELP ods_dashboard_api_http_request_duration_seconds HTTP request latency.",
+            "# TYPE ods_dashboard_api_http_request_duration_seconds histogram",
+        ])
+        for method, route in sorted(duration_count):
+            labels = f'method="{_label_value(method)}",route="{_label_value(route)}"'
+            for bucket in _DURATION_BUCKETS:
+                count = duration_buckets.get((method, route, bucket), 0)
+                lines.append(
+                    "ods_dashboard_api_http_request_duration_seconds_bucket"
+                    f'{{{labels},le="{bucket:g}"}} {count}',
+                )
+            count = duration_count[(method, route)]
+            lines.append(
+                "ods_dashboard_api_http_request_duration_seconds_bucket"
+                f'{{{labels},le="+Inf"}} {count}',
+            )
+            lines.append(
+                f"ods_dashboard_api_http_request_duration_seconds_sum{{{labels}}} "
+                f"{duration_sum[(method, route)]:.9g}",
+            )
+            lines.append(
+                f"ods_dashboard_api_http_request_duration_seconds_count{{{labels}}} {count}",
+            )
+        return "\n".join(lines) + "\n"
+
+
+class ApiMetricsMiddleware:
+    """ASGI middleware that records one observation per completed request."""
+
+    def __init__(self, app, registry: ApiMetrics) -> None:
+        self.app = app
+        self.registry = registry
+
+    async def __call__(self, scope, receive, send) -> None:
+        if scope["type"] != "http" or scope.get("path") == "/metrics":
+            await self.app(scope, receive, send)
+            return
+
+        self.registry.request_started()
+        started = time.perf_counter()
+        status_code = 500
+
+        async def capture_status(message):
+            nonlocal status_code
+            if message["type"] == "http.response.start":
+                status_code = message["status"]
+            await send(message)
+
+        try:
+            await self.app(scope, receive, capture_status)
+        finally:
+            route = scope.get("route")
+            route_template = getattr(route, "path", "unmatched")
+            self.registry.request_finished(
+                scope.get("method", "UNKNOWN"),
+                route_template,
+                status_code,
+                time.perf_counter() - started,
+            )
+
+
+api_metrics = ApiMetrics()

ods/extensions/services/dashboard-api/main.py

@@ -28,7 +28,7 @@
 from typing import Any, Optional
 
 import httpx
-from fastapi import FastAPI, Depends, HTTPException, Body
+from fastapi import FastAPI, Depends, HTTPException, Body, Response
 from fastapi.middleware.cors import CORSMiddleware
 
 # --- Local modules ---
@@ -61,6 +61,7 @@
     shutdown_clients as shutdown_agent_clients,
 )
 from agent_monitor import collect_metrics
+from api_metrics import ApiMetricsMiddleware, api_metrics
 from routers import (
     workflows, features, setup, updates, agents, privacy, extensions,
     gpu as gpu_router, resources, voice, models as models_router, model_state as model_state_router,
@@ -1060,6 +1061,8 @@ async def _lifespan(app: FastAPI):
     lifespan=_lifespan,
 )
 
+app.add_middleware(ApiMetricsMiddleware, registry=api_metrics)
+
 # --- CORS ---
 
 def get_allowed_origins():
@@ -1126,6 +1129,15 @@ async def health():
     return {"status": "ok", "timestamp": datetime.now(timezone.utc).isoformat()}
 
 
+@app.get("/metrics", include_in_schema=False)
+async def prometheus_metrics():
+    """Expose process-local API request metrics for Prometheus scrapers."""
+    return Response(
+        content=api_metrics.render(),
+        media_type="text/plain; version=0.0.4",
+    )
+
+
 @app.get("/api/host-agent/diagnostics", dependencies=[Depends(verify_api_key)])
 async def host_agent_diagnostics():
     """Report how dashboard-api resolves and reaches the host agent."""

ods/extensions/services/dashboard-api/tests/test_api_metrics.py

@@ -0,0 +1,54 @@
+"""Tests for the dashboard API Prometheus request metrics."""
+
+from api_metrics import ApiMetrics, api_metrics
+
+
+def test_registry_renders_counter_gauge_and_cumulative_histogram():
+    registry = ApiMetrics()
+    registry.request_started()
+    registry.request_finished("get", "/api/items/{item_id}", 201, 0.02)
+
+    output = registry.render()
+
+    assert "ods_dashboard_api_http_requests_in_flight 0" in output
+    assert (
+        'ods_dashboard_api_http_requests_total{method="GET",'
+        'route="/api/items/{item_id}",status="201"} 1'
+    ) in output
+    assert (
+        'ods_dashboard_api_http_request_duration_seconds_bucket{method="GET",'
+        'route="/api/items/{item_id}",le="0.01"} 0'
+    ) in output
+    assert (
+        'ods_dashboard_api_http_request_duration_seconds_bucket{method="GET",'
+        'route="/api/items/{item_id}",le="0.025"} 1'
+    ) in output
+
+
+def test_metrics_endpoint_uses_route_templates_and_excludes_scrapes(test_client):
+    api_metrics.reset()
+
+    assert test_client.get("/health").status_code == 200
+    assert test_client.get(
+        "/api/extensions/example-service/progress",
+        headers=test_client.auth_headers,
+    ).status_code == 200
+    assert test_client.get("/not-a-real-resource/123").status_code == 404
+    response = test_client.get("/metrics")
+
+    assert response.status_code == 200
+    assert response.headers["content-type"].startswith("text/plain; version=0.0.4")
+    assert 'route="/health",status="200"} 1' in response.text
+    assert 'route="/api/extensions/{service_id}/progress",status="200"} 1' in response.text
+    assert 'route="/api/extensions/example-service/progress"' not in response.text
+    assert 'route="unmatched",status="404"} 1' in response.text
+    assert 'route="/not-a-real-resource/123"' not in response.text
+    assert 'route="/metrics"' not in response.text
+
+
+def test_registry_escapes_prometheus_label_values():
+    registry = ApiMetrics()
+    registry.request_started()
+    registry.request_finished("GET", '/quoted/"value"', 200, 0.001)
+
+    assert 'route="/quoted/\\"value\\""' in registry.render()