diff --git a/ods/extensions/services/dashboard-api/README.md b/ods/extensions/services/dashboard-api/README.md index b709b7aed..b36678a8b 100644 --- a/ods/extensions/services/dashboard-api/README.md +++ b/ods/extensions/services/dashboard-api/README.md @@ -14,6 +14,7 @@ It runs at `http://localhost:3002` and is the single backend used by the React d - **Service health**: Health checks for all ODS services via Docker network - **LLM metrics**: Tokens/second, lifetime tokens, loaded model, context size - **System metrics**: CPU usage, RAM usage, uptime, disk space +- **API observability**: Prometheus request counts, in-flight gauge, and latency histograms with bounded route labels - **Workflow management**: n8n workflow catalog — install, enable, disable, track executions - **Feature discovery**: Hardware-aware feature recommendations with VRAM tier detection - **Setup wizard**: First-run setup, persona selection, diagnostic tests @@ -44,6 +45,7 @@ Environment variables (set in `.env`): | Method | Path | Auth | Description | |--------|------|------|-------------| | `GET` | `/health` | No | Health check | +| `GET` | `/metrics` | No | Prometheus API request metrics (route templates, status, and latency) | | `GET` | `/gpu` | Yes | GPU metrics (VRAM, temp, utilization) | | `GET` | `/services` | Yes | All service health statuses | | `GET` | `/disk` | Yes | Disk usage | diff --git a/ods/extensions/services/dashboard-api/api_metrics.py b/ods/extensions/services/dashboard-api/api_metrics.py new file mode 100644 index 000000000..653f530d8 --- /dev/null +++ b/ods/extensions/services/dashboard-api/api_metrics.py @@ -0,0 +1,146 @@ +"""Low-cardinality Prometheus metrics for the dashboard API.""" + +import threading +import time +from collections import defaultdict + + +_DURATION_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0) + + +def _label_value(value: str) -> str: + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +class ApiMetrics: + """Thread-safe process-local request metrics registry.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._started_at = time.time() + self._in_flight = 0 + self._requests: dict[tuple[str, str, int], int] = defaultdict(int) + self._duration_count: dict[tuple[str, str], int] = defaultdict(int) + self._duration_sum: dict[tuple[str, str], float] = defaultdict(float) + self._duration_buckets: dict[tuple[str, str, float], int] = defaultdict(int) + + def request_started(self) -> None: + with self._lock: + self._in_flight += 1 + + def request_finished( + self, + method: str, + route: str, + status_code: int, + duration_seconds: float, + ) -> None: + method = method.upper() + route = route or "unmatched" + duration_seconds = max(0.0, duration_seconds) + key = (method, route) + with self._lock: + self._in_flight -= 1 + self._requests[(method, route, status_code)] += 1 + self._duration_count[key] += 1 + self._duration_sum[key] += duration_seconds + for bucket in _DURATION_BUCKETS: + if duration_seconds <= bucket: + self._duration_buckets[(method, route, bucket)] += 1 + + def reset(self) -> None: + """Clear observations; intended for isolated tests.""" + with self._lock: + self._started_at = time.time() + self._in_flight = 0 + self._requests.clear() + self._duration_count.clear() + self._duration_sum.clear() + self._duration_buckets.clear() + + def render(self) -> str: + with self._lock: + started_at = self._started_at + in_flight = self._in_flight + requests = dict(self._requests) + duration_count = dict(self._duration_count) + duration_sum = dict(self._duration_sum) + duration_buckets = dict(self._duration_buckets) + + lines = [ + "# HELP ods_dashboard_api_process_start_time_seconds Start time of this API process.", + "# TYPE ods_dashboard_api_process_start_time_seconds gauge", + f"ods_dashboard_api_process_start_time_seconds {started_at:.3f}", + "# HELP ods_dashboard_api_http_requests_in_flight Requests currently being served.", + "# TYPE ods_dashboard_api_http_requests_in_flight gauge", + f"ods_dashboard_api_http_requests_in_flight {in_flight}", + "# HELP ods_dashboard_api_http_requests_total Completed HTTP requests.", + "# TYPE ods_dashboard_api_http_requests_total counter", + ] + for (method, route, status), count in sorted(requests.items()): + labels = f'method="{_label_value(method)}",route="{_label_value(route)}",status="{status}"' + lines.append(f"ods_dashboard_api_http_requests_total{{{labels}}} {count}") + + lines.extend([ + "# HELP ods_dashboard_api_http_request_duration_seconds HTTP request latency.", + "# TYPE ods_dashboard_api_http_request_duration_seconds histogram", + ]) + for method, route in sorted(duration_count): + labels = f'method="{_label_value(method)}",route="{_label_value(route)}"' + for bucket in _DURATION_BUCKETS: + count = duration_buckets.get((method, route, bucket), 0) + lines.append( + "ods_dashboard_api_http_request_duration_seconds_bucket" + f'{{{labels},le="{bucket:g}"}} {count}', + ) + count = duration_count[(method, route)] + lines.append( + "ods_dashboard_api_http_request_duration_seconds_bucket" + f'{{{labels},le="+Inf"}} {count}', + ) + lines.append( + f"ods_dashboard_api_http_request_duration_seconds_sum{{{labels}}} " + f"{duration_sum[(method, route)]:.9g}", + ) + lines.append( + f"ods_dashboard_api_http_request_duration_seconds_count{{{labels}}} {count}", + ) + return "\n".join(lines) + "\n" + + +class ApiMetricsMiddleware: + """ASGI middleware that records one observation per completed request.""" + + def __init__(self, app, registry: ApiMetrics) -> None: + self.app = app + self.registry = registry + + async def __call__(self, scope, receive, send) -> None: + if scope["type"] != "http" or scope.get("path") == "/metrics": + await self.app(scope, receive, send) + return + + self.registry.request_started() + started = time.perf_counter() + status_code = 500 + + async def capture_status(message): + nonlocal status_code + if message["type"] == "http.response.start": + status_code = message["status"] + await send(message) + + try: + await self.app(scope, receive, capture_status) + finally: + route = scope.get("route") + route_template = getattr(route, "path", "unmatched") + self.registry.request_finished( + scope.get("method", "UNKNOWN"), + route_template, + status_code, + time.perf_counter() - started, + ) + + +api_metrics = ApiMetrics() diff --git a/ods/extensions/services/dashboard-api/main.py b/ods/extensions/services/dashboard-api/main.py index 8b5e06d2b..706c92ef4 100644 --- a/ods/extensions/services/dashboard-api/main.py +++ b/ods/extensions/services/dashboard-api/main.py @@ -28,7 +28,7 @@ from typing import Any, Optional import httpx -from fastapi import FastAPI, Depends, HTTPException, Body +from fastapi import FastAPI, Depends, HTTPException, Body, Response from fastapi.middleware.cors import CORSMiddleware # --- Local modules --- @@ -61,6 +61,7 @@ shutdown_clients as shutdown_agent_clients, ) from agent_monitor import collect_metrics +from api_metrics import ApiMetricsMiddleware, api_metrics from routers import ( workflows, features, setup, updates, agents, privacy, extensions, gpu as gpu_router, resources, voice, models as models_router, model_state as model_state_router, @@ -1060,6 +1061,8 @@ async def _lifespan(app: FastAPI): lifespan=_lifespan, ) +app.add_middleware(ApiMetricsMiddleware, registry=api_metrics) + # --- CORS --- def get_allowed_origins(): @@ -1126,6 +1129,15 @@ async def health(): return {"status": "ok", "timestamp": datetime.now(timezone.utc).isoformat()} +@app.get("/metrics", include_in_schema=False) +async def prometheus_metrics(): + """Expose process-local API request metrics for Prometheus scrapers.""" + return Response( + content=api_metrics.render(), + media_type="text/plain; version=0.0.4", + ) + + @app.get("/api/host-agent/diagnostics", dependencies=[Depends(verify_api_key)]) async def host_agent_diagnostics(): """Report how dashboard-api resolves and reaches the host agent.""" diff --git a/ods/extensions/services/dashboard-api/tests/test_api_metrics.py b/ods/extensions/services/dashboard-api/tests/test_api_metrics.py new file mode 100644 index 000000000..a73d96f2f --- /dev/null +++ b/ods/extensions/services/dashboard-api/tests/test_api_metrics.py @@ -0,0 +1,54 @@ +"""Tests for the dashboard API Prometheus request metrics.""" + +from api_metrics import ApiMetrics, api_metrics + + +def test_registry_renders_counter_gauge_and_cumulative_histogram(): + registry = ApiMetrics() + registry.request_started() + registry.request_finished("get", "/api/items/{item_id}", 201, 0.02) + + output = registry.render() + + assert "ods_dashboard_api_http_requests_in_flight 0" in output + assert ( + 'ods_dashboard_api_http_requests_total{method="GET",' + 'route="/api/items/{item_id}",status="201"} 1' + ) in output + assert ( + 'ods_dashboard_api_http_request_duration_seconds_bucket{method="GET",' + 'route="/api/items/{item_id}",le="0.01"} 0' + ) in output + assert ( + 'ods_dashboard_api_http_request_duration_seconds_bucket{method="GET",' + 'route="/api/items/{item_id}",le="0.025"} 1' + ) in output + + +def test_metrics_endpoint_uses_route_templates_and_excludes_scrapes(test_client): + api_metrics.reset() + + assert test_client.get("/health").status_code == 200 + assert test_client.get( + "/api/extensions/example-service/progress", + headers=test_client.auth_headers, + ).status_code == 200 + assert test_client.get("/not-a-real-resource/123").status_code == 404 + response = test_client.get("/metrics") + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/plain; version=0.0.4") + assert 'route="/health",status="200"} 1' in response.text + assert 'route="/api/extensions/{service_id}/progress",status="200"} 1' in response.text + assert 'route="/api/extensions/example-service/progress"' not in response.text + assert 'route="unmatched",status="404"} 1' in response.text + assert 'route="/not-a-real-resource/123"' not in response.text + assert 'route="/metrics"' not in response.text + + +def test_registry_escapes_prometheus_label_values(): + registry = ApiMetrics() + registry.request_started() + registry.request_finished("GET", '/quoted/"value"', 200, 0.001) + + assert 'route="/quoted/\\"value\\""' in registry.render()