Merge pull request #3835 from mariadb-corporation/tracing-log-json-requests

Save json payload of request in tracing

Author: mariadb-AlexanderPresniakov

cmapi/cmapi_server/__main__.py

@@ -6,6 +6,7 @@
 
 import logging
 import os
+import sys
 import threading
 import time
 from datetime import datetime
@@ -19,20 +20,20 @@
 from tracing.sentry import maybe_init_sentry
 from tracing.traceparent_backend import TraceparentBackend
 from tracing.tracer import get_tracer
-config_cmapi_server_logging()
-from tracing.trace_tool import register_tracing_tools
 
+config_cmapi_server_logging()
 from cmapi_server import helpers
-from cmapi_server.constants import DEFAULT_MCS_CONF_PATH, CMAPI_CONF_PATH
-from cmapi_server.controllers.dispatcher import dispatcher, jsonify_error, jsonify_404
+from cmapi_server.constants import CMAPI_CONF_PATH, DEFAULT_MCS_CONF_PATH
+from cmapi_server.controllers.dispatcher import dispatcher, jsonify_404, jsonify_error
 from cmapi_server.failover_agent import FailoverAgent
+from cmapi_server.invariant_checks import run_invariant_checks
 from cmapi_server.managers.application import AppManager
-from cmapi_server.managers.process import MCSProcessManager
 from cmapi_server.managers.certificate import CertificateManager
-from cmapi_server.invariant_checks import run_invariant_checks
+from cmapi_server.managers.process import MCSProcessManager
 from failover.node_monitor import NodeMonitor
 from mcs_node_control.models.dbrm_socket import SOCK_TIMEOUT, DBRMSocketHandler
 from mcs_node_control.models.node_config import NodeConfig
+from tracing.trace_tool import register_tracing_tools
 
 
 def worker(app):

cmapi/tracing/trace_tool.py

@@ -2,12 +2,21 @@
 CherryPy tool that uses the tracer to start a span for each request.
 """
 import socket
-from typing import Dict
+import json
+import logging
+from typing import Dict, Any
 
 import cherrypy
 
+from tracing.utils import swallow_exceptions
 from tracing.tracer import get_tracer
 
+logger = logging.getLogger("tracer")
+
+# Limit for raw JSON string preview (in characters)
+_PREVIEW_MAX_CHARS = 512
+
+@swallow_exceptions
 def _on_request_start() -> None:
     req = cherrypy.request
     tracer = get_tracer()
@@ -30,32 +39,39 @@ def _on_request_start() -> None:
     else:
         span_name = f"{method} {path}"
 
-    ctx = tracer.start_as_current_span(span_name, kind="SERVER")
+    attrs = {
+        'http.method': getattr(req, 'method', ''),
+        'http.path': getattr(req, 'path_info', ''),
+        'client.ip': requester_host,
+        'instance.hostname': socket.gethostname(),
+    }
+    attrs.update(_record_incoming_json_preview(req))
+
+    ctx = tracer.start_as_current_span(span_name, kind="SERVER", attributes=attrs)
     span = ctx.__enter__()
-    span.set_attribute('http.method', getattr(req, 'method', ''))
-    span.set_attribute('http.path', getattr(req, 'path_info', ''))
-    span.set_attribute('client.ip', requester_host)
-    span.set_attribute('instance.hostname', socket.gethostname())
     safe_headers = {k: v for k, v in headers.items() if k.lower() not in {'authorization', 'x-api-key'}}
     span.set_attribute('sentry.incoming_headers', safe_headers)
     req._trace_span_ctx = ctx
     req._trace_span = span
 
     tracer.inject_traceparent(cherrypy.response.headers)
 
-
+@swallow_exceptions
 def _on_request_end() -> None:
     req = cherrypy.request
     try:
         status_str = str(cherrypy.response.status)
         status_code = int(status_str.split()[0])
     except Exception:
         status_code = None
-    tracer = get_tracer()
-    tracer.notify_request_finished(status_code)
+
     span = getattr(req, "_trace_span", None)
     if span is not None and status_code is not None:
         span.set_attribute('http.status_code', status_code)
+
+    tracer = get_tracer()
+    tracer.notify_request_finished(status_code)
+
     ctx = getattr(req, "_trace_span_ctx", None)
     if ctx is not None:
         try:
@@ -66,8 +82,27 @@ def _on_request_end() -> None:
 
 
 def register_tracing_tools() -> None:
-    cherrypy.tools.trace = cherrypy.Tool("on_start_resource", _on_request_start, priority=10)
+    cherrypy.tools.trace = cherrypy.Tool("on_start_resource", _on_request_start, priority=70)
     cherrypy.tools.trace_end = cherrypy.Tool("on_end_resource", _on_request_end, priority=80)
 
+@swallow_exceptions
+def _record_incoming_json_preview(req) -> dict[str, Any]:
+    """If request Content-Type is JSON, attach a compact preview to span.
+    Returns dict of span attributes to set
+    """
+    attrs = {}
+    content_type = str(getattr(req, 'headers', {}).get('Content-Type', '') or '')
+    attrs['http.request.content_type'] = content_type
+    if 'json' not in content_type.lower():
+        return attrs
 
-
+    parsed_json = getattr(req, 'json', None)
+    if parsed_json is None:
+        logger.debug('request.json is not available')
+        return attrs
+    normalized = json.dumps(parsed_json, ensure_ascii=False, sort_keys=True)
+    if len(normalized) > _PREVIEW_MAX_CHARS:
+        normalized = normalized[:_PREVIEW_MAX_CHARS] + '...<truncated>'
+    attrs['http.request.json'] = normalized
+    attrs['http.request.body.size'] = len(normalized)
+    return attrs

cmapi/tracing/tracer.py

@@ -46,7 +46,12 @@ def clear_backends(self) -> None:
         self._backends.clear()
 
     @contextmanager
-    def start_as_current_span(self, name: str, kind: str = "INTERNAL") -> Iterator[TraceSpan]:
+    def start_as_current_span(
+        self,
+        name: str,
+        kind: str = "INTERNAL",
+        attributes: Optional[Dict[str, Any]] = None,
+    ) -> Iterator[TraceSpan]:
         trace_id = _current_trace_id.get() or rand_16_hex()
         parent_span = _current_span_id.get()
         new_span_id = rand_8_hex()
@@ -63,7 +68,7 @@ def start_as_current_span(self, name: str, kind: str = "INTERNAL") -> Iterator[T
             trace_id=trace_id,
             span_id=new_span_id,
             parent_span_id=parent_span,
-            attributes={"span.kind": kind, "span.name": name},
+            attributes={"span.kind": kind, "span.name": name, **(attributes or {})},
             tracer=self,
         )