Merge branch 'stable-23.10' into feat/MCOL-6194-failover-conf

Author: mariadb-AlexanderPresniakov

.drone.jsonnet

@@ -396,23 +396,13 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
     name: "dockerfile",
     depends_on: ["publish pkg", "publish cmapi build"],
     image: "alpine/git:2.49.0",
+    volumes: [pipeline._volumes.mdb],
     environment: {
-      DOCKER_BRANCH_REF: "${DRONE_SOURCE_BRANCH}",
-      DOCKER_REF_AUX: branch_ref,
+      DRONE_SOURCE_BRANCH: "${DRONE_SOURCE_BRANCH}",
     },
     commands: [
-      // compute branch.
-      'echo "$$DOCKER_REF"',
-      'echo "$$DOCKER_BRANCH_REF"',
-      // if DOCKER_REF is empty, try to see whether docker repository has a branch named as one we PR.
-      'export DOCKER_REF=$${DOCKER_REF:-$$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-docker --h --sort origin "refs/heads/$$DOCKER_BRANCH_REF" | grep -E -o "[^/]+$$")}',
-      'echo "$$DOCKER_REF"',
-      // DOCKER_REF can be empty if there is no appropriate branch in docker repository.
-      // assign what is appropriate by default.
-      "export DOCKER_REF=$${DOCKER_REF:-$$DOCKER_REF_AUX}",
-      'echo "$$DOCKER_REF"',
-      "git clone --branch $$DOCKER_REF --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-docker docker",
-      "touch docker/.secrets",
+      "apk add bash && " +
+      get_build_command("clone_docker_repo.sh"),
     ],
   },
   dockerhub:: {
@@ -485,7 +475,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
       MCS_IMAGE_NAME: "mariadb/enterprise-columnstore-dev:" + container_tags[0],
     },
     commands: [
-      "echo $$DOCKER_PASSWORD | docker login --username $$DOCKER_LOGIN --password-stdin",
       "apk add bash && " +
       get_build_command("run_multi_node_mtr.sh") +
       " --columnstore-image-name $${MCS_IMAGE_NAME} " +

build/clone_docker_repo.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -eo pipefail
+
+SCRIPT_LOCATION=$(dirname "$0")
+source "$SCRIPT_LOCATION"/utils.sh
+
+
+# Determine which branch to use
+DOCKER_REF="${DOCKER_REF:-}"
+DOCKER_BRANCH_REF="${DRONE_SOURCE_BRANCH:-stable-23.10}"
+DOCKER_REF_AUX="stable-23.10"
+
+message "Checking for docker repository branch: $DOCKER_BRANCH_REF"
+
+# Try to find matching branch in docker repository
+if [[ -z "$DOCKER_REF" ]]; then
+    DOCKER_REF=$(git ls-remote https://github.com/mariadb-corporation/mariadb-columnstore-docker --heads --sort origin "refs/heads/$DOCKER_BRANCH_REF" 2>/dev/null | grep -E -o "[^/]+$" || true)
+fi
+
+# Fall back to default branch if no match found
+if [[ -z "$DOCKER_REF" ]]; then
+    DOCKER_REF="$DOCKER_REF_AUX"
+fi
+
+message "Cloning docker repository from branch: $DOCKER_REF"
+
+# Remove existing docker folder if it exists to avoid git clone conflicts
+if [[ -d "docker" ]]; then
+    message "Removing existing docker folder..."
+    rm -rf docker
+fi
+
+# Clone the docker repository
+git clone --branch "$DOCKER_REF" --depth 1 https://github.com/mariadb-corporation/mariadb-columnstore-docker docker
+
+# Create empty .secrets file
+touch docker/.secrets
+
+message "Docker repository cloned successfully"

build/run_multi_node_mtr.sh

@@ -5,6 +5,7 @@ set -eo pipefail
 SCRIPT_LOCATION=$(dirname "$0")
 source "$SCRIPT_LOCATION"/utils.sh
 
+#usage example: sudo ./run_multi_node_mtr.sh  --columnstore-image-name mariadb/enterprise-columnstore-dev:stable-23.10-pull-request2240  --distro rockylinux:8
 optparse.define short=i long=columnstore-image-name desc="Name of columnstore docker image" variable=MCS_IMAGE_NAME
 optparse.define short=d long=distro desc="Linux distro for which multinode mtr is executed" variable=DISTRO
 source $(optparse.build)
@@ -35,12 +36,76 @@ fi
 
 message "Running multinode mtr tests..."
 
+# Detect docker compose command (new syntax vs old)
+if command -v docker &> /dev/null && docker compose version &> /dev/null; then
+    DOCKER_COMPOSE="docker compose"
+elif command -v docker-compose &> /dev/null; then
+    DOCKER_COMPOSE="docker-compose"
+else
+    error "Docker compose is not installed!"
+    exit 1
+fi
+
+# Handle Docker Hub authentication
+message "Checking Docker Hub authentication..."
+if [[ -n "${DOCKER_LOGIN:-}" ]] && [[ -n "${DOCKER_PASSWORD:-}" ]]; then
+    # CI environment - use credentials from secrets
+    message "Using Docker credentials from environment variables"
+    echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_LOGIN" --password-stdin
+    if [[ $? -ne 0 ]]; then
+        error "Docker login failed with provided credentials"
+        exit 1
+    fi
+else
+    message "No Docker credentials in env variables found, checking if already logged in..."
+    
+    # Try to verify docker login by checking auth for docker.io
+    if docker system info 2>&1 | grep -q "Username:"; then
+        message "Already logged in to Docker Hub"
+    else
+        message "Not logged in to Docker Hub"
+        message "You need to login to Docker Hub to pull enterprise-columnstore-dev images"
+        message ""
+        
+        # Prompt for username
+        read -p "Docker Hub Username: " docker_username
+        
+        # Run docker login with username (will prompt for password)
+        docker login -u "$docker_username"
+        
+        if [[ $? -ne 0 ]]; then
+            error "Docker login failed or was cancelled"
+            exit 1
+        fi
+        
+        message "Docker login successful"
+    fi
+fi
+
+# Check if docker folder exists and has required files
+if [[ ! -d "docker" ]] || [[ ! -f "docker/Dockerfile" ]]; then
+    message "Docker folder not found or incomplete, cloning mariadb-columnstore-docker repository..."
+    "$SCRIPT_LOCATION"/clone_docker_repo.sh
+fi
+
 cd docker
+
+# Clean up any existing containers from previous runs
+if docker ps -a --format '{{.Names}}' | grep -q -E '^(mcs1|mcs2|mcs3)$'; then
+    message "Found existing containers from previous run, cleaning up..."
+    $DOCKER_COMPOSE down -v 2>/dev/null || true
+    docker rm -f mcs1 mcs2 mcs3 2>/dev/null || true
+    message "Cleanup completed"
+fi
+
 cp .env_example .env
 sed -i "/^MCS_IMAGE_NAME=/s|=.*|=${MCS_IMAGE_NAME}|" .env
 sed -i "/^MAXSCALE=/s|=.*|=false|" .env
 
-docker-compose up -d
+message "Starting containers..."
+$DOCKER_COMPOSE up -d
+
+message "Provisioning cluster..."
 docker exec mcs1 provision mcs1 mcs2 mcs3
 docker cp ../mysql-test/columnstore mcs1:"${MTR_PATH}/suite/"
 docker exec -t mcs1 chown -R mysql:mysql "${MTR_PATH}"

cmapi/cmapi_server/__main__.py

@@ -6,6 +6,7 @@
 
 import logging
 import os
+import sys
 import threading
 import time
 from datetime import datetime
@@ -19,21 +20,21 @@
 from tracing.sentry import maybe_init_sentry
 from tracing.traceparent_backend import TraceparentBackend
 from tracing.tracer import get_tracer
-config_cmapi_server_logging()
-from tracing.trace_tool import register_tracing_tools
 
+config_cmapi_server_logging()
 from cmapi_server import helpers
-from cmapi_server.constants import DEFAULT_MCS_CONF_PATH, CMAPI_CONF_PATH
-from cmapi_server.controllers.dispatcher import dispatcher, jsonify_error, jsonify_404
+from cmapi_server.constants import CMAPI_CONF_PATH, DEFAULT_MCS_CONF_PATH
+from cmapi_server.controllers.dispatcher import dispatcher, jsonify_404, jsonify_error
 from cmapi_server.failover_agent import FailoverAgent
+from cmapi_server.invariant_checks import run_invariant_checks
 from cmapi_server.managers.application import AppManager
-from cmapi_server.managers.process import MCSProcessManager
 from cmapi_server.managers.certificate import CertificateManager
-from cmapi_server.invariant_checks import run_invariant_checks
+from cmapi_server.managers.process import MCSProcessManager
 from failover.node_monitor import NodeMonitor
 from failover.config import Config
 from mcs_node_control.models.dbrm_socket import SOCK_TIMEOUT, DBRMSocketHandler
 from mcs_node_control.models.node_config import NodeConfig
+from tracing.trace_tool import register_tracing_tools
 
 
 def worker(app):

cmapi/tracing/trace_tool.py

@@ -2,12 +2,21 @@
 CherryPy tool that uses the tracer to start a span for each request.
 """
 import socket
-from typing import Dict
+import json
+import logging
+from typing import Dict, Any
 
 import cherrypy
 
+from tracing.utils import swallow_exceptions
 from tracing.tracer import get_tracer
 
+logger = logging.getLogger("tracer")
+
+# Limit for raw JSON string preview (in characters)
+_PREVIEW_MAX_CHARS = 512
+
+@swallow_exceptions
 def _on_request_start() -> None:
     req = cherrypy.request
     tracer = get_tracer()
@@ -30,32 +39,39 @@ def _on_request_start() -> None:
     else:
         span_name = f"{method} {path}"
 
-    ctx = tracer.start_as_current_span(span_name, kind="SERVER")
+    attrs = {
+        'http.method': getattr(req, 'method', ''),
+        'http.path': getattr(req, 'path_info', ''),
+        'client.ip': requester_host,
+        'instance.hostname': socket.gethostname(),
+    }
+    attrs.update(_record_incoming_json_preview(req))
+
+    ctx = tracer.start_as_current_span(span_name, kind="SERVER", attributes=attrs)
     span = ctx.__enter__()
-    span.set_attribute('http.method', getattr(req, 'method', ''))
-    span.set_attribute('http.path', getattr(req, 'path_info', ''))
-    span.set_attribute('client.ip', requester_host)
-    span.set_attribute('instance.hostname', socket.gethostname())
     safe_headers = {k: v for k, v in headers.items() if k.lower() not in {'authorization', 'x-api-key'}}
     span.set_attribute('sentry.incoming_headers', safe_headers)
     req._trace_span_ctx = ctx
     req._trace_span = span
 
     tracer.inject_traceparent(cherrypy.response.headers)
 
-
+@swallow_exceptions
 def _on_request_end() -> None:
     req = cherrypy.request
     try:
         status_str = str(cherrypy.response.status)
         status_code = int(status_str.split()[0])
     except Exception:
         status_code = None
-    tracer = get_tracer()
-    tracer.notify_request_finished(status_code)
+
     span = getattr(req, "_trace_span", None)
     if span is not None and status_code is not None:
         span.set_attribute('http.status_code', status_code)
+
+    tracer = get_tracer()
+    tracer.notify_request_finished(status_code)
+
     ctx = getattr(req, "_trace_span_ctx", None)
     if ctx is not None:
         try:
@@ -66,8 +82,27 @@ def _on_request_end() -> None:
 
 
 def register_tracing_tools() -> None:
-    cherrypy.tools.trace = cherrypy.Tool("on_start_resource", _on_request_start, priority=10)
+    cherrypy.tools.trace = cherrypy.Tool("on_start_resource", _on_request_start, priority=70)
     cherrypy.tools.trace_end = cherrypy.Tool("on_end_resource", _on_request_end, priority=80)
 
+@swallow_exceptions
+def _record_incoming_json_preview(req) -> dict[str, Any]:
+    """If request Content-Type is JSON, attach a compact preview to span.
+    Returns dict of span attributes to set
+    """
+    attrs = {}
+    content_type = str(getattr(req, 'headers', {}).get('Content-Type', '') or '')
+    attrs['http.request.content_type'] = content_type
+    if 'json' not in content_type.lower():
+        return attrs
 
-
+    parsed_json = getattr(req, 'json', None)
+    if parsed_json is None:
+        logger.debug('request.json is not available')
+        return attrs
+    normalized = json.dumps(parsed_json, ensure_ascii=False, sort_keys=True)
+    if len(normalized) > _PREVIEW_MAX_CHARS:
+        normalized = normalized[:_PREVIEW_MAX_CHARS] + '...<truncated>'
+    attrs['http.request.json'] = normalized
+    attrs['http.request.body.size'] = len(normalized)
+    return attrs

cmapi/tracing/tracer.py

@@ -46,7 +46,12 @@ def clear_backends(self) -> None:
         self._backends.clear()
 
     @contextmanager
-    def start_as_current_span(self, name: str, kind: str = "INTERNAL") -> Iterator[TraceSpan]:
+    def start_as_current_span(
+        self,
+        name: str,
+        kind: str = "INTERNAL",
+        attributes: Optional[Dict[str, Any]] = None,
+    ) -> Iterator[TraceSpan]:
         trace_id = _current_trace_id.get() or rand_16_hex()
         parent_span = _current_span_id.get()
         new_span_id = rand_8_hex()
@@ -63,7 +68,7 @@ def start_as_current_span(self, name: str, kind: str = "INTERNAL") -> Iterator[T
             trace_id=trace_id,
             span_id=new_span_id,
             parent_span_id=parent_span,
-            attributes={"span.kind": kind, "span.name": name},
+            attributes={"span.kind": kind, "span.name": name, **(attributes or {})},
             tracer=self,
         )