fix(evaluation): validate OceanBase targets and resume identity

Author: Teingi

evaluation/coding/swebench_pro/README.md

@@ -105,6 +105,14 @@ processing configuration in a database-wide manifest, so separate Scopes in the
 isolate these configurations. The runner rejects OFF/ON configurations that point to the same OceanBase host,
 port, and database, even when they use different credentials.
 
+OceanBase URLs must put credentials and the host/port in the authority and the database name in the path.
+Query options such as `charset=utf8mb4` remain supported. The runner rejects query parameters that override
+connection identity (`host`, `port`, `user`, `username`, `password`, `passwd`, `db`, `database`, `unix_socket`),
+load external connection defaults (`read_default_file`, `read_default_group`), or run initialization SQL
+(`init_command`, `sql_mode`). Query keys are checked after URL decoding, regardless of case. Percent-encode
+`@` inside passwords and any `#`, tab, carriage return, or newline characters because URL parsers interpret
+their raw forms differently. Usernames such as `user@tenant` remain supported.
+
 When deriving the configurations from `.env`, serialize each actual URL, not the masked `SecretStr`
 representation, and replace its database name with the corresponding newly created evaluation database. Keep the
 file private (mode `0600`) and delete it after the run. The runner does not create or drop external databases.

evaluation/coding/swebench_pro/src/powercontext_eval_swebench_pro/powercontext_sut.py

@@ -52,7 +52,7 @@
 from pathlib import Path
 from types import MappingProxyType
 from typing import Any, Protocol
-from urllib.parse import quote, quote_plus, unquote, urlsplit
+from urllib.parse import parse_qsl, quote, quote_plus, unquote, urlsplit
 
 from powercontext_eval_swebench_pro import docker_pressure
 from powercontext_eval_swebench_pro.artifacts import ArtifactStore
@@ -89,6 +89,23 @@
 PLUGIN_ID = "powercontext@powercontext"
 _SAFE_RUN_ID = re.compile(r"[a-z0-9][a-z0-9-]{0,62}")
 _SAFE_DOCKER_NETWORK = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}")
+_OCEANBASE_QUERY_OVERRIDES = frozenset(
+    {
+        "host",
+        "port",
+        "user",
+        "username",
+        "password",
+        "passwd",
+        "db",
+        "database",
+        "unix_socket",
+        "read_default_file",
+        "read_default_group",
+        "init_command",
+        "sql_mode",
+    }
+)
 _SHA = re.compile(r"[0-9a-f]{40}")
 _INVALID_DOCKER_COPY_SYMLINK = re.compile(r'invalid symlink "[^"\r\n]+" -> "[^"\r\n]+"')
 _DOCKER_NETWORK_CONTROL_LOCK = threading.Lock()
@@ -2940,8 +2957,24 @@ def validated_database_config(config: Mapping[str, object] | None) -> Mapping[st
         raise UnsafeSutConfiguration("Evaluation database configuration is invalid")
     try:
         json.dumps(dict(config), allow_nan=False)
-        if kind == "oceanbase" and (not urlsplit(url).hostname or not urlsplit(url).path.strip("/")):
-            raise ValueError
+        if kind == "oceanbase":
+            parsed = urlsplit(url)
+            # Keep urllib's authority/query interpretation aligned with the
+            # database driver's URL parser; encoded password bytes are safe.
+            if (
+                not parsed.hostname
+                or not parsed.path.strip("/")
+                or any(character in url for character in "#\r\n\t")
+                or "@" in (parsed.password or "")
+            ):
+                raise ValueError
+            # Query overrides, option files and initial SQL can change the
+            # credentials or target beyond the authority/path we audit.
+            if any(
+                key.casefold() in _OCEANBASE_QUERY_OVERRIDES
+                for key, _ in parse_qsl(parsed.query, keep_blank_values=True)
+            ):
+                raise ValueError
     except (TypeError, ValueError):
         raise UnsafeSutConfiguration("Evaluation database configuration is invalid") from None
     return MappingProxyType(dict(config))

evaluation/coding/swebench_pro/tests/unit/test_cli.py

@@ -31,13 +31,22 @@
 from powercontext_eval_swebench_pro.runner import MinimalRunResult, RunConfig
 
 
-def test_swebench_pro_run_derives_portable_paths_from_explicit_root(monkeypatch, tmp_path: Path) -> None:
+@pytest.mark.parametrize(
+    ("username", "password"),
+    [("eval", "private-test-password"), ("eval@tenant", "private-test-password%40%23%0A%0D%09")],
+)
+def test_swebench_pro_run_derives_portable_paths_from_explicit_root(
+    monkeypatch, tmp_path: Path, username: str, password: str
+) -> None:
     calls: list[tuple[object, object]] = []
     instance = object()
     databases = {
         arm: {
             "kind": "oceanbase",
-            "url": f"mysql+aoceanbase://eval:private-test-password@localhost/test_{arm.value}",
+            "url": (
+                f"mysql+aoceanbase://{username}:{password}@localhost/test_{arm.value}"
+                "?charset=utf8mb4&connect_timeout=10"
+            ),
         }
         for arm in Arm
     }
@@ -120,6 +129,33 @@ def require(self, instance_id: str) -> object:
                 },
             }
         ),
+        *(
+            json.dumps(
+                {
+                    arm.value: {
+                        "kind": "oceanbase",
+                        "url": f"mysql+aoceanbase://eval@localhost/test_{arm.value}?charset=utf8mb4&{query}",
+                    }
+                    for arm in Arm
+                }
+            )
+            for query in (
+                "password=private-test-password",
+                "db=shared-private-test-password",
+            )
+        ),
+        *(
+            json.dumps(
+                {
+                    arm.value: {
+                        "kind": "oceanbase",
+                        "url": f"mysql+aoceanbase://eval:private-test-password{character}suffix@localhost/test_{arm.value}",
+                    }
+                    for arm in Arm
+                }
+            )
+            for character in ("@", "\n", "\r", "\t")
+        ),
     ],
     ids=[
         "missing-file",
@@ -130,6 +166,12 @@ def require(self, instance_id: str) -> object:
         "invalid-arm",
         "invalid-kind",
         "same-db",
+        "query-password",
+        "query-shared-db",
+        "raw-at-password",
+        "raw-newline-password",
+        "raw-carriage-return-password",
+        "raw-tab-password",
     ],
 )
 def test_swebench_pro_run_rejects_invalid_database_config_without_exposing_secrets(
@@ -148,7 +190,7 @@ def unexpected_catalog_load(_path: Path) -> None:
         [
             "run",
             "--root",
-            "/srv/evaluation",
+            str(tmp_path / "evaluation"),
             "--instance-id",
             "instance_owner__repo-b",
             "--database-config",
@@ -162,6 +204,7 @@ def unexpected_catalog_load(_path: Path) -> None:
     assert "Database configuration" in output
     assert "private-test-password" not in output
     assert "another-private-password" not in output
+    assert not (tmp_path / "evaluation").exists()
 
 
 def test_swebench_pro_run_defaults_optional_integrations_off(monkeypatch) -> None:

evaluation/coding/swebench_pro/tests/unit/test_runner_phases.py

@@ -291,6 +291,50 @@ def test_run_configuration_rejects_shared_oceanbase_before_gold(tmp_path: Path)
     assert not (tmp_path / "eval").exists()
 
 
+@pytest.mark.parametrize(
+    "option",
+    [
+        "host",
+        "port",
+        "user",
+        "username",
+        "password",
+        "passwd",
+        "db",
+        "database",
+        "unix_socket",
+        "read_default_file",
+        "read_default_group",
+        "init_command",
+        "sql_mode",
+        "PASSWORD",
+        "%70assword",
+        "%64%62",
+        "Read_Default_File",
+        "%69nit_command",
+        "unused=value#fragment&password",
+        "password=first&password",
+        "PaSsWd",
+        "UNIX_SOCKET",
+    ],
+)
+def test_run_configuration_rejects_oceanbase_query_overrides_before_gold(tmp_path: Path, option: str) -> None:
+    password = "private-query-value"
+    on_url = f"mysql+aoceanbase://eval@database.test/on?charset=utf8mb4&{option}={password}"
+    with pytest.raises(UnsafeSutConfiguration) as raised:
+        replace(
+            _config(tmp_path),
+            database_configs={
+                Arm.OFF: {"kind": "oceanbase", "url": "mysql+aoceanbase://eval:off-secret@database.test/off"},
+                Arm.ON: {"kind": "oceanbase", "url": on_url},
+            },
+        )
+
+    assert password not in str(raised.value)
+    assert on_url not in str(raised.value)
+    assert not (tmp_path / "eval").exists()
+
+
 def _run_with_fakes(
     tmp_path: Path,
     monkeypatch: pytest.MonkeyPatch,

evaluation/memory/locomo_plus/README.md

@@ -101,8 +101,14 @@ so separate runs remain isolated even when they share a database and the same `-
 that database for resume; dispose of a dedicated evaluation database only after its runs are no longer needed.
 
 `run.json` and the summaries record the actual backend and a database target fingerprint. Connection credentials and
-the raw database URL are excluded. The fingerprint binds the run to the host, port, tenant/user, database and routing
-parameters (or the local database path); changing the password alone does not change it. Reusing a results directory
+the raw database URL are excluded. OceanBase fingerprints use the installed dialect's effective host, port,
+tenant/user and database, including query-string overrides such as `db`, `user`, `host` and `port`. When a Unix socket
+is configured, its resolved path replaces the TCP host and port. Passwords in either the URL authority or query string,
+TLS/authentication material and connection tuning options do not affect the fingerprint. The runtime overrides
+`init_command` with its fixed transaction initialization command. Duplicate query parameters, `read_default_file`,
+`read_default_group` and `sql_mode` are rejected before opening services because their routing is ambiguous or can
+depend on external configuration or SQL. SQLite and seekdb fingerprints continue to identify the local database path.
+Reusing a results directory
 with a different database target is rejected before opening the database or calling models. Pending Memory runs
 also verify that their saved Scopes still exist, including judge-only retries; missing Scopes are not recreated.
 Completed runs can regenerate their summary without opening the database; use `replay` for explicitly offline checks.
@@ -205,6 +211,9 @@ location with the same settings to resume. Successful cases are retained; failed
 failure reuses the frozen generated answer. Configuration or dataset changes require a separate run.
 Runs without a saved database fingerprint and Scope namespace cannot be resumed; their saved results remain
 available through `replay`. Preserve `ingestion.json` alongside `run.json` to retain the original Scope identities.
+OceanBase manifests must also record `database_fingerprint_version: oceanbase-target-v2`. Unversioned OceanBase
+fingerprints are not reinterpreted or migrated: their saved results remain replayable, but a new run needs a new
+output directory. This version requirement does not apply to SQLite or seekdb manifests.
 The run command exits with status `1` if cases fail to execute or remain unobserved. A completed evaluation with an
 incorrect model answer still exits with status `0`; answer quality is recorded in the report.
 

evaluation/memory/locomo_plus/runner.py

@@ -32,7 +32,9 @@
 from pydantic import ValidationError
 from pydantic_ai import capture_run_messages
 from pydantic_ai.messages import ModelMessagesTypeAdapter
-from sqlalchemy.engine import make_url
+from pyobvector import AsyncOceanBaseDialect
+from sqlalchemy.engine import URL, make_url
+from sqlalchemy.exc import ArgumentError
 
 from evaluation.memory.locomo.dataset import LoCoMoSession
 from evaluation.memory.locomo.metrics import retrieval_metrics
@@ -280,28 +282,68 @@ def _database_config(settings: ServerSettings, output_directory: Path) -> Databa
     return settings.database
 
 
+def _oceanbase_target(url: URL) -> dict[str, Any]:
+    if any(not isinstance(value, str) for value in url.query.values()):
+        raise ValueError("OceanBase URL query parameters must not be repeated")  # noqa: TRY003
+    if {key.casefold() for key in url.query} & {"read_default_file", "read_default_group", "sql_mode"}:
+        raise ValueError("OceanBase external defaults and SQL mode overrides are not supported for resumable runs")  # noqa: TRY003
+    try:
+        # URL query values override the translated authority/path in the installed dialect.
+        _, options = AsyncOceanBaseDialect().create_connect_args(url)
+        username = options.get("user")
+        database = options.get("db")
+        host = options.get("host")
+        port = int(options.get("port", 3306))
+        if (
+            not isinstance(host, str)
+            or not host
+            or not all(isinstance(value, str) and value for value in (username, database))
+            or not 1 <= port <= 65535
+        ):
+            raise ValueError  # noqa: TRY301
+    except (ArgumentError, TypeError, ValueError):
+        raise ValueError(  # noqa: TRY003
+            "OceanBase connection target must explicitly identify its host, port, user and database"
+        ) from None
+    socket = options.get("unix_socket")
+    # Passwords, TLS/authentication material and tuning options do not identify the database.
+    # init_command is replaced by the runtime's fixed SET autocommit = 0 command.
+    return {
+        "driver": url.drivername,
+        "username": username,
+        "database": database,
+        "host": None if socket else host.lower(),
+        "port": None if socket else port,
+        "unix_socket": str(Path(socket).resolve()) if socket else None,
+    }
+
+
 def _database_identity(database: DatabaseConfig) -> dict[str, str]:
+    if database.kind == "oceanbase":
+        target = _oceanbase_target(make_url(database.url.get_secret_value()))
+        return {
+            "database_kind": database.kind,
+            "database_fingerprint_version": "oceanbase-target-v2",
+            "database_fingerprint": _digest(json.dumps(target, sort_keys=True)),
+        }
     target: dict[str, Any]
     if database.kind == "seekdb":
         target = {"path": str(database.path.resolve()), "database": database.database}
     else:
-        value = database.url if database.kind == "sqlite" else database.url.get_secret_value()
-        url = make_url(value)
-        if database.kind == "sqlite":
-            name = url.database
-            if name and name != ":memory:" and not name.startswith("file:"):
-                name = str(Path(name).resolve())
-            url = url.set(database=name)
-        # The OceanBase username can select a tenant. Password rotation does not change database identity.
+        url = make_url(database.url)
+        name = url.database
+        if name and name != ":memory:" and not name.startswith("file:"):
+            name = str(Path(name).resolve())
+        url = url.set(database=name)
         target = {
             "driver": url.drivername,
             "host": (url.host or "").lower(),
-            "port": url.port or (3306 if database.kind == "oceanbase" else None),
+            "port": url.port or None,
             "username": url.username,
             "database": url.database,
             "query": dict(url.query),
         }
-        if database.kind == "sqlite" and (url.database or "").startswith("file:"):
+        if (url.database or "").startswith("file:"):
             target["working_directory"] = str(Path.cwd().resolve())
     return {
         "database_kind": database.kind,