From b0c6d274b4c7d73b912e435886e185057a2dcb70 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Mon, 28 Sep 2026 18:13:27 +0800 Subject: [PATCH 01/37] feat(db): add versioned migration prototype --- .github/workflows/master.yml | 7 + pyproject.toml | 2 + scripts/database_migration_probe.py | 69 +++ .../persistence/migrations/__init__.py | 25 ++ .../builtin/persistence/migrations/bundle.py | 106 +++++ .../builtin/persistence/migrations/locking.py | 47 ++ .../builtin/persistence/migrations/models.py | 62 +++ .../persistence/migrations/oceanbase.py | 98 ++++ .../builtin/persistence/migrations/sqlite.py | 359 +++++++++++++++ .../test_database_migration_backends.py | 131 ++++++ .../persistence/test_database_migrations.py | 417 ++++++++++++++++++ tests/fixtures/database_migrations/README.md | 70 +++ tests/fixtures/database_migrations/env.py | 26 ++ .../database_migrations/manifest.json | 25 ++ .../schemas/pre_dream.json | 27 ++ .../database_migrations/schemas/v1_1_0.json | 27 ++ .../versions/p0001_baseline.py | 34 ++ .../versions/p0002_citations.py | 27 ++ .../versions/p0003_tag_families.py | 40 ++ tests/test_database_migration_probe.py | 50 +++ uv.lock | 48 +- 21 files changed, 1692 insertions(+), 5 deletions(-) create mode 100644 scripts/database_migration_probe.py create mode 100644 src/powercontext/builtin/persistence/migrations/__init__.py create mode 100644 src/powercontext/builtin/persistence/migrations/bundle.py create mode 100644 src/powercontext/builtin/persistence/migrations/locking.py create mode 100644 src/powercontext/builtin/persistence/migrations/models.py create mode 100644 src/powercontext/builtin/persistence/migrations/oceanbase.py create mode 100644 src/powercontext/builtin/persistence/migrations/sqlite.py create mode 100644 tests/builtin/persistence/test_database_migration_backends.py create mode 100644 tests/builtin/persistence/test_database_migrations.py create mode 100644 tests/fixtures/database_migrations/README.md create mode 100644 tests/fixtures/database_migrations/env.py create mode 100644 tests/fixtures/database_migrations/manifest.json create mode 100644 tests/fixtures/database_migrations/schemas/pre_dream.json create mode 100644 tests/fixtures/database_migrations/schemas/v1_1_0.json create mode 100644 tests/fixtures/database_migrations/versions/p0001_baseline.py create mode 100644 tests/fixtures/database_migrations/versions/p0002_citations.py create mode 100644 tests/fixtures/database_migrations/versions/p0003_tag_families.py create mode 100644 tests/test_database_migration_probe.py diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 00e2b4ab02..25284210b2 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -86,6 +86,13 @@ jobs: - name: Test code indexing with embedded seekdb run: make code-seekdb-test + - name: Probe Alembic migrations with embedded seekdb + env: + POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" + run: >- + uv run --locked --extra seekdb pytest -q + tests/builtin/persistence/test_database_migration_backends.py -k seekdb + windows-unit-portability: timeout-minutes: 20 runs-on: windows-latest diff --git a/pyproject.toml b/pyproject.toml index 237f3e0e22..b2548fef5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,9 @@ code = [ ] builtin = [ "aiosqlite>=0.22,<1", + "alembic>=1.16,<2", "apscheduler>=3.11,<4", + "filelock>=3.16,<4", "jsonschema>=4.23,<5", "packaging>=24,<27", "pydantic-ai-slim[anthropic,openai]>=2.27.1,<3", diff --git a/scripts/database_migration_probe.py b/scripts/database_migration_probe.py new file mode 100644 index 0000000000..4c1b8a738a --- /dev/null +++ b/scripts/database_migration_probe.py @@ -0,0 +1,69 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exercise the RFC 1771 SQLite prototype against an explicitly chosen scratch DB. + +This repository-only command does not load Server settings or a business runtime. +The four-table fixture is intentionally unable to adopt a complete deployment. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from powercontext.builtin.persistence.migrations import MigrationBundle, MigrationError, SQLiteMigrationRunner + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("status", "plan", "apply", "verify")) + parser.add_argument("--sqlite-path", type=Path, required=True, help="Explicit disposable database path.") + parser.add_argument("--plan-id") + parser.add_argument("--resume") + parser.add_argument("--maintenance-confirmed", action="store_true") + parser.add_argument("--yes", action="store_true") + args = parser.parse_args() + bundle = MigrationBundle(Path(__file__).resolve().parents[1] / "tests/fixtures/database_migrations") + runner = SQLiteMigrationRunner(args.sqlite_path, bundle) + try: + if args.action in {"status", "plan"}: + result = runner.plan() + elif args.action == "verify": + result = runner.verify() + else: + plan = runner.plan() + plan_id = args.plan_id + accepted = args.yes + if plan.state != "ready" and not accepted and sys.stdin.isatty(): + print(plan.model_dump_json(indent=2)) + accepted = input("Apply this plan to the scratch database? [y/N] ").strip().lower() == "y" + plan_id = plan.plan_id + result = runner.apply( + plan_id=plan_id, + accepted=accepted, + maintenance_confirmed=args.maintenance_confirmed, + resume=args.resume, + ) + print(result.model_dump_json()) + except MigrationError as error: + print(json.dumps({"error": error.code, "message": str(error)}), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/powercontext/builtin/persistence/migrations/__init__.py b/src/powercontext/builtin/persistence/migrations/__init__.py new file mode 100644 index 0000000000..0faa915296 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline migration primitives; not yet wired into application startup. + +Bundles explicitly describe the objects they own. The experimental fixture bundle +is an isolated four-table prototype, not a migration for a deployed PowerContext DB. +""" + +from .bundle import MigrationBundle +from .models import MigrationError, MigrationPlan, MigrationResult +from .sqlite import SQLiteMigrationRunner + +__all__ = ["MigrationBundle", "MigrationError", "MigrationPlan", "MigrationResult", "SQLiteMigrationRunner"] diff --git a/src/powercontext/builtin/persistence/migrations/bundle.py b/src/powercontext/builtin/persistence/migrations/bundle.py new file mode 100644 index 0000000000..f12046bcf8 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/bundle.py @@ -0,0 +1,106 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A checked, linear Alembic history with explicitly recognized schema shapes.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from alembic.config import Config +from alembic.script import ScriptDirectory +from sqlalchemy import Connection + +from .models import MigrationError, digest + + +class MigrationBundle: + """Load a trusted local bundle. No revision imports current application metadata. + + The manifest records exact SQLite schema fingerprints, including alternate + shapes produced by a supported historical initializer. Unlisted objects or + partially applied DDL cannot be mistaken for a supported baseline. + """ + + def __init__(self, directory: Path) -> None: + self.directory = directory.resolve() + self.script = ScriptDirectory(str(self.directory)) + revisions = list(reversed(list(self.script.walk_revisions()))) + previous = None + for item in revisions: + if item.down_revision != previous or item.dependencies or len(item.revision) > 32: + raise MigrationError("invalid_bundle", "Migration history must be a single linear chain.") + previous = item.revision + if not revisions: + raise MigrationError("invalid_bundle", "Migration history is empty.") + self.revisions = tuple(item.revision for item in revisions) + self.head = self.revisions[-1] + manifest = json.loads((self.directory / "manifest.json").read_text()) + self.fingerprints: dict[str, list[str]] = manifest["sqlite_fingerprints"] + self.baselines: dict[str, str] = manifest["sqlite_baselines"] + if set(self.fingerprints) != set(self.revisions) or not set(self.baselines.values()) <= set(self.revisions): + raise MigrationError("invalid_bundle", "Schema inventory does not cover the revision chain.") + if any(fingerprint not in self.fingerprints[revision] for fingerprint, revision in self.baselines.items()): + raise MigrationError("invalid_bundle", "A baseline does not match its declared revision.") + self.resources = { + path.relative_to(self.directory).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(self.directory.rglob("*")) + if path.is_file() and path.suffix in {".py", ".sql", ".json"} + } + self.checksums = {} + for item in revisions: + dependencies = manifest["revision_resources"][item.revision] + inputs = {name: self.resources[name] for name in ["env.py", *dependencies]} + inputs[Path(item.path).relative_to(self.directory).as_posix()] = hashlib.sha256( + Path(item.path).read_bytes() + ).hexdigest() + self.checksums[item.revision] = digest(inputs) + self.checksum = digest(self.resources) + + def config(self, connection: Connection) -> Config: + config = Config() + config.set_main_option("script_location", str(self.directory).replace("%", "%%")) + config.attributes["connection"] = connection + return config + + def pending(self, revision: str | None) -> tuple[str, ...]: + if revision is None: + return self.revisions + if revision not in self.revisions: + raise MigrationError("incompatible_schema", "Database revision is unknown to this binary.") + return self.revisions[self.revisions.index(revision) + 1 :] + + def verify(self, revision: str | None, fingerprint: str) -> None: + if revision is None: + allowed = [digest([])] + else: + self.pending(revision) + allowed = self.fingerprints[revision] + if fingerprint not in allowed: + raise MigrationError("incompatible_schema", "Actual schema does not match the recorded revision.") + + +def sqlite_inventory(connection: Connection, *, exclude: frozenset[str] = frozenset()) -> list[list[Any]]: + """Include tables, indexes, triggers, views and constraint-bearing SQL. + + This deliberately uses exact SQLite definitions. A semantically equivalent + but unregistered schema requires review, never an optimistic stamp. + """ + rows = connection.exec_driver_sql( + "SELECT type, name, tbl_name, sql FROM sqlite_schema WHERE substr(name, 1, 7) != 'sqlite_' ORDER BY type, name" + ) + return [list(row) for row in rows if row[1] not in exclude and row[2] not in exclude] diff --git a/src/powercontext/builtin/persistence/migrations/locking.py b/src/powercontext/builtin/persistence/migrations/locking.py new file mode 100644 index 0000000000..0c793808d2 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/locking.py @@ -0,0 +1,47 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Locks are held for the complete operation, including backup and verification.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +from filelock import FileLock, SoftFileLock, Timeout + +from .models import MigrationError + + +@contextmanager +def local_migration_lock(target: Path) -> Iterator[None]: + """Lock a canonical database file or embedded-engine directory before opening it. + + Keep the lock inode after release: removing it permits waiters on the old + inode and newcomers on its replacement to enter concurrently. + """ + target = target.expanduser().resolve() + target.parent.mkdir(parents=True, exist_ok=True) + lock = FileLock(str(target.with_name(target.name + ".pc-migration.lock")), timeout=0, mode=0o600) + if isinstance(lock, SoftFileLock): + raise MigrationError("migration_lock_unsupported", "This platform does not provide an OS-backed file lock.") + try: + lock.acquire() + except Timeout as error: + raise MigrationError("migration_locked", "Another process holds the migration lock.") from error + try: + yield + finally: + lock.release() diff --git a/src/powercontext/builtin/persistence/migrations/models.py b/src/powercontext/builtin/persistence/migrations/models.py new file mode 100644 index 0000000000..0a9681fa3b --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/models.py @@ -0,0 +1,62 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Reviewable migration plans and payload-free operator errors.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict + + +def digest(value: Any) -> str: + """Hash a canonical JSON value without including it in diagnostics.""" + return hashlib.sha256( + json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode() + ).hexdigest() + + +class MigrationError(RuntimeError): + def __init__(self, code: str, message: str) -> None: + self.code = code + super().__init__(f"{code}: {message}") + + +class MigrationPlan(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + plan_id: str + database_id: str + source_revision: str | None + target_revision: str + schema_fingerprint: str + bundle_checksum: str + configuration_digest: str + state: Literal["uninitialized", "migration_required", "ready", "recovery_required"] + revisions: tuple[str, ...] + adopt_baseline: bool = False + resume_run_id: str | None = None + + +class MigrationResult(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + state: Literal["ready"] = "ready" + revision: str + changed: bool + run_id: str | None = None + backup_ref: str | None = None diff --git a/src/powercontext/builtin/persistence/migrations/oceanbase.py b/src/powercontext/builtin/persistence/migrations/oceanbase.py new file mode 100644 index 0000000000..374bf25e41 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/oceanbase.py @@ -0,0 +1,98 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Probe and hold a session lock on a pinned OceanBase connection. + +This is one prerequisite for a nontransactional-DDL executor, not that executor. +Deployment acceptance must additionally test its actual node/proxy routing. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass + +from sqlalchemy import text +from sqlalchemy.exc import DBAPIError +from sqlalchemy.ext.asyncio import AsyncConnection + +from .models import MigrationError, digest + + +@dataclass(frozen=True) +class PinnedOceanBaseLock: + connection: AsyncConnection + name: str + session_id: int + + async def verify(self) -> None: + """Call before and after each DDL; a reconnected session is not trusted.""" + if self.connection.invalidated or self.connection.closed: + raise MigrationError("migration_lock_lost", "The lock connection was lost; inspect DDL before recovery.") + current = await self.connection.scalar(text("SELECT CONNECTION_ID()")) + owner = await self.connection.scalar(text("SELECT IS_USED_LOCK(:name)"), {"name": self.name}) + if current != self.session_id or owner != self.session_id: + raise MigrationError("migration_lock_lost", "The pinned session no longer owns the migration lock.") + + +@asynccontextmanager +async def oceanbase_migration_lock( + connection: AsyncConnection, contender: AsyncConnection, *, database_id: str +) -> AsyncIterator[PinnedOceanBaseLock]: + """Test exclusion and commit survival using two independently routed sessions. + + Both connections must be dedicated to maintenance. The caller supplies a + stable tenant/schema identity, not a hostname that changes through a proxy. + A rejected capability never falls back to a process-local lock. + """ + name = "pc-migrate-" + digest(database_id)[:48] + owned = False + try: + try: + owner_id = await connection.scalar(text("SELECT CONNECTION_ID()")) + other_id = await contender.scalar(text("SELECT CONNECTION_ID()")) + if owner_id == other_id: + raise MigrationError("migration_lock_unsupported", "Lock probing needs two distinct physical sessions.") + acquired = await connection.scalar(text("SELECT GET_LOCK(:name, 0)"), {"name": name}) + if acquired == 0: + raise MigrationError("migration_locked", "Another session holds the database migration lock.") + if acquired != 1: + raise MigrationError("migration_lock_unsupported", "Named-lock acquisition is unavailable.") + owned = True + await connection.commit() + competing = await contender.scalar(text("SELECT GET_LOCK(:name, 0)"), {"name": name}) + if competing == 1: + await contender.scalar(text("SELECT RELEASE_LOCK(:name)"), {"name": name}) + await contender.commit() + if competing != 0: + raise MigrationError( + "migration_lock_unsupported", "The lock does not exclude a second session across commit." + ) + if not isinstance(owner_id, int): + raise MigrationError("migration_lock_unsupported", "The backend did not return a physical session ID.") + lock = PinnedOceanBaseLock(connection, name, owner_id) + await lock.verify() + except DBAPIError as error: + raise MigrationError( + "migration_lock_unsupported", "Named-lock capability could not be verified." + ) from error + yield lock + finally: + if owned and not connection.invalidated and not connection.closed: + # Never reconnect simply to release: a replacement session owns no lock. + # Releasing a lock must not commit unfinished work in the caller. + await connection.rollback() + await connection.scalar(text("SELECT RELEASE_LOCK(:name)"), {"name": name}) + await connection.commit() diff --git a/src/powercontext/builtin/persistence/migrations/sqlite.py b/src/powercontext/builtin/persistence/migrations/sqlite.py new file mode 100644 index 0000000000..f25b18b48d --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/sqlite.py @@ -0,0 +1,359 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Explicit offline SQLite migrations with atomic steps and durable recovery. + +This executor is for an explicitly supplied bundle. It does not establish that +the rest of the PowerContext service or its projections are ready. +""" + +from __future__ import annotations + +import hashlib +import os +import sqlite3 +import threading +from collections.abc import Iterator, Mapping +from contextlib import closing, contextmanager +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from uuid import uuid4 + +from alembic import command +from sqlalchemy import Connection, create_engine, text +from sqlalchemy.exc import OperationalError + +from .bundle import MigrationBundle, sqlite_inventory +from .locking import local_migration_lock +from .models import MigrationError, MigrationPlan, MigrationResult, digest + +_CONTROL_DDL = { + "pc_schema_revision": ( + "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " + "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num))" + ), + "pc_migration_runs": ( + "CREATE TABLE pc_migration_runs (run_id TEXT PRIMARY KEY NOT NULL, plan_json TEXT NOT NULL, " + "backup_path TEXT NOT NULL, backup_checksum TEXT NOT NULL, " + "state TEXT NOT NULL CHECK (state IN ('running', 'complete')), started_at TEXT NOT NULL)" + ), + "pc_migration_steps": ( + "CREATE TABLE pc_migration_steps (run_id TEXT NOT NULL, revision TEXT NOT NULL, checksum TEXT NOT NULL, " + "PRIMARY KEY (run_id, revision), FOREIGN KEY (run_id) REFERENCES pc_migration_runs(run_id))" + ), +} +_CONTROL_NAMES = frozenset(_CONTROL_DDL) +# Alembic's environment/op proxies are process-global, even for different DBs. +_ALEMBIC_CONTEXT_LOCK = threading.RLock() + + +class SQLiteMigrationRunner: + """A synchronous maintenance executor. Call outside the business event loop. + + File locks serialize cooperating migrators; BEGIN IMMEDIATE detects a current + SQLite writer. Neither proves that an idle old process has stopped, so the + operator's maintenance confirmation is required for every changing run. + """ + + def __init__( + self, database: Path, bundle: MigrationBundle, *, configuration: Mapping[str, Any] | None = None + ) -> None: + self.database = database.expanduser().resolve() + self.bundle = bundle + self.configuration_digest = digest(configuration or {}) + + def plan(self) -> MigrationPlan: + """Inspect without creating the database, its parent, locks or version rows.""" + if MigrationBundle(self.bundle.directory).checksum != self.bundle.checksum: + raise MigrationError("plan_changed", "Migration resources changed after the bundle was loaded.") + if not self.database.exists(): + return self._plan(None, digest([]), initialized=False) + self._check_file() + with self._connect(read_only=True) as connection: + return self._inspect(connection) + + def verify(self) -> MigrationResult: + plan = self.plan() + if plan.state != "ready": + raise MigrationError(plan.state, "The bundle is not fully migrated; inspect its plan before applying.") + return MigrationResult(revision=self.bundle.head, changed=False) + + def apply( + self, + *, + plan_id: str | None = None, + maintenance_confirmed: bool = False, + accepted: bool = False, + resume: str | None = None, + ) -> MigrationResult: + """Require an accepted plan; resume keeps its original plan and backup.""" + initial = self.plan() + if initial.state == "ready" and resume is None: + return MigrationResult(revision=self.bundle.head, changed=False) + if not accepted or plan_id is None: + raise MigrationError("confirmation_required", "Accept the displayed plan ID explicitly.") + if not maintenance_confirmed: + raise MigrationError("maintenance_required", "Stop all application, SDK and background writers first.") + if initial.state == "recovery_required" and resume != initial.resume_run_id: + raise MigrationError("recovery_required", "Resume the recorded run; do not start a replacement run.") + if resume is not None and resume != initial.resume_run_id: + raise MigrationError("recovery_required", "The requested run is not the pending run.") + if initial.plan_id != plan_id: + raise MigrationError("plan_changed", "The reviewed plan no longer matches the database or bundle.") + with local_migration_lock(self.database): + # Revalidate before opening in write mode, including a missing target. + if self.plan() != initial: + raise MigrationError("plan_changed", "State changed while acquiring the migration lock.") + with self._connect(read_only=False) as connection: + connection.exec_driver_sql("PRAGMA foreign_keys=OFF") + connection.commit() + self._begin(connection) + locked = self._inspect(connection) + if locked != initial: + raise MigrationError("plan_changed", "State changed before acquiring the SQLite write transaction.") + if resume: + run_id, backup = self._resume(connection, resume, initial) + connection.commit() + else: + # A separate read connection can back up the committed WAL + # while this connection prevents writers with BEGIN IMMEDIATE. + backup, checksum = self._backup() + run_id = uuid4().hex + self._bootstrap(connection, initial, run_id, backup, checksum) + connection.commit() + self._upgrade(connection, run_id) + self._begin(connection) + self._verify_schema(connection, self.bundle.head) + connection.execute( + text("UPDATE pc_migration_runs SET state='complete' WHERE run_id=:run"), {"run": run_id} + ) + connection.commit() + return MigrationResult(revision=self.bundle.head, changed=True, run_id=run_id, backup_ref=str(backup)) + + def _check_file(self) -> None: + if not self.database.is_file() or self.database.stat().st_nlink != 1: + raise MigrationError("unsupported_target", "Use a regular database file without hard-link aliases.") + + @contextmanager + def _connect(self, *, read_only: bool) -> Iterator[Connection]: + mode = "ro" if read_only else "rwc" + uri = self.database.as_uri() + "?mode=" + mode + engine = create_engine( + "sqlite://", creator=lambda: sqlite3.connect(uri, uri=True, isolation_level=None, timeout=0) + ) + try: + with engine.connect() as connection: + if read_only: + connection.exec_driver_sql("PRAGMA query_only=ON") + connection.exec_driver_sql("BEGIN") + yield connection + finally: + engine.dispose() + + def _inspect(self, connection: Connection) -> MigrationPlan: + inventory = sqlite_inventory(connection) + controls = [row for row in inventory if row[1] in _CONTROL_NAMES or row[2] in _CONTROL_NAMES] + initialized = bool(controls) + if initialized: + expected = sorted([["table", name, name, ddl] for name, ddl in _CONTROL_DDL.items()]) + if sorted(controls) != expected: + raise MigrationError( + "recovery_required", "Migration bookkeeping is incomplete or has an unknown shape." + ) + fingerprint = digest(sqlite_inventory(connection, exclude=_CONTROL_NAMES)) + if not initialized: + if not inventory: + return self._plan(None, fingerprint, initialized=False) + revision = self.bundle.baselines.get(fingerprint) + if revision is None: + raise MigrationError("unknown_baseline", "Schema is not a recognized baseline; no changes were made.") + self._verify_integrity(connection) + return self._plan(revision, fingerprint, initialized=False) + revisions = list(connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalars()) + if len(revisions) > 1: + raise MigrationError("incompatible_schema", "Multiple database revision heads are unsupported.") + revision = revisions[0] if revisions else None + self._verify_schema(connection, revision) + self._verify_history(connection, revision) + recovery = self._pending_run(connection, revision) + if recovery is not None: + return recovery + return self._plan(revision, fingerprint, initialized=True) + + def _verify_history(self, connection: Connection, revision: str | None) -> None: + receipts = list(connection.exec_driver_sql("SELECT revision, checksum FROM pc_migration_steps")) + expected = self.bundle.revisions[: self.bundle.revisions.index(revision) + 1] if revision else () + if sorted(item[0] for item in receipts) != sorted(expected): + raise MigrationError("recovery_required", "Revision receipts do not match the database version.") + for recorded_revision, checksum in receipts: + if self.bundle.checksums.get(recorded_revision) != checksum: + raise MigrationError("checksum_conflict", "An applied revision is absent or its source has changed.") + + def _pending_run(self, connection: Connection, revision: str | None) -> MigrationPlan | None: + runs = list(connection.exec_driver_sql("SELECT run_id, plan_json FROM pc_migration_runs WHERE state='running'")) + if len(runs) > 1: + raise MigrationError("recovery_required", "Multiple unfinished runs require manual inspection.") + if runs: + run_id, plan_json = runs[0] + original = MigrationPlan.model_validate_json(plan_json) + if ( + original.bundle_checksum != self.bundle.checksum + or original.configuration_digest != self.configuration_digest + ): + raise MigrationError("plan_changed", "Resume requires the original migration bundle and configuration.") + if original.database_id != self._database_id(): + raise MigrationError("plan_changed", "The recovery record belongs to a different database target.") + completed_count = self.bundle.revisions.index(revision) + 1 if revision is not None else 0 + source_count = self.bundle.revisions.index(original.source_revision) + 1 if original.source_revision else 0 + if completed_count < source_count: + raise MigrationError("recovery_required", "Recorded progress precedes the original source revision.") + return original.model_copy(update={"state": "recovery_required", "resume_run_id": run_id}) + return None + + def _database_id(self) -> str: + # The deployment identity is the canonical path, never a DSN or payload. + return digest({"kind": "sqlite", "path": str(self.database)}) + + def _plan(self, revision: str | None, fingerprint: str, *, initialized: bool) -> MigrationPlan: + pending = self.bundle.pending(revision) + adopt = not initialized and revision is not None + state = "migration_required" if pending or adopt else "ready" + if not initialized and revision is None: + state = "uninitialized" + fields = { + "database_id": self._database_id(), + "source_revision": revision, + "target_revision": self.bundle.head, + "schema_fingerprint": fingerprint, + "bundle_checksum": self.bundle.checksum, + "configuration_digest": self.configuration_digest, + "state": state, + "revisions": pending, + "adopt_baseline": adopt, + } + return MigrationPlan.model_validate({"plan_id": digest(fields), **fields}) + + @staticmethod + def _begin(connection: Connection) -> None: + try: + connection.exec_driver_sql("BEGIN IMMEDIATE") + except OperationalError as error: + if isinstance(error.orig, sqlite3.OperationalError) and error.orig.sqlite_errorcode in { + sqlite3.SQLITE_BUSY, + sqlite3.SQLITE_LOCKED, + }: + raise MigrationError("active_writers", "SQLite is busy; stop writers before retrying.") from error + raise + + def _backup(self) -> tuple[Path, str]: + directory = self.database.with_name(self.database.name + ".pc-migration-backups") + backup = directory / f"{uuid4().hex}.sqlite3" + try: + directory.mkdir(mode=0o700, exist_ok=True) + os.close(os.open(backup, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)) + with ( + closing(sqlite3.connect(self.database.as_uri() + "?mode=ro", uri=True)) as source, + closing(sqlite3.connect(backup)) as destination, + ): + source.backup(destination) + if destination.execute("PRAGMA integrity_check").fetchall() != [("ok",)]: + raise MigrationError("backup_required", "The consistent SQLite backup failed integrity checks.") + with backup.open("rb") as persisted: + os.fsync(persisted.fileno()) + if os.name != "nt": + directory_fd = os.open(directory, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + return backup, _file_checksum(backup) + except (OSError, sqlite3.Error) as error: + raise MigrationError( + "backup_required", "Could not create and verify a consistent SQLite backup." + ) from error + + def _bootstrap(self, connection: Connection, plan: MigrationPlan, run_id: str, backup: Path, checksum: str) -> None: + if plan.adopt_baseline or plan.state == "uninitialized": + for ddl in _CONTROL_DDL.values(): + connection.exec_driver_sql(ddl) + if plan.adopt_baseline and plan.source_revision is not None: + # Adoption follows exact baseline recognition and a verified backup. + with _ALEMBIC_CONTEXT_LOCK: + command.stamp(self.bundle.config(connection), plan.source_revision) + connection.execute( + text("INSERT INTO pc_migration_runs VALUES (:run, :plan, :backup, :checksum, 'running', :started)"), + { + "run": run_id, + "plan": plan.model_dump_json(), + "backup": str(backup), + "checksum": checksum, + "started": datetime.now(UTC).isoformat(), + }, + ) + if plan.adopt_baseline and plan.source_revision: + for revision in self.bundle.revisions[: self.bundle.revisions.index(plan.source_revision) + 1]: + connection.execute( + text("INSERT INTO pc_migration_steps VALUES (:run, :revision, :checksum)"), + {"run": run_id, "revision": revision, "checksum": self.bundle.checksums[revision]}, + ) + + def _resume(self, connection: Connection, run_id: str, plan: MigrationPlan) -> tuple[str, Path]: + path, checksum, plan_json = connection.execute( + text("SELECT backup_path, backup_checksum, plan_json FROM pc_migration_runs WHERE run_id=:run"), + {"run": run_id}, + ).one() + original = MigrationPlan.model_validate_json(plan_json) + if original.plan_id != plan.plan_id: + raise MigrationError("plan_changed", "The run does not match the accepted plan.") + backup = Path(path) + try: + valid = backup.is_file() and _file_checksum(backup) == checksum + except OSError as error: + raise MigrationError("backup_required", "The original recovery point cannot be read.") from error + if not valid: + raise MigrationError("backup_required", "The original recovery point is missing or has changed.") + return run_id, backup + + def _upgrade(self, connection: Connection, run_id: str) -> None: + current = connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one_or_none() + connection.commit() + for revision in self.bundle.pending(current): + self._begin(connection) + self._verify_schema(connection, current) + with _ALEMBIC_CONTEXT_LOCK: + command.upgrade(self.bundle.config(connection), revision) + self._verify_schema(connection, revision) + connection.execute( + text("INSERT INTO pc_migration_steps VALUES (:run, :revision, :checksum)"), + {"run": run_id, "revision": revision, "checksum": self.bundle.checksums[revision]}, + ) + connection.commit() + current = revision + + def _verify_schema(self, connection: Connection, revision: str | None) -> None: + self.bundle.verify(revision, digest(sqlite_inventory(connection, exclude=_CONTROL_NAMES))) + self._verify_integrity(connection) + + @staticmethod + def _verify_integrity(connection: Connection) -> None: + if connection.exec_driver_sql("PRAGMA foreign_key_check").first() is not None: + raise MigrationError("verification_failed", "Foreign-key violations block migration readiness.") + if connection.exec_driver_sql("PRAGMA quick_check").scalars().all() != ["ok"]: + raise MigrationError("verification_failed", "SQLite integrity verification failed.") + + +def _file_checksum(path: Path) -> str: + with path.open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() diff --git a/tests/builtin/persistence/test_database_migration_backends.py b/tests/builtin/persistence/test_database_migration_backends.py new file mode 100644 index 0000000000..cfc37465e8 --- /dev/null +++ b/tests/builtin/persistence/test_database_migration_backends.py @@ -0,0 +1,131 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Opt-in real seekDB/OceanBase DDL probes; a skip is not backend acceptance. + +OceanBase requires an empty dedicated database whose name starts with +pc_migration_probe_. The test leaves its schema for inspection and never drops +an existing database. Recreate the scratch database before running it again. +""" + +from __future__ import annotations + +import asyncio +import importlib.metadata +import json +import os +import platform +from pathlib import Path + +import pytest +from alembic import command +from sqlalchemy import Connection, inspect, text +from sqlalchemy.engine import make_url +from sqlalchemy.ext.asyncio import AsyncEngine + +from powercontext.builtin.persistence.migrations import MigrationBundle +from powercontext.builtin.persistence.migrations.locking import local_migration_lock +from powercontext.builtin.persistence.migrations.oceanbase import oceanbase_migration_lock +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile +from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile + +FIXTURE = Path(__file__).resolve().parents[2] / "fixtures/database_migrations" + + +def exercise_revisions(connection: Connection, bundle: MigrationBundle) -> None: + assert inspect(connection).get_table_names() == [], "Use an empty disposable database" + assert inspect(connection).get_view_names() == [], "Use an empty disposable database" + print( + json.dumps({ + "python": platform.python_version(), + "sqlalchemy": importlib.metadata.version("sqlalchemy"), + "alembic": importlib.metadata.version("alembic"), + "pyobvector": importlib.metadata.version("pyobvector"), + "driver": connection.dialect.driver, + "server": connection.exec_driver_sql("SELECT VERSION()").scalar_one(), + }) + ) + command.upgrade(bundle.config(connection), "p0001") + connection.execute( + text( + "INSERT INTO pc_artifacts (scope_id, family, artifact_id, revision, content) VALUES ('s','memory','m',1,:data)" + ), + {"data": b'{"content":"preserve"}'}, + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_heads (scope_id,family,artifact_id,revision) VALUES ('s','memory','m',1)" + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_tags (scope_id,family,artifact_id,target_type,target_id,tag_key_hash,tag_key,tag,assigned_at) " + "VALUES ('s','memory','m','artifact','m',UNHEX(REPEAT('00',32)),'keep','Keep','2026-09-28 12:00:00')" + ) + connection.commit() + command.upgrade(bundle.config(connection), "p0002") + connection.commit() + command.upgrade(bundle.config(connection), "p0003") + connection.commit() + command.upgrade(bundle.config(connection), "head") + assert connection.exec_driver_sql("SELECT content,memory_citations FROM pc_artifacts").all() == [ + (b'{"content":"preserve"}', None) + ] + assert connection.exec_driver_sql("SELECT tag FROM pc_artifact_tags").all() == [("Keep",)] + names = {row["name"] for row in inspect(connection).get_check_constraints("pc_artifact_tags")} + assert "ck_pc_artifact_tags_family" not in names + assert "ck_pc_artifact_tags_target" in names + assert {row["name"] for row in inspect(connection).get_indexes("pc_artifact_tags")} >= { + "ix_pc_artifact_tags_family_key", + "ix_pc_artifact_tags_key", + } + + +async def exercise_oceanbase(engine: AsyncEngine, database_name: str) -> None: + async with ( + engine.connect() as connection, + engine.connect() as contender, + oceanbase_migration_lock(connection, contender, database_id=database_name) as lock, + ): + await connection.run_sync(lambda sync: exercise_revisions(sync, MigrationBundle(FIXTURE))) + await lock.verify() + + +@pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekDB probe not enabled") +def test_real_seekdb_revision_operations(tmp_path: Path) -> None: + async def scenario() -> None: + path = tmp_path / "seekdb" + # The OS lock precedes engine startup and outlives engine shutdown. + with local_migration_lock(path): + async with SeekDBProfile.open(SeekDBConfig(path=path), tables=()) as profile: + async with profile.database.engine.connect() as connection: + await connection.run_sync(lambda sync: exercise_revisions(sync, MigrationBundle(FIXTURE))) + + asyncio.run(scenario()) + + +@pytest.mark.skipif( + not os.environ.get("POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL"), + reason="dedicated real OceanBase target not configured", +) +def test_real_oceanbase_revision_operations_and_named_lock() -> None: + value = os.environ["POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL"] + url = make_url(value) + assert url.database and url.database.startswith("pc_migration_probe_"), ( + "Only a dedicated scratch database is allowed" + ) + config = OceanBaseConfig.model_validate({"url": value}) + + async def scenario() -> None: + async with OceanBaseProfile.open(config, tables=()) as profile: + await exercise_oceanbase(profile.database.engine, str(url.database)) + + asyncio.run(scenario()) diff --git a/tests/builtin/persistence/test_database_migrations.py b/tests/builtin/persistence/test_database_migrations.py new file mode 100644 index 0000000000..e0d8a26b7f --- /dev/null +++ b/tests/builtin/persistence/test_database_migrations.py @@ -0,0 +1,417 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Observable safety and recovery behavior using actual SQLite DDL and files.""" + +from __future__ import annotations + +import json +import shutil +import sqlite3 +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + +import pytest +from alembic import command +from sqlalchemy import Connection, create_engine, inspect, text + +from powercontext.builtin.persistence.migrations import MigrationBundle, MigrationError, SQLiteMigrationRunner + +FIXTURE = Path(__file__).resolve().parents[2] / "fixtures/database_migrations" + + +@pytest.fixture +def bundle() -> MigrationBundle: + return MigrationBundle(FIXTURE) + + +def load_schema(database: Path, name: str = "pre_dream") -> None: + statements = json.loads((FIXTURE / "schemas" / f"{name}.json").read_text())["sqlite"] + with sqlite3.connect(database) as connection: + for statement in statements: + connection.execute(statement) + + +def seed(connection: Connection) -> None: + connection.execute( + text( + "INSERT INTO pc_artifacts (scope_id, family, artifact_id, revision, content) VALUES ('s','memory','m',1,:body)" + ), + {"body": b'{"content":"original"}'}, + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_heads (scope_id,family,artifact_id,revision) VALUES ('s','memory','m',1)" + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_tags (scope_id,family,artifact_id,target_type,target_id,tag_key_hash,tag_key,tag,assigned_at) " + "VALUES ('s','memory','m','artifact','m',X'00','keep','Keep','2026-09-28 12:00:00')" + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_candidate_versions " + "(scope_id,candidate_id,version,family,proposal,source_refs,artifact_refs) " + "VALUES ('s','c',1,'memory',X'7B7D',X'5B5D',X'5B5D')" + ) + + +def populate(database: Path) -> None: + engine = create_engine("sqlite:///" + str(database)) + try: + with engine.begin() as connection: + seed(connection) + finally: + engine.dispose() + + +def apply(runner: SQLiteMigrationRunner, **kwargs: Any): + return runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True, **kwargs) + + +def rows(database: Path, query: str): + with sqlite3.connect(database) as connection: + return connection.execute(query).fetchall() + + +def test_read_only_commands_do_not_create_missing_database_or_parent(tmp_path: Path, bundle: MigrationBundle) -> None: + database = tmp_path / "absent" / "nested" / "pc.sqlite3" + runner = SQLiteMigrationRunner(database, bundle) + assert runner.plan().state == "uninitialized" + with pytest.raises(MigrationError, match="uninitialized"): + runner.verify() + with pytest.raises(MigrationError, match="confirmation_required"): + runner.apply() + assert not list(tmp_path.iterdir()) + + +def test_empty_initialization_and_noop_apply(tmp_path: Path, bundle: MigrationBundle) -> None: + database = tmp_path / "new.sqlite3" + runner = SQLiteMigrationRunner(database, bundle) + result = apply(runner) + assert result.changed and result.run_id and result.backup_ref + assert runner.verify().revision == "p0003" + before = database.read_bytes() + files = sorted(str(path) for path in tmp_path.rglob("*")) + assert not runner.apply().changed + assert database.read_bytes() == before + assert sorted(str(path) for path in tmp_path.rglob("*")) == files + assert rows(database, "SELECT COUNT(*) FROM pc_migration_runs") == [(1,)] + + +def test_distinct_databases_can_migrate_in_one_process_without_crossing_contexts( + tmp_path: Path, bundle: MigrationBundle +) -> None: + runners = [SQLiteMigrationRunner(tmp_path / f"database-{index}.sqlite3", bundle) for index in range(4)] + with ThreadPoolExecutor(max_workers=4) as executor: + results = list(executor.map(apply, runners)) + assert len({result.run_id for result in results}) == 4 + for runner in runners: + assert runner.verify().state == "ready" + assert rows(runner.database, "SELECT COUNT(*) FROM pc_migration_runs") == [(1,)] + + +@pytest.mark.parametrize("source", ["pre_dream", "v1_1_0"]) +def test_known_historical_tables_preserve_data_and_indexes( + tmp_path: Path, bundle: MigrationBundle, source: str +) -> None: + database = tmp_path / "legacy.sqlite3" + load_schema(database, source) + populate(database) + runner = SQLiteMigrationRunner(database, bundle) + original = database.read_bytes() + plan = runner.plan() + assert plan.adopt_baseline + with pytest.raises(MigrationError, match="migration_required"): + runner.verify() + assert database.read_bytes() == original + result = apply(runner) + assert result.backup_ref + backup = Path(result.backup_ref) + assert SQLiteMigrationRunner(backup, bundle).plan().schema_fingerprint == plan.schema_fingerprint + assert rows(backup, "SELECT content FROM pc_artifacts") == [(b'{"content":"original"}',)] + assert rows(database, "SELECT content FROM pc_artifacts") == [(b'{"content":"original"}',)] + assert rows(database, "SELECT tag, tag_key_hash FROM pc_artifact_tags") == [("Keep", b"\x00")] + assert rows(database, "SELECT proposal, memory_citations FROM pc_artifact_candidate_versions") == [(b"{}", None)] + with sqlite3.connect(database) as connection: + connection.execute("PRAGMA foreign_keys=ON") + connection.execute( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content) VALUES ('s','topic-memory','t',1,X'7B7D')" + ) + connection.execute( + "INSERT INTO pc_artifact_heads (scope_id,family,artifact_id,revision) VALUES ('s','topic-memory','t',1)" + ) + connection.execute( + "INSERT INTO pc_artifact_tags VALUES ('s','topic-memory','t','artifact','t',X'01','new','New','2026-09-28')" + ) + with pytest.raises(sqlite3.IntegrityError): + connection.execute("UPDATE pc_artifact_tags SET target_id='not-the-artifact' WHERE artifact_id='t'") + assert connection.execute("PRAGMA foreign_key_check").fetchall() == [] + indexes = {item[1] for item in connection.execute("PRAGMA index_list(pc_artifact_tags)")} + assert {"ix_pc_artifact_tags_family_key", "ix_pc_artifact_tags_key"} <= indexes + assert runner.verify().state == "ready" + + +@pytest.mark.parametrize( + "ddl", + [ + "CREATE TABLE surprise (id INTEGER)", + "CREATE TABLE pc_artifact_tags_topic_memory (id INTEGER)", + "CREATE TABLE pc_schema_revision (version_num TEXT)", + ], +) +def test_unknown_or_partial_schema_is_rejected_without_writes( + tmp_path: Path, bundle: MigrationBundle, ddl: str +) -> None: + database = tmp_path / "unknown.sqlite3" + load_schema(database) + with sqlite3.connect(database) as connection: + connection.execute(ddl) + before = database.read_bytes() + runner = SQLiteMigrationRunner(database, bundle) + with pytest.raises(MigrationError, match=r"unknown_baseline|recovery_required"): + apply(runner) + assert database.read_bytes() == before + assert len(list(tmp_path.iterdir())) == 1 + + +def test_unknown_newer_revision_never_downgrades(tmp_path: Path, bundle: MigrationBundle) -> None: + database = tmp_path / "newer.sqlite3" + runner = SQLiteMigrationRunner(database, bundle) + apply(runner) + with sqlite3.connect(database) as connection: + connection.execute("UPDATE pc_schema_revision SET version_num='future-release'") + before = database.read_bytes() + with pytest.raises(MigrationError, match="incompatible_schema"): + runner.apply(accepted=True, maintenance_confirmed=True, plan_id="anything") + assert database.read_bytes() == before + + +def test_changed_configuration_requires_a_new_plan(tmp_path: Path, bundle: MigrationBundle) -> None: + database = tmp_path / "new.sqlite3" + reviewed = SQLiteMigrationRunner(database, bundle, configuration={"projection": "fts"}).plan() + runner = SQLiteMigrationRunner(database, bundle, configuration={"projection": "hybrid"}) + with pytest.raises(MigrationError, match="plan_changed"): + runner.apply(plan_id=reviewed.plan_id, accepted=True, maintenance_confirmed=True) + assert not database.exists() + + +def test_confirmation_and_stopped_writes_are_both_required(tmp_path: Path, bundle: MigrationBundle) -> None: + runner = SQLiteMigrationRunner(tmp_path / "new.sqlite3", bundle) + with pytest.raises(MigrationError, match="confirmation_required"): + runner.apply(accepted=True, maintenance_confirmed=True) + with pytest.raises(MigrationError, match="maintenance_required"): + runner.apply(plan_id=runner.plan().plan_id, accepted=True) + assert not list(tmp_path.iterdir()) + + +def test_backup_failure_precedes_schema_writes(tmp_path: Path, bundle: MigrationBundle) -> None: + database = tmp_path / "legacy.sqlite3" + load_schema(database) + populate(database) + database.with_name(database.name + ".pc-migration-backups").write_text("not a directory") + before = database.read_bytes() + with pytest.raises(MigrationError, match="backup_required"): + apply(SQLiteMigrationRunner(database, bundle)) + assert database.read_bytes() == before + assert not rows(database, "SELECT name FROM sqlite_schema WHERE name='pc_schema_revision'") + + +def test_consistent_backup_includes_committed_wal_data_and_can_be_restored( + tmp_path: Path, bundle: MigrationBundle +) -> None: + database = tmp_path / "wal.sqlite3" + load_schema(database) + populate(database) + with sqlite3.connect(database) as writer: + writer.execute("PRAGMA journal_mode=WAL") + writer.execute("PRAGMA wal_autocheckpoint=0") + writer.execute("UPDATE pc_artifact_tags SET tag='Committed in WAL'") + writer.commit() + assert Path(str(database) + "-wal").stat().st_size > 0 + result = apply(SQLiteMigrationRunner(database, bundle)) + assert result.backup_ref + backup = Path(result.backup_ref) + assert rows(backup, "SELECT tag FROM pc_artifact_tags") == [("Committed in WAL",)] + assert "memory_citations" not in {row[1] for row in rows(backup, "PRAGMA table_info(pc_artifacts)")} + restored = tmp_path / "restored.sqlite3" + shutil.copy2(backup, restored) + assert SQLiteMigrationRunner(restored, bundle).plan().source_revision == "p0001" + assert rows(restored, "PRAGMA integrity_check") == [("ok",)] + + +def test_current_writer_blocks_even_with_confirmation(tmp_path: Path, bundle: MigrationBundle) -> None: + database = tmp_path / "busy.sqlite3" + load_schema(database) + with sqlite3.connect(database) as writer: + writer.execute("BEGIN IMMEDIATE") + with pytest.raises(MigrationError, match="active_writers"): + apply(SQLiteMigrationRunner(database, bundle)) + assert not rows(database, "SELECT name FROM sqlite_schema WHERE name='pc_schema_revision'") + assert not database.with_name(database.name + ".pc-migration-backups").exists() + + +@pytest.mark.parametrize("revision", ["p0001", "p0002", "p0003"]) +@pytest.mark.parametrize("after_commit", [False, True]) +def test_interruption_at_each_revision_boundary_resumes_one_run_and_backup( + tmp_path: Path, bundle: MigrationBundle, monkeypatch: pytest.MonkeyPatch, revision: str, after_commit: bool +) -> None: + database = tmp_path / "interrupted.sqlite3" + runner = SQLiteMigrationRunner(database, bundle) + original_plan = runner.plan() + real_upgrade = command.upgrade + real_commit = Connection.commit + interrupted = False + + def upgrade(config, target, **kwargs): + real_upgrade(config, target, **kwargs) + if target == revision: + raise InterruptedError("Injected after real DDL, before transaction commit") # noqa: TRY003 + + def commit(connection): + nonlocal interrupted + present = connection.exec_driver_sql("SELECT name FROM sqlite_schema WHERE name='pc_migration_steps'").first() + completed = ( + present + and connection.execute( + text("SELECT revision FROM pc_migration_steps WHERE revision=:revision"), {"revision": revision} + ).first() + ) + real_commit(connection) + if completed and not interrupted: + interrupted = True + raise InterruptedError("Injected immediately after atomic commit") # noqa: TRY003 + + with monkeypatch.context() as injection: + if after_commit: + injection.setattr(Connection, "commit", commit) + else: + injection.setattr(command, "upgrade", upgrade) + with pytest.raises(InterruptedError): + apply(runner) + recovery = runner.plan() + assert recovery.state == "recovery_required" + assert recovery.plan_id == original_plan.plan_id + assert recovery.resume_run_id + before_backups = list(tmp_path.glob("*.pc-migration-backups/*.sqlite3")) + assert len(before_backups) == 1 + with pytest.raises(MigrationError, match="recovery_required"): + apply(runner) + with pytest.raises(MigrationError, match="recovery_required"): + runner.verify() + result = apply(runner, resume=recovery.resume_run_id) + assert result.run_id == recovery.resume_run_id + assert Path(result.backup_ref) == before_backups[0] + assert list(tmp_path.glob("*.pc-migration-backups/*.sqlite3")) == before_backups + assert rows(database, "SELECT COUNT(*) FROM pc_migration_runs") == [(1,)] + assert rows(database, "SELECT COUNT(*) FROM pc_migration_steps") == [(3,)] + assert runner.verify().state == "ready" + + +@pytest.mark.parametrize("resource", ["versions/p0002_citations.py", "schemas/pre_dream.json"]) +def test_applied_revision_checksum_conflict_is_rejected(tmp_path: Path, resource: str) -> None: + copied = tmp_path / "bundle" + shutil.copytree(FIXTURE, copied) + database = tmp_path / "pc.sqlite3" + apply(SQLiteMigrationRunner(database, MigrationBundle(copied))) + script = copied / resource + script.write_text(script.read_text() + "\n") + with pytest.raises(MigrationError, match="checksum_conflict"): + SQLiteMigrationRunner(database, MigrationBundle(copied)).verify() + + +def test_resume_requires_the_original_backup( + tmp_path: Path, bundle: MigrationBundle, monkeypatch: pytest.MonkeyPatch +) -> None: + runner = SQLiteMigrationRunner(tmp_path / "pc.sqlite3", bundle) + real_upgrade = command.upgrade + + def interrupted(config, target, **kwargs): + real_upgrade(config, target, **kwargs) + raise InterruptedError + + with monkeypatch.context() as injection: + injection.setattr(command, "upgrade", interrupted) + with pytest.raises(InterruptedError): + apply(runner) + plan = runner.plan() + backups = list(tmp_path.glob("*.pc-migration-backups/*.sqlite3")) + assert len(backups) == 1 + backups[0].write_bytes(b"not the original recovery point") + before = runner.database.read_bytes() + with pytest.raises(MigrationError, match="backup_required"): + apply(runner, resume=plan.resume_run_id) + assert runner.database.read_bytes() == before + assert list(tmp_path.glob("*.pc-migration-backups/*.sqlite3")) == backups + + +def test_integrity_failure_blocks_readiness(tmp_path: Path, bundle: MigrationBundle) -> None: + database = tmp_path / "corrupt.sqlite3" + apply(SQLiteMigrationRunner(database, bundle)) + with sqlite3.connect(database) as connection: + connection.execute( + "INSERT INTO pc_artifact_tags VALUES ('missing','memory','a','artifact','a',X'00','k','t','2026-09-28')" + ) + before = database.read_bytes() + with pytest.raises(MigrationError, match="verification_failed"): + SQLiteMigrationRunner(database, bundle).verify() + assert database.read_bytes() == before + + +def test_another_process_cannot_migrate_through_a_symlink_then_lock_owner_exit_releases( + tmp_path: Path, bundle: MigrationBundle +) -> None: + database = tmp_path / "pc.sqlite3" + load_schema(database) + alias = tmp_path / "alias.sqlite3" + alias.symlink_to(database) + code = ( + "from pathlib import Path; import sys; " + "from powercontext.builtin.persistence.migrations.locking import local_migration_lock; " + "lock=local_migration_lock(Path(sys.argv[1])); lock.__enter__(); " + "print('locked', flush=True); sys.stdin.read()" + ) + process = subprocess.Popen( + [sys.executable, "-c", code, str(database)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True + ) + try: + assert process.stdout and process.stdout.readline().strip() == "locked" + with pytest.raises(MigrationError, match="migration_locked"): + apply(SQLiteMigrationRunner(alias, bundle)) + finally: + process.terminate() + process.wait(timeout=10) + assert apply(SQLiteMigrationRunner(database, bundle)).state == "ready" + + +def test_batch_rebuild_preserves_registered_triggers_and_foreign_keys(bundle: MigrationBundle) -> None: + engine = create_engine("sqlite://") + with engine.connect() as connection: + connection.exec_driver_sql("BEGIN IMMEDIATE") + command.upgrade(bundle.config(connection), "p0002") + seed(connection) + connection.exec_driver_sql("CREATE TABLE tag_audit (tag TEXT)") + connection.exec_driver_sql( + "CREATE TRIGGER registered_tag_audit AFTER UPDATE ON pc_artifact_tags " + "BEGIN INSERT INTO tag_audit VALUES (new.tag); END" + ) + command.upgrade(bundle.config(connection), "p0003") + connection.exec_driver_sql("UPDATE pc_artifact_tags SET tag='Changed'") + assert connection.exec_driver_sql("SELECT tag FROM tag_audit").all() == [("Changed",)] + assert connection.exec_driver_sql("PRAGMA foreign_key_check").all() == [] + assert {item["name"] for item in inspect(connection).get_check_constraints("pc_artifact_tags")} == { + "ck_pc_artifact_tags_target" + } + connection.commit() + engine.dispose() diff --git a/tests/fixtures/database_migrations/README.md b/tests/fixtures/database_migrations/README.md new file mode 100644 index 0000000000..c814520648 --- /dev/null +++ b/tests/fixtures/database_migrations/README.md @@ -0,0 +1,70 @@ +# Database migration prototype + +Implementation of the Phase A migration primitives in [RFC #1771](https://github.com/oceanbase/powercontext/pull/1771), tracked by [#1756](https://github.com/oceanbase/powercontext/issues/1756). + +This is a repository-only prototype for **disposable databases**. Its `ready` result means that this bundle's four tables and migration receipts verify. It does not mean that a PowerContext Server, its data migrations, or its search projections are ready. The production `powercontext server` commands and startup behavior are unchanged. + +## Fixed historical inputs + +The JSON resources contain the original compiled definitions of `pc_artifacts`, `pc_artifact_heads`, `pc_artifact_candidate_versions`, and `pc_artifact_tags`, including indexes, constraints, and MySQL collations. They do not import current application metadata. + +| Input | Source | Purpose | +| --- | --- | --- | +| `schemas/pre_dream.json` | [495cc93d67387879382c98706f4975f638faec15](https://github.com/oceanbase/powercontext/commit/495cc93d67387879382c98706f4975f638faec15), before Dream citations | Upgrade existing rows through both change classes | +| `schemas/v1_1_0.json` | [powercontext-v1.1.0 / 86e687f13fec62151ee5657a90733586fd5158dc](https://github.com/oceanbase/powercontext/commit/86e687f13fec62151ee5657a90733586fd5158dc) | Recognize the already-upgraded released table shapes | + +Both resources record the source Git blob and the SQLAlchemy compiler version. They were captured from the original commit's `tables.py` **and `limits.py`**, selecting these four original tables and compiling `CreateTable` plus each named `CreateIndex` for SQLite and MySQL. The pre-Dream fixture was not synthesized by subtracting columns from today's models. These are table subsets, not complete released database images. + +The single Alembic chain is: + +1. `p0001`: create the frozen pre-Dream structures. +2. `p0002`: add nullable `memory_citations` columns to immutable artifact and candidate revisions. +3. `p0003`: remove `ck_pc_artifact_tags_family`; use an Alembic batch rebuild on SQLite and explicit constraint DDL on MySQL dialects. Preserve the target constraint, foreign key, indexes, rows, and registered triggers. + +`manifest.json` records exact SQLite schema fingerprints at each revision and the two supported unversioned baselines. Exact matching intentionally rejects unregistered tables, indexes, triggers, views, temporary-table residue, and alternate DDL spellings. The released table definitions and the migrated definitions have separate admitted fingerprints. No generic `stamp head` fallback exists. + +Revision receipts bind each script, its declared historical resources, and the Alembic environment. A reviewed plan also binds the complete bundle, configuration digest, canonical database path, source fingerprint, and target. Editing a recorded revision or its declared resources prevents acceptance. + +## SQLite executor + +Run from the repository after `uv sync --locked`: + +```bash +uv run --locked python scripts/database_migration_probe.py plan --sqlite-path /tmp/pc-migration-probe.sqlite3 +uv run --locked python scripts/database_migration_probe.py apply --sqlite-path /tmp/pc-migration-probe.sqlite3 --maintenance-confirmed --yes --plan-id PLAN_ID +uv run --locked python scripts/database_migration_probe.py verify --sqlite-path /tmp/pc-migration-probe.sqlite3 +``` + +Replace `PLAN_ID` with the returned ID. An interactive apply prints a fresh plan and asks once; non-interactive apply requires both the exact ID and `--yes`. The command never discovers `.env` files or opens the business runtime. `status`, `plan`, and `verify` open an existing target read-only and never create a missing target, parent, lock, or bookkeeping table. A no-op apply needs no confirmation, lock, backup, or new run. + +Changing runs require the operator to stop all writers. A canonical-path OS lock excludes other migrators across processes, including symlink aliases; hard-linked SQLite files are rejected. Each schema transaction begins with `BEGIN IMMEDIATE`, which also rejects a currently active database writer. This check cannot prove that an idle legacy process has stopped. The maintenance confirmation remains an operator responsibility. + +Before bookkeeping or business DDL, the executor uses SQLite's online backup API from a separate read connection while holding the database write reservation. Backups include committed WAL data, are created with restrictive permissions, and pass integrity verification. They are retained beside the target in `.pc-migration-backups/`; migration does not automatically delete them or restore over the database. + +Bookkeeping bootstrap is atomic on SQLite. Each subsequent revision, version update, schema/foreign-key verification, and completion receipt commits together. Foreign-key enforcement is disabled only on the dedicated migration connection before its explicit transaction; `foreign_key_check` runs before commit. An interrupted committed run blocks readiness until explicitly resumed: + +```bash +uv run --locked python scripts/database_migration_probe.py status --sqlite-path /tmp/pc-migration-probe.sqlite3 +uv run --locked python scripts/database_migration_probe.py apply --sqlite-path /tmp/pc-migration-probe.sqlite3 --maintenance-confirmed --yes --plan-id ORIGINAL_PLAN_ID --resume RUN_ID +``` + +Recovery requires the original bundle, configuration, target, and unmodified backup. It retains the run ID and recovery point, skipping only atomically completed revisions. A crash before bootstrap commits can leave an unreferenced backup file; it is preserved, while the rolled-back database can be planned again. + +## Backend probes and tests + +```bash +uv run --locked pytest -q tests/builtin/persistence/test_database_migrations.py tests/test_database_migration_probe.py +POWERCONTEXT_TEST_MIGRATION_SEEKDB=1 uv run --locked --extra seekdb pytest -q tests/builtin/persistence/test_database_migration_backends.py -k seekdb +``` + +The seekDB probe holds the directory lock before engine startup until after shutdown and runs actual Alembic DDL on a new temporary instance. The main workflow runs it on Linux with Python 3.11 and 3.14. The probe establishes additive and constraint-change behavior, data/index preservation, and repeated `upgrade head`; it does not establish backup or interrupted-DDL recovery support. + +For OceanBase, provision an **empty dedicated database** named `pc_migration_probe_` and supply its `mysql+aoceanbase` URL through `POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL`. Run the same backend test file with `-k oceanbase`. It refuses other database names and nonempty targets, leaves the resulting schema for inspection, and does not provision or drop databases. No deployment credentials are embedded in the fixtures or test output. + +The OceanBase lock primitive tests exclusion with two distinct physical sessions, ownership after commit, and the pinned session after DDL. It refuses unsupported locking rather than falling back to a local mutex. This is one capability probe; it does not prove exclusion across every node/proxy route. The probe has no production backup or nontransactional-DDL recovery executor. + +SQLite tests exercise empty initialization, both historical table subsets, payload preservation, new tag families, constraints/indexes/triggers, consistent WAL backup and restore, no-op behavior, read-only inspection, explicit confirmation, plan changes, active writers, concurrent processes, lock-owner exit, interruption before/after every revision commit, original-backup reuse, checksum conflicts, unknown/newer schemas, and integrity failures. Backend tests skipped for missing configuration are not counted as acceptance. + +## Before production enablement + +The remaining RFC phases require complete released database fixtures and ownership inventory; full application revision chains; required data/projection task scheduling; seekDB consistent backup and OceanBase external recovery evidence; nontransactional-DDL postcondition recovery; all-profile startup/SDK/Worker gates; the public `server db-migrate` interface and `processing-migrate` compatibility; and mandatory schema-change CI checks. These primitives do not bypass those release gates. PR #1716 remains independent. diff --git a/tests/fixtures/database_migrations/env.py b/tests/fixtures/database_migrations/env.py new file mode 100644 index 0000000000..90315cee59 --- /dev/null +++ b/tests/fixtures/database_migrations/env.py @@ -0,0 +1,26 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run a revision on the executor's pinned connection; never open another engine.""" + +from alembic import context + +connection = context.config.attributes["connection"] +context.configure( + connection=connection, + version_table="pc_schema_revision", + transactional_ddl=connection.dialect.name == "sqlite", +) +with context.begin_transaction(): + context.run_migrations() diff --git a/tests/fixtures/database_migrations/manifest.json b/tests/fixtures/database_migrations/manifest.json new file mode 100644 index 0000000000..ce51a824ee --- /dev/null +++ b/tests/fixtures/database_migrations/manifest.json @@ -0,0 +1,25 @@ +{ + "revision_resources": { + "p0001": [ + "schemas/pre_dream.json" + ], + "p0002": [], + "p0003": [] + }, + "sqlite_baselines": { + "e1cae089a8a565291a7a1f782cea1f6c6e1a6cdd9eab75ef0c4ca61dfda453cb": "p0001", + "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d": "p0003" + }, + "sqlite_fingerprints": { + "p0001": [ + "e1cae089a8a565291a7a1f782cea1f6c6e1a6cdd9eab75ef0c4ca61dfda453cb" + ], + "p0002": [ + "2e487ed26e6dfec3f61fe71e304e5bfd2f33c449485ffbc2d8da02ba7defca7f" + ], + "p0003": [ + "9958fa0413ed3c16442df27dc48057d438e2cf3f7e4312c5a914cccd9d3acaab", + "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d" + ] + } +} diff --git a/tests/fixtures/database_migrations/schemas/pre_dream.json b/tests/fixtures/database_migrations/schemas/pre_dream.json new file mode 100644 index 0000000000..e278c6e1f3 --- /dev/null +++ b/tests/fixtures/database_migrations/schemas/pre_dream.json @@ -0,0 +1,27 @@ +{ + "source_commit": "495cc93d67387879382c98706f4975f638faec15", + "tables_blob": "bd617564eb7d7a6a35e2e7d09002c010bdf3a651", + "sqlalchemy_version": "2.0.51", + "tables": [ + "pc_artifacts", + "pc_artifact_heads", + "pc_artifact_candidate_versions", + "pc_artifact_tags" + ], + "sqlite": [ + "CREATE TABLE pc_artifacts (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text TEXT, \n\tlifecycle_state VARCHAR(16) DEFAULT 'active' NOT NULL, \n\treplacement_artifact_id VARCHAR(128), \n\tgovernance_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated')\n)", + "CREATE TABLE pc_artifact_candidate_versions (\n\tscope_id VARCHAR(256) NOT NULL, \n\tcandidate_id VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tproposal BLOB NOT NULL, \n\tsource_refs BLOB NOT NULL, \n\tartifact_refs BLOB NOT NULL, \n\ttarget_family VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\ttarget_revision INTEGER, \n\treason TEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_artifact_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\ttarget_type VARCHAR(12) NOT NULL, \n\ttarget_id VARCHAR(128) NOT NULL, \n\ttag_key_hash BLOB NOT NULL, \n\ttag_key VARCHAR(128) NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_family CHECK (family IN ('memory', 'experience', 'skill', 'handoff')), \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)" + ], + "mysql": [ + "CREATE TABLE pc_artifacts (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent MEDIUMBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text MEDIUMTEXT, \n\tlifecycle_state VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'active', \n\treplacement_artifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tgovernance_generation BIGINT NOT NULL DEFAULT '0', \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated')\n)", + "CREATE TABLE pc_artifact_candidate_versions (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tcandidate_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tproposal MEDIUMBLOB NOT NULL, \n\tsource_refs MEDIUMBLOB NOT NULL, \n\tartifact_refs MEDIUMBLOB NOT NULL, \n\ttarget_family VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\ttarget_artifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\ttarget_revision INTEGER, \n\treason MEDIUMTEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_artifact_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttarget_type VARCHAR(12) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttarget_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttag_key_hash BINARY(32) NOT NULL, \n\ttag_key VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_family CHECK (family IN ('memory', 'experience', 'skill', 'handoff')), \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)" + ] +} diff --git a/tests/fixtures/database_migrations/schemas/v1_1_0.json b/tests/fixtures/database_migrations/schemas/v1_1_0.json new file mode 100644 index 0000000000..f728735586 --- /dev/null +++ b/tests/fixtures/database_migrations/schemas/v1_1_0.json @@ -0,0 +1,27 @@ +{ + "source_commit": "86e687f13fec62151ee5657a90733586fd5158dc", + "tables_blob": "530d3f13de425e796e059b428a7b57c7dd774a19", + "sqlalchemy_version": "2.0.51", + "tables": [ + "pc_artifacts", + "pc_artifact_heads", + "pc_artifact_candidate_versions", + "pc_artifact_tags" + ], + "sqlite": [ + "CREATE TABLE pc_artifacts (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tmemory_citations BLOB, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text TEXT, \n\tlifecycle_state VARCHAR(16) DEFAULT 'active' NOT NULL, \n\treplacement_artifact_id VARCHAR(128), \n\tgovernance_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated')\n)", + "CREATE TABLE pc_artifact_candidate_versions (\n\tscope_id VARCHAR(256) NOT NULL, \n\tcandidate_id VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tproposal BLOB NOT NULL, \n\tsource_refs BLOB NOT NULL, \n\tartifact_refs BLOB NOT NULL, \n\tmemory_citations BLOB, \n\ttarget_family VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\ttarget_revision INTEGER, \n\treason TEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_artifact_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\ttarget_type VARCHAR(12) NOT NULL, \n\ttarget_id VARCHAR(128) NOT NULL, \n\ttag_key_hash BLOB NOT NULL, \n\ttag_key VARCHAR(128) NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)" + ], + "mysql": [ + "CREATE TABLE pc_artifacts (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent MEDIUMBLOB NOT NULL, \n\tmemory_citations MEDIUMBLOB, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text MEDIUMTEXT, \n\tlifecycle_state VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'active', \n\treplacement_artifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tgovernance_generation BIGINT NOT NULL DEFAULT '0', \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated')\n)", + "CREATE TABLE pc_artifact_candidate_versions (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tcandidate_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tproposal MEDIUMBLOB NOT NULL, \n\tsource_refs MEDIUMBLOB NOT NULL, \n\tartifact_refs MEDIUMBLOB NOT NULL, \n\tmemory_citations MEDIUMBLOB, \n\ttarget_family VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\ttarget_artifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\ttarget_revision INTEGER, \n\treason MEDIUMTEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_artifact_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttarget_type VARCHAR(12) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttarget_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttag_key_hash BINARY(32) NOT NULL, \n\ttag_key VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)" + ] +} diff --git a/tests/fixtures/database_migrations/versions/p0001_baseline.py b/tests/fixtures/database_migrations/versions/p0001_baseline.py new file mode 100644 index 0000000000..f3b0d4d7a0 --- /dev/null +++ b/tests/fixtures/database_migrations/versions/p0001_baseline.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Four actual PowerContext tables frozen before Dream citations. + +This isolated prototype is not the complete schema of any PowerContext release. +The resource provenance is recorded alongside the frozen SQL. +""" + +import json +from pathlib import Path + +from alembic import op + +revision = "p0001" +down_revision = None + + +def upgrade() -> None: + dialect = op.get_bind().dialect.name + statements = json.loads((Path(__file__).parents[1] / "schemas" / "pre_dream.json").read_text())[dialect] + for statement in statements: + op.execute(statement) diff --git a/tests/fixtures/database_migrations/versions/p0002_citations.py b/tests/fixtures/database_migrations/versions/p0002_citations.py new file mode 100644 index 0000000000..0bfa2d58a5 --- /dev/null +++ b/tests/fixtures/database_migrations/versions/p0002_citations.py @@ -0,0 +1,27 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add immutable revision citations using fixed historical types.""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.mysql import MEDIUMBLOB + +revision = "p0002" +down_revision = "p0001" + + +def upgrade() -> None: + for table in ("pc_artifacts", "pc_artifact_candidate_versions"): + op.add_column(table, sa.Column("memory_citations", sa.LargeBinary().with_variant(MEDIUMBLOB(), "mysql"))) diff --git a/tests/fixtures/database_migrations/versions/p0003_tag_families.py b/tests/fixtures/database_migrations/versions/p0003_tag_families.py new file mode 100644 index 0000000000..90e722eb4e --- /dev/null +++ b/tests/fixtures/database_migrations/versions/p0003_tag_families.py @@ -0,0 +1,40 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Remove the family whitelist while preserving the target constraint and indexes.""" + +from alembic import op + +revision = "p0003" +down_revision = "p0002" + + +def upgrade() -> None: + connection = op.get_bind() + if connection.dialect.name == "sqlite": + # Alembic reflects indexes and named CHECKs, but not application triggers. + triggers = ( + connection + .exec_driver_sql( + "SELECT sql FROM sqlite_schema WHERE type='trigger' AND tbl_name='pc_artifact_tags' ORDER BY name" + ) + .scalars() + .all() + ) + with op.batch_alter_table("pc_artifact_tags", recreate="always") as batch: + batch.drop_constraint("ck_pc_artifact_tags_family", type_="check") + for statement in triggers: + connection.exec_driver_sql(statement) + else: + op.drop_constraint("ck_pc_artifact_tags_family", "pc_artifact_tags", type_="check") diff --git a/tests/test_database_migration_probe.py b/tests/test_database_migration_probe.py new file mode 100644 index 0000000000..76e6a4257b --- /dev/null +++ b/tests/test_database_migration_probe.py @@ -0,0 +1,50 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The repository probe requires explicit non-interactive plan acceptance.""" + +import json +import subprocess +import sys +from pathlib import Path + + +def test_probe_plan_confirmation_and_noop(tmp_path: Path) -> None: + script = Path(__file__).resolve().parents[1] / "scripts/database_migration_probe.py" + database = tmp_path / "probe.sqlite3" + + def run(action: str, *options: str): + return subprocess.run( + [sys.executable, str(script), action, "--sqlite-path", str(database), *options], + input="", + capture_output=True, + text=True, + timeout=20, + check=False, + ) + + plan = run("plan") + assert plan.returncode == 0 + plan_id = json.loads(plan.stdout)["plan_id"] + assert not database.exists() + for flags in [(), ("--yes",), ("--plan-id", plan_id)]: + rejected = run("apply", "--maintenance-confirmed", *flags) + assert rejected.returncode == 1 + assert json.loads(rejected.stderr)["error"] == "confirmation_required" + assert not database.exists() + applied = run("apply", "--maintenance-confirmed", "--yes", "--plan-id", plan_id) + assert applied.returncode == 0, applied.stderr + assert json.loads(applied.stdout)["changed"] + assert run("verify").returncode == 0 + assert not json.loads(run("apply").stdout)["changed"] diff --git a/uv.lock b/uv.lock index 131d963c53..b423534462 100644 --- a/uv.lock +++ b/uv.lock @@ -40,6 +40,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/b7/e3bf5133d697a08128598c8d0abc5e16377b51465a33756de24fa7dee953/aiosqlite-0.22.1-py3-none-any.whl", hash = "sha256:21c002eb13823fad740196c5a2e9d8e62f6243bd9e7e4a1f87fb5e44ecb4fceb", size = 17405, upload-time = "2025-12-23T19:25:42.139Z" }, ] +[[package]] +name = "alembic" +version = "1.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/aa/02910bdb8e2f1444f6654d5b296cd827d126f82209050ee7b1000f92ac4b/alembic-1.20.0.tar.gz", hash = "sha256:db505480647bc60386c5369402f4a57a506b7539c9e9ef5e270d45cbbe4939bf", size = 2093272, upload-time = "2026-09-11T19:09:11.126Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/27/78a89b55b0904d222183164e079b4ca56208e94eff1d35ad1f1ad5be9b06/alembic-1.20.0-py3-none-any.whl", hash = "sha256:77eb101048d95f982c0353e9233404889dcd7a6fc244c107836c0e2fc9cf7d9d", size = 268719, upload-time = "2026-09-11T19:09:12.88Z" }, +] + [[package]] name = "annotated-doc" version = "0.0.4" @@ -1202,7 +1216,7 @@ name = "importlib-metadata" version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp", marker = "python_full_version < '3.12'" }, + { name = "zipp" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ @@ -1935,6 +1949,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/26/22/598f988f293ce391e88baddeeaa1acaea625822653193935301e7149b5c6/logfire_api-4.38.0-py3-none-any.whl", hash = "sha256:6c0612d9200727b64b42eba6068d0d6afd45308f4b844ffd6290c2d33aa6a6f6", size = 140109, upload-time = "2026-07-20T12:15:31.973Z" }, ] +[[package]] +name = "mako" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/09/e07c4b5579a79f4b16f8d4f29f6c54514ac787c4ad506b8c4f28a0e6b0bf/mako-1.4.3.tar.gz", hash = "sha256:cd6537fe88d5fec315c55c2f8529bc4ce7a9a352ad7db3eeaa6a66e2dd4ec37a", size = 412799, upload-time = "2026-09-22T20:54:31.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/a0/053d6af3e8f871e0073b4a36732d9e65be77a72e5434c31b94f6af78a6bb/mako-1.4.3-py3-none-any.whl", hash = "sha256:723296007c870bfd6b3f0c3230dba7198096e5269297ebf5e4eff9e7ffa39d4f", size = 80164, upload-time = "2026-09-22T20:54:33.128Z" }, +] + [[package]] name = "markdown-it-py" version = "4.2.0" @@ -2369,8 +2395,8 @@ name = "opendalfs" version = "0.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fsspec", marker = "python_full_version >= '3.12'" }, - { name = "opendal", marker = "python_full_version >= '3.12'" }, + { name = "fsspec" }, + { name = "opendal" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0d/23/fdb4d34f08ac68a27b2d553256e13b15ce139916d36bab33f9a0e71e8770/opendalfs-0.1.0.tar.gz", hash = "sha256:5ceaeccc0852ef10c8ef2ce20e5927dbc18083909c06550ad36f81bcacb5774d", size = 75112, upload-time = "2026-08-26T05:30:33.497Z" } wheels = [ @@ -2693,7 +2719,9 @@ dependencies = [ [package.optional-dependencies] builtin = [ { name = "aiosqlite" }, + { name = "alembic" }, { name = "apscheduler" }, + { name = "filelock" }, { name = "jsonschema" }, { name = "packaging" }, { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, @@ -2729,7 +2757,9 @@ code = [ ] seekdb = [ { name = "aiosqlite" }, + { name = "alembic" }, { name = "apscheduler" }, + { name = "filelock" }, { name = "jsonschema" }, { name = "packaging" }, { name = "pydantic-ai-slim", extra = ["anthropic", "openai"] }, @@ -2743,9 +2773,11 @@ seekdb = [ ] server = [ { name = "aiosqlite" }, + { name = "alembic" }, { name = "apscheduler" }, { name = "fastapi" }, { name = "fastmcp" }, + { name = "filelock" }, { name = "httpx" }, { name = "jinja2" }, { name = "jsonschema" }, @@ -2807,11 +2839,17 @@ requires-dist = [ { name = "aiosqlite", marker = "extra == 'builtin'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'seekdb'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'server'", specifier = ">=0.22,<1" }, + { name = "alembic", marker = "extra == 'builtin'", specifier = ">=1.16,<2" }, + { name = "alembic", marker = "extra == 'seekdb'", specifier = ">=1.16,<2" }, + { name = "alembic", marker = "extra == 'server'", specifier = ">=1.16,<2" }, { name = "apscheduler", marker = "extra == 'builtin'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'seekdb'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'server'", specifier = ">=3.11,<4" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115,<1" }, { name = "fastmcp", marker = "extra == 'server'", specifier = ">=3.4,<4" }, + { name = "filelock", marker = "extra == 'builtin'", specifier = ">=3.16,<4" }, + { name = "filelock", marker = "extra == 'seekdb'", specifier = ">=3.16,<4" }, + { name = "filelock", marker = "extra == 'server'", specifier = ">=3.16,<4" }, { name = "httpx", marker = "extra == 'server'", specifier = ">=0.28,<1" }, { name = "httpx", extras = ["socks"], marker = "extra == 'cli'", specifier = ">=0.28,<1" }, { name = "httpx", extras = ["socks"], marker = "extra == 'client'", specifier = ">=0.28,<1" }, @@ -2912,8 +2950,8 @@ name = "powercontext-connector-opendal" version = "0.0.1" source = { editable = "integrations/opendal" } dependencies = [ - { name = "opendalfs", marker = "python_full_version >= '3.12'" }, - { name = "powercontext", extra = ["client"], marker = "python_full_version >= '3.12'" }, + { name = "opendalfs" }, + { name = "powercontext", extra = ["client"] }, ] [package.metadata] From 847f70694fccf8e9fc23eac5bb1e09dfa8335329 Mon Sep 17 00:00:00 2001 From: xufeng <110882560+Zxf-xufeng@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:08:20 +0800 Subject: [PATCH 02/37] fix(db): repair migration prototype CI checks --- e2e/bub/uv.lock | 6 ++++++ tests/conftest.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/e2e/bub/uv.lock b/e2e/bub/uv.lock index 977ecfbd4b..2bfb95b23e 100644 --- a/e2e/bub/uv.lock +++ b/e2e/bub/uv.lock @@ -1620,11 +1620,17 @@ requires-dist = [ { name = "aiosqlite", marker = "extra == 'builtin'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'seekdb'", specifier = ">=0.22,<1" }, { name = "aiosqlite", marker = "extra == 'server'", specifier = ">=0.22,<1" }, + { name = "alembic", marker = "extra == 'builtin'", specifier = ">=1.16,<2" }, + { name = "alembic", marker = "extra == 'seekdb'", specifier = ">=1.16,<2" }, + { name = "alembic", marker = "extra == 'server'", specifier = ">=1.16,<2" }, { name = "apscheduler", marker = "extra == 'builtin'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'seekdb'", specifier = ">=3.11,<4" }, { name = "apscheduler", marker = "extra == 'server'", specifier = ">=3.11,<4" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115,<1" }, { name = "fastmcp", marker = "extra == 'server'", specifier = ">=3.4,<4" }, + { name = "filelock", marker = "extra == 'builtin'", specifier = ">=3.16,<4" }, + { name = "filelock", marker = "extra == 'seekdb'", specifier = ">=3.16,<4" }, + { name = "filelock", marker = "extra == 'server'", specifier = ">=3.16,<4" }, { name = "httpx", marker = "extra == 'server'", specifier = ">=0.28,<1" }, { name = "httpx", extras = ["socks"], marker = "extra == 'cli'", specifier = ">=0.28,<1" }, { name = "httpx", extras = ["socks"], marker = "extra == 'client'", specifier = ">=0.28,<1" }, diff --git a/tests/conftest.py b/tests/conftest.py index 4b0a6f7e13..1a0a095b5e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,7 @@ import pytest _REAL_E2E_ROOT = Path(__file__).parent / "e2e" / "real_experience_skill" +_DATABASE_MIGRATION_ENV = Path(__file__).parent / "fixtures" / "database_migrations" / "env.py" @pytest.fixture @@ -70,6 +71,10 @@ def pytest_addoption(parser: pytest.Parser) -> None: def pytest_ignore_collect(collection_path: Path, config: pytest.Config) -> bool | None: + # Alembic executes env.py as migration configuration; doctest collection + # must not import it as a test module. + if collection_path.resolve() == _DATABASE_MIGRATION_ENV.resolve(): + return True if config.getoption("run_real_e2e"): return None try: From 40594b0ca4ea1b2b0e10b49bbfd40c06cc8237d4 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Tue, 29 Sep 2026 17:04:22 +0800 Subject: [PATCH 03/37] docs: explain database migration design --- docs/en/docs/operate/database-migrations.md | 200 ++++++++++++++++++++ docs/en/docs/operate/index.md | 1 + docs/en/docs/operate/meta.json | 1 + docs/zh/docs/operate/database-migrations.md | 165 ++++++++++++++++ docs/zh/docs/operate/index.md | 1 + docs/zh/docs/operate/meta.json | 1 + 6 files changed, 369 insertions(+) create mode 100644 docs/en/docs/operate/database-migrations.md create mode 100644 docs/zh/docs/operate/database-migrations.md diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md new file mode 100644 index 0000000000..8b97e72e42 --- /dev/null +++ b/docs/en/docs/operate/database-migrations.md @@ -0,0 +1,200 @@ +--- +title: Database migration design +description: Understand unified migration, shared-database upgrades, backup choices, service lifecycle, and compatibility checks. +--- + +# Database migration design + +PowerContext's unified migration design uses Alembic for schema versions and separates installing software from migrating +existing databases. This page explains the operational contract in the +[unified migration RFC](https://github.com/oceanbase/powercontext/pull/1771), tracked by +[issue #1756](https://github.com/oceanbase/powercontext/issues/1756). + +## Current availability + +**Unified migration is still a prototype. `server db-migrate`, `BackupProvider`, `service start/stop/restart`, and +`--manage-service` below are proposed, not released commands.** The prototype handles an isolated four-table fixture, +not business startup. Its `ready` is not complete Server readiness, and its execution records do not mean the final +single-table persistence design is implemented. + +The current foreground entry point is `powercontext server run`; personal service commands provide +`install/status/uninstall`. For existing Artifact processing maintenance, follow +[Migrate Artifact processing state](artifact-processing-migration.md) using `server processing-migrate`. Do not substitute +the prototype for production migration. See [Deploy the Server](deploy-server.md) for service deployment. + +## Versions and upgrade boundaries + +The final framework adds one migration control table, `pc_schema_revision(version_num)`, using standard Alembic version +handling, without generic run, step, or task ledgers. Each managed schema change adds an immutable revision executed in +dependency order. A release without schema changes needs no new script. Application versions, API versions, and schema +revisions are distinct. Data tasks, validators, and impact manifests ship with the package; logs and backup records stay +outside the target database. + +Installing/upgrading software does not migrate existing databases. Both local and remote databases require explicit +migration. Truly empty local databases may initialize under a lock where configuration permits; remote/shared-database +deployments use a separate initialization Job. An incompatible existing database blocks new business startup with +`migration_required`. Once enabled, all managed-schema PRs require migrations and verification. Framework development is +independent of PR #1716. + +## Maintenance with one confirmation + +Interactive users can go directly to proposed `apply` without assembling individual steps. This illustrates the interface +and cannot yet be executed: + +```bash +powercontext server db-migrate apply --env-file deployment.env --manage-service +``` + +It combines target, source/target versions, affected tables/APIs, legacy tasks, downtime requirements, backup, and service +scope. Select on one screen and confirm once, followed by stopped writes, locked reinspection, backup policy, schema/data +conversion, index rebuilds, and final verification. No per-SQL confirmation is required. Read-only verification with no +persistent changes returns no changes without backup or service lifecycle actions. + +Optional proposed read-only `status` and `plan` provide previews; material changes require renewed confirmation. +Automation explicitly supplies the plan and policy. For example, after manual backup and deployment-managed shutdown +of all writers: + +```bash +powercontext server db-migrate plan --env-file deployment.env --backup manual +powercontext server db-migrate apply --env-file deployment.env --plan-id PLAN_ID --backup manual --backup-confirmed --maintenance-confirmed --yes +``` + +`--yes` accepts the plan, not a manual-backup declaration or no-backup risk. Plans bind target, versions, schema/resource +digests, compatibility, backup choice, and service scope. Missing required consent returns `confirmation_required`. +No backup choice bypasses locking, active writers, or data checks. + +## Multiple Servers sharing one database + +Multi-node means one shared database; this design does not cover independent per-node databases. Run one migration Job +per database. + +1. Prepare the new program separately. Stop new requests and task production, then drain in-flight work and tasks that + require the old handler according to the plan. +2. Stop all API, Worker, scheduler, and SDK writers. Suspend scaling, automatic restarts, and schedules that could launch + old programs. +3. One new-version Job runs `apply`, takes the database-wide lock, rechecks the plan and legacy tasks, then handles backup + and migration. +4. Start new nodes after migration returns `ready`; each checks schema, task formats, and required capabilities before + passing readiness and accepting traffic. +5. Migration failure keeps maintenance active. If the database is ready but a node cannot start, handle startup separately + without rerunning migration or automatically downgrading. + +Migration locks exclude other migrators, not arbitrary external clients. `--maintenance-confirmed` is an operator +statement that writes are stopped. Reject execution when those conditions cannot be established. PC upgrades do not +restart the OceanBase database cluster. SQLite/embedded seekDB retain the existing single-host `all` role restriction; migration does not enable cross-host shared directories or multi-node deployment for them. Shared-database multi-node services use an already supported remote deployment topology. + +## Backup choices and native methods + +| Choice | Behavior | Proposed options | +| --- | --- | --- | +| PC-managed | Recommend a supported native method, warn that large data may require considerable time/space, await completion and adapter checks | `--backup auto` | +| Already backed up manually | Record only the declaration, without checking files, jobs, time, target, or recoverability; reference optional | `--backup manual --backup-confirmed`, optionally `--backup-ref BACKUP_ID` | +| No backup | Warn that original data may be unrecoverable after failure; require explicit acceptance | `--backup skip --accept-no-backup` | + +Manual mode reports `user_confirmed`, not `verified`; skip reports `skipped`; genuinely empty initialization reports +`not_required`. Failed, incomplete, or insufficiently supported automatic backup stops before migration writes. Changing +policy requires renewed confirmation, never automatic skipping. + +A separate `BackupProvider` is a database maintenance capability implemented by backend adapters, reusing Profile +configuration, identity, and engine lifecycle. `capabilities(context)` reports support, `create_backup(context)` starts +backup, `inspect_backup(ref)` inspects PC-managed backups, and `restore_plan(ref)` provides explicit recovery steps. +Manual and skip paths do not call it to verify user backups. + +| Backend | Recommended method | Limitations | +| --- | --- | --- | +| SQLite | After stopping writes, use SQLite Online Backup API to create an independent file, checking opening and integrity | Include committed WAL; copying only the main file is insufficient. Restoration also checks extensions, indexes, and business data | +| seekDB | Version/object/recovery-validated `FORK DATABASE` for a same-instance pre-migration recovery point | Not multiple per-table forks by default; shared storage provides no disk-failure protection | +| OceanBase | Native physical backup and log archiving through configured administration or backup services, querying completion | Check jobs and log coverage; tenant restoration may affect other applications, so never default to overwriting a shared tenant | + +Native references: [SQLite Backup API](https://sqlite.org/backup.html), +[OceanBase backup architecture](https://en.oceanbase.com/docs/common-oceanbase-database-10000000001168918). + +seekDB V1.2.0 describes a common database snapshot, excluding some non-table objects and cross-database foreign keys. +Fork uses copy-on-write and shared underlying storage. After acceptance, it may cover failed migrations while storage +and engine remain healthy without an additional physical backup; users arrange independent disaster recovery as needed. +Multiple `FORK TABLE` operations are not the default substitute for database-wide consistency. +[seekDB V1.2.0](https://github.com/oceanbase/seekdb/releases/tag/v1.2.0), +[Fork mechanism](https://en.oceanbase.com/blog/fork-table-ready-for-agents). + +Before enabling auto fork, verify the actual embedded version, index/constraint coverage, subsequent source DDL, engine +restart, and full restoration. `SeekDBConfig.database` currently fixes the database to `test`, so changing the connection +to a fork database cannot be promised. Implement restoration to the original name or separately provide configurable +targets and coordinated switching. `MERGE TABLE` does not automatically roll back schema. Unsupported coverage/recovery +returns `backup_unsupported`; users may select manual or skip. Do not copy a running directory or automatically upgrade +the database engine. + +## Service shutdown and restart + +An incompatible upgrade follows: stop old writers → backup policy → migration → verification → start new version → +readiness → restore traffic. Installing software is separate from replacing processes. Preserve an independently +runnable old environment when deferring the switch. + +Proposed `service stop/start/restart` manages only the current user's PC-owned local service, retaining registration and +configuration. `--manage-service` displays and confirms the target executable, configuration, and original running state, +then drains/stops it and suppresses restarts. Only successful migration permits updating the confirmed launch definition +and restarting a previously running service. Previously stopped services remain stopped; failure does not restart old +services. Do not guess versions from PATH or use potentially autostarting `service install` during maintenance. + +Deployment systems manage clusters and external services. For manual maintenance, first use proposed +`powercontext service stop`, run `apply` successfully, ensure the launch definition points at the new environment, then +use `powercontext service start`. `restart` does not replace migration while stopped. For foreground deployment, after +migration succeeds and the old process exits, use the existing command from the new environment: + +```bash +powercontext server run --env-file deployment.env --role all +``` + +## Release decisions for tables, APIs, and legacy tasks + +Authors ship impact manifests; operators choose deployment methods from this evidence, not merely from “changed a table” +or “internal API” labels: + +| Declaration | Operational decision | +| --- | --- | +| `affected_objects` | Tables, constraints, indexes, files, large copies, resources, and irreversible transformations | +| `execution_mode` and binary/schema support | Ordinary deployment without persistent changes; stopped-write maintenance for incompatibility; online migration requires implemented and verified backend/mixed-version support | +| `api_changes` | Callers released together or independently, retained/deprecated APIs, earliest removal version/date | +| `task_formats` | Queued, running, delayed, retry, and dead-letter formats; compatible consumption, idempotent conversion, or draining | + +Operators may choose stricter downtime, but cannot select online mode to override known incompatibility. Generic online +DDL is not initially promised. Same-package internal entry points can be replaced with callers; independently deployed +Workers/SDKs still need compatibility arrangements. Breaking public API replacements, such as Artifact APIs, mark old +contracts `deprecated` and specify alternatives/removal. Storage-only changes with unchanged API contracts need no +deprecation. Retained APIs can translate requests/responses to the current implementation without every handler supporting +two table layouts. + +**Check legacy task formats during upgrades.** Read-only `plan` inventories queues, Leases, and payload formats, rechecking +under the lock after writes stop, after migration, and before Worker startup. Use frozen recognizers where version fields +are absent. Unknown/unobservable sources block affected migration/Workers rather than falsely reporting no tasks. +Unknown formats return `unsupported_task_format`; incomplete draining returns `legacy_tasks_pending`. Stop producers, +let old Workers drain within bounds, then stop Workers before DDL; alternatively use declared compatible consumption or +idempotent conversion. Preserve task IDs, idempotency keys, Scope/identity, retries, and domain receipts. Never delete +queues or mark unfinished work successful. Resolve retained tasks before deleting handlers; release plans coordinate +external API consumers. + +## Combining existing storage layers + +| Existing layer | Upgrade responsibility | +| --- | --- | +| Profile | Reuse connection configuration/engine lifecycle; split inspection/maintenance from `create_tables` side effects | +| `AsyncDatabase` | Dedicated maintenance connection and transactions; `AsyncConnection.run_sync` supplies the same connection to Alembic, keeping locks and DDL together | +| Alembic | Frozen revisions execute DDL and advance the sole version table after verification | +| Repository | Reuse stable domain operations once schema permits; historical conversions prefer frozen SQL/mappings over changing current repositories | +| Indexes | Structure definitions enter versioned resources; controlled projection rebuilding and checks replace implicit startup schema writes | +| Startup gates | Check schema, tasks, and required capabilities before composing repositories, indexes, and Workers | + +For a required column, a revision first adds it as nullable, an idempotent script backfills in batches, verification permits +a later constraint-tightening revision, and affected indexes rebuild before new repositories start. seekDB/OceanBase DDL +may commit implicitly; one Python transaction cannot make the full upgrade atomic. Commit existing domain receipts with +their data batches and rebuild indexes from authoritative data. + +## Interruption and recovery + +After interruption, inspect through read-only `status`, `verify`, and `plan`; retry only when actual state proves it safe. +One version table offers no generic run-ID resume. Ambiguous state returns `recovery_required`. Auto retains the original +recovery point and external records rather than replacing them with a partial-state backup; manual remains unverified; +skip cannot promise original-data recovery. Every mode checks actual schema, data, and tasks without blind stamping. + +Restoration is independent and explicit with writes stopped. Restore the database, required files, and matching program, +then verify. Application rollback is not database rollback. Report migration `ready` separately from startup success; +do not immediately delete recovery points. Production support remains subject to the RFC's real-backend acceptance. diff --git a/docs/en/docs/operate/index.md b/docs/en/docs/operate/index.md index b05f22c974..e28b03098f 100644 --- a/docs/en/docs/operate/index.md +++ b/docs/en/docs/operate/index.md @@ -10,6 +10,7 @@ Start with the operational task you need: | Task | Guide | | --- | --- | | Keep a personal Server running, deploy a container, or enable authentication | [Deploy the Server](deploy-server.md) | +| Understand unified database upgrades, backups, and service lifecycle | [Database migration design](database-migrations.md) | | Upgrade or switch background supervision mode | [Migrate Artifact processing state](artifact-processing-migration.md) | | Inspect logs, metrics, and traces | [Observability](observability.md) | | Send traces to an analysis service | [Phoenix](trace-with-phoenix.md), [Langfuse](trace-with-langfuse.md) | diff --git a/docs/en/docs/operate/meta.json b/docs/en/docs/operate/meta.json index 5e04efad00..3bb4d17220 100644 --- a/docs/en/docs/operate/meta.json +++ b/docs/en/docs/operate/meta.json @@ -4,6 +4,7 @@ "index", "deploy-server", "connect-remote-server", + "database-migrations", "artifact-processing-migration", "observability", "trace-with-phoenix", diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md new file mode 100644 index 0000000000..f6d7f4867e --- /dev/null +++ b/docs/zh/docs/operate/database-migrations.md @@ -0,0 +1,165 @@ +--- +title: 数据库迁移设计 +description: 了解统一迁移的设计、共享数据库升级、备份选择、服务启停与兼容性检查。 +--- + +# 数据库迁移设计 + +PowerContext 的统一数据库迁移方案使用 Alembic 管理结构版本，安装新版与迁移已有数据库分开。本文介绍 +[统一迁移 RFC](https://github.com/oceanbase/powercontext/pull/1771) 的运维契约，关联 +[issue #1756](https://github.com/oceanbase/powercontext/issues/1756)。 + +## 当前可用范围 + +**统一迁移仍处于原型阶段。本文中的 `server db-migrate`、`BackupProvider`、`service start/stop/restart` +及 `--manage-service` 是拟议设计，尚不能作为已发布命令使用。** 原型只验证隔离的四表测试库，不接入业务启动， +其 `ready` 不代表完整 Server 可用；原型中的执行记录也不代表最终单表持久化设计已落地。 + +当前前台启动入口是 `powercontext server run`，个人服务命令提供 `install/status/uninstall`。 +已有 Artifact 处理状态的维护请使用[迁移 Artifact 处理状态](artifact-processing-migration.md)中的 +`server processing-migrate`，不要用原型替代生产迁移。服务部署参考[部署 Server](deploy-server.md)。 + +## 版本与更新边界 + +最终框架只新增 `pc_schema_revision(version_num)` 一张迁移控制表，沿用 Alembic 的版本读写；不新增通用执行、 +步骤、任务台账。每次受管结构变更新增独立且不可改写的 revision，沿依赖链执行；没有结构变化的发布无需新脚本。 +业务版本、API 版本和 schema revision 分开管理。数据任务、验证器与影响声明随包发布，日志和备份记录保存在库外。 + +安装或升级软件包不迁移已有库；本地库、远端库均通过显式命令迁移。真正空白的本地库允许在配置许可且持锁确认后 +初始化；远端库和多节点共享库使用单独初始化任务。已有库不兼容时新版拒绝业务启动，提示 `migration_required`。 +框架启用后，所有受管结构变更 PR 都必须提交 migration 和验证；框架与 PR #1716 独立推进。 + +## 一次确认完成维护 + +交互用户可以直接执行拟议的 `apply`，不用先手工拼装多个迁移步骤。以下示例供理解接口，当前不可直接执行： + +```bash +powercontext server db-migrate apply --env-file deployment.env --manage-service +``` + +它汇总目标库、来源／目标版本、受影响的表和接口、旧任务、停机要求、备份与启停范围。用户在同一页面选择后确认一次， +随后执行停写、取锁复核、备份策略、结构与数据转换、索引重建和最终验证；不逐条 SQL 确认。 +只有只读检查且没有持久化变更时，返回“无需变更”，不备份、不启停服务。 + +希望提前审阅时使用拟议的只读 `status`、`plan`；出现实质变化时需要重新确认。自动化场景明确提供计划与策略， +例如用户自行备份、部署系统已停止全部写入者后： + +```bash +powercontext server db-migrate plan --env-file deployment.env --backup manual +powercontext server db-migrate apply --env-file deployment.env --plan-id PLAN_ID --backup manual --backup-confirmed --maintenance-confirmed --yes +``` + +`--yes` 仅接受该计划，不等于手动备份声明或接受无备份风险。计划绑定数据库、版本、结构和脚本摘要、兼容性、 +备份选择与启停范围；缺失所需确认返回 `confirmation_required`。任何备份选择都不能绕过锁、活跃写入者或数据检查。 + +## 多个 Server 共用一个数据库 + +本方案的多节点指共享一个数据库；不设计每个节点拥有独立数据库的升级。每个数据库只运行一个迁移 Job。 + +1. 在独立环境准备新版程序；禁用新请求和任务生产，按计划排空在途操作及必须由旧处理器消费的任务。 +2. 停止全部 API、Worker、调度器和 SDK 写入者，暂停自动扩容、自动重启及会拉起旧程序的调度。 +3. 一个新版迁移 Job 执行 `apply`，取得数据库级锁，复核计划和旧任务后处理备份并迁移。 +4. 迁移返回 `ready` 后启动新版节点，各节点仍须检查 schema、任务格式及必要能力，通过 readiness 才恢复流量。 +5. 迁移失败保持维护；数据库已就绪但某个节点启动失败时，单独处理启动问题，不重跑迁移或自动降级。 + +迁移锁只排斥其他迁移进程，不能自动阻止任意外部客户端写入。`--maintenance-confirmed` 是运维停写声明。 +无法建立停写条件时拒绝执行。PC 升级不重启 OceanBase 数据库集群。 SQLite／嵌入式 seekDB 仍受现有单机 `all` 角色限制，迁移设计不使其获得跨主机共享目录或多节点部署能力；共享数据库的多节点服务使用已有支持的远端部署方式。 + +## 备份选择与原生方法 + +| 选择 | 行为 | 拟议参数 | +| --- | --- | --- | +| PC 备份 | 推荐支持的原生方法，提示大数据量可能耗时较久、占用更多空间，等待完成与适配器检查 | `--backup auto` | +| 已手动备份 | 仅记录用户声明，不检查文件、备份任务、时间、目标或可恢复性；引用可选 | `--backup manual --backup-confirmed`，可加 `--backup-ref BACKUP_ID` | +| 不备份 | 提示失败后可能无法恢复原数据，明确接受风险才继续 | `--backup skip --accept-no-backup` | + +手动模式显示 `user_confirmed`，不显示 `verified`；跳过显示 `skipped`；真正空库显示 `not_required`。 +PC 自动备份失败、未完成或不支持所需对象时，迁移停在写入之前。改变策略需重新确认，不能自动跳过。 + +独立 `BackupProvider` 属于数据库维护能力，复用 Profile 的配置、目标身份和引擎生命周期，由后端适配器实现： +`capabilities(context)` 报告能力，`create_backup(context)` 发起备份，`inspect_backup(ref)` 查询 PC 备份， +`restore_plan(ref)` 提供显式恢复步骤。手动和跳过分支不调用它核验用户备份。 + +| 后端 | 推荐方法 | 限制 | +| --- | --- | --- | +| SQLite | 停写后使用 SQLite Online Backup API 创建独立数据库文件，检查可打开及完整性 | 包含已提交 WAL；不能只复制主文件；恢复验证还包含扩展、索引和业务数据 | +| seekDB | 经过版本、对象覆盖和恢复验收的 `FORK DATABASE`，保存同实例迁移前恢复点 | 不默认逐表 fork；同实例共享存储，不提供磁盘损坏保护 | +| OceanBase | 数据库原生物理备份和日志归档，通过配置好的管理接口或备份平台发起并查询完成 | 检查任务和日志覆盖；租户级恢复可能影响其他应用，不能默认覆盖整个共享租户 | + +原生能力参考：[SQLite Backup API](https://sqlite.org/backup.html)、 +[OceanBase 备份架构](https://en.oceanbase.com/docs/common-oceanbase-database-10000000001168918)。 + +seekDB V1.2.0 发布说明提供整库共同快照能力，部分非表对象及跨库外键不复制。Fork 使用写时复制并共享底层存储， +因此经验证后可以用于引擎／存储健康情况下的迁移失败恢复，不要求同时另做物理备份；独立灾备由用户按需安排。 +逐表 `FORK TABLE` 不能默认替代整库一致性保护。 +[seekDB V1.2.0](https://github.com/oceanbase/seekdb/releases/tag/v1.2.0)、 +[Fork 机制](https://en.oceanbase.com/blog/fork-table-ready-for-agents)。 + +自动 fork 启用前必须验收实际嵌入式版本、索引／约束覆盖、源库后续 DDL、引擎重启和完整恢复。 +当前 `SeekDBConfig.database` 固定为 `test`，不能直接承诺修改连接指向 fork 库；需要实现恢复到原库名， +或另行提供可配置目标及统一切换。`MERGE TABLE` 不用于自动回滚 schema。对象覆盖或恢复路径不满足时返回 +`backup_unsupported`，可重新选择手动或跳过，不复制运行中的目录，也不自动升级数据库引擎。 + +## 服务启停与重启 + +不兼容升级采用：停止旧写入者 → 备份策略 → 迁移 → 验证 → 启动新版 → readiness → 恢复流量。 +安装新版不等于切换进程；延后切换时保留可独立运行的旧环境。 + +拟议的 `service stop/start/restart` 只管理当前用户归 PC 所有的本机服务，并保留注册和配置。 +`--manage-service` 显示并确认目标可执行程序、配置及原运行状态，自动排空停服、抑制重启；只在迁移成功后 +更新已确认的启动定义并恢复原本在运行的服务。原本停止的服务保持停止，失败不自动拉起旧服务。 +不按 PATH 猜新版本，不使用可能立即启动服务的 `service install` 作为维护中间步骤。 + +集群和外部服务由部署系统启停。手工维护时，拟议命令是先 `powercontext service stop`，执行 `apply` 成功后， +确认服务定义已指向新版，再运行 `powercontext service start`。`restart` 不能替代停服期间的数据库迁移。 +前台服务在迁移成功且旧进程退出后，通过现有命令从新版环境启动： + +```bash +powercontext server run --env-file deployment.env --role all +``` + +## 表、接口与旧任务的发布判断 + +变更作者提供随包影响清单，操作者据此选择部署方式，不能仅靠“改了表”或“内部接口”猜测： + +| 声明 | 运维判断 | +| --- | --- | +| `affected_objects` | 表、约束、索引和外部文件范围，大表复制、资源与不可逆转换 | +| `execution_mode` 与程序／schema 支持范围 | 无持久化变更可普通发布；不兼容变更默认停写维护；在线迁移必须有已实现且验证过的后端与混合版本支持 | +| `api_changes` | 是否内部同步发布，是否独立调用方，旧接口保留／弃用及最早移除版本或日期 | +| `task_formats` | 待执行、运行中、延迟、重试和死信任务的版本，兼容消费／幂等转换／排空方式 | + +操作者可选择更保守的停机方式，不能选择“在线”来绕过已知不兼容。首期不承诺通用在线 DDL。 +内部同包入口可随调用方一起替换；独立部署的 Worker／SDK 即使称为内部也需要兼容安排。 +Artifact 等公共 API 不兼容替换时标记 `deprecated`，说明替代入口和移除计划；仅表变化而 API 契约不变无需弃用。 +保留的旧 API 可转换请求／响应后共用当前实现，不要求每个 handler 同时兼容新旧表。 + +**升级检查旧格式任务。** `plan` 只读统计相关队列、Lease 和 payload 格式，停写取锁后、迁移后及 Worker 启动前复查。 +没有格式版本字段时用冻结识别器；未知或不可观察的来源阻止相关迁移／Worker，不假报无任务。 +未知格式返回 `unsupported_task_format`，未排空返回 `legacy_tasks_pending`。先停生产，再让旧 Worker 有边界地排空， +最后停 Worker 才执行 DDL；或按计划兼容消费／幂等转换。保留任务 ID、幂等键、Scope／身份、重试和领域收据， +不删队列或把未完成任务标成功。移除旧处理器前必须处理遗留任务；外部 API 调用方升级由发布计划协调。 + +## 与现有存储层结合 + +| 现有层 | 升级中的职责 | +| --- | --- | +| Profile | 复用连接配置和引擎生命周期，拆出无 `create_tables` 副作用的 inspection／maintenance 入口 | +| `AsyncDatabase` | 提供专用维护连接及事务；`AsyncConnection.run_sync` 将同一连接交给 Alembic，锁和 DDL 不换连接 | +| Alembic | 冻结 revision 执行 DDL，验证后推进唯一版本表 | +| Repository | schema 满足后复用稳定领域操作；历史转换优先冻结 SQL／映射，不依赖不断变化的当前 Repository | +| 索引 | 结构定义进入版本资源，受控重建派生数据并检查，普通启动不隐式改表 | +| 启动门禁 | 先检查 schema、任务和必要能力，再装配 Repository、索引和 Worker | + +例如新增必填列：先 revision 加可空列，数据脚本分批幂等回填，校验后下一 revision 收紧约束，重建相关索引， +最后才启用新版 Repository。seekDB／OceanBase DDL 可隐式提交，不能用一个 Python 事务假装整个升级原子化。 +已有领域收据与对应数据批次一起提交，索引从权威数据重建。 + +## 中断与恢复 + +中断后只读检查 `status`、`verify`、`plan`，仅在实际状态证明可安全重试时继续。单一版本表不提供通用按 run ID +续跑；含混状态返回 `recovery_required`。auto 保留原始恢复点和库外记录，不用部分迁移后的备份覆盖它；manual +仍不核验，skip 不承诺恢复原数据。所有模式均检查真实结构、数据和任务，不盲目 stamp。 + +恢复独立、显式执行并保持停写；恢复数据库、必要文件及匹配程序后验证。应用回退不等于数据库回滚。 +迁移 `ready` 与服务启动结果分别报告，不在成功后立即删除恢复点。生产支持仍以 RFC 的实际后端验收为准。 diff --git a/docs/zh/docs/operate/index.md b/docs/zh/docs/operate/index.md index ed6a6b4af0..ae59d768e5 100644 --- a/docs/zh/docs/operate/index.md +++ b/docs/zh/docs/operate/index.md @@ -10,6 +10,7 @@ description: 持续运行 Server，查找部署、监控与恢复步骤。 | 任务 | 指南 | | --- | --- | | 持续运行个人 Server、部署容器或启用认证 | [部署 Server](deploy-server.md) | +| 了解统一数据库升级、备份与服务启停设计 | [数据库迁移设计](database-migrations.md) | | 升级或切换后台监督模式 | [迁移 Artifact 处理状态](artifact-processing-migration.md) | | 查看日志、指标和 Trace | [可观测性](observability.md) | | 将 Trace 发送到分析服务 | [Phoenix](trace-with-phoenix.md)、[Langfuse](trace-with-langfuse.md) | diff --git a/docs/zh/docs/operate/meta.json b/docs/zh/docs/operate/meta.json index f8a7b06d97..d0b1c9ebbe 100644 --- a/docs/zh/docs/operate/meta.json +++ b/docs/zh/docs/operate/meta.json @@ -4,6 +4,7 @@ "index", "deploy-server", "connect-remote-server", + "database-migrations", "artifact-processing-migration", "observability", "trace-with-phoenix", From 0584b760a2a93ba5f0c37911767c856b523fbdc8 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Tue, 29 Sep 2026 17:44:27 +0800 Subject: [PATCH 04/37] test: check runtime health after cancelled usage write --- tests/e2e/test_topic_memory_generic_api.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/e2e/test_topic_memory_generic_api.py b/tests/e2e/test_topic_memory_generic_api.py index ed55fd3dfa..333003e235 100644 --- a/tests/e2e/test_topic_memory_generic_api.py +++ b/tests/e2e/test_topic_memory_generic_api.py @@ -558,20 +558,23 @@ async def fail_record(repository, connection, *args): def test_cancelling_a_request_during_a_stalled_usage_write_leaves_the_runtime_healthy(tmp_path, monkeypatch): """Cancelling must propagate and must not wedge the runtime or its recorder. - Usage no longer runs inside the request, so a cancelled request neither waits - for it nor rolls it back; what must hold is that cancellation propagates and - the runtime keeps accepting work afterwards. + Usage no longer runs inside the request. Its best-effort write may succeed or + fail after cancellation; the runtime must still accept new work afterwards. """ async def scenario(): release = asyncio.Event() entered = asyncio.Event() + completed = asyncio.Event() original = StatisticsRepository.record async def stalled_record(repository, connection, *args): entered.set() await release.wait() - return await original(repository, connection, *args) + try: + return await original(repository, connection, *args) + finally: + completed.set() # As above: the stalled write must be reached rather than dropped for # spending its budget on a slow machine. @@ -594,7 +597,9 @@ async def stalled_record(repository, connection, *args): with pytest.raises(asyncio.CancelledError): await pending release.set() - await _await_usage_record(tmp_path / "topics.db", scope) + # The recorder is best-effort and may reject an interrupted write. + # Wait for its own bounded attempt, then test the business path. + await asyncio.wait_for(completed.wait(), 35) assert (await client.post(path, json=payload)).status_code == 201 asyncio.run(scenario()) From fb616ca50acc9eb90b57dfe27871ce4409a81b99 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Tue, 29 Sep 2026 17:59:36 +0800 Subject: [PATCH 05/37] test: align usage lock test with write budget --- tests/builtin/runtime/test_model_usage_recorder.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/builtin/runtime/test_model_usage_recorder.py b/tests/builtin/runtime/test_model_usage_recorder.py index 74be76a101..d92ff6d06c 100644 --- a/tests/builtin/runtime/test_model_usage_recorder.py +++ b/tests/builtin/runtime/test_model_usage_recorder.py @@ -206,9 +206,11 @@ async def scenario() -> None: path = tmp_path / "retry.db" config = SQLiteConfig(url=f"sqlite+aiosqlite:///{path}", busy_timeout_ms=5_000) async with _database(config) as database: - # One record's budget is spent in slices, so a writer that lets go - # partway through still yields a recorded usage row. - recorder = _ModelUsageRecorder(database, StatisticsRepository(), write_timeout_seconds=2.0) + # The held lock is released inside the write budget. Wait longer + # than that budget before asserting the committed usage row. + recorder = _ModelUsageRecorder( + database, StatisticsRepository(), write_timeout_seconds=5.0, flush_timeout_seconds=6.0 + ) holder = sqlite3.connect(path) try: holder.execute("BEGIN IMMEDIATE") From 2bcc5e6f3069cd8b3c3ad5edf2078cc9df0eaeda Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Tue, 29 Sep 2026 18:22:22 +0800 Subject: [PATCH 06/37] test: wait for SQLite writer release after cancellation --- tests/e2e/test_topic_memory_generic_api.py | 24 +++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/e2e/test_topic_memory_generic_api.py b/tests/e2e/test_topic_memory_generic_api.py index 333003e235..32dfbd3c6d 100644 --- a/tests/e2e/test_topic_memory_generic_api.py +++ b/tests/e2e/test_topic_memory_generic_api.py @@ -122,6 +122,26 @@ async def _await_usage_record(database, scope): await asyncio.sleep(0.02) +async def _await_sqlite_writer(database): + """Wait for the preceding transaction to release SQLite's write lock.""" + + async with asyncio.timeout(35): + while True: + connection = sqlite3.connect(database, timeout=0) + try: + try: + connection.execute("BEGIN IMMEDIATE") + except sqlite3.OperationalError as error: + if error.sqlite_errorcode not in (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED): + raise + else: + connection.rollback() + return + finally: + connection.close() + await asyncio.sleep(0.02) + + @pytest.mark.parametrize("vector", [False, True]) def test_topic_lifecycle_tags_filtered_pages_and_publication_survive_restart(tmp_path, vector): embedding = Embeddings() if vector else None @@ -598,8 +618,10 @@ async def stalled_record(repository, connection, *args): await pending release.set() # The recorder is best-effort and may reject an interrupted write. - # Wait for its own bounded attempt, then test the business path. + # The repository call finishes before its transaction releases the + # write lock. Wait for that release before testing the business path. await asyncio.wait_for(completed.wait(), 35) + await _await_sqlite_writer(tmp_path / "topics.db") assert (await client.post(path, json=payload)).status_code == 201 asyncio.run(scenario()) From 7478cead84d5e34535d36af90b31f81d29f3e1a6 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Tue, 29 Sep 2026 18:55:10 +0800 Subject: [PATCH 07/37] fix(db): finish transaction cleanup after cancellation --- .../builtin/persistence/database.py | 57 +++++++++++++++---- .../persistence/test_sqlite_profile.py | 56 ++++++++++++++++++ 2 files changed, 101 insertions(+), 12 deletions(-) diff --git a/src/powercontext/builtin/persistence/database.py b/src/powercontext/builtin/persistence/database.py index 77a6a415dc..e019823c4a 100644 --- a/src/powercontext/builtin/persistence/database.py +++ b/src/powercontext/builtin/persistence/database.py @@ -17,9 +17,11 @@ from __future__ import annotations import asyncio +import sys import time from collections.abc import AsyncIterator from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext +from types import TracebackType from aiosqlite import Connection as SQLiteConnection from sqlalchemy.exc import OperationalError @@ -87,19 +89,27 @@ async def transaction(self) -> AsyncIterator[AsyncConnection]: self._active_transactions += 1 try: guard = self._shared_connection_lock if self._shared_connection_lock is not None else nullcontext() - async with guard, self._engine.begin() as connection: - if connection.dialect.name == "mysql": - # The MySQL dialect's begin hook is a no-op. Explicitly start - # the owned transaction even when the server session uses autocommit. - await connection.exec_driver_sql("START TRANSACTION") - if self._shared_connection_lock is not None: - self._transaction_owner = owner - self._shared_connection = connection + async with guard: + context = self._engine.begin() + connection = await context.__aenter__() try: - yield connection - finally: - self._transaction_owner = None - self._shared_connection = None + if connection.dialect.name == "mysql": + # The MySQL dialect's begin hook is a no-op. Explicitly start + # the owned transaction even when the server session uses autocommit. + await connection.exec_driver_sql("START TRANSACTION") + if self._shared_connection_lock is not None: + self._transaction_owner = owner + self._shared_connection = connection + try: + yield connection + finally: + self._transaction_owner = None + self._shared_connection = None + except BaseException: + await _finish_engine_transaction(context, *sys.exc_info()) + raise + else: + await _finish_engine_transaction(context, None, None, None) finally: async with self._state_changed: self._active_transactions -= 1 @@ -195,6 +205,29 @@ async def close(self) -> None: self._state_changed.notify_all() +async def _finish_engine_transaction( + context: AbstractAsyncContextManager[AsyncConnection], + error_type: type[BaseException] | None, + error: BaseException | None, + traceback: TracebackType | None, +) -> None: + """Finish commit or rollback before a cancelled caller releases its guard.""" + + cleanup = asyncio.create_task(context.__aexit__(error_type, error, traceback)) + cancelled = False + while True: + try: + await asyncio.shield(cleanup) + break + except asyncio.CancelledError: + cancelled = True + if cleanup.done(): + break + cleanup.result() + if cancelled: + raise asyncio.CancelledError + + @asynccontextmanager async def _sqlite_model_usage_transaction(connection: AsyncConnection, deadline: float) -> AsyncIterator[None]: driver = (await connection.get_raw_connection()).driver_connection diff --git a/tests/builtin/persistence/test_sqlite_profile.py b/tests/builtin/persistence/test_sqlite_profile.py index ec52477672..3f4885eec6 100644 --- a/tests/builtin/persistence/test_sqlite_profile.py +++ b/tests/builtin/persistence/test_sqlite_profile.py @@ -15,16 +15,19 @@ from __future__ import annotations import asyncio +import sqlite3 import pytest from pydantic import ValidationError from sqlalchemy import func, insert, select from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncTransaction from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import ( ARTIFACT_LINEAGE_SOURCES_TABLE, ARTIFACTS_TABLE, + SCOPES_TABLE, SHARED_TABLES, ) @@ -46,6 +49,59 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_repeated_cancellation_waits_for_transaction_rollback_before_releasing_writer(tmp_path, monkeypatch) -> None: + async def scenario() -> None: + database_path = tmp_path / "cancelled.db" + body_started = asyncio.Event() + rollback_started = asyncio.Event() + finish_rollback = asyncio.Event() + original_exit = AsyncTransaction.__aexit__ + + async def delayed_exit(transaction, error_type, error, traceback): + rollback_started.set() + await finish_rollback.wait() + return await original_exit(transaction, error_type, error, traceback) + + async with SQLiteProfile.open( + SQLiteConfig(url=f"sqlite+aiosqlite:///{database_path}"), tables=(SCOPES_TABLE,) + ) as profile: + + async def cancelled_write() -> None: + async with profile.database.transaction() as connection: + await connection.exec_driver_sql("UPDATE pc_scopes SET version = version") + body_started.set() + await asyncio.Event().wait() + + with monkeypatch.context() as injected: + # Simulate a slow driver rollback and a second cancellation + # while the transaction is returning its connection. + injected.setattr(AsyncTransaction, "__aexit__", delayed_exit) + task = asyncio.create_task(cancelled_write()) + try: + await asyncio.wait_for(body_started.wait(), 2) + task.cancel() + await asyncio.wait_for(rollback_started.wait(), 2) + task.cancel() + await asyncio.sleep(0) + assert not task.done() + finally: + finish_rollback.set() + task.cancel() + await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 2) + assert task.cancelled() + + writer = sqlite3.connect(database_path, timeout=0) + try: + writer.execute("BEGIN IMMEDIATE") + writer.rollback() + finally: + writer.close() + async with profile.database.transaction() as connection: + assert await connection.scalar(select(func.count()).select_from(SCOPES_TABLE)) == 0 + + asyncio.run(scenario()) + + def test_sqlite_pragmas_enforce_lineage_source_foreign_keys() -> None: async def scenario() -> None: async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: From 06cc7527db8633769b5cabbb99395ce91c8951dd Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Tue, 29 Sep 2026 19:24:57 +0800 Subject: [PATCH 08/37] test: release stalled writer before awaiting cancellation --- tests/e2e/test_topic_memory_generic_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/e2e/test_topic_memory_generic_api.py b/tests/e2e/test_topic_memory_generic_api.py index 32dfbd3c6d..38e586995c 100644 --- a/tests/e2e/test_topic_memory_generic_api.py +++ b/tests/e2e/test_topic_memory_generic_api.py @@ -614,9 +614,11 @@ async def stalled_record(repository, connection, *args): pending = asyncio.create_task(client.post(path, json=payload)) await asyncio.wait_for(entered.wait(), 5) pending.cancel() - with pytest.raises(asyncio.CancelledError): - await pending + # Let the recorder release its SQLite lock so cancellation can + # finish the request transaction's rollback before returning. release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(pending, 35) # The recorder is best-effort and may reject an interrupted write. # The repository call finishes before its transaction releases the # write lock. Wait for that release before testing the business path. From b779129055d8f0331ca246e48db941525dbb1f8b Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Tue, 29 Sep 2026 20:33:06 +0800 Subject: [PATCH 09/37] chore(ci): isolate topic memory test stall --- .github/workflows/master.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 25284210b2..0c9d0beda8 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -39,11 +39,11 @@ jobs: run: make contract-test tests: - timeout-minutes: 40 + timeout-minutes: 8 runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] + python-version: ["3.14"] fail-fast: false defaults: run: @@ -59,11 +59,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Run unit tests - run: make unit-test - - - name: Run end-to-end tests - run: make e2e-test + - name: Diagnose Topic Memory end-to-end stall + run: timeout 180s uv run python -m pytest -vv -x -o faulthandler_timeout=30 tests/e2e/test_topic_memory_generic_api.py code-seekdb: timeout-minutes: 20 From 05f2b0da11f86743a4cda761aa029121db3a8a23 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Tue, 29 Sep 2026 20:48:34 +0800 Subject: [PATCH 10/37] chore(test): capture stalled asyncio tasks in CI --- tests/e2e/test_topic_memory_generic_api.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/test_topic_memory_generic_api.py b/tests/e2e/test_topic_memory_generic_api.py index 38e586995c..92a55c6afd 100644 --- a/tests/e2e/test_topic_memory_generic_api.py +++ b/tests/e2e/test_topic_memory_generic_api.py @@ -17,6 +17,7 @@ import asyncio import logging import sqlite3 +import sys import httpx import pytest @@ -588,6 +589,13 @@ async def scenario(): completed = asyncio.Event() original = StatisticsRepository.record + def dump_tasks(): + for task in asyncio.all_tasks(): + print(f"DIAGNOSTIC TASK {task.get_name()} {task!r}", file=sys.stderr) + task.print_stack(limit=20, file=sys.stderr) + + asyncio.get_running_loop().call_later(10, dump_tasks) + async def stalled_record(repository, connection, *args): entered.set() await release.wait() From 7ec166974a44e08f86ea0d470de4c96e55a16f29 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Tue, 29 Sep 2026 20:54:49 +0800 Subject: [PATCH 11/37] chore(ci): stream asyncio task diagnostics --- .github/workflows/master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 0c9d0beda8..dcd7240199 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -60,7 +60,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Diagnose Topic Memory end-to-end stall - run: timeout 180s uv run python -m pytest -vv -x -o faulthandler_timeout=30 tests/e2e/test_topic_memory_generic_api.py + run: timeout 180s uv run python -m pytest -s -vv -x -o faulthandler_timeout=30 tests/e2e/test_topic_memory_generic_api.py code-seekdb: timeout-minutes: 20 From 24a7ba9faa800f6e1d97789a6e9c8afe92056b43 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Tue, 29 Sep 2026 21:03:14 +0800 Subject: [PATCH 12/37] fix(ci): restore full checks before applying driver cleanup fix --- .github/workflows/master.yml | 11 ++-- .../builtin/persistence/database.py | 57 ++++--------------- .../persistence/test_sqlite_profile.py | 56 ------------------ tests/e2e/test_topic_memory_generic_api.py | 51 +++-------------- 4 files changed, 26 insertions(+), 149 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index dcd7240199..25284210b2 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -39,11 +39,11 @@ jobs: run: make contract-test tests: - timeout-minutes: 8 + timeout-minutes: 40 runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.14"] + python-version: ["3.11", "3.12", "3.13", "3.14"] fail-fast: false defaults: run: @@ -59,8 +59,11 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Diagnose Topic Memory end-to-end stall - run: timeout 180s uv run python -m pytest -s -vv -x -o faulthandler_timeout=30 tests/e2e/test_topic_memory_generic_api.py + - name: Run unit tests + run: make unit-test + + - name: Run end-to-end tests + run: make e2e-test code-seekdb: timeout-minutes: 20 diff --git a/src/powercontext/builtin/persistence/database.py b/src/powercontext/builtin/persistence/database.py index e019823c4a..77a6a415dc 100644 --- a/src/powercontext/builtin/persistence/database.py +++ b/src/powercontext/builtin/persistence/database.py @@ -17,11 +17,9 @@ from __future__ import annotations import asyncio -import sys import time from collections.abc import AsyncIterator from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext -from types import TracebackType from aiosqlite import Connection as SQLiteConnection from sqlalchemy.exc import OperationalError @@ -89,27 +87,19 @@ async def transaction(self) -> AsyncIterator[AsyncConnection]: self._active_transactions += 1 try: guard = self._shared_connection_lock if self._shared_connection_lock is not None else nullcontext() - async with guard: - context = self._engine.begin() - connection = await context.__aenter__() + async with guard, self._engine.begin() as connection: + if connection.dialect.name == "mysql": + # The MySQL dialect's begin hook is a no-op. Explicitly start + # the owned transaction even when the server session uses autocommit. + await connection.exec_driver_sql("START TRANSACTION") + if self._shared_connection_lock is not None: + self._transaction_owner = owner + self._shared_connection = connection try: - if connection.dialect.name == "mysql": - # The MySQL dialect's begin hook is a no-op. Explicitly start - # the owned transaction even when the server session uses autocommit. - await connection.exec_driver_sql("START TRANSACTION") - if self._shared_connection_lock is not None: - self._transaction_owner = owner - self._shared_connection = connection - try: - yield connection - finally: - self._transaction_owner = None - self._shared_connection = None - except BaseException: - await _finish_engine_transaction(context, *sys.exc_info()) - raise - else: - await _finish_engine_transaction(context, None, None, None) + yield connection + finally: + self._transaction_owner = None + self._shared_connection = None finally: async with self._state_changed: self._active_transactions -= 1 @@ -205,29 +195,6 @@ async def close(self) -> None: self._state_changed.notify_all() -async def _finish_engine_transaction( - context: AbstractAsyncContextManager[AsyncConnection], - error_type: type[BaseException] | None, - error: BaseException | None, - traceback: TracebackType | None, -) -> None: - """Finish commit or rollback before a cancelled caller releases its guard.""" - - cleanup = asyncio.create_task(context.__aexit__(error_type, error, traceback)) - cancelled = False - while True: - try: - await asyncio.shield(cleanup) - break - except asyncio.CancelledError: - cancelled = True - if cleanup.done(): - break - cleanup.result() - if cancelled: - raise asyncio.CancelledError - - @asynccontextmanager async def _sqlite_model_usage_transaction(connection: AsyncConnection, deadline: float) -> AsyncIterator[None]: driver = (await connection.get_raw_connection()).driver_connection diff --git a/tests/builtin/persistence/test_sqlite_profile.py b/tests/builtin/persistence/test_sqlite_profile.py index 3f4885eec6..ec52477672 100644 --- a/tests/builtin/persistence/test_sqlite_profile.py +++ b/tests/builtin/persistence/test_sqlite_profile.py @@ -15,19 +15,16 @@ from __future__ import annotations import asyncio -import sqlite3 import pytest from pydantic import ValidationError from sqlalchemy import func, insert, select from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncTransaction from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import ( ARTIFACT_LINEAGE_SOURCES_TABLE, ARTIFACTS_TABLE, - SCOPES_TABLE, SHARED_TABLES, ) @@ -49,59 +46,6 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_repeated_cancellation_waits_for_transaction_rollback_before_releasing_writer(tmp_path, monkeypatch) -> None: - async def scenario() -> None: - database_path = tmp_path / "cancelled.db" - body_started = asyncio.Event() - rollback_started = asyncio.Event() - finish_rollback = asyncio.Event() - original_exit = AsyncTransaction.__aexit__ - - async def delayed_exit(transaction, error_type, error, traceback): - rollback_started.set() - await finish_rollback.wait() - return await original_exit(transaction, error_type, error, traceback) - - async with SQLiteProfile.open( - SQLiteConfig(url=f"sqlite+aiosqlite:///{database_path}"), tables=(SCOPES_TABLE,) - ) as profile: - - async def cancelled_write() -> None: - async with profile.database.transaction() as connection: - await connection.exec_driver_sql("UPDATE pc_scopes SET version = version") - body_started.set() - await asyncio.Event().wait() - - with monkeypatch.context() as injected: - # Simulate a slow driver rollback and a second cancellation - # while the transaction is returning its connection. - injected.setattr(AsyncTransaction, "__aexit__", delayed_exit) - task = asyncio.create_task(cancelled_write()) - try: - await asyncio.wait_for(body_started.wait(), 2) - task.cancel() - await asyncio.wait_for(rollback_started.wait(), 2) - task.cancel() - await asyncio.sleep(0) - assert not task.done() - finally: - finish_rollback.set() - task.cancel() - await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), 2) - assert task.cancelled() - - writer = sqlite3.connect(database_path, timeout=0) - try: - writer.execute("BEGIN IMMEDIATE") - writer.rollback() - finally: - writer.close() - async with profile.database.transaction() as connection: - assert await connection.scalar(select(func.count()).select_from(SCOPES_TABLE)) == 0 - - asyncio.run(scenario()) - - def test_sqlite_pragmas_enforce_lineage_source_foreign_keys() -> None: async def scenario() -> None: async with SQLiteProfile.open(SQLiteConfig(), tables=SHARED_TABLES) as profile: diff --git a/tests/e2e/test_topic_memory_generic_api.py b/tests/e2e/test_topic_memory_generic_api.py index 92a55c6afd..ed55fd3dfa 100644 --- a/tests/e2e/test_topic_memory_generic_api.py +++ b/tests/e2e/test_topic_memory_generic_api.py @@ -17,7 +17,6 @@ import asyncio import logging import sqlite3 -import sys import httpx import pytest @@ -123,26 +122,6 @@ async def _await_usage_record(database, scope): await asyncio.sleep(0.02) -async def _await_sqlite_writer(database): - """Wait for the preceding transaction to release SQLite's write lock.""" - - async with asyncio.timeout(35): - while True: - connection = sqlite3.connect(database, timeout=0) - try: - try: - connection.execute("BEGIN IMMEDIATE") - except sqlite3.OperationalError as error: - if error.sqlite_errorcode not in (sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED): - raise - else: - connection.rollback() - return - finally: - connection.close() - await asyncio.sleep(0.02) - - @pytest.mark.parametrize("vector", [False, True]) def test_topic_lifecycle_tags_filtered_pages_and_publication_survive_restart(tmp_path, vector): embedding = Embeddings() if vector else None @@ -579,30 +558,20 @@ async def fail_record(repository, connection, *args): def test_cancelling_a_request_during_a_stalled_usage_write_leaves_the_runtime_healthy(tmp_path, monkeypatch): """Cancelling must propagate and must not wedge the runtime or its recorder. - Usage no longer runs inside the request. Its best-effort write may succeed or - fail after cancellation; the runtime must still accept new work afterwards. + Usage no longer runs inside the request, so a cancelled request neither waits + for it nor rolls it back; what must hold is that cancellation propagates and + the runtime keeps accepting work afterwards. """ async def scenario(): release = asyncio.Event() entered = asyncio.Event() - completed = asyncio.Event() original = StatisticsRepository.record - def dump_tasks(): - for task in asyncio.all_tasks(): - print(f"DIAGNOSTIC TASK {task.get_name()} {task!r}", file=sys.stderr) - task.print_stack(limit=20, file=sys.stderr) - - asyncio.get_running_loop().call_later(10, dump_tasks) - async def stalled_record(repository, connection, *args): entered.set() await release.wait() - try: - return await original(repository, connection, *args) - finally: - completed.set() + return await original(repository, connection, *args) # As above: the stalled write must be reached rather than dropped for # spending its budget on a slow machine. @@ -622,16 +591,10 @@ async def stalled_record(repository, connection, *args): pending = asyncio.create_task(client.post(path, json=payload)) await asyncio.wait_for(entered.wait(), 5) pending.cancel() - # Let the recorder release its SQLite lock so cancellation can - # finish the request transaction's rollback before returning. - release.set() with pytest.raises(asyncio.CancelledError): - await asyncio.wait_for(pending, 35) - # The recorder is best-effort and may reject an interrupted write. - # The repository call finishes before its transaction releases the - # write lock. Wait for that release before testing the business path. - await asyncio.wait_for(completed.wait(), 35) - await _await_sqlite_writer(tmp_path / "topics.db") + await pending + release.set() + await _await_usage_record(tmp_path / "topics.db", scope) assert (await client.post(path, json=payload)).status_code == 201 asyncio.run(scenario()) From 1b3a9040a7df63a3eb85414eabbe8243c16516a4 Mon Sep 17 00:00:00 2001 From: heyamei690-hash gua Date: Tue, 29 Sep 2026 10:49:10 +0800 Subject: [PATCH 13/37] fix(persistence): finish connection cleanup before a cancellation propagates Repeatedly cancelling a transaction could keep its pool slot checked out. Two driver-level gaps combine. SQLAlchemy can drive both a graceful close and a stop on the same aiosqlite connection, whose stop is not idempotent and enqueues a second request behind a worker that already exited. And a statement cancelled mid-await leaves its native cursor held by the exception stack, so the rollback that follows fails with "Connection closed" and the slot is never returned. Close the cursor tracked for the connection before retiring it, make the driver stop idempotent per connection, interrupt the statement SQLite is still running, and finish the whole exit inside a managed task so a repeated cancellation cannot strand the cleanup. Only the transaction that owns the connection runs it, so a nested exit cannot close its caller's cursors. A clean exit stays in the caller's task, which the seekdb shutdown ordering depends on. Without the cursor step the new regression fails: the only pool slot is still checked out after the tenth cancelled transaction. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../builtin/persistence/database.py | 94 ++++++- .../builtin/persistence/sqlite/profile.py | 68 ++++- .../persistence/test_sqlite_profile.py | 248 ++++++++++++++++++ tests/e2e/test_topic_memory_generic_api.py | 69 +++-- 4 files changed, 446 insertions(+), 33 deletions(-) diff --git a/src/powercontext/builtin/persistence/database.py b/src/powercontext/builtin/persistence/database.py index 77a6a415dc..a9624b8bdb 100644 --- a/src/powercontext/builtin/persistence/database.py +++ b/src/powercontext/builtin/persistence/database.py @@ -18,10 +18,11 @@ import asyncio import time -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable, Collection from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext from aiosqlite import Connection as SQLiteConnection +from aiosqlite import Cursor as SQLiteCursor from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine @@ -75,12 +76,30 @@ def engine(self) -> AsyncEngine: async def transaction(self) -> AsyncIterator[AsyncConnection]: """Yield a connection in a transaction owned by the calling use case.""" - owner = asyncio.current_task() - if self._shared_connection is not None and self._transaction_owner is owner: - # Nested lookups on a single-connection profile must join their - # caller's transaction, not acquire or commit that connection again. + if self._shared_connection is not None and self._transaction_owner is asyncio.current_task(): + # Nested lookups on a single-connection profile join their caller's + # transaction, not acquire or commit that connection again. Only the + # owner runs cleanup, so a nested exit cannot close the caller's cursors. yield self._shared_connection return + context = self._transaction() + connection = await context.__aenter__() + sqlite_stop = None + sqlite_cursors: Collection[SQLiteCursor] | None = None + try: + if connection.dialect.name == "sqlite": + sqlite_stop = connection.info.get("_powercontext_sqlite_stop") + sqlite_cursors = connection.info.get("_powercontext_sqlite_cursors") + yield connection + except BaseException as error: + await _finish_transaction(context, connection, error, sqlite_stop, sqlite_cursors) + raise + else: + await context.__aexit__(None, None, None) + + @asynccontextmanager + async def _transaction(self) -> AsyncIterator[AsyncConnection]: + owner = asyncio.current_task() async with self._state_changed: if self._closed or self._closing: raise DatabaseClosedError @@ -195,6 +214,71 @@ async def close(self) -> None: self._state_changed.notify_all() +async def _retire_cancelled_connection( + connection: AsyncConnection, + error: BaseException, + sqlite_cursors: Collection[SQLiteCursor] | None, +) -> None: + """Close a cancelled connection's native cursor, then discard a dead driver.""" + + raw = await connection.get_raw_connection() + driver = raw.driver_connection + if sqlite_cursors and isinstance(driver, SQLiteConnection) and driver._running: + # A cancelled await leaves the native cursor held by the exception stack; + # rollback cannot complete until it closes. + for cursor in tuple(sqlite_cursors): + await cursor.close() + if _driver_is_dead(driver, connection.dialect.name): + # A cancelled invalidate can close the driver before updating SQLAlchemy's state. + await connection.invalidate(error) + + +def _driver_is_dead(driver: object, dialect_name: str) -> bool: + """Report whether a cancelled driver already exited and must not be reused.""" + + if isinstance(driver, SQLiteConnection): + return not driver._running + if dialect_name == "mysql": + from aiomysql import Connection as MySQLConnection + + return isinstance(driver, MySQLConnection) and driver.closed + return False + + +async def _finish_transaction( + context: AbstractAsyncContextManager[AsyncConnection], + connection: AsyncConnection, + error: BaseException, + sqlite_stop: Callable[[], asyncio.Future[object] | None] | None, + sqlite_cursors: Collection[SQLiteCursor] | None = None, +) -> None: + """Retire a failed transaction before its error propagates to the caller.""" + + async def finish() -> None: + try: + if isinstance(error, asyncio.CancelledError) and not connection.closed: + if not connection.invalidated: + await _retire_cancelled_connection(connection, error, sqlite_cursors) + if connection.invalidated and sqlite_stop is not None: + stopped = sqlite_stop() + if stopped is not None: + await stopped + finally: + await context.__aexit__(type(error), error, error.__traceback__) + + finishing = asyncio.create_task(finish(), name="powercontext-transaction-cleanup") + interrupted = None + # Keep connection ownership until cleanup finishes, even under repeated cancellation. + while not finishing.done(): + try: + await asyncio.shield(finishing) + except asyncio.CancelledError as cancellation: + interrupted = cancellation + finishing.result() + if interrupted is not None: + raise interrupted + + @asynccontextmanager async def _sqlite_model_usage_transaction(connection: AsyncConnection, deadline: float) -> AsyncIterator[None]: driver = (await connection.get_raw_connection()).driver_connection diff --git a/src/powercontext/builtin/persistence/sqlite/profile.py b/src/powercontext/builtin/persistence/sqlite/profile.py index d2a22aa32e..0f59fc3914 100644 --- a/src/powercontext/builtin/persistence/sqlite/profile.py +++ b/src/powercontext/builtin/persistence/sqlite/profile.py @@ -17,21 +17,23 @@ from __future__ import annotations import asyncio -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable from contextlib import asynccontextmanager from pathlib import Path -from typing import Literal -from weakref import WeakKeyDictionary +from typing import Any, Literal, cast +from weakref import WeakKeyDictionary, WeakSet +import aiosqlite import sqlite_vec -from aiosqlite import Connection +from aiosqlite import Connection, Cursor +from aiosqlite.context import Result from pydantic import BaseModel, ConfigDict, Field, field_validator from sqlalchemy import Table, event -from sqlalchemy.engine import make_url +from sqlalchemy.engine import AdaptedConnection, ExceptionContext, make_url from sqlalchemy.engine.interfaces import DBAPIConnection from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine -from sqlalchemy.pool import StaticPool +from sqlalchemy.pool import ConnectionPoolEntry, StaticPool from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.schema import create_tables @@ -123,7 +125,11 @@ def _configure_sqlite( load_vector_extension: bool, ) -> None: @event.listens_for(engine.sync_engine, "connect") - def set_pragmas(dbapi_connection: DBAPIConnection, _connection_record: object) -> None: + def set_pragmas(dbapi_connection: DBAPIConnection, connection_record: ConnectionPoolEntry) -> None: + driver = cast(Connection, cast(AdaptedConnection, dbapi_connection).driver_connection) + connection_record.info["_powercontext_sqlite_cursors"] = _track_sqlite_cursors(driver) + if aiosqlite.__version__ == "0.22.1": + connection_record.info["_powercontext_sqlite_stop"] = _make_sqlite_stop_idempotent(driver) if load_vector_extension: dbapi_connection.run_async(_load_sqlite_vec) cursor = dbapi_connection.cursor() @@ -133,6 +139,54 @@ def set_pragmas(dbapi_connection: DBAPIConnection, _connection_record: object) - finally: cursor.close() + @event.listens_for(engine.sync_engine, "handle_error") + def interrupt_cancelled_statement(context: ExceptionContext) -> None: + connection = context.connection + if ( + isinstance(context.original_exception, asyncio.CancelledError) + and connection is not None + and not connection.closed + and not connection.invalidated + ): + driver = cast(Connection, connection.connection.driver_connection) + if driver._running and driver._connection is not None: + # Cancelling an await does not stop SQLite's native statement or disconnect its worker. + driver._connection.interrupt() + context.is_disconnect = False + + +def _track_sqlite_cursors(connection: Connection) -> WeakSet[Cursor]: + original_cursor = connection.cursor + cursors: WeakSet[Cursor] = WeakSet() + + async def open_cursor() -> Cursor: + result = await original_cursor() + cursors.add(result) + return result + + def cursor() -> Result[Cursor]: + return Result(open_cursor()) + + connection.cursor = cursor + return cursors + + +def _make_sqlite_stop_idempotent(connection: Connection) -> Callable[[], asyncio.Future[Any] | None]: + original_stop = connection.stop + requested = False + stopped: asyncio.Future[Any] | None = None + + def stop_once() -> asyncio.Future[Any] | None: + nonlocal requested, stopped + # aiosqlite 0.22.1 can enqueue a second stop after its worker has exited. + if not requested: + stopped = original_stop() + requested = True + return asyncio.shield(stopped) if stopped is not None else None + + setattr(connection, "stop", stop_once) # noqa: B010 - patch one connection; the shared aiosqlite class stays untouched + return stop_once + async def _load_sqlite_vec(connection: Connection) -> None: await connection.enable_load_extension(True) diff --git a/tests/builtin/persistence/test_sqlite_profile.py b/tests/builtin/persistence/test_sqlite_profile.py index ec52477672..250fed8440 100644 --- a/tests/builtin/persistence/test_sqlite_profile.py +++ b/tests/builtin/persistence/test_sqlite_profile.py @@ -15,13 +15,21 @@ from __future__ import annotations import asyncio +from functools import partial +from threading import Event as ThreadEvent +from typing import cast +import aiosqlite +import anyio import pytest from pydantic import ValidationError from sqlalchemy import func, insert, select from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine +from sqlalchemy.pool import QueuePool from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.sqlite import profile as sqlite_profile from powercontext.builtin.persistence.tables import ( ARTIFACT_LINEAGE_SOURCES_TABLE, ARTIFACTS_TABLE, @@ -29,6 +37,246 @@ ) +def test_sqlite_stop_waiters_finish_when_another_waiter_is_cancelled(tmp_path) -> None: + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'stop.db'}") + async with SQLiteProfile.open(config, tables=()) as profile, profile.database.engine.connect() as connection: + driver = cast(aiosqlite.Connection, (await connection.get_raw_connection()).driver_connection) + entered = ThreadEvent() + release = ThreadEvent() + + def hold_worker() -> None: + entered.set() + assert release.wait(5) + + blocked = asyncio.create_task(driver._execute(hold_worker)) + try: + assert await asyncio.to_thread(entered.wait, 2) + first, second = driver.stop(), driver.stop() + assert first is not None and second is not None + first.cancel() + release.set() + await asyncio.wait_for(blocked, 2) + await asyncio.wait_for(second, 2) + assert not second.cancelled() + finally: + release.set() + await asyncio.gather(blocked, return_exceptions=True) + await connection.invalidate() + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("phase", ["rollback", "close"]) +def test_transaction_finishes_cleanup_before_propagating_repeated_cancellation(tmp_path, monkeypatch, phase) -> None: + async def scenario() -> None: + entered = asyncio.Event() + cleaning = asyncio.Event() + release = asyncio.Event() + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'cleanup.db'}") + async with SQLiteProfile.open(config, tables=()) as profile: + database = profile.database + pool = cast(QueuePool, database.engine.pool) + async with database.transaction() as connection: + await connection.exec_driver_sql("CREATE TABLE probe (value INTEGER)") + + async def operation() -> None: + async with database.transaction() as connection: + if phase == "rollback": + driver = cast(aiosqlite.Connection, (await connection.get_raw_connection()).driver_connection) + original_rollback = driver.rollback + + async def delayed_rollback() -> None: + if not cleaning.is_set(): + cleaning.set() + await release.wait() + await original_rollback() + + monkeypatch.setattr(driver, "rollback", delayed_rollback) + else: + original_close = AsyncConnection.close + + async def delayed_close(closing: AsyncConnection) -> None: + if closing is connection: + cleaning.set() + await release.wait() + await original_close(closing) + + monkeypatch.setattr(AsyncConnection, "close", delayed_close) + await connection.exec_driver_sql("INSERT INTO probe VALUES (1)") + entered.set() + await asyncio.Event().wait() + + pending = asyncio.create_task(operation()) + try: + await asyncio.wait_for(entered.wait(), 2) + pending.cancel() + await asyncio.wait_for(cleaning.wait(), 2) + pending.cancel() + done, _ = await asyncio.wait({pending}, timeout=0.02) + assert not done, "cancellation returned before transaction cleanup finished" + assert pool.checkedout() == 1 + assert database._active_transactions == 1 + finally: + release.set() + if not pending.done(): + pending.cancel() + await asyncio.wait_for(asyncio.gather(pending, return_exceptions=True), 2) + assert pending.cancelled() + assert pool.checkedout() == 0 + async with database.transaction() as connection: + assert (await connection.exec_driver_sql("SELECT count(*) FROM probe")).scalar_one() == 0 + + asyncio.run(scenario()) + + +def test_shared_transaction_keeps_its_lock_until_cancelled_rollback_finishes(monkeypatch) -> None: + async def scenario() -> None: + entered, cleaning, release = asyncio.Event(), asyncio.Event(), asyncio.Event() + contender_started, contender_entered = asyncio.Event(), asyncio.Event() + async with SQLiteProfile.open(SQLiteConfig(), tables=()) as profile: + database = profile.database + async with database.transaction() as connection: + await connection.exec_driver_sql("CREATE TABLE probe (value INTEGER)") + + async def operation() -> None: + async with database.transaction() as connection: + async with database.transaction() as nested: + assert nested is connection + await nested.exec_driver_sql("INSERT INTO probe VALUES (1)") + driver = cast(aiosqlite.Connection, (await connection.get_raw_connection()).driver_connection) + original = driver.rollback + + async def delayed_rollback() -> None: + if not cleaning.is_set(): + cleaning.set() + await release.wait() + await original() + + monkeypatch.setattr(driver, "rollback", delayed_rollback) + entered.set() + await asyncio.Event().wait() + + async def contender() -> None: + contender_started.set() + async with database.transaction() as connection: + contender_entered.set() + assert (await connection.exec_driver_sql("SELECT count(*) FROM probe")).scalar_one() == 0 + await connection.exec_driver_sql("INSERT INTO probe VALUES (2)") + + pending = asyncio.create_task(operation()) + competing = None + try: + await asyncio.wait_for(entered.wait(), 2) + pending.cancel() + await asyncio.wait_for(cleaning.wait(), 2) + competing = asyncio.create_task(contender()) + await asyncio.wait_for(contender_started.wait(), 2) + pending.cancel() + done, _ = await asyncio.wait({competing}, timeout=0.02) + assert not done + assert not contender_entered.is_set() + finally: + release.set() + if not pending.done(): + pending.cancel() + await asyncio.wait_for(asyncio.gather(pending, return_exceptions=True), 2) + assert pending.cancelled() + assert competing is not None + await asyncio.wait_for(competing, 2) + async with database.transaction() as connection: + assert (await connection.exec_driver_sql("SELECT value FROM probe")).scalar_one() == 2 + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("phase", ["body", "sql"]) +def test_repeated_transaction_cancellation_does_not_exhaust_the_pool(tmp_path, monkeypatch, phase) -> None: + monkeypatch.setattr( + sqlite_profile, + "create_async_engine", + partial(create_async_engine, pool_size=1, max_overflow=0, pool_timeout=0.5), + ) + + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'repeated.db'}") + async with SQLiteProfile.open(config, tables=()) as profile: + database = profile.database + pool = cast(QueuePool, database.engine.pool) + async with database.transaction() as connection: + await connection.exec_driver_sql("CREATE TABLE probe (value INTEGER)") + for _ in range(10): + entered = asyncio.Event() + release = ThreadEvent() + + async def operation(entered: asyncio.Event = entered, release: ThreadEvent = release) -> None: + async with database.transaction() as connection: + await connection.exec_driver_sql("INSERT INTO probe VALUES (1)") + if phase == "sql": + driver = cast( + aiosqlite.Connection, (await connection.get_raw_connection()).driver_connection + ) + loop = asyncio.get_running_loop() + + def wait_for_release() -> int: + loop.call_soon_threadsafe(entered.set) + assert release.wait(5) + return 1 + + await driver.create_function("wait_for_release", 0, wait_for_release) + await connection.exec_driver_sql("SELECT wait_for_release()") + else: + entered.set() + await asyncio.Event().wait() + + try: + async with anyio.create_task_group() as group: + group.start_soon(operation) + await asyncio.wait_for(entered.wait(), 2) + group.cancel_scope.cancel() + release.set() + finally: + release.set() + assert pool.checkedout() == 0 + await asyncio.wait_for(database.ping(), 2) + async with database.transaction() as connection: + assert (await connection.exec_driver_sql("SELECT count(*) FROM probe")).scalar_one() == 0 + + asyncio.run(scenario()) + + +def test_cancelled_checkout_does_not_keep_the_only_pool_slot(tmp_path, monkeypatch) -> None: + monkeypatch.setattr( + sqlite_profile, + "create_async_engine", + partial(create_async_engine, pool_size=1, max_overflow=0, pool_timeout=0.5), + ) + + async def scenario() -> None: + entered = asyncio.Event() + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'checkout.db'}") + async with SQLiteProfile.open(config, tables=()) as profile: + database = profile.database + pool = cast(QueuePool, database.engine.pool) + + async def blocked_checkout() -> None: + entered.set() + async with database.transaction(): + pytest.fail("the held pool slot was unexpectedly available") + + async with database.transaction(): + pending = asyncio.create_task(blocked_checkout()) + await asyncio.wait_for(entered.wait(), 2) + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(pending, 2) + assert pool.checkedout() == 1 + assert pool.checkedout() == 0 + await asyncio.wait_for(database.ping(), 2) + + asyncio.run(scenario()) + + def test_sqlite_config_requires_the_async_dialect() -> None: with pytest.raises(ValidationError, match=r"sqlite\+aiosqlite"): SQLiteConfig(url="sqlite:///:memory:") diff --git a/tests/e2e/test_topic_memory_generic_api.py b/tests/e2e/test_topic_memory_generic_api.py index ed55fd3dfa..5a08f20ac1 100644 --- a/tests/e2e/test_topic_memory_generic_api.py +++ b/tests/e2e/test_topic_memory_generic_api.py @@ -31,6 +31,7 @@ from powercontext.builtin.persistence.statistics import StatisticsRepository from powercontext.builtin.persistence.tag_schema import ensure_topic_memory_tag_schema from powercontext.builtin.runtime import InferenceConfig, RuntimeConfig +from powercontext.builtin.runtime._model_usage import _ModelUsageRecorder from powercontext.server.factory import create_server_app from powercontext.server.settings import AccessControlConfig, BearerAuthConfig, McpConfig, ServerSettings @@ -107,8 +108,8 @@ def _topic_embedding_requests(database, scope=None): return connection.execute(query, parameters).fetchone()[0] -async def _await_usage_record(database, scope): - """Wait until the scope's usage row is visible. +async def _await_usage_record(database, scope, *, minimum_requests=1): + """Wait until at least the requested number of usage requests is visible. A visible row means the recorder's transaction committed, so it no longer holds SQLite's write lock. A later write that upgrades from a read does not @@ -118,7 +119,7 @@ async def _await_usage_record(database, scope): """ async with asyncio.timeout(5): - while _topic_embedding_requests(database, scope) == 0: # noqa: ASYNC110 - bounded observation of committed database state + while _topic_embedding_requests(database, scope) < minimum_requests: # noqa: ASYNC110 - bounded observation of committed database state await asyncio.sleep(0.02) @@ -555,26 +556,32 @@ async def fail_record(repository, connection, *args): assert not [record for record in caplog.records if record.levelno >= logging.ERROR] -def test_cancelling_a_request_during_a_stalled_usage_write_leaves_the_runtime_healthy(tmp_path, monkeypatch): - """Cancelling must propagate and must not wedge the runtime or its recorder. - - Usage no longer runs inside the request, so a cancelled request neither waits - for it nor rolls it back; what must hold is that cancellation propagates and - the runtime keeps accepting work afterwards. - """ +@pytest.mark.parametrize("record_fails", [False, True], ids=["normal", "storage-failure"]) +def test_cancelling_a_request_during_a_stalled_usage_write_leaves_the_runtime_healthy( + tmp_path, monkeypatch, record_fails +): + """Cancellation may lose a usage record, but later work must still be recorded.""" async def scenario(): release = asyncio.Event() entered = asyncio.Event() - original = StatisticsRepository.record + settled = asyncio.Event() + failed = asyncio.Event() + original_record = StatisticsRepository.record + original_settle = _ModelUsageRecorder._settle async def stalled_record(repository, connection, *args): entered.set() await release.wait() - return await original(repository, connection, *args) + if record_fails: + failed.set() + raise ValueError("injected closed usage connection") # noqa: TRY003 + return await original_record(repository, connection, *args) + + def observe_settlement(recorder, sequence): + original_settle(recorder, sequence) + settled.set() - # As above: the stalled write must be reached rather than dropped for - # spending its budget on a slow machine. app = _app(tmp_path, UsageEmbeddings(), model_usage_write_timeout_seconds=30.0) async with ( app.router.lifespan_context(app), @@ -583,19 +590,39 @@ async def stalled_record(repository, connection, *args): created_scope = await client.post( "/v1/scopes", json={"title": "Cancelled", "summary": "Cancelled", "idempotency_key": "cancelled"} ) + assert created_scope.status_code == 201, created_scope.text scope = created_scope.json()["scope_id"] path = f"/v1/scopes/{scope}/artifacts" payload = {"family": "topic-memory", "content": _content("cancelled")} with monkeypatch.context() as injected: injected.setattr(StatisticsRepository, "record", stalled_record) + injected.setattr(_ModelUsageRecorder, "_settle", observe_settlement) pending = asyncio.create_task(client.post(path, json=payload)) - await asyncio.wait_for(entered.wait(), 5) - pending.cancel() - with pytest.raises(asyncio.CancelledError): - await pending - release.set() - await _await_usage_record(tmp_path / "topics.db", scope) - assert (await client.post(path, json=payload)).status_code == 201 + try: + await asyncio.wait_for(entered.wait(), 5) + pending.cancel() + done, _ = await asyncio.wait({pending}, timeout=5) + assert pending in done, "cancelled request did not finish" + with pytest.raises(asyncio.CancelledError): + await pending + assert not settled.is_set(), "request cancellation stopped the independent recorder" + finally: + release.set() + if not pending.done(): + pending.cancel() + done, _ = await asyncio.wait({pending}, timeout=5) + assert pending in done, "request cleanup did not finish" + await asyncio.gather(pending, return_exceptions=True) + # Settlement follows transaction cleanup even when the usage row is dropped. + await asyncio.wait_for(settled.wait(), 5) + assert failed.is_set() == record_fails + previous_requests = _topic_embedding_requests(tmp_path / "topics.db", scope) + recovered = await client.post(path, json={"family": "topic-memory", "content": _content("recovered")}) + assert recovered.status_code == 201, recovered.text + saved = await client.get(recovered.headers["Location"]) + assert saved.status_code == 200, saved.text + assert saved.json()["content"] == _content("recovered") + await _await_usage_record(tmp_path / "topics.db", scope, minimum_requests=previous_requests + 1) asyncio.run(scenario()) From 39bf56e31f2a742f6435ac490c141f5d09c8b2e9 Mon Sep 17 00:00:00 2001 From: heyamei690-hash gua Date: Tue, 29 Sep 2026 16:04:22 +0800 Subject: [PATCH 14/37] fix(persistence): roll back an interrupted transaction instead of committing part of it Interrupting a SQLite statement rolls the whole native transaction back, so writes that already reported success are gone while SQLAlchemy still trusts the transaction. Keeping `is_disconnect = False` avoided discarding the connection -- an in-memory StaticPool holds its entire database in it -- but it also let a caller that caught the error keep writing and commit a partial result: insert row 3, interrupt a long UPDATE, insert row 4, and only row 4 survives. Record the interruption on the connection and refuse to commit when the transaction exits, raising PendingRollbackError the way the unmodified baseline does. The connection still returns to the pool, so an in-memory database survives. * `test_interrupted_write_cannot_commit_earlier_work[file|memory]` fails without the guard. Co-Authored-By: Claude Sonnet 4.6 --- .../builtin/persistence/database.py | 16 ++++-- .../builtin/persistence/sqlite/profile.py | 6 +++ .../persistence/test_sqlite_profile.py | 51 ++++++++++++++++++- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/src/powercontext/builtin/persistence/database.py b/src/powercontext/builtin/persistence/database.py index a9624b8bdb..9b08d56da1 100644 --- a/src/powercontext/builtin/persistence/database.py +++ b/src/powercontext/builtin/persistence/database.py @@ -23,7 +23,7 @@ from aiosqlite import Connection as SQLiteConnection from aiosqlite import Cursor as SQLiteCursor -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import OperationalError, PendingRollbackError from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine from powercontext.builtin.persistence.errors import DatabaseClosedError @@ -92,10 +92,20 @@ async def transaction(self) -> AsyncIterator[AsyncConnection]: sqlite_cursors = connection.info.get("_powercontext_sqlite_cursors") yield connection except BaseException as error: + if connection.dialect.name == "sqlite": + connection.info.pop("_powercontext_sqlite_interrupted", None) await _finish_transaction(context, connection, error, sqlite_stop, sqlite_cursors) raise - else: - await context.__aexit__(None, None, None) + if connection.dialect.name == "sqlite" and connection.info.pop("_powercontext_sqlite_interrupted", False): + # An interrupted statement rolled SQLite's whole native transaction + # back, so writes that already reported success are gone. Committing + # now would persist only the later ones; fail the transaction instead. + interrupted = PendingRollbackError( + "The transaction was interrupted, so its earlier writes were rolled back.", code="8s2b" + ) + await _finish_transaction(context, connection, interrupted, sqlite_stop, sqlite_cursors) + raise interrupted + await context.__aexit__(None, None, None) @asynccontextmanager async def _transaction(self) -> AsyncIterator[AsyncConnection]: diff --git a/src/powercontext/builtin/persistence/sqlite/profile.py b/src/powercontext/builtin/persistence/sqlite/profile.py index 0f59fc3914..a131843e92 100644 --- a/src/powercontext/builtin/persistence/sqlite/profile.py +++ b/src/powercontext/builtin/persistence/sqlite/profile.py @@ -152,6 +152,12 @@ def interrupt_cancelled_statement(context: ExceptionContext) -> None: if driver._running and driver._connection is not None: # Cancelling an await does not stop SQLite's native statement or disconnect its worker. driver._connection.interrupt() + # SQLite rolls the entire native transaction back when an INSERT, + # UPDATE or DELETE is interrupted, so writes that already reported + # success are gone while SQLAlchemy still trusts the transaction. + # Record it here; the transaction must not commit later work on top + # of that rollback. + connection.info["_powercontext_sqlite_interrupted"] = True context.is_disconnect = False diff --git a/tests/builtin/persistence/test_sqlite_profile.py b/tests/builtin/persistence/test_sqlite_profile.py index 250fed8440..fcd913f413 100644 --- a/tests/builtin/persistence/test_sqlite_profile.py +++ b/tests/builtin/persistence/test_sqlite_profile.py @@ -24,7 +24,7 @@ import pytest from pydantic import ValidationError from sqlalchemy import func, insert, select -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import IntegrityError, PendingRollbackError from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine from sqlalchemy.pool import QueuePool @@ -277,6 +277,55 @@ async def blocked_checkout() -> None: asyncio.run(scenario()) +@pytest.mark.parametrize("in_memory", [False, True], ids=["file", "memory"]) +def test_interrupted_write_cannot_commit_earlier_work(tmp_path, in_memory) -> None: + """An interrupted write rolls SQLite's whole native transaction back. + + SQLite discards the entire transaction when an INSERT, UPDATE or DELETE is + interrupted, so a caller that catches the error and keeps writing must not + be able to commit the later work on top of that rollback. + """ + + release = ThreadEvent() + + async def scenario() -> None: + url = "sqlite+aiosqlite:///:memory:" if in_memory else f"sqlite+aiosqlite:///{tmp_path / 'atomicity.db'}" + + def slow(value: int) -> int: + release.wait(10) + return value + + async with SQLiteProfile.open(SQLiteConfig(url=url), tables=()) as profile: + database = profile.database + async with database.transaction() as connection: + await connection.exec_driver_sql("CREATE TABLE probe (value INTEGER)") + + try: + async with database.transaction() as connection: + await connection.exec_driver_sql("INSERT INTO probe VALUES (3)") + driver = cast(aiosqlite.Connection, (await connection.get_raw_connection()).driver_connection) + await driver.create_function("slow", 1, slow) + try: + async with asyncio.timeout(0.5): + await connection.exec_driver_sql("UPDATE probe SET value = slow(value)") + except TimeoutError: + release.set() + await connection.exec_driver_sql("INSERT INTO probe VALUES (4)") + except PendingRollbackError: + pass + else: + pytest.fail("an interrupted transaction committed instead of failing") + + async with database.transaction() as connection: + rows = (await connection.exec_driver_sql("SELECT value FROM probe")).fetchall() + assert [row[0] for row in rows] == [] + + try: + asyncio.run(scenario()) + finally: + release.set() + + def test_sqlite_config_requires_the_async_dialect() -> None: with pytest.raises(ValidationError, match=r"sqlite\+aiosqlite"): SQLiteConfig(url="sqlite:///:memory:") From b246d2e8ca594b18970b91d4be11f94b8462fc20 Mon Sep 17 00:00:00 2001 From: heyamei690-hash gua Date: Tue, 29 Sep 2026 18:04:38 +0800 Subject: [PATCH 15/37] fix(persistence): scope the interrupted-transaction mark to its own transaction The mark lives on the pooled connection, and the usage recorder runs its own transaction without going through `AsyncDatabase.transaction()`, so an interrupt recorded there was never consumed. It survived on the connection until an unrelated transaction picked that connection up and raised PendingRollbackError for a rollback it had nothing to do with -- which is how this first showed up, as a topic-memory supervisor test failing on 3.11. Clear any leftover mark when a transaction starts, so only an interrupt raised inside it can fail it. * `test_interrupt_marked_outside_the_transaction_does_not_fail_it` fails without the cleanup. Co-Authored-By: Claude Sonnet 4.6 --- .../builtin/persistence/database.py | 6 ++++++ .../persistence/test_sqlite_profile.py | 21 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/powercontext/builtin/persistence/database.py b/src/powercontext/builtin/persistence/database.py index 9b08d56da1..3cddd2eaf3 100644 --- a/src/powercontext/builtin/persistence/database.py +++ b/src/powercontext/builtin/persistence/database.py @@ -86,6 +86,12 @@ async def transaction(self) -> AsyncIterator[AsyncConnection]: connection = await context.__aenter__() sqlite_stop = None sqlite_cursors: Collection[SQLiteCursor] | None = None + if connection.dialect.name == "sqlite": + sqlite_stop = connection.info.get("_powercontext_sqlite_stop") + sqlite_cursors = connection.info.get("_powercontext_sqlite_cursors") + # An interrupt recorded by a statement outside this transaction (the + # usage recorder owns its own) must not fail this one. + connection.info.pop("_powercontext_sqlite_interrupted", None) try: if connection.dialect.name == "sqlite": sqlite_stop = connection.info.get("_powercontext_sqlite_stop") diff --git a/tests/builtin/persistence/test_sqlite_profile.py b/tests/builtin/persistence/test_sqlite_profile.py index fcd913f413..3566b846c0 100644 --- a/tests/builtin/persistence/test_sqlite_profile.py +++ b/tests/builtin/persistence/test_sqlite_profile.py @@ -326,6 +326,27 @@ def slow(value: int) -> int: release.set() +def test_interrupt_marked_outside_the_transaction_does_not_fail_it(tmp_path) -> None: + """A mark left by a statement outside transaction() must not fail a later one. + + The usage recorder owns its own transaction and never passes through + ``AsyncDatabase.transaction()``, so a mark an interrupt left on that pooled + connection survives until another transaction picks the connection up. That + later transaction has nothing to do with the interrupt and must commit. + """ + + async def scenario() -> None: + config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'stale-mark.db'}") + async with SQLiteProfile.open(config, tables=()) as profile: + database = profile.database + async with database.engine.connect() as connection: + connection.info["_powercontext_sqlite_interrupted"] = True + async with database.transaction() as connection: + await connection.exec_driver_sql("SELECT 1") + + asyncio.run(scenario()) + + def test_sqlite_config_requires_the_async_dialect() -> None: with pytest.raises(ValidationError, match=r"sqlite\+aiosqlite"): SQLiteConfig(url="sqlite:///:memory:") From 76c9c7170588456aad704c047d222c5e7995d73c Mon Sep 17 00:00:00 2001 From: heyamei690-hash gua Date: Tue, 29 Sep 2026 18:39:09 +0800 Subject: [PATCH 16/37] fix(persistence): read transaction state without touching a dead connection Two follow-ups from review. `connection.info` re-resolves the DBAPI connection, so once a cancelled statement invalidated it through SQLAlchemy's normal path, reading the interruption mark from the failure path raised PendingRollbackError and skipped the rest of `_finish_transaction()`. That left the transaction counted as active and kept `database.close()` from finishing. Hold the info dict itself instead; it stays usable after the connection is gone. The mark was also only checked before COMMIT ran. A commit cancelled while in flight records the mark afterwards, so it survived on the pooled connection and failed the next borrower's transaction. Clear it in a `finally` around the exit, which covers cancellation during COMMIT or ROLLBACK too. * `test_invalidated_connection_still_reaches_transaction_cleanup` fails without the first part. Co-Authored-By: Claude Sonnet 4.6 --- .../builtin/persistence/database.py | 30 ++++++++++----- .../persistence/test_sqlite_profile.py | 37 +++++++++++++++++++ 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/src/powercontext/builtin/persistence/database.py b/src/powercontext/builtin/persistence/database.py index 3cddd2eaf3..762a7e913c 100644 --- a/src/powercontext/builtin/persistence/database.py +++ b/src/powercontext/builtin/persistence/database.py @@ -20,6 +20,7 @@ import time from collections.abc import AsyncIterator, Callable, Collection from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext +from typing import Any from aiosqlite import Connection as SQLiteConnection from aiosqlite import Cursor as SQLiteCursor @@ -86,23 +87,25 @@ async def transaction(self) -> AsyncIterator[AsyncConnection]: connection = await context.__aenter__() sqlite_stop = None sqlite_cursors: Collection[SQLiteCursor] | None = None + info: dict[str, Any] | None = None if connection.dialect.name == "sqlite": - sqlite_stop = connection.info.get("_powercontext_sqlite_stop") - sqlite_cursors = connection.info.get("_powercontext_sqlite_cursors") + # Hold the info dict itself. `connection.info` re-resolves the DBAPI + # connection and raises once it has been invalidated, and cleanup has + # to survive that. + info = connection.info + sqlite_stop = info.get("_powercontext_sqlite_stop") + sqlite_cursors = info.get("_powercontext_sqlite_cursors") # An interrupt recorded by a statement outside this transaction (the # usage recorder owns its own) must not fail this one. - connection.info.pop("_powercontext_sqlite_interrupted", None) + info.pop("_powercontext_sqlite_interrupted", None) try: - if connection.dialect.name == "sqlite": - sqlite_stop = connection.info.get("_powercontext_sqlite_stop") - sqlite_cursors = connection.info.get("_powercontext_sqlite_cursors") yield connection except BaseException as error: - if connection.dialect.name == "sqlite": - connection.info.pop("_powercontext_sqlite_interrupted", None) + if info is not None: + info.pop("_powercontext_sqlite_interrupted", None) await _finish_transaction(context, connection, error, sqlite_stop, sqlite_cursors) raise - if connection.dialect.name == "sqlite" and connection.info.pop("_powercontext_sqlite_interrupted", False): + if info is not None and info.pop("_powercontext_sqlite_interrupted", False): # An interrupted statement rolled SQLite's whole native transaction # back, so writes that already reported success are gone. Committing # now would persist only the later ones; fail the transaction instead. @@ -111,7 +114,14 @@ async def transaction(self) -> AsyncIterator[AsyncConnection]: ) await _finish_transaction(context, connection, interrupted, sqlite_stop, sqlite_cursors) raise interrupted - await context.__aexit__(None, None, None) + try: + await context.__aexit__(None, None, None) + finally: + # The commit itself can be cancelled, and its error handler records the + # mark after the check above; clear it so the next borrower of this + # pooled connection cannot inherit an interrupt that was not theirs. + if info is not None: + info.pop("_powercontext_sqlite_interrupted", None) @asynccontextmanager async def _transaction(self) -> AsyncIterator[AsyncConnection]: diff --git a/tests/builtin/persistence/test_sqlite_profile.py b/tests/builtin/persistence/test_sqlite_profile.py index 3566b846c0..97a7bbcbc3 100644 --- a/tests/builtin/persistence/test_sqlite_profile.py +++ b/tests/builtin/persistence/test_sqlite_profile.py @@ -28,6 +28,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine from sqlalchemy.pool import QueuePool +from powercontext.builtin.persistence.database import AsyncDatabase from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.sqlite import profile as sqlite_profile from powercontext.builtin.persistence.tables import ( @@ -347,6 +348,42 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_invalidated_connection_still_reaches_transaction_cleanup() -> None: + """Cleanup must not read ``connection.info`` directly. + + A cancelled statement can invalidate the connection through SQLAlchemy's + normal path, and from then on ``connection.info`` raises + PendingRollbackError. Reading it inside the failure path skipped the rest of + the cleanup and left the transaction counted as active, which also kept + database shutdown from finishing. + """ + + class Boom(Exception): + pass + + async def scenario() -> None: + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + database = AsyncDatabase.attach(engine) + try: + async with database.transaction() as connection: + await connection.exec_driver_sql("CREATE TABLE probe (value INTEGER)") + + async def operation() -> None: + async with database.transaction() as connection: + await connection.invalidate() + raise Boom + + with pytest.raises(Boom): + await operation() + + assert database._active_transactions == 0 + await asyncio.wait_for(database.close(), 5) + finally: + await engine.dispose() + + asyncio.run(scenario()) + + def test_sqlite_config_requires_the_async_dialect() -> None: with pytest.raises(ValidationError, match=r"sqlite\+aiosqlite"): SQLiteConfig(url="sqlite:///:memory:") From e0e765649bd89c4004a70c6dc8a7b54d51c9dcb7 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Wed, 30 Sep 2026 10:00:44 +0800 Subject: [PATCH 17/37] test: separate native lock waits from cancellation deadlines --- .../runtime/test_model_usage_recorder.py | 17 +++++++++++++---- tests/e2e/test_topic_memory_generic_api.py | 4 +++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/builtin/runtime/test_model_usage_recorder.py b/tests/builtin/runtime/test_model_usage_recorder.py index d92ff6d06c..4d928eb93e 100644 --- a/tests/builtin/runtime/test_model_usage_recorder.py +++ b/tests/builtin/runtime/test_model_usage_recorder.py @@ -178,18 +178,25 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_file_writer_lock_does_not_consume_busy_timeout(tmp_path: Path) -> None: +def test_file_writer_lock_does_not_consume_busy_timeout(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: async def scenario() -> None: config = SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'locked.db'}", busy_timeout_ms=5_000) async with _database(config) as database: - recorder = _ModelUsageRecorder(database, StatisticsRepository(), write_timeout_seconds=0.04) + recorder = _ModelUsageRecorder( + database, StatisticsRepository(), write_timeout_seconds=0.2, flush_timeout_seconds=2.0 + ) try: async with database.transaction() as connection: await connection.execute(update(SCOPES_TABLE).values(title="locked")) + # Exclude first-checkout setup from the lock-wait measurement. + async with database.engine.connect(): + pass start = asyncio.get_running_loop().time() _offer(recorder) await recorder.flush() - assert asyncio.get_running_loop().time() - start < 0.5 + # Allow scheduling overhead while still rejecting a full + # five-second native busy wait. Flush must actually settle. + assert asyncio.get_running_loop().time() - start < 3.0 assert await _rows(database) == () await _assert_connection_restored(database) _offer(recorder) @@ -198,7 +205,9 @@ async def scenario() -> None: finally: await recorder.close() - asyncio.run(scenario()) + with caplog.at_level(logging.WARNING): + asyncio.run(scenario()) + assert "Model usage flush timed out" not in caplog.text def test_writer_lock_released_inside_the_budget_still_records(tmp_path: Path) -> None: diff --git a/tests/e2e/test_topic_memory_generic_api.py b/tests/e2e/test_topic_memory_generic_api.py index 5a08f20ac1..8e22ac3a91 100644 --- a/tests/e2e/test_topic_memory_generic_api.py +++ b/tests/e2e/test_topic_memory_generic_api.py @@ -582,7 +582,9 @@ def observe_settlement(recorder, sequence): original_settle(recorder, sequence) settled.set() - app = _app(tmp_path, UsageEmbeddings(), model_usage_write_timeout_seconds=30.0) + # Cancelling an await does not end SQLite's native busy-handler wait. + # Keep that wait below the cancellation assertion's five-second bound. + app = _app(tmp_path, UsageEmbeddings(), busy_timeout_ms=100, model_usage_write_timeout_seconds=30.0) async with ( app.router.lifespan_context(app), httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="http://test") as client, From 4f99df144fed1009cc20140a549c6c82a42e1b77 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Wed, 30 Sep 2026 17:17:26 +0800 Subject: [PATCH 18/37] feat(db): align migration prototype with single-table RFC --- .github/workflows/master.yml | 3 +- docs/en/docs/operate/database-migrations.md | 288 +++---- docs/zh/docs/operate/database-migrations.md | 171 ++-- scripts/database_migration_probe.py | 12 +- .../builtin/persistence/migrations/backup.py | 767 ++++++++++++++++++ .../builtin/persistence/migrations/bundle.py | 45 +- .../persistence/migrations/deployment.py | 57 ++ .../builtin/persistence/migrations/models.py | 13 +- .../persistence/migrations/resources/env.py | 26 + .../migrations/resources/manifest.json | 42 + .../resources/schemas/pre_dream.json | 27 + .../migrations/resources/schemas/v1_1_0.json | 27 + .../resources/versions/p0001_baseline.py | 34 + .../resources/versions/p0002_citations.py | 27 + .../resources/versions/p0003_tag_families.py | 48 ++ .../builtin/persistence/migrations/sqlite.py | 589 +++++++++----- src/powercontext/server/cli.py | 2 + src/powercontext/server/database_migration.py | 303 +++++++ src/powercontext/service/adapters/base.py | 15 + src/powercontext/service/adapters/launchd.py | 13 + src/powercontext/service/adapters/systemd.py | 46 ++ src/powercontext/service/adapters/windows.py | 34 +- src/powercontext/service/cli.py | 36 +- src/powercontext/service/controller.py | 291 ++++++- src/powercontext/service/probe.py | 33 +- .../test_database_backup_providers.py | 366 +++++++++ .../persistence/test_database_migrations.py | 278 ++++++- tests/fixtures/database_migrations/README.md | 69 +- .../database_migrations/manifest.json | 17 + .../versions/p0003_tag_families.py | 8 + .../native/test_personal_service_lifecycle.py | 36 + tests/test_database_migration_cli.py | 290 +++++++ tests/test_service.py | 13 + tests/test_service_maintenance.py | 286 +++++++ 34 files changed, 3718 insertions(+), 594 deletions(-) create mode 100644 src/powercontext/builtin/persistence/migrations/backup.py create mode 100644 src/powercontext/builtin/persistence/migrations/deployment.py create mode 100644 src/powercontext/builtin/persistence/migrations/resources/env.py create mode 100644 src/powercontext/builtin/persistence/migrations/resources/manifest.json create mode 100644 src/powercontext/builtin/persistence/migrations/resources/schemas/pre_dream.json create mode 100644 src/powercontext/builtin/persistence/migrations/resources/schemas/v1_1_0.json create mode 100644 src/powercontext/builtin/persistence/migrations/resources/versions/p0001_baseline.py create mode 100644 src/powercontext/builtin/persistence/migrations/resources/versions/p0002_citations.py create mode 100644 src/powercontext/builtin/persistence/migrations/resources/versions/p0003_tag_families.py create mode 100644 src/powercontext/server/database_migration.py create mode 100644 tests/builtin/persistence/test_database_backup_providers.py create mode 100644 tests/test_database_migration_cli.py create mode 100644 tests/test_service_maintenance.py diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 25284210b2..321e818dd2 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -91,7 +91,8 @@ jobs: POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" run: >- uv run --locked --extra seekdb pytest -q - tests/builtin/persistence/test_database_migration_backends.py -k seekdb + tests/builtin/persistence/test_database_migration_backends.py + tests/builtin/persistence/test_database_backup_providers.py -k seekdb windows-unit-portability: timeout-minutes: 20 diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index 8b97e72e42..ea35548ed4 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -1,200 +1,164 @@ --- -title: Database migration design -description: Understand unified migration, shared-database upgrades, backup choices, service lifecycle, and compatibility checks. +title: Database migrations +description: Understand migration commands, backup choices, service lifecycle, and current acceptance boundaries. --- -# Database migration design +# Database migrations -PowerContext's unified migration design uses Alembic for schema versions and separates installing software from migrating -existing databases. This page explains the operational contract in the -[unified migration RFC](https://github.com/oceanbase/powercontext/pull/1771), tracked by +Unified migration uses Alembic and adds only `pc_schema_revision(version_num)` as its control table. Installing software +and migrating a database are separate operations. Managed schema changes ship as immutable revisions executed explicitly. +The design follows [RFC #1771](https://github.com/oceanbase/powercontext/pull/1771) and [issue #1756](https://github.com/oceanbase/powercontext/issues/1756). ## Current availability -**Unified migration is still a prototype. `server db-migrate`, `BackupProvider`, `service start/stop/restart`, and -`--manage-service` below are proposed, not released commands.** The prototype handles an isolated four-table fixture, -not business startup. Its `ready` is not complete Server readiness, and its execution records do not mean the final -single-table persistence design is implemented. +This branch provides `powercontext server db-migrate status/plan/apply/verify`. They read the same deployment settings as +the Server and do not accept arbitrary test-bundle paths. Frozen resources ship in the wheel for Phase A acceptance of a +four-table Artifact migration. Only persistent SQLite databases with registered schema shapes are supported. A complete +Server database containing unmanaged objects returns `unknown_baseline`; unified seekdb and OceanBase migration returns +`unsupported_backend`. -The current foreground entry point is `powercontext server run`; personal service commands provide -`install/status/uninstall`. For existing Artifact processing maintenance, follow -[Migrate Artifact processing state](artifact-processing-migration.md) using `server processing-migrate`. Do not substitute -the prototype for production migration. See [Deploy the Server](deploy-server.md) for service deployment. +`ready` means this migration bundle passed its schema and data checks. Output explicitly includes +`readiness_scope=registered_bundle` and `server_ready=false`: it does not establish complete Server, index, legacy-task, or +cluster readiness. Do not substitute it for a production upgrade or initialization of the complete business database. +Full startup gates and historical baselines require acceptance across all three backends before integration. -## Versions and upgrade boundaries +The existing `server processing-migrate` remains available. Follow +[Migrate Artifact processing state](artifact-processing-migration.md); it does not yet forward to the unified entry point. +See [Deploy the Server](deploy-server.md) for service deployment. -The final framework adds one migration control table, `pc_schema_revision(version_num)`, using standard Alembic version -handling, without generic run, step, or task ledgers. Each managed schema change adds an immutable revision executed in -dependency order. A release without schema changes needs no new script. Application versions, API versions, and schema -revisions are distinct. Data tasks, validators, and impact manifests ship with the package; logs and backup records stay -outside the target database. +## Read-only inspection and one confirmation -Installing/upgrading software does not migrate existing databases. Both local and remote databases require explicit -migration. Truly empty local databases may initialize under a lock where configuration permits; remote/shared-database -deployments use a separate initialization Job. An incompatible existing database blocks new business startup with -`migration_required`. Once enabled, all managed-schema PRs require migrations and verification. Framework development is -independent of PR #1716. +Inspect the target or preview a plan. These commands never create a missing database, parent directory, or control table: -## Maintenance with one confirmation +```bash +powercontext server db-migrate status --env-file deployment.env +powercontext server db-migrate plan --env-file deployment.env --backup auto +powercontext server db-migrate verify --env-file deployment.env +``` + +Plans bind target identity, source/target revisions, actual schema, frozen resources, configuration digests, backup policy, +and service scope. Diagnostics do not print configuration or credential values. Expected failures return stable JSON +categories and a nonzero exit status. -Interactive users can go directly to proposed `apply` without assembling individual steps. This illustrates the interface -and cannot yet be executed: +Within the supported acceptance scope, interactive users can invoke `apply` directly. PC-managed backup is the default. +The command displays the plan, backup method, maintenance impact, and stopped-write responsibility, then asks for one +confirmation before execution. It does not ask about each SQL statement: ```bash -powercontext server db-migrate apply --env-file deployment.env --manage-service +powercontext server db-migrate apply --env-file deployment.env --backup auto ``` -It combines target, source/target versions, affected tables/APIs, legacy tasks, downtime requirements, backup, and service -scope. Select on one screen and confirm once, followed by stopped writes, locked reinspection, backup policy, schema/data -conversion, index rebuilds, and final verification. No per-SQL confirmation is required. Read-only verification with no -persistent changes returns no changes without backup or service lifecycle actions. +That confirmation declares all other APIs, Workers, SDK clients, schedules, and automatic restarts stopped. Shared-database +operators also coordinate subsequent node upgrades. PC cannot discover arbitrary external clients. Material plan changes +block execution and require a new review. -Optional proposed read-only `status` and `plan` provide previews; material changes require renewed confirmation. -Automation explicitly supplies the plan and policy. For example, after manual backup and deployment-managed shutdown -of all writers: +Automation explicitly supplies the policy, plan, maintenance declaration, and policy-specific consent. After making a +manual backup and stopping every writer: ```bash powercontext server db-migrate plan --env-file deployment.env --backup manual -powercontext server db-migrate apply --env-file deployment.env --plan-id PLAN_ID --backup manual --backup-confirmed --maintenance-confirmed --yes +powercontext server db-migrate apply --env-file deployment.env --backup manual \ + --plan-id PLAN_ID --backup-confirmed --maintenance-confirmed --yes ``` -`--yes` accepts the plan, not a manual-backup declaration or no-backup risk. Plans bind target, versions, schema/resource -digests, compatibility, backup choice, and service scope. Missing required consent returns `confirmation_required`. -No backup choice bypasses locking, active writers, or data checks. +`--yes` accepts the plan; it does not declare a manual backup, accept no-backup risk, or confirm all nodes stopped writing. +`--shared-database` binds shared-database responsibilities to the plan and does not enable cross-host SQLite deployment. +When the bundle is already verified and no persistent change is needed, `apply` returns `changed=false` without backup or +service lifecycle actions. Switching the application version remains a separate deployment operation. -## Multiple Servers sharing one database +## Backup choices -Multi-node means one shared database; this design does not cover independent per-node databases. Run one migration Job -per database. +| Policy | Behavior | Options | +| --- | --- | --- | +| PC-managed | Use a supported native method, await completion and checks; warn that large data may need considerable time and free space | `--backup auto` | +| Manually backed up | Record only the declaration without checking files, jobs, time, target, or recoverability | `--backup manual --backup-confirmed`, optionally `--backup-ref BACKUP_ID` | +| No backup | Explicitly accept that failure may make original data unrecoverable | `--backup skip --accept-no-backup` | + +Automatic backup reports `completed`, manual reports `user_confirmed`, skip reports `skipped`, and genuinely empty +initialization reports `not_required`. Unsupported, failed, or incomplete auto backup stops before migration writes. +It returns `backup_unsupported` or a backup failure category and never silently skips. Changing policy requires a new plan +confirmation. + +A separate `BackupProvider` reuses backend settings and maintenance connections. SQLite uses Online Backup API to create +an independent file, including committed WAL, and checks integrity. The seekdb/OceanBase cluster design prefers native +`FORK DATABASE`. `FORK TABLE` is eligible only when every affected object and dependency, cross-table consistency, and +restoration path have passed adapter acceptance. PC does not automatically replace unavailable remote Fork with physical +backup. + +| Product | First `FORK TABLE` version | First `FORK DATABASE` version | +| --- | --- | --- | +| OceanBase AI Database | V4.6.2 | V4.6.2 | +| seekdb | V1.1.0 | V1.2.0 | -1. Prepare the new program separately. Stop new requests and task production, then drain in-flight work and tasks that - require the old handler according to the plan. -2. Stop all API, Worker, scheduler, and SDK writers. Suspend scaling, automatic restarts, and schedules that could launch - old programs. -3. One new-version Job runs `apply`, takes the database-wide lock, rechecks the plan and legacy tasks, then handles backup - and migration. -4. Start new nodes after migration returns `ready`; each checks schema, task formats, and required capabilities before - passing readiness and accepting traffic. -5. Migration failure keeps maintenance active. If the database is ready but a node cannot start, handle startup separately - without rerunning migration or automatically downgrading. +Version eligibility is only the start: capability checks also consider actual product, tenant mode, permissions, objects, +subsequent DDL limits, and restoration. Several table Forks do not automatically constitute one database snapshot. Fork +shares underlying storage and does not protect against disk failure. Remote unified migration has not completed full +acceptance; an implemented provider is not a production-upgrade guarantee. -Migration locks exclude other migrators, not arbitrary external clients. `--maintenance-confirmed` is an operator -statement that writes are stopped. Reject execution when those conditions cannot be established. PC upgrades do not -restart the OceanBase database cluster. SQLite/embedded seekDB retain the existing single-host `all` role restriction; migration does not enable cross-host shared directories or multi-node deployment for them. Shared-database multi-node services use an already supported remote deployment topology. +Retain old business tables, Fork recovery points, and SQLite backups during this upgrade. Migration success, normal +startup, and retry do not delete them. A later independent revision removes old tables; explicit maintenance removes +recovery points after checking dependencies, retention windows, and recovery obligations. -## Backup choices and native methods +## Stopping, starting, and upgrading services -| Choice | Behavior | Proposed options | -| --- | --- | --- | -| PC-managed | Recommend a supported native method, warn that large data may require considerable time/space, await completion and adapter checks | `--backup auto` | -| Already backed up manually | Record only the declaration, without checking files, jobs, time, target, or recoverability; reference optional | `--backup manual --backup-confirmed`, optionally `--backup-ref BACKUP_ID` | -| No backup | Warn that original data may be unrecoverable after failure; require explicit acceptance | `--backup skip --accept-no-backup` | +Personal services provide these commands while retaining registration and settings: -Manual mode reports `user_confirmed`, not `verified`; skip reports `skipped`; genuinely empty initialization reports -`not_required`. Failed, incomplete, or insufficiently supported automatic backup stops before migration writes. Changing -policy requires renewed confirmation, never automatic skipping. +```bash +powercontext service stop +powercontext service start +powercontext service restart +``` -A separate `BackupProvider` is a database maintenance capability implemented by backend adapters, reusing Profile -configuration, identity, and engine lifecycle. `capabilities(context)` reports support, `create_backup(context)` starts -backup, `inspect_backup(ref)` inspects PC-managed backups, and `restore_plan(ref)` provides explicit recovery steps. -Manual and skip paths do not call it to verify user backups. +They manage only the current user's PC-owned local service, not a cluster or other clients. `stop` suppresses automatic +restart during maintenance; `start` restores startup. One `restart` cannot replace stopped-write migration. -| Backend | Recommended method | Limitations | -| --- | --- | --- | -| SQLite | After stopping writes, use SQLite Online Backup API to create an independent file, checking opening and integrity | Include committed WAL; copying only the main file is insufficient. Restoration also checks extensions, indexes, and business data | -| seekDB | Version/object/recovery-validated `FORK DATABASE` for a same-instance pre-migration recovery point | Not multiple per-table forks by default; shared storage provides no disk-failure protection | -| OceanBase | Native physical backup and log archiving through configured administration or backup services, querying completion | Check jobs and log coverage; tenant restoration may affect other applications, so never default to overwriting a shared tenant | - -Native references: [SQLite Backup API](https://sqlite.org/backup.html), -[OceanBase backup architecture](https://en.oceanbase.com/docs/common-oceanbase-database-10000000001168918). - -seekDB V1.2.0 describes a common database snapshot, excluding some non-table objects and cross-database foreign keys. -Fork uses copy-on-write and shared underlying storage. After acceptance, it may cover failed migrations while storage -and engine remain healthy without an additional physical backup; users arrange independent disaster recovery as needed. -Multiple `FORK TABLE` operations are not the default substitute for database-wide consistency. -[seekDB V1.2.0](https://github.com/oceanbase/seekdb/releases/tag/v1.2.0), -[Fork mechanism](https://en.oceanbase.com/blog/fork-table-ready-for-agents). - -Before enabling auto fork, verify the actual embedded version, index/constraint coverage, subsequent source DDL, engine -restart, and full restoration. `SeekDBConfig.database` currently fixes the database to `test`, so changing the connection -to a fork database cannot be promised. Implement restoration to the original name or separately provide configurable -targets and coordinated switching. `MERGE TABLE` does not automatically roll back schema. Unsupported coverage/recovery -returns `backup_unsupported`; users may select manual or skip. Do not copy a running directory or automatically upgrade -the database engine. - -## Service shutdown and restart - -An incompatible upgrade follows: stop old writers → backup policy → migration → verification → start new version → -readiness → restore traffic. Installing software is separate from replacing processes. Preserve an independently -runnable old environment when deferring the switch. - -Proposed `service stop/start/restart` manages only the current user's PC-owned local service, retaining registration and -configuration. `--manage-service` displays and confirms the target executable, configuration, and original running state, -then drains/stops it and suppresses restarts. Only successful migration permits updating the confirmed launch definition -and restarting a previously running service. Previously stopped services remain stopped; failure does not restart old -services. Do not guess versions from PATH or use potentially autostarting `service install` during maintenance. - -Deployment systems manage clusters and external services. For manual maintenance, first use proposed -`powercontext service stop`, run `apply` successfully, ensure the launch definition points at the new environment, then -use `powercontext service start`. `restart` does not replace migration while stopped. For foreground deployment, after -migration succeeds and the old process exits, use the existing command from the new environment: +The complete managed-upgrade design drains requests, stops the service and automatic restarts, takes the migration lock, +reinspects, applies backup policy, migrates, verifies, switches the launch definition to the accepted new executable, and +starts a previously running service with readiness checks. Migration failure keeps it stopped; a previously stopped +service stays stopped. Database verification and service startup are reported separately. Resolve startup failures without +rerunning a successful migration. + +`--manage-service` always covers only the accepted local service. It cannot replace `--maintenance-confirmed` for other +writers and shared nodes. The Phase A partial bundle does not establish complete Server readiness: when persistent changes +are needed, `--manage-service` returns `service_unsupported` rather than switching a business service. + +For manual deployment, stop the old service first. After a supported complete migration succeeds, ensure the launch +definition points to the new environment before starting it. For foreground deployment, exit the old process and use the +existing command from the new environment: ```bash powercontext server run --env-file deployment.env --role all ``` -## Release decisions for tables, APIs, and legacy tasks - -Authors ship impact manifests; operators choose deployment methods from this evidence, not merely from “changed a table” -or “internal API” labels: - -| Declaration | Operational decision | -| --- | --- | -| `affected_objects` | Tables, constraints, indexes, files, large copies, resources, and irreversible transformations | -| `execution_mode` and binary/schema support | Ordinary deployment without persistent changes; stopped-write maintenance for incompatibility; online migration requires implemented and verified backend/mixed-version support | -| `api_changes` | Callers released together or independently, retained/deprecated APIs, earliest removal version/date | -| `task_formats` | Queued, running, delayed, retry, and dead-letter formats; compatible consumption, idempotent conversion, or draining | - -Operators may choose stricter downtime, but cannot select online mode to override known incompatibility. Generic online -DDL is not initially promised. Same-package internal entry points can be replaced with callers; independently deployed -Workers/SDKs still need compatibility arrangements. Breaking public API replacements, such as Artifact APIs, mark old -contracts `deprecated` and specify alternatives/removal. Storage-only changes with unchanged API contracts need no -deprecation. Retained APIs can translate requests/responses to the current implementation without every handler supporting -two table layouts. - -**Check legacy task formats during upgrades.** Read-only `plan` inventories queues, Leases, and payload formats, rechecking -under the lock after writes stop, after migration, and before Worker startup. Use frozen recognizers where version fields -are absent. Unknown/unobservable sources block affected migration/Workers rather than falsely reporting no tasks. -Unknown formats return `unsupported_task_format`; incomplete draining returns `legacy_tasks_pending`. Stop producers, -let old Workers drain within bounds, then stop Workers before DDL; alternatively use declared compatible consumption or -idempotent conversion. Preserve task IDs, idempotency keys, Scope/identity, retries, and domain receipts. Never delete -queues or mark unfinished work successful. Resolve retained tasks before deleting handlers; release plans coordinate -external API consumers. - -## Combining existing storage layers - -| Existing layer | Upgrade responsibility | -| --- | --- | -| Profile | Reuse connection configuration/engine lifecycle; split inspection/maintenance from `create_tables` side effects | -| `AsyncDatabase` | Dedicated maintenance connection and transactions; `AsyncConnection.run_sync` supplies the same connection to Alembic, keeping locks and DDL together | -| Alembic | Frozen revisions execute DDL and advance the sole version table after verification | -| Repository | Reuse stable domain operations once schema permits; historical conversions prefer frozen SQL/mappings over changing current repositories | -| Indexes | Structure definitions enter versioned resources; controlled projection rebuilding and checks replace implicit startup schema writes | -| Startup gates | Check schema, tasks, and required capabilities before composing repositories, indexes, and Workers | - -For a required column, a revision first adds it as nullable, an idempotent script backfills in batches, verification permits -a later constraint-tightening revision, and affected indexes rebuild before new repositories start. seekDB/OceanBase DDL -may commit implicitly; one Python transaction cannot make the full upgrade atomic. Commit existing domain receipts with -their data batches and rebuild indexes from authoritative data. - -## Interruption and recovery - -After interruption, inspect through read-only `status`, `verify`, and `plan`; retry only when actual state proves it safe. -One version table offers no generic run-ID resume. Ambiguous state returns `recovery_required`. Auto retains the original -recovery point and external records rather than replacing them with a partial-state backup; manual remains unverified; -skip cannot promise original-data recovery. Every mode checks actual schema, data, and tasks without blind stamping. - -Restoration is independent and explicit with writes stopped. Restore the database, required files, and matching program, -then verify. Application rollback is not database rollback. Report migration `ready` separately from startup success; -do not immediately delete recovery points. Production support remains subject to the RFC's real-backend acceptance. +## Shared databases, APIs, and legacy tasks + +For multiple Servers sharing one database, run one migration Job per database. Deployment tooling stops traffic and task +production, drains requests and tasks requiring old handlers, stops all writers, and suspends scaling and automatic +restarts. Coordinate subsequent node upgrades; each node passes its schema, task, and capability checks before traffic +returns. A migration lock excludes other migrators, not unknown external clients. + +Release authors provide affected objects, execution mode, binary/schema compatibility, API changes, and task formats. +Breaking public API replacements require deprecation, alternatives, and removal plans. Same-package internal entry points +can change together with callers. Retained API adapters can translate requests/responses into one current implementation +without every handler supporting two table layouts. + +Legacy tasks require inspection in the read-only plan, after stopped-write locking, after migration, and before Worker +startup. Unknown formats or unobservable sources must not be reported as an empty queue. Use declared compatible +consumption, idempotent conversion, or bounded draining. Preserve task IDs, idempotency keys, Scope, retries, and domain +receipts. The Phase A bundle does not yet cover these complete business checks. + +## Storage layers and recovery boundaries + +Maintenance reuses configuration and keeps Alembic and locking on a dedicated connection. Frozen historical SQL does not +depend on current Repositories or application metadata. Repositories run after schema checks; versioned resources and +controlled rebuilds manage full-text/vector indexes. seekdb/OceanBase DDL may commit implicitly, so one Python transaction +cannot make the entire upgrade atomic. + +One version table offers no generic run-ID resume. After interruption, inspect actual schema/data through read-only +commands. Ambiguous state returns `recovery_required`; never blindly stamp or replace the original recovery point with a +partial-state backup. External summaries record execution and backup information. + +Restoration is separate, explicit, and performed with writes stopped. Restore the database, required files, and matching +program before verification. Application rollback is not database rollback. Manual mode does not claim verified recovery; +skip cannot promise recovery of original data. diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index f6d7f4867e..e423512741 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -1,165 +1,106 @@ --- -title: 数据库迁移设计 -description: 了解统一迁移的设计、共享数据库升级、备份选择、服务启停与兼容性检查。 +title: 数据库迁移 +description: 了解数据库迁移命令、备份策略、服务启停及当前验收范围。 --- -# 数据库迁移设计 +# 数据库迁移 -PowerContext 的统一数据库迁移方案使用 Alembic 管理结构版本，安装新版与迁移已有数据库分开。本文介绍 -[统一迁移 RFC](https://github.com/oceanbase/powercontext/pull/1771) 的运维契约，关联 -[issue #1756](https://github.com/oceanbase/powercontext/issues/1756)。 +统一迁移使用 Alembic 管理版本，只新增 `pc_schema_revision(version_num)` 一张控制表。安装或更新程序与迁移数据库分开；结构变化由随包发布、不可改写的 revision 显式执行。 +设计关联 [RFC #1771](https://github.com/oceanbase/powercontext/pull/1771) 和 [issue #1756](https://github.com/oceanbase/powercontext/issues/1756)。 ## 当前可用范围 -**统一迁移仍处于原型阶段。本文中的 `server db-migrate`、`BackupProvider`、`service start/stop/restart` -及 `--manage-service` 是拟议设计，尚不能作为已发布命令使用。** 原型只验证隔离的四表测试库，不接入业务启动， -其 `ready` 不代表完整 Server 可用；原型中的执行记录也不代表最终单表持久化设计已落地。 +本分支提供 `powercontext server db-migrate status/plan/apply/verify`，读取与 Server 相同的部署配置，不接受任意测试 bundle 路径。随 wheel 发布的冻结资源目前用于阶段 A 的 Artifact 四表迁移验收；只有已登记结构的持久 SQLite 数据库可以执行。完整 Server 数据库含有尚未纳管的对象时返回 `unknown_baseline`，seekdb 和 OceanBase 统一迁移返回 `unsupported_backend`。 -当前前台启动入口是 `powercontext server run`，个人服务命令提供 `install/status/uninstall`。 -已有 Artifact 处理状态的维护请使用[迁移 Artifact 处理状态](artifact-processing-migration.md)中的 -`server processing-migrate`，不要用原型替代生产迁移。服务部署参考[部署 Server](deploy-server.md)。 +输出中的 `ready` 仅表示当前迁移 bundle 已通过结构和数据验证；`readiness_scope=registered_bundle`、`server_ready=false` 明确区分它与完整 Server、索引、旧任务及集群可用。不要以此替代生产升级或完整业务库初始化。普通业务启动的全面版本门禁和完整历史基线仍须在三后端验收后接入。 -## 版本与更新边界 +原有 `server processing-migrate` 保留，请按[迁移 Artifact 处理状态](artifact-processing-migration.md)使用。它尚未自动转发到统一入口。服务部署参考[部署 Server](deploy-server.md)。 -最终框架只新增 `pc_schema_revision(version_num)` 一张迁移控制表，沿用 Alembic 的版本读写；不新增通用执行、 -步骤、任务台账。每次受管结构变更新增独立且不可改写的 revision，沿依赖链执行；没有结构变化的发布无需新脚本。 -业务版本、API 版本和 schema revision 分开管理。数据任务、验证器与影响声明随包发布，日志和备份记录保存在库外。 +## 只读检查与一次确认 -安装或升级软件包不迁移已有库；本地库、远端库均通过显式命令迁移。真正空白的本地库允许在配置许可且持锁确认后 -初始化；远端库和多节点共享库使用单独初始化任务。已有库不兼容时新版拒绝业务启动，提示 `migration_required`。 -框架启用后，所有受管结构变更 PR 都必须提交 migration 和验证；框架与 PR #1716 独立推进。 +先检查目标或预览计划；这些命令不会创建缺失数据库、父目录或控制表： -## 一次确认完成维护 +```bash +powercontext server db-migrate status --env-file deployment.env +powercontext server db-migrate plan --env-file deployment.env --backup auto +powercontext server db-migrate verify --env-file deployment.env +``` + +计划绑定目标身份、源／目标 revision、实际结构摘要、冻结脚本摘要、配置摘要、备份策略和服务范围。配置与凭据值不输出到诊断中；预期错误返回稳定 JSON 类别和非零退出码。 -交互用户可以直接执行拟议的 `apply`，不用先手工拼装多个迁移步骤。以下示例供理解接口，当前不可直接执行： +在支持范围内，终端用户可直接执行 `apply`，默认 PC 自动备份。命令展示计划、备份方法、维护影响及停写责任，确认一次后执行，不逐条 SQL 询问： ```bash -powercontext server db-migrate apply --env-file deployment.env --manage-service +powercontext server db-migrate apply --env-file deployment.env --backup auto ``` -它汇总目标库、来源／目标版本、受影响的表和接口、旧任务、停机要求、备份与启停范围。用户在同一页面选择后确认一次， -随后执行停写、取锁复核、备份策略、结构与数据转换、索引重建和最终验证；不逐条 SQL 确认。 -只有只读检查且没有持久化变更时，返回“无需变更”，不备份、不启停服务。 +这次确认同时声明：所有其他 API、Worker、SDK、调度和自动重启入口均已停写；共享库场景还由操作者协调后续节点升级。PC 无法发现任意外部客户端。计划发生实质变化时拒绝执行，需要重新审阅。 -希望提前审阅时使用拟议的只读 `status`、`plan`；出现实质变化时需要重新确认。自动化场景明确提供计划与策略， -例如用户自行备份、部署系统已停止全部写入者后： +自动化必须显式指定策略和计划，并分别声明维护及备份选择。例如用户自行完成备份并停止全部写入者后： ```bash powercontext server db-migrate plan --env-file deployment.env --backup manual -powercontext server db-migrate apply --env-file deployment.env --plan-id PLAN_ID --backup manual --backup-confirmed --maintenance-confirmed --yes +powercontext server db-migrate apply --env-file deployment.env --backup manual \ + --plan-id PLAN_ID --backup-confirmed --maintenance-confirmed --yes ``` -`--yes` 仅接受该计划，不等于手动备份声明或接受无备份风险。计划绑定数据库、版本、结构和脚本摘要、兼容性、 -备份选择与启停范围；缺失所需确认返回 `confirmation_required`。任何备份选择都不能绕过锁、活跃写入者或数据检查。 +`--yes` 仅接受计划，不隐含手动备份声明、无备份风险或所有节点已停写。`--shared-database` 将共享库维护责任加入计划，不使 SQLite 获得跨主机部署能力。数据库已达到 bundle 目标且无需持久化变更时，`apply` 返回 `changed=false`，不备份、不启停服务；程序版本切换仍须单独安排。 -## 多个 Server 共用一个数据库 +## 备份选择 -本方案的多节点指共享一个数据库；不设计每个节点拥有独立数据库的升级。每个数据库只运行一个迁移 Job。 - -1. 在独立环境准备新版程序；禁用新请求和任务生产，按计划排空在途操作及必须由旧处理器消费的任务。 -2. 停止全部 API、Worker、调度器和 SDK 写入者，暂停自动扩容、自动重启及会拉起旧程序的调度。 -3. 一个新版迁移 Job 执行 `apply`，取得数据库级锁，复核计划和旧任务后处理备份并迁移。 -4. 迁移返回 `ready` 后启动新版节点，各节点仍须检查 schema、任务格式及必要能力，通过 readiness 才恢复流量。 -5. 迁移失败保持维护；数据库已就绪但某个节点启动失败时，单独处理启动问题，不重跑迁移或自动降级。 +| 策略 | 行为 | 参数 | +| --- | --- | --- | +| PC 自动备份 | 使用支持的数据库原生方法，等待完成并检查；提示大数据量可能耗时较久、占用更多空间 | `--backup auto` | +| 用户已手动备份 | 仅记录声明，不核验文件、任务、时间、目标或可恢复性 | `--backup manual --backup-confirmed`，可加 `--backup-ref BACKUP_ID` | +| 不备份 | 明确接受失败后可能无法恢复原数据的风险 | `--backup skip --accept-no-backup` | -迁移锁只排斥其他迁移进程，不能自动阻止任意外部客户端写入。`--maintenance-confirmed` 是运维停写声明。 -无法建立停写条件时拒绝执行。PC 升级不重启 OceanBase 数据库集群。 SQLite／嵌入式 seekDB 仍受现有单机 `all` 角色限制，迁移设计不使其获得跨主机共享目录或多节点部署能力；共享数据库的多节点服务使用已有支持的远端部署方式。 +自动备份报告 `completed`，手动备份报告 `user_confirmed`，跳过报告 `skipped`；真正空库初始化报告 `not_required`。自动备份不支持、失败或未完成时停在迁移写入之前，返回 `backup_unsupported` 或备份失败类别，不能自动改成跳过。选择其他策略后需要重新确认计划。 -## 备份选择与原生方法 +独立 `BackupProvider` 复用后端配置和维护连接。SQLite 使用 Online Backup API 创建独立文件，包含已提交 WAL，并检查文件完整性。seekdb 和 OceanBase 集群的设计采用原生 `FORK DATABASE`，在所有受影响表、依赖和跨表一致性均能覆盖且恢复路径经过验收时可选 `FORK TABLE`。PC 不将不支持 Fork 的远端引擎自动切换到物理备份。 -| 选择 | 行为 | 拟议参数 | +| 产品 | `FORK TABLE` 起始版本 | `FORK DATABASE` 起始版本 | | --- | --- | --- | -| PC 备份 | 推荐支持的原生方法，提示大数据量可能耗时较久、占用更多空间，等待完成与适配器检查 | `--backup auto` | -| 已手动备份 | 仅记录用户声明，不检查文件、备份任务、时间、目标或可恢复性；引用可选 | `--backup manual --backup-confirmed`，可加 `--backup-ref BACKUP_ID` | -| 不备份 | 提示失败后可能无法恢复原数据，明确接受风险才继续 | `--backup skip --accept-no-backup` | - -手动模式显示 `user_confirmed`，不显示 `verified`；跳过显示 `skipped`；真正空库显示 `not_required`。 -PC 自动备份失败、未完成或不支持所需对象时，迁移停在写入之前。改变策略需重新确认，不能自动跳过。 +| OceanBase AI 数据库 | V4.6.2 | V4.6.2 | +| seekdb | V1.1.0 | V1.2.0 | -独立 `BackupProvider` 属于数据库维护能力，复用 Profile 的配置、目标身份和引擎生命周期，由后端适配器实现： -`capabilities(context)` 报告能力，`create_backup(context)` 发起备份，`inspect_backup(ref)` 查询 PC 备份， -`restore_plan(ref)` 提供显式恢复步骤。手动和跳过分支不调用它核验用户备份。 +版本满足只是能力检查的起点，还需检查实际产品、租户模式、权限、对象覆盖、后续 DDL 限制和恢复路径。多个表 Fork 不自动代表整库共同快照；Fork 共享底层存储，不提供磁盘损坏保护。当前远端统一迁移尚未通过完整验收，存在 provider 不等于可用生产升级。 -| 后端 | 推荐方法 | 限制 | -| --- | --- | --- | -| SQLite | 停写后使用 SQLite Online Backup API 创建独立数据库文件，检查可打开及完整性 | 包含已提交 WAL；不能只复制主文件；恢复验证还包含扩展、索引和业务数据 | -| seekDB | 经过版本、对象覆盖和恢复验收的 `FORK DATABASE`，保存同实例迁移前恢复点 | 不默认逐表 fork；同实例共享存储，不提供磁盘损坏保护 | -| OceanBase | 数据库原生物理备份和日志归档，通过配置好的管理接口或备份平台发起并查询完成 | 检查任务和日志覆盖；租户级恢复可能影响其他应用，不能默认覆盖整个共享租户 | +旧业务表、Fork 恢复点和 SQLite 备份本次保留，不在迁移成功、普通启动或重试时删除。旧表删除由后续版本单独 revision 实现；恢复点由显式维护操作清理，均需验证依赖、保留窗口及恢复责任。 -原生能力参考：[SQLite Backup API](https://sqlite.org/backup.html)、 -[OceanBase 备份架构](https://en.oceanbase.com/docs/common-oceanbase-database-10000000001168918)。 +## 服务停止、启动与升级 -seekDB V1.2.0 发布说明提供整库共同快照能力，部分非表对象及跨库外键不复制。Fork 使用写时复制并共享底层存储， -因此经验证后可以用于引擎／存储健康情况下的迁移失败恢复，不要求同时另做物理备份；独立灾备由用户按需安排。 -逐表 `FORK TABLE` 不能默认替代整库一致性保护。 -[seekDB V1.2.0](https://github.com/oceanbase/seekdb/releases/tag/v1.2.0)、 -[Fork 机制](https://en.oceanbase.com/blog/fork-table-ready-for-agents)。 +本机个人服务提供以下命令，保留注册和配置： -自动 fork 启用前必须验收实际嵌入式版本、索引／约束覆盖、源库后续 DDL、引擎重启和完整恢复。 -当前 `SeekDBConfig.database` 固定为 `test`，不能直接承诺修改连接指向 fork 库；需要实现恢复到原库名， -或另行提供可配置目标及统一切换。`MERGE TABLE` 不用于自动回滚 schema。对象覆盖或恢复路径不满足时返回 -`backup_unsupported`，可重新选择手动或跳过，不复制运行中的目录，也不自动升级数据库引擎。 +```bash +powercontext service stop +powercontext service start +powercontext service restart +``` -## 服务启停与重启 +这些命令只管理当前用户归 PC 所有的服务，不管理集群或其他客户端。`stop` 禁止维护期间自动拉起旧服务；`start` 恢复启动。涉及迁移时不能用一次 `restart` 替代停写维护。 -不兼容升级采用：停止旧写入者 → 备份策略 → 迁移 → 验证 → 启动新版 → readiness → 恢复流量。 -安装新版不等于切换进程；延后切换时保留可独立运行的旧环境。 +完整升级的托管设计为：排空请求、停服并禁止自动重启 → 取迁移锁并复核 → 按策略备份 → 迁移 → 验证 → 将启动定义切换到已确认新版程序 → 启动原本运行的服务 → readiness。迁移失败保持停服；原本停止的服务保持停止。数据库验证成功而服务启动失败时分别报告，处理启动问题而不重跑迁移。 -拟议的 `service stop/start/restart` 只管理当前用户归 PC 所有的本机服务，并保留注册和配置。 -`--manage-service` 显示并确认目标可执行程序、配置及原运行状态，自动排空停服、抑制重启；只在迁移成功后 -更新已确认的启动定义并恢复原本在运行的服务。原本停止的服务保持停止，失败不自动拉起旧服务。 -不按 PATH 猜新版本，不使用可能立即启动服务的 `service install` 作为维护中间步骤。 +`--manage-service` 的范围始终只是已确认的本机服务，不能代替 `--maintenance-confirmed` 对其他写入者和共享库节点的声明。阶段 A 的有限 bundle 未完成完整 Server readiness 验收；需要持久化变更时，`--manage-service` 返回 `service_unsupported`，不能据此自动切换生产服务。 -集群和外部服务由部署系统启停。手工维护时，拟议命令是先 `powercontext service stop`，执行 `apply` 成功后， -确认服务定义已指向新版，再运行 `powercontext service start`。`restart` 不能替代停服期间的数据库迁移。 -前台服务在迁移成功且旧进程退出后，通过现有命令从新版环境启动： +手动管理时，先停旧服务，在已支持的完整迁移成功后确认服务启动定义指向新版，再启动。前台部署则退出旧进程，使用新版环境的现有入口： ```bash powercontext server run --env-file deployment.env --role all ``` -## 表、接口与旧任务的发布判断 - -变更作者提供随包影响清单，操作者据此选择部署方式，不能仅靠“改了表”或“内部接口”猜测： - -| 声明 | 运维判断 | -| --- | --- | -| `affected_objects` | 表、约束、索引和外部文件范围，大表复制、资源与不可逆转换 | -| `execution_mode` 与程序／schema 支持范围 | 无持久化变更可普通发布；不兼容变更默认停写维护；在线迁移必须有已实现且验证过的后端与混合版本支持 | -| `api_changes` | 是否内部同步发布，是否独立调用方，旧接口保留／弃用及最早移除版本或日期 | -| `task_formats` | 待执行、运行中、延迟、重试和死信任务的版本，兼容消费／幂等转换／排空方式 | - -操作者可选择更保守的停机方式，不能选择“在线”来绕过已知不兼容。首期不承诺通用在线 DDL。 -内部同包入口可随调用方一起替换；独立部署的 Worker／SDK 即使称为内部也需要兼容安排。 -Artifact 等公共 API 不兼容替换时标记 `deprecated`，说明替代入口和移除计划；仅表变化而 API 契约不变无需弃用。 -保留的旧 API 可转换请求／响应后共用当前实现，不要求每个 handler 同时兼容新旧表。 +## 共享数据库、接口与旧任务 -**升级检查旧格式任务。** `plan` 只读统计相关队列、Lease 和 payload 格式，停写取锁后、迁移后及 Worker 启动前复查。 -没有格式版本字段时用冻结识别器；未知或不可观察的来源阻止相关迁移／Worker，不假报无任务。 -未知格式返回 `unsupported_task_format`，未排空返回 `legacy_tasks_pending`。先停生产，再让旧 Worker 有边界地排空， -最后停 Worker 才执行 DDL；或按计划兼容消费／幂等转换。保留任务 ID、幂等键、Scope／身份、重试和领域收据， -不删队列或把未完成任务标成功。移除旧处理器前必须处理遗留任务；外部 API 调用方升级由发布计划协调。 +多个 Server 共用一个数据库时，每库只执行一个迁移 Job。部署系统停止流量和任务生产、排空在途请求及需由旧处理器消费的任务、停止全部写入者，并暂停扩容和自动重启。迁移后协调全部相关节点升级，各节点通过自身 schema、任务和能力检查后才恢复流量。迁移锁只排斥迁移进程，不阻止未知外部客户端。 -## 与现有存储层结合 +发布作者应提供受影响对象、执行模式、程序／schema 兼容范围、API 变化和任务格式声明。公共 API 不兼容替换需标记弃用并说明替代入口和移除计划；同包内部入口可与调用方一起替换。保留的 API 可转换请求／响应后共用当前实现，不要求每个 handler 支持两套表结构。 -| 现有层 | 升级中的职责 | -| --- | --- | -| Profile | 复用连接配置和引擎生命周期，拆出无 `create_tables` 副作用的 inspection／maintenance 入口 | -| `AsyncDatabase` | 提供专用维护连接及事务；`AsyncConnection.run_sync` 将同一连接交给 Alembic，锁和 DDL 不换连接 | -| Alembic | 冻结 revision 执行 DDL，验证后推进唯一版本表 | -| Repository | schema 满足后复用稳定领域操作；历史转换优先冻结 SQL／映射，不依赖不断变化的当前 Repository | -| 索引 | 结构定义进入版本资源，受控重建派生数据并检查，普通启动不隐式改表 | -| 启动门禁 | 先检查 schema、任务和必要能力，再装配 Repository、索引和 Worker | +旧格式任务需要在只读计划、停写取锁后、迁移后及 Worker 启动前检查。未知格式或不可观察来源不能假报无任务；应按声明兼容消费、幂等转换或有界排空，并保留任务 ID、幂等键、Scope、重试和领域收据。这些完整业务检查尚未由阶段 A bundle 覆盖。 -例如新增必填列：先 revision 加可空列，数据脚本分批幂等回填，校验后下一 revision 收紧约束，重建相关索引， -最后才启用新版 Repository。seekDB／OceanBase DDL 可隐式提交，不能用一个 Python 事务假装整个升级原子化。 -已有领域收据与对应数据批次一起提交，索引从权威数据重建。 +## 存储层与恢复边界 -## 中断与恢复 +维护执行器复用配置，使用专用连接承载 Alembic 和锁；冻结历史 SQL 不依赖当前 Repository 或应用 metadata。Repository 仅在结构满足后运行；全文／向量索引由版本化资源和受控重建管理。seekdb／OceanBase DDL 可能隐式提交，不能用 Python 事务承诺整个迁移原子化。 -中断后只读检查 `status`、`verify`、`plan`，仅在实际状态证明可安全重试时继续。单一版本表不提供通用按 run ID -续跑；含混状态返回 `recovery_required`。auto 保留原始恢复点和库外记录，不用部分迁移后的备份覆盖它；manual -仍不核验，skip 不承诺恢复原数据。所有模式均检查真实结构、数据和任务，不盲目 stamp。 +单版本表不提供通用 run ID 续跑。中断后先用只读命令检查真实结构和数据；含混状态返回 `recovery_required`，不盲目 stamp、不用部分状态的备份替换原恢复点。库外摘要记录执行结果及备份信息。 -恢复独立、显式执行并保持停写；恢复数据库、必要文件及匹配程序后验证。应用回退不等于数据库回滚。 -迁移 `ready` 与服务启动结果分别报告，不在成功后立即删除恢复点。生产支持仍以 RFC 的实际后端验收为准。 +恢复独立、显式执行，并保持停写。恢复数据库、必要文件和匹配程序后重新验证；应用回退不等于数据库回滚。manual 不承诺已验证可恢复，skip 不承诺恢复原数据。 diff --git a/scripts/database_migration_probe.py b/scripts/database_migration_probe.py index 4c1b8a738a..0f48fc742e 100644 --- a/scripts/database_migration_probe.py +++ b/scripts/database_migration_probe.py @@ -33,7 +33,9 @@ def main() -> int: parser.add_argument("action", choices=("status", "plan", "apply", "verify")) parser.add_argument("--sqlite-path", type=Path, required=True, help="Explicit disposable database path.") parser.add_argument("--plan-id") - parser.add_argument("--resume") + parser.add_argument("--backup", choices=("auto", "manual", "skip"), default="auto") + parser.add_argument("--backup-confirmed", action="store_true") + parser.add_argument("--accept-no-backup", action="store_true") parser.add_argument("--maintenance-confirmed", action="store_true") parser.add_argument("--yes", action="store_true") args = parser.parse_args() @@ -41,11 +43,11 @@ def main() -> int: runner = SQLiteMigrationRunner(args.sqlite_path, bundle) try: if args.action in {"status", "plan"}: - result = runner.plan() + result = runner.plan(backup_policy=args.backup) elif args.action == "verify": result = runner.verify() else: - plan = runner.plan() + plan = runner.plan(backup_policy=args.backup) plan_id = args.plan_id accepted = args.yes if plan.state != "ready" and not accepted and sys.stdin.isatty(): @@ -56,7 +58,9 @@ def main() -> int: plan_id=plan_id, accepted=accepted, maintenance_confirmed=args.maintenance_confirmed, - resume=args.resume, + backup_policy=args.backup, + backup_confirmed=args.backup_confirmed, + accept_no_backup=args.accept_no_backup, ) print(result.model_dump_json()) except MigrationError as error: diff --git a/src/powercontext/builtin/persistence/migrations/backup.py b/src/powercontext/builtin/persistence/migrations/backup.py new file mode 100644 index 0000000000..86b933c333 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/backup.py @@ -0,0 +1,767 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Native recovery points on dedicated maintenance connections. + +The protocol is synchronous so async backends can use ``AsyncConnection.run_sync``. +No provider starts a business profile, verifies a user's manual backup, deletes a +recovery point, or restores a database automatically. Fork feature availability +and acceptance of the complete PC recovery workflow are separate facts. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import sqlite3 +from contextlib import closing +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Literal, Protocol + +from pydantic import BaseModel, ConfigDict +from sqlalchemy import Connection, text +from sqlalchemy.exc import SQLAlchemyError + +from .models import MigrationError, digest + +BackupMethod = Literal["sqlite_online", "fork_database", "fork_table"] +ForkProduct = Literal["seekdb", "oceanbase_ai"] + + +class BackupContext(BaseModel): + """The accepted target and maintenance window, with no credentials or rows.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + database_id: str + source_revision: str | None + bundle_checksum: str + maintenance_window_id: str + objects: tuple[str, ...] = () + writers_stopped: bool = False + + +class BackupCapabilities(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + available: bool + method: BackupMethod | None = None + product: str + server_version: str | None = None + server_identity_digest: str | None = None + table_fork_available: bool = False + database_fork_available: bool = False + table_fork_minimum: str | None = None + database_fork_minimum: str | None = None + reasons: tuple[str, ...] = () + limitations: tuple[str, ...] = () + requires_stopped_writers: bool = True + same_instance: bool = False + experimental: bool = False + recovery_workflow_accepted: bool = False + + +class BackupRef(BaseModel): + """Persist outside the target database and preserve across migration retries.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + ref_id: str + method: BackupMethod + database_id: str + source_revision: str | None + bundle_checksum: str + maintenance_window_id: str + location: str + objects: tuple[str, ...] + object_mapping: dict[str, str] = {} + checksum: str | None = None + schema_fingerprint: str | None = None + server_version: str | None = None + server_identity_digest: str | None = None + snapshot_boundary: str + created_at: str + + +class BackupInspection(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + state: Literal["pending", "completed", "failed"] + check_level: Literal["none", "metadata_checked", "integrity_checked"] = "none" + reasons: tuple[str, ...] = () + # Neither an SQL success nor integrity_check is an actual recovery exercise. + recovery_verified: bool = False + + +class RestorePlan(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + database_id: str + scope: tuple[str, ...] + steps: tuple[str, ...] + requirements: tuple[str, ...] + automatic: bool = False + recovery_verified: bool = False + + +class BackupProvider(Protocol): + def capabilities(self, context: BackupContext) -> BackupCapabilities: ... + + def create_backup(self, context: BackupContext) -> BackupRef: ... + + def inspect_backup(self, ref: BackupRef) -> BackupInspection: ... + + def restore_plan(self, ref: BackupRef) -> RestorePlan: ... + + +class SQLiteBackupProvider: + """Create an independent Online Backup API file including committed WAL data.""" + + def __init__(self, database: Path, *, directory: Path | None = None) -> None: + self.database = database.expanduser().resolve() + self.directory = (directory or self.database.with_name(self.database.name + ".pc-migration-backups")).resolve() + + def capabilities(self, context: BackupContext) -> BackupCapabilities: + reasons: tuple[str, ...] = () + try: + if not self.database.is_file(): + reasons = ("The target is not an existing SQLite file.",) + else: + with closing(self._read_database(self.database)) as connection: + connection.execute("PRAGMA schema_version").fetchone() + except (OSError, sqlite3.Error): + reasons = ("The existing SQLite target cannot be read.",) + return BackupCapabilities( + available=not reasons, + method="sqlite_online" if not reasons else None, + product="sqlite", + server_version=sqlite3.sqlite_version, + reasons=reasons, + limitations=( + "Large databases may take a long time and require additional storage.", + "Extensions, external files, and business invariants require separate restore verification.", + ), + recovery_workflow_accepted=True, + ) + + def create_backup(self, context: BackupContext) -> BackupRef: + if not context.writers_stopped: + raise MigrationError("active_writers", "Stop all SQLite writers before creating the recovery point.") + ref_id = _backup_ref_id(context, "sqlite_online") + target = self.directory / f"{ref_id}.sqlite3" + manifest = self.directory / f"{ref_id}.json" + if target.exists() or manifest.exists(): + return self._original_backup(context, manifest) + capabilities = self.capabilities(context) + if not capabilities.available: + raise MigrationError("backup_unsupported", "; ".join(capabilities.reasons)) + try: + self.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + os.close(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)) + with closing(self._read_database(self.database)) as source, closing(sqlite3.connect(target)) as destination: + source.backup(destination) + if destination.execute("PRAGMA integrity_check").fetchall() != [("ok",)]: + raise MigrationError("backup_failed", "The SQLite recovery point failed integrity checks.") + _fsync_file(target) + ref = BackupRef( + ref_id=ref_id, + method="sqlite_online", + **context.model_dump(exclude={"writers_stopped"}), + location=str(target), + checksum=_file_checksum(target), + server_version=sqlite3.sqlite_version, + snapshot_boundary="Committed state copied by SQLite Online Backup API while writers are stopped.", + created_at=datetime.now(UTC).isoformat(), + ) + # Exclusive creation prevents a retry from replacing the first recovery point. + with manifest.open("x", encoding="utf-8") as output: + os.chmod(manifest, 0o600) + output.write(ref.model_dump_json(indent=2)) + output.flush() + os.fsync(output.fileno()) + _fsync_directory(self.directory) + except (OSError, sqlite3.Error) as error: + # A partial file is retained as evidence, never reported as completed. + raise MigrationError("backup_failed", "Could not create and verify the SQLite recovery point.") from error + else: + return ref + + def _original_backup(self, context: BackupContext, manifest: Path) -> BackupRef: + try: + ref = BackupRef.model_validate_json(manifest.read_text(encoding="utf-8")) + if not _matches_context(ref, context) or self.inspect_backup(ref).state != "completed": + raise MigrationError("backup_failed", "The original recovery point is missing or has changed.") + except (OSError, ValueError) as error: + raise MigrationError( + "backup_failed", "The original recovery point cannot be reused; do not replace it with a new snapshot." + ) from error + else: + return ref + + def inspect_backup(self, ref: BackupRef) -> BackupInspection: + if ref.method != "sqlite_online" or re.fullmatch(r"[a-f0-9]{32}", ref.ref_id) is None: + return BackupInspection(state="failed", reasons=("The backup method does not match this provider.",)) + target = Path(ref.location) + try: + expected = self.directory / f"{ref.ref_id}.sqlite3" + manifest = self.directory / f"{ref.ref_id}.json" + if target.resolve() != expected or not target.is_file() or not manifest.is_file(): + return BackupInspection( + state="failed", reasons=("The original recovery point or manifest is missing.",) + ) + persisted = BackupRef.model_validate_json(manifest.read_text(encoding="utf-8")) + if persisted != ref or ref.checksum != _file_checksum(target): + return BackupInspection( + state="failed", reasons=("The original recovery point or manifest has changed.",) + ) + with closing(self._read_database(target)) as connection: + if connection.execute("PRAGMA integrity_check").fetchall() != [("ok",)]: + return BackupInspection(state="failed", reasons=("The recovery point failed integrity checks.",)) + except (OSError, sqlite3.Error, ValueError): + return BackupInspection(state="failed", reasons=("The original recovery point cannot be inspected.",)) + return BackupInspection(state="completed", check_level="integrity_checked") + + def restore_plan(self, ref: BackupRef) -> RestorePlan: + if self.inspect_backup(ref).state != "completed": + raise MigrationError("backup_failed", "Inspect the original SQLite recovery point before planning restore.") + return RestorePlan( + database_id=ref.database_id, + scope=ref.objects, + requirements=( + "Keep every writer stopped and hold the database maintenance lock.", + "Use a SQLite version and extensions compatible with the backup schema.", + "Explicit approval is required before replacing the configured database.", + ), + steps=( + "Preserve the current database and its WAL/SHM files as failure evidence.", + "Restore the checked backup into a separate SQLite file with the Online Backup API.", + "Check integrity, schema revision, required extensions, queues, and business invariants.", + "With all connections closed, explicitly replace the configured database and remove only stale WAL/SHM sidecars.", + "Start a compatible application and verify readiness before restoring traffic; retain the original backup.", + ), + ) + + @staticmethod + def _read_database(path: Path) -> sqlite3.Connection: + return sqlite3.connect(path.as_uri() + "?mode=ro", uri=True) + + +class ForkWorkflowAcceptance(BaseModel): + """Release-owned evidence for a tested engine, schema, DDL, and recovery path. + + This is supplied by a backend adapter, never by CLI flags or user confirmation. + The default provider has no accepted workflow: a feature version alone cannot + enable PC automatic backup. Evidence must cover restoring the configured name, + especially embedded seekdb's fixed ``test`` database. + """ + + model_config = ConfigDict(frozen=True, extra="forbid") + + product: ForkProduct + server_version: str + server_identity_digest: str + method: Literal["fork_database", "fork_table"] + schema_fingerprint: str + bundle_checksum: str + covered_objects: tuple[str, ...] + evidence_reference: str + restore_steps: tuple[str, ...] + ddl_verified: bool = False + restart_verified: bool = False + restore_to_original_database: bool = False + cross_table_consistency_verified: bool = False + events_not_supported: bool = False + + +class ForkBackupProvider: + """Native Fork recovery points, using an already pinned maintenance session. + + The caller owns engine configuration and lifecycle. Capabilities perform only + reads. No unknown engine, unsupported object, missing grant, or unaccepted + recovery path falls back to copying files or to physical backup. + """ + + def __init__( + self, + connection: Connection, + *, + database_name: str, + expected_product: ForkProduct, + directory: Path, + acceptance: ForkWorkflowAcceptance | None = None, + ) -> None: + self.connection = connection + self.database_name = database_name + self.expected_product = expected_product + self.directory = directory.expanduser().resolve() + self.acceptance = acceptance + + def capabilities(self, context: BackupContext) -> BackupCapabilities: + table_minimum, database_minimum = _fork_minimums(self.expected_product) + common: dict[str, Any] = { + "table_fork_minimum": _format_version(table_minimum), + "database_fork_minimum": _format_version(database_minimum), + "same_instance": True, + "experimental": True, + "limitations": ( + "Fork retains a same-instance recovery point, not independent disaster recovery.", + "Large databases may take a long time; retained objects and later writes consume storage.", + "Fork objects remain after migration and retry; cleanup requires a later explicit operation.", + "Metadata checks do not prove that this recovery point was successfully restored.", + ), + } + product: ForkProduct | None = None + version: tuple[int, int, int] | None = None + identity_digest: str | None = None + table_available = False + database_available = False + try: + product, version, identity_digest = self._server_identity() + if product != self.expected_product or version is None: + return BackupCapabilities( + available=False, + product=product or "unknown", + server_version=None if version is None else _format_version(version), + server_identity_digest=identity_digest, + reasons=("The actual server product or version does not match the selected backend.",), + **common, + ) + table_available = version >= table_minimum + database_available = version >= database_minimum + reasons = self._eligibility(context, product, version, identity_digest, table_available, database_available) + method = None if reasons else self.acceptance.method if self.acceptance is not None else None + return BackupCapabilities( + available=not reasons, + method=method, + product=product, + server_version=_format_version(version), + server_identity_digest=identity_digest, + table_fork_available=table_available, + database_fork_available=database_available, + reasons=reasons, + recovery_workflow_accepted=not reasons, + **common, + ) + except (SQLAlchemyError, ValueError, TypeError): + return BackupCapabilities( + available=False, + product=product or "unknown", + server_version=None if version is None else _format_version(version), + server_identity_digest=identity_digest, + table_fork_available=table_available, + database_fork_available=database_available, + reasons=("Server product, permissions, objects, or recovery conditions could not be verified.",), + **common, + ) + + def create_backup(self, context: BackupContext) -> BackupRef: + if not context.writers_stopped: + raise MigrationError("active_writers", "Stop every writer before creating a Fork recovery point.") + if self.acceptance is not None: + original_id = _backup_ref_id(context, self.acceptance.method) + original_manifest = self.directory / f"{original_id}.json" + if original_manifest.exists(): + return self._original_backup(context, original_manifest) + capability = self.capabilities(context) + if not capability.available or capability.method is None: + raise MigrationError("backup_unsupported", "; ".join(capability.reasons)) + ref_id = _backup_ref_id(context, capability.method) + backup_name = "pc_backup_" + ref_id + mapping: dict[str, str] = {} + manifest = self.directory / f"{ref_id}.json" + try: + self.directory.mkdir(mode=0o700, parents=True, exist_ok=True) + # Reserve the external name before any nontransactional DDL. On + # failure its pending record names every object that may exist. + inventory, fingerprint = self._inventory() + if capability.method == "fork_database": + mapping = {name: f"{backup_name}.{name}" for name in inventory} + statements = ( + f"FORK DATABASE {_quote_identifier(self.database_name)} TO {_quote_identifier(backup_name)}", + ) + location = backup_name + boundary = "All database base tables share the engine's Fork snapshot." + else: + mapping = {name: f"{self.database_name}.{backup_name}_{index}" for index, name in enumerate(inventory)} + statements = tuple( + f"FORK TABLE {_qualified(self.database_name, name)} TO {_qualified(self.database_name, backup_name + '_' + str(index))}" + for index, name in enumerate(inventory) + ) + location = self.database_name + boundary = ( + "Separate table snapshots; consistency relies on stopped writers and accepted dependency coverage." + ) + ref = BackupRef( + ref_id=ref_id, + method=capability.method, + **context.model_dump(exclude={"writers_stopped", "objects"}), + location=location, + objects=tuple(inventory), + object_mapping=mapping, + schema_fingerprint=fingerprint, + server_version=capability.server_version, + server_identity_digest=capability.server_identity_digest, + snapshot_boundary=boundary, + created_at=datetime.now(UTC).isoformat(), + ) + self._write_manifest(manifest, ref, state="pending", exclusive=True) + for statement in statements: + self.connection.exec_driver_sql(statement) + # Native Fork is nontransactional; the dedicated connection must + # not carry business DML which could be accidentally committed. + self.connection.commit() + self._write_manifest(manifest, ref, state="created", exclusive=False) + inspection = self.inspect_backup(ref) + if inspection.state != "completed": + raise MigrationError( + "backup_failed", "The Fork recovery point is incomplete; preserve all Fork objects." + ) + except (OSError, SQLAlchemyError, ValueError) as error: + raise MigrationError( + "backup_failed", + "Fork creation or verification failed; preserve the external manifest and Fork objects.", + ) from error + else: + return ref + + def _original_backup(self, context: BackupContext, manifest: Path) -> BackupRef: + try: + content = json.loads(manifest.read_text(encoding="utf-8")) + ref = BackupRef.model_validate(content["backup"]) + if not _matches_context(ref, context) or self.inspect_backup(ref).state != "completed": + raise MigrationError( + "backup_failed", "The original Fork point is incomplete or has changed; preserve it." + ) + except (OSError, ValueError, KeyError, TypeError) as error: + raise MigrationError("backup_failed", "The original Fork manifest cannot be reused safely.") from error + else: + return ref + + def inspect_backup(self, ref: BackupRef) -> BackupInspection: + if ref.method not in {"fork_database", "fork_table"} or re.fullmatch(r"[a-f0-9]{32}", ref.ref_id) is None: + return BackupInspection(state="failed", reasons=("The backup method does not match this provider.",)) + try: + manifest = self.directory / f"{ref.ref_id}.json" + content = json.loads(manifest.read_text(encoding="utf-8")) + if BackupRef.model_validate(content["backup"]) != ref: + return BackupInspection(state="failed", reasons=("The original Fork manifest has changed.",)) + if content["state"] == "pending": + return BackupInspection( + state="pending", reasons=("Fork completion was not recorded; inspect before recovery.",) + ) + if content["state"] != "created": + return BackupInspection(state="failed", reasons=("Fork creation did not reach the completed state.",)) + product, version, identity_digest = self._server_identity() + if ( + product != self.expected_product + or version is None + or _format_version(version) != ref.server_version + or identity_digest != ref.server_identity_digest + ): + return BackupInspection(state="failed", reasons=("The Fork server identity or version has changed.",)) + for destination in ref.object_mapping.values(): + schema, table_name = destination.split(".", 1) + result = self.connection.execute( + text( + "SELECT TABLE_NAME FROM information_schema.TABLES WHERE TABLE_SCHEMA=:schema AND TABLE_NAME=:table" + ), + {"schema": schema, "table": table_name}, + ).all() + if result != [(table_name,)]: + return BackupInspection( + state="failed", reasons=("A retained Fork object is missing or inaccessible.",) + ) + # Opening the snapshot is a metadata/access check, not a restore exercise. + self.connection.exec_driver_sql( + f"SELECT 1 FROM {_qualified(schema, table_name)} LIMIT 0" # noqa: S608 -- quoted identifiers + ) + if set(ref.object_mapping) != set(ref.objects): + return BackupInspection( + state="failed", reasons=("The Fork mapping does not cover its declared scope.",) + ) + return BackupInspection(state="completed", check_level="metadata_checked") + except (OSError, SQLAlchemyError, ValueError, KeyError, TypeError): + return BackupInspection( + state="failed", reasons=("The retained Fork point or manifest cannot be inspected.",) + ) + + def restore_plan(self, ref: BackupRef) -> RestorePlan: + if self.inspect_backup(ref).state != "completed": + raise MigrationError("backup_failed", "The Fork recovery point is not ready for a restore plan.") + acceptance = self.acceptance + if ( + acceptance is None + or not acceptance.restore_to_original_database + or acceptance.server_identity_digest != ref.server_identity_digest + or acceptance.schema_fingerprint != ref.schema_fingerprint + or acceptance.bundle_checksum != ref.bundle_checksum + or acceptance.method != ref.method + or set(acceptance.covered_objects) != set(ref.objects) + ): + raise MigrationError( + "backup_unsupported", "No accepted restore path to the configured database is available." + ) + return RestorePlan( + database_id=ref.database_id, + scope=ref.objects, + requirements=( + "Keep all writers stopped and preserve the original Fork objects.", + "Restore only the confirmed database and dependency scope, never the entire shared tenant.", + "Use the engine version, permissions, and object types covered by adapter acceptance.", + "Obtain explicit approval before replacing any current object; application downgrade alone is insufficient.", + ), + steps=acceptance.restore_steps, + ) + + def _server_identity(self) -> tuple[ForkProduct | None, tuple[int, int, int] | None, str]: + version = self.connection.exec_driver_sql("SELECT VERSION()").scalar_one() + comment = self.connection.exec_driver_sql("SELECT @@version_comment").scalar_one() + product, number = identify_fork_server(str(version), str(comment)) + return product, number, digest([version, comment]) + + def _eligibility( + self, + context: BackupContext, + product: ForkProduct, + version: tuple[int, int, int], + identity_digest: str, + table_available: bool, + database_available: bool, + ) -> tuple[str, ...]: + if not table_available: + return ("The actual engine predates this product's Fork Table minimum version.",) + if self.connection.dialect.name not in {"mysql", "oceanbase"}: + return ("Fork requires a supported MySQL-compatible maintenance connection.",) + current = self.connection.exec_driver_sql("SELECT DATABASE()").scalar_one() + if current != self.database_name or self.database_name.lower() in { + "oceanbase", + "mysql", + "information_schema", + "sys", + }: + return ("The maintenance connection does not target an eligible user database.",) + if product == "oceanbase_ai": + rows = self.connection.exec_driver_sql("SHOW VARIABLES LIKE 'ob_compatibility_mode'").all() + if not rows or str(rows[0][1]).upper() != "MYSQL": + return ("OceanBase Fork requires a verified MySQL user tenant.",) + inventory, fingerprint = self._inventory() + if not set(context.objects).issubset(inventory): + return ("The affected object and dependency scope is not covered by eligible base tables.",) + return self._acceptance_reasons( + context, product, version, identity_digest, inventory, fingerprint, database_available + ) + + def _acceptance_reasons( + self, + context: BackupContext, + product: ForkProduct, + version: tuple[int, int, int], + identity_digest: str, + inventory: tuple[str, ...], + fingerprint: str, + database_available: bool, + ) -> tuple[str, ...]: + acceptance = self.acceptance + if acceptance is None: + return ( + "PC has no accepted object, DDL, restart, and configured-target recovery workflow for this engine.", + ) + if ( + acceptance.product != product + or acceptance.server_version != _format_version(version) + or acceptance.server_identity_digest != identity_digest + or acceptance.schema_fingerprint != fingerprint + or acceptance.bundle_checksum != context.bundle_checksum + or set(acceptance.covered_objects) != set(inventory) + or not acceptance.evidence_reference + or not acceptance.restore_steps + or not acceptance.ddl_verified + or not acceptance.restart_verified + or not acceptance.restore_to_original_database + ): + return ("Adapter recovery acceptance does not match this actual engine, schema, or required DDL.",) + if acceptance.method == "fork_database" and not database_available: + return ("The actual engine predates this product's Fork Database minimum version.",) + if acceptance.method == "fork_table" and not acceptance.cross_table_consistency_verified: + return ("Table Fork dependency coverage and stopped-write consistency have not been accepted.",) + if not self._has_required_grants(acceptance.method): + return ( + "Required Fork SELECT/CREATE permissions could not be verified from the current user's direct grants.", + ) + return () + + def _inventory(self) -> tuple[tuple[str, ...], str]: + tables = self.connection.execute( + text( + "SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA=:schema ORDER BY TABLE_NAME" + ), + {"schema": self.database_name}, + ).all() + if any(str(row[1]).upper() != "BASE TABLE" for row in tables): + raise ValueError("Non-table objects are not covered by this Fork adapter") # noqa: TRY003 + for query in ( + "SELECT COUNT(*) FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=:schema", + "SELECT COUNT(*) FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA=:schema", + "SELECT COUNT(*) FROM information_schema.EVENTS WHERE EVENT_SCHEMA=:schema", + ): + try: + found = self.connection.execute(text(query), {"schema": self.database_name}).scalar_one() + except SQLAlchemyError as error: + # A verified seekdb adapter may account for its unsupported + # Event Scheduler. Unknown catalogs and permission errors do + # not mean that there are no affected objects. + arguments = getattr(getattr(error, "orig", None), "args", ()) + code = arguments[0] if arguments else None + if ( + "information_schema.EVENTS " in query + and code in {1109, 1146} + and self.expected_product == "seekdb" + and self.acceptance is not None + and self.acceptance.events_not_supported + ): + continue + raise + if found: + raise ValueError("Non-table objects need a separately accepted reconstruction path") # noqa: TRY003 + definitions: dict[str, str] = {} + for row in tables: + name = str(row[0]) + definition = self.connection.exec_driver_sql( + f"SHOW CREATE TABLE {_qualified(self.database_name, name)}" + ).one() + definitions[name] = re.sub(r"\bAUTO_INCREMENT=\d+\b", "", str(definition[1])) + return tuple(definitions), digest(definitions) + + def _has_required_grants(self, method: BackupMethod) -> bool: + grants = [str(row[0]) for row in self.connection.exec_driver_sql("SHOW GRANTS").all()] + source = False + create = False + for grant in grants: + match = re.match(r"GRANT (.+?) ON (.+?) TO ", grant, re.IGNORECASE) + if match is None: + continue + privileges, scope = match.groups() + privileges = privileges.upper().split(", ") + all_privileges = "ALL PRIVILEGES" in privileges + # Role grants and wildcard patterns are deliberately not interpreted. + # Failing closed is preferable to guessing effective permissions. + if scope in {"*.*", f"{_quote_identifier(self.database_name)}.*"}: + source = source or all_privileges or "SELECT" in privileges + if scope == "*.*" or (method == "fork_table" and scope == f"{_quote_identifier(self.database_name)}.*"): + create = create or all_privileges or "CREATE" in privileges + return source and create + + @staticmethod + def _write_manifest(path: Path, ref: BackupRef, *, state: str, exclusive: bool) -> None: + content = json.dumps({"state": state, "backup": ref.model_dump()}, indent=2, ensure_ascii=False) + if exclusive: + with path.open("x", encoding="utf-8") as output: + os.chmod(path, 0o600) + output.write(content) + output.flush() + os.fsync(output.fileno()) + else: + temporary = path.with_suffix(".pending") + with temporary.open("x", encoding="utf-8") as output: + os.chmod(temporary, 0o600) + output.write(content) + output.flush() + os.fsync(output.fileno()) + temporary.replace(path) + _fsync_directory(path.parent) + + +def identify_fork_server(version: str, comment: str) -> tuple[ForkProduct | None, tuple[int, int, int] | None]: + """Read product-qualified engine versions, never the leading MySQL version. + + Actual embedded seekdb 1.4 reports MySQL 5.7 and OceanBase 4.3 in these + strings; neither is the seekdb feature version. + """ + identity = version + " " + comment + if "seekdb" in identity.lower(): + patterns = ( + r"seekdb[\s_-]*(?:[vr])?(\d+)\.(\d+)\.(\d+)", + r"seekdb\s*\(r(\d+)\.(\d+)\.(\d+)", + ) + product: ForkProduct = "seekdb" + elif "oceanbase" in identity.lower(): + patterns = (r"OceanBase(?:_CE|_EE|\s+AI)?[\s_-]*(?:v)?(\d+)\.(\d+)\.(\d+)",) + product = "oceanbase_ai" + else: + return None, None + for pattern in patterns: + match = re.search(pattern, identity, re.IGNORECASE) + if match is not None: + return product, (int(match[1]), int(match[2]), int(match[3])) + return product, None + + +def _fork_minimums(product: ForkProduct) -> tuple[tuple[int, int, int], tuple[int, int, int]]: + return ((1, 1, 0), (1, 2, 0)) if product == "seekdb" else ((4, 6, 2), (4, 6, 2)) + + +def _format_version(version: tuple[int, int, int]) -> str: + return ".".join(str(part) for part in version) + + +def _quote_identifier(value: str) -> str: + if not value or "\x00" in value or "." in value: + raise ValueError("Invalid database or table identifier") # noqa: TRY003 + return "`" + value.replace("`", "``") + "`" + + +def _qualified(database: str, table_name: str) -> str: + return _quote_identifier(database) + "." + _quote_identifier(table_name) + + +def _file_checksum(path: Path) -> str: + with path.open("rb") as source: + return hashlib.file_digest(source, "sha256").hexdigest() + + +def _backup_ref_id(context: BackupContext, method: BackupMethod) -> str: + # Reuse even if the process died after backup completion but before the + # executor persisted its reference. A new maintenance window gets a new ID. + return digest({ + "database_id": context.database_id, + "maintenance_window_id": context.maintenance_window_id, + "method": method, + })[:32] + + +def _matches_context(ref: BackupRef, context: BackupContext) -> bool: + return ( + ref.database_id == context.database_id + and ref.source_revision == context.source_revision + and ref.bundle_checksum == context.bundle_checksum + and ref.maintenance_window_id == context.maintenance_window_id + and set(context.objects).issubset(ref.objects) + ) + + +def _fsync_file(path: Path) -> None: + with path.open("rb") as source: + os.fsync(source.fileno()) + + +def _fsync_directory(path: Path) -> None: + if os.name != "nt": + descriptor = os.open(path, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) diff --git a/src/powercontext/builtin/persistence/migrations/bundle.py b/src/powercontext/builtin/persistence/migrations/bundle.py index f12046bcf8..d957935229 100644 --- a/src/powercontext/builtin/persistence/migrations/bundle.py +++ b/src/powercontext/builtin/persistence/migrations/bundle.py @@ -19,6 +19,7 @@ import hashlib import json from pathlib import Path +from tempfile import TemporaryDirectory from typing import Any from alembic.config import Config @@ -38,7 +39,20 @@ class MigrationBundle: def __init__(self, directory: Path) -> None: self.directory = directory.resolve() - self.script = ScriptDirectory(str(self.directory)) + # Execute exactly the reviewed bytes, even if the installed resources + # change during a long backup. Alembic reloads scripts on every command. + resources = { + path.relative_to(self.directory).as_posix(): path.read_bytes() + for path in sorted(self.directory.rglob("*")) + if path.is_file() and path.suffix in {".py", ".sql", ".json"} + } + self._snapshot = TemporaryDirectory(prefix="pc-migration-bundle-") + self.snapshot_directory = Path(self._snapshot.name).resolve() + for name, content in resources.items(): + target = self.snapshot_directory / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + self.script = ScriptDirectory(str(self.snapshot_directory)) revisions = list(reversed(list(self.script.walk_revisions()))) previous = None for item in revisions: @@ -49,31 +63,44 @@ def __init__(self, directory: Path) -> None: raise MigrationError("invalid_bundle", "Migration history is empty.") self.revisions = tuple(item.revision for item in revisions) self.head = self.revisions[-1] - manifest = json.loads((self.directory / "manifest.json").read_text()) + manifest = json.loads(resources["manifest.json"]) + self.affected_objects = tuple(manifest.get("affected_objects", ())) + self.retained_objects = tuple(manifest.get("retained_objects", ())) + self.retained_fingerprints = manifest.get("retained_fingerprints", {}) + self.retained_revisions = manifest.get("retained_revisions", {}) + if set(self.retained_objects) != set(self.retained_revisions) or not set( + self.retained_revisions.values() + ) <= set(self.revisions): + raise MigrationError("invalid_bundle", "Retained objects must declare their introducing revision.") self.fingerprints: dict[str, list[str]] = manifest["sqlite_fingerprints"] self.baselines: dict[str, str] = manifest["sqlite_baselines"] if set(self.fingerprints) != set(self.revisions) or not set(self.baselines.values()) <= set(self.revisions): raise MigrationError("invalid_bundle", "Schema inventory does not cover the revision chain.") if any(fingerprint not in self.fingerprints[revision] for fingerprint, revision in self.baselines.items()): raise MigrationError("invalid_bundle", "A baseline does not match its declared revision.") - self.resources = { - path.relative_to(self.directory).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() - for path in sorted(self.directory.rglob("*")) - if path.is_file() and path.suffix in {".py", ".sql", ".json"} - } + self.resources = {name: hashlib.sha256(content).hexdigest() for name, content in resources.items()} self.checksums = {} for item in revisions: dependencies = manifest["revision_resources"][item.revision] inputs = {name: self.resources[name] for name in ["env.py", *dependencies]} - inputs[Path(item.path).relative_to(self.directory).as_posix()] = hashlib.sha256( + inputs[Path(item.path).relative_to(self.snapshot_directory).as_posix()] = hashlib.sha256( Path(item.path).read_bytes() ).hexdigest() self.checksums[item.revision] = digest(inputs) self.checksum = digest(self.resources) + def assert_unchanged(self) -> None: + resources = { + path.relative_to(self.directory).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(self.directory.rglob("*")) + if path.is_file() and path.suffix in {".py", ".sql", ".json"} + } + if resources != self.resources: + raise MigrationError("plan_changed", "Migration resources changed after the bundle was loaded.") + def config(self, connection: Connection) -> Config: config = Config() - config.set_main_option("script_location", str(self.directory).replace("%", "%%")) + config.set_main_option("script_location", str(self.snapshot_directory).replace("%", "%%")) config.attributes["connection"] = connection return config diff --git a/src/powercontext/builtin/persistence/migrations/deployment.py b/src/powercontext/builtin/persistence/migrations/deployment.py new file mode 100644 index 0000000000..f9da0c4a57 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/deployment.py @@ -0,0 +1,57 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Packaged acceptance bundle, isolated from side-effectful runtime factories. + +This release does not register full Server baselines or remote executors. A +complete deployment must not be optimistically stamped from a four-table test. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from sqlalchemy.engine import make_url + +from powercontext.builtin.persistence.sqlite import SQLiteConfig + +from .bundle import MigrationBundle +from .models import MigrationError +from .sqlite import SQLiteMigrationRunner + +if TYPE_CHECKING: + from powercontext.server.settings import ServerSettings + + +def production_bundle() -> MigrationBundle: + """Load installed resources; readiness is scoped to this acceptance bundle.""" + return MigrationBundle(Path(__file__).parent / "resources") + + +def deployment_runner(settings: ServerSettings) -> SQLiteMigrationRunner: + """Use the real configured identity without opening a business profile.""" + database = settings.database + if not isinstance(database, SQLiteConfig): + raise MigrationError( + "unsupported_backend", "Remote and seekdb deployment migration awaits full-backend acceptance." + ) + url = make_url(database.url) + if database.is_in_memory or url.query or not url.database: + raise MigrationError( + "unsupported_target", "Use an explicitly configured regular SQLite file without URI options." + ) + return SQLiteMigrationRunner( + Path(url.database), production_bundle(), configuration=database.model_dump(exclude={"url", "echo"}) + ) diff --git a/src/powercontext/builtin/persistence/migrations/models.py b/src/powercontext/builtin/persistence/migrations/models.py index 0a9681fa3b..5e682b46bb 100644 --- a/src/powercontext/builtin/persistence/migrations/models.py +++ b/src/powercontext/builtin/persistence/migrations/models.py @@ -49,7 +49,15 @@ class MigrationPlan(BaseModel): state: Literal["uninitialized", "migration_required", "ready", "recovery_required"] revisions: tuple[str, ...] adopt_baseline: bool = False - resume_run_id: str | None = None + backup_policy: Literal["auto", "manual", "skip"] = "auto" + backup_available: bool = True + backup_reason: tuple[str, ...] = () + affected_objects: tuple[str, ...] = () + retained_objects: tuple[str, ...] = () + shared_database: bool = False + service: dict[str, Any] | None = None + execution_mode: Literal["offline"] = "offline" + readiness_scope: Literal["registered_bundle"] = "registered_bundle" class MigrationResult(BaseModel): @@ -58,5 +66,6 @@ class MigrationResult(BaseModel): state: Literal["ready"] = "ready" revision: str changed: bool - run_id: str | None = None backup_ref: str | None = None + backup_state: Literal["completed", "user_confirmed", "skipped", "not_required"] = "not_required" + readiness_scope: Literal["registered_bundle"] = "registered_bundle" diff --git a/src/powercontext/builtin/persistence/migrations/resources/env.py b/src/powercontext/builtin/persistence/migrations/resources/env.py new file mode 100644 index 0000000000..90315cee59 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/env.py @@ -0,0 +1,26 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Run a revision on the executor's pinned connection; never open another engine.""" + +from alembic import context + +connection = context.config.attributes["connection"] +context.configure( + connection=connection, + version_table="pc_schema_revision", + transactional_ddl=connection.dialect.name == "sqlite", +) +with context.begin_transaction(): + context.run_migrations() diff --git a/src/powercontext/builtin/persistence/migrations/resources/manifest.json b/src/powercontext/builtin/persistence/migrations/resources/manifest.json new file mode 100644 index 0000000000..d3c36a44e4 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/manifest.json @@ -0,0 +1,42 @@ +{ + "revision_resources": { + "p0001": [ + "schemas/pre_dream.json" + ], + "p0002": [], + "p0003": [] + }, + "sqlite_baselines": { + "e1cae089a8a565291a7a1f782cea1f6c6e1a6cdd9eab75ef0c4ca61dfda453cb": "p0001", + "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d": "p0003" + }, + "sqlite_fingerprints": { + "p0001": [ + "e1cae089a8a565291a7a1f782cea1f6c6e1a6cdd9eab75ef0c4ca61dfda453cb" + ], + "p0002": [ + "2e487ed26e6dfec3f61fe71e304e5bfd2f33c449485ffbc2d8da02ba7defca7f" + ], + "p0003": [ + "9958fa0413ed3c16442df27dc48057d438e2cf3f7e4312c5a914cccd9d3acaab", + "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d" + ] + }, + "affected_objects": [ + "pc_artifacts", + "pc_artifact_heads", + "pc_artifact_candidate_versions", + "pc_artifact_tags" + ], + "retained_objects": [ + "pc_retained_p0002_artifact_tags" + ], + "retained_fingerprints": { + "pc_retained_p0002_artifact_tags": [ + "33052f9e6ae051c52e7ec13d53033b6c11e96d7f5b2c1df40047a7344e471eee" + ] + }, + "retained_revisions": { + "pc_retained_p0002_artifact_tags": "p0003" + } +} diff --git a/src/powercontext/builtin/persistence/migrations/resources/schemas/pre_dream.json b/src/powercontext/builtin/persistence/migrations/resources/schemas/pre_dream.json new file mode 100644 index 0000000000..e278c6e1f3 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/schemas/pre_dream.json @@ -0,0 +1,27 @@ +{ + "source_commit": "495cc93d67387879382c98706f4975f638faec15", + "tables_blob": "bd617564eb7d7a6a35e2e7d09002c010bdf3a651", + "sqlalchemy_version": "2.0.51", + "tables": [ + "pc_artifacts", + "pc_artifact_heads", + "pc_artifact_candidate_versions", + "pc_artifact_tags" + ], + "sqlite": [ + "CREATE TABLE pc_artifacts (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text TEXT, \n\tlifecycle_state VARCHAR(16) DEFAULT 'active' NOT NULL, \n\treplacement_artifact_id VARCHAR(128), \n\tgovernance_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated')\n)", + "CREATE TABLE pc_artifact_candidate_versions (\n\tscope_id VARCHAR(256) NOT NULL, \n\tcandidate_id VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tproposal BLOB NOT NULL, \n\tsource_refs BLOB NOT NULL, \n\tartifact_refs BLOB NOT NULL, \n\ttarget_family VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\ttarget_revision INTEGER, \n\treason TEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_artifact_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\ttarget_type VARCHAR(12) NOT NULL, \n\ttarget_id VARCHAR(128) NOT NULL, \n\ttag_key_hash BLOB NOT NULL, \n\ttag_key VARCHAR(128) NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_family CHECK (family IN ('memory', 'experience', 'skill', 'handoff')), \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)" + ], + "mysql": [ + "CREATE TABLE pc_artifacts (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent MEDIUMBLOB NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text MEDIUMTEXT, \n\tlifecycle_state VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'active', \n\treplacement_artifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tgovernance_generation BIGINT NOT NULL DEFAULT '0', \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated')\n)", + "CREATE TABLE pc_artifact_candidate_versions (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tcandidate_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tproposal MEDIUMBLOB NOT NULL, \n\tsource_refs MEDIUMBLOB NOT NULL, \n\tartifact_refs MEDIUMBLOB NOT NULL, \n\ttarget_family VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\ttarget_artifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\ttarget_revision INTEGER, \n\treason MEDIUMTEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_artifact_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttarget_type VARCHAR(12) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttarget_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttag_key_hash BINARY(32) NOT NULL, \n\ttag_key VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_family CHECK (family IN ('memory', 'experience', 'skill', 'handoff')), \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)" + ] +} diff --git a/src/powercontext/builtin/persistence/migrations/resources/schemas/v1_1_0.json b/src/powercontext/builtin/persistence/migrations/resources/schemas/v1_1_0.json new file mode 100644 index 0000000000..f728735586 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/schemas/v1_1_0.json @@ -0,0 +1,27 @@ +{ + "source_commit": "86e687f13fec62151ee5657a90733586fd5158dc", + "tables_blob": "530d3f13de425e796e059b428a7b57c7dd774a19", + "sqlalchemy_version": "2.0.51", + "tables": [ + "pc_artifacts", + "pc_artifact_heads", + "pc_artifact_candidate_versions", + "pc_artifact_tags" + ], + "sqlite": [ + "CREATE TABLE pc_artifacts (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent BLOB NOT NULL, \n\tmemory_citations BLOB, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text TEXT, \n\tlifecycle_state VARCHAR(16) DEFAULT 'active' NOT NULL, \n\treplacement_artifact_id VARCHAR(128), \n\tgovernance_generation BIGINT DEFAULT '0' NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated')\n)", + "CREATE TABLE pc_artifact_candidate_versions (\n\tscope_id VARCHAR(256) NOT NULL, \n\tcandidate_id VARCHAR(128) NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tproposal BLOB NOT NULL, \n\tsource_refs BLOB NOT NULL, \n\tartifact_refs BLOB NOT NULL, \n\tmemory_citations BLOB, \n\ttarget_family VARCHAR(128), \n\ttarget_artifact_id VARCHAR(128), \n\ttarget_revision INTEGER, \n\treason TEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_artifact_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) NOT NULL, \n\tfamily VARCHAR(128) NOT NULL, \n\tartifact_id VARCHAR(128) NOT NULL, \n\ttarget_type VARCHAR(12) NOT NULL, \n\ttarget_id VARCHAR(128) NOT NULL, \n\ttag_key_hash BLOB NOT NULL, \n\ttag_key VARCHAR(128) NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)" + ], + "mysql": [ + "CREATE TABLE pc_artifacts (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tcontent MEDIUMBLOB NOT NULL, \n\tmemory_citations MEDIUMBLOB, \n\tPRIMARY KEY (scope_id, family, artifact_id, revision)\n)", + "CREATE TABLE pc_artifact_heads (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\trevision INTEGER NOT NULL, \n\tsearchable_text MEDIUMTEXT, \n\tlifecycle_state VARCHAR(16) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'active', \n\treplacement_artifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\tgovernance_generation BIGINT NOT NULL DEFAULT '0', \n\tPRIMARY KEY (scope_id, family, artifact_id), \n\tFOREIGN KEY(scope_id, family, artifact_id, revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_heads_revision_positive CHECK (revision > 0), \n\tCONSTRAINT ck_pc_artifact_heads_lifecycle_state CHECK (lifecycle_state IN ('active', 'deprecated', 'retired')), \n\tCONSTRAINT ck_pc_artifact_heads_governance_generation_nonnegative CHECK (governance_generation >= 0), \n\tCONSTRAINT ck_pc_artifact_heads_replacement_deprecated CHECK (replacement_artifact_id IS NULL OR lifecycle_state = 'deprecated')\n)", + "CREATE TABLE pc_artifact_candidate_versions (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tcandidate_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tversion INTEGER NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tproposal MEDIUMBLOB NOT NULL, \n\tsource_refs MEDIUMBLOB NOT NULL, \n\tartifact_refs MEDIUMBLOB NOT NULL, \n\tmemory_citations MEDIUMBLOB, \n\ttarget_family VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\ttarget_artifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin, \n\ttarget_revision INTEGER, \n\treason MEDIUMTEXT, \n\tPRIMARY KEY (scope_id, candidate_id, version), \n\tFOREIGN KEY(scope_id, target_family, target_artifact_id, target_revision) REFERENCES pc_artifacts (scope_id, family, artifact_id, revision) ON DELETE RESTRICT, \n\tCONSTRAINT ck_pc_artifact_candidate_versions_version_positive CHECK (version > 0), \n\tCONSTRAINT ck_pc_artifact_candidate_versions_target_complete CHECK ((target_family IS NULL AND target_artifact_id IS NULL AND target_revision IS NULL) OR (target_family IS NOT NULL AND target_artifact_id IS NOT NULL AND target_revision > 0))\n)", + "CREATE TABLE pc_artifact_tags (\n\tscope_id VARCHAR(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tfamily VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\tartifact_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttarget_type VARCHAR(12) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttarget_id VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttag_key_hash BINARY(32) NOT NULL, \n\ttag_key VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, \n\ttag VARCHAR(64) NOT NULL, \n\tassigned_at DATETIME NOT NULL, \n\tPRIMARY KEY (scope_id, family, artifact_id, target_type, target_id, tag_key_hash), \n\tFOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads (scope_id, family, artifact_id) ON DELETE CASCADE, \n\tCONSTRAINT ck_pc_artifact_tags_target CHECK ((target_type = 'artifact' AND target_id = artifact_id) OR (target_type = 'memory_entry' AND family = 'memory'))\n)", + "CREATE INDEX ix_pc_artifact_tags_family_key ON pc_artifact_tags (scope_id, family, tag_key_hash, target_type, artifact_id, target_id)", + "CREATE INDEX ix_pc_artifact_tags_key ON pc_artifact_tags (scope_id, tag_key_hash, family, target_type, artifact_id, target_id)" + ] +} diff --git a/src/powercontext/builtin/persistence/migrations/resources/versions/p0001_baseline.py b/src/powercontext/builtin/persistence/migrations/resources/versions/p0001_baseline.py new file mode 100644 index 0000000000..f3b0d4d7a0 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/versions/p0001_baseline.py @@ -0,0 +1,34 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Four actual PowerContext tables frozen before Dream citations. + +This isolated prototype is not the complete schema of any PowerContext release. +The resource provenance is recorded alongside the frozen SQL. +""" + +import json +from pathlib import Path + +from alembic import op + +revision = "p0001" +down_revision = None + + +def upgrade() -> None: + dialect = op.get_bind().dialect.name + statements = json.loads((Path(__file__).parents[1] / "schemas" / "pre_dream.json").read_text())[dialect] + for statement in statements: + op.execute(statement) diff --git a/src/powercontext/builtin/persistence/migrations/resources/versions/p0002_citations.py b/src/powercontext/builtin/persistence/migrations/resources/versions/p0002_citations.py new file mode 100644 index 0000000000..0bfa2d58a5 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/versions/p0002_citations.py @@ -0,0 +1,27 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add immutable revision citations using fixed historical types.""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.mysql import MEDIUMBLOB + +revision = "p0002" +down_revision = "p0001" + + +def upgrade() -> None: + for table in ("pc_artifacts", "pc_artifact_candidate_versions"): + op.add_column(table, sa.Column("memory_citations", sa.LargeBinary().with_variant(MEDIUMBLOB(), "mysql"))) diff --git a/src/powercontext/builtin/persistence/migrations/resources/versions/p0003_tag_families.py b/src/powercontext/builtin/persistence/migrations/resources/versions/p0003_tag_families.py new file mode 100644 index 0000000000..cc98f5e684 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/resources/versions/p0003_tag_families.py @@ -0,0 +1,48 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Remove the family whitelist while preserving the target constraint and indexes.""" + +from alembic import op + +revision = "p0003" +down_revision = "p0002" + + +def upgrade() -> None: + connection = op.get_bind() + if connection.dialect.name == "sqlite": + # Keep a frozen copy of the replaced layout and rows. Cleanup belongs + # to a later, independently reviewed maintenance revision. + original = connection.exec_driver_sql( + "SELECT sql FROM sqlite_schema WHERE type='table' AND name='pc_artifact_tags'" + ).scalar_one() + retained = original.replace("CREATE TABLE pc_artifact_tags", "CREATE TABLE pc_retained_p0002_artifact_tags", 1) + connection.exec_driver_sql(retained) + connection.exec_driver_sql("INSERT INTO pc_retained_p0002_artifact_tags SELECT * FROM pc_artifact_tags") + # Alembic reflects indexes and named CHECKs, but not application triggers. + triggers = ( + connection + .exec_driver_sql( + "SELECT sql FROM sqlite_schema WHERE type='trigger' AND tbl_name='pc_artifact_tags' ORDER BY name" + ) + .scalars() + .all() + ) + with op.batch_alter_table("pc_artifact_tags", recreate="always") as batch: + batch.drop_constraint("ck_pc_artifact_tags_family", type_="check") + for statement in triggers: + connection.exec_driver_sql(statement) + else: + op.drop_constraint("ck_pc_artifact_tags_family", "pc_artifact_tags", type_="check") diff --git a/src/powercontext/builtin/persistence/migrations/sqlite.py b/src/powercontext/builtin/persistence/migrations/sqlite.py index f25b18b48d..9041d44080 100644 --- a/src/powercontext/builtin/persistence/migrations/sqlite.py +++ b/src/powercontext/builtin/persistence/migrations/sqlite.py @@ -12,82 +12,137 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Explicit offline SQLite migrations with atomic steps and durable recovery. +"""Explicit SQLite maintenance with Alembic's single standard version table. -This executor is for an explicitly supplied bundle. It does not establish that -the rest of the PowerContext service or its projections are ready. +External evidence preserves the original recovery point. Retry inspects the +actual revision and schema; there is no generic run ID or task checkpoint. +Readiness applies only to the registered bundle, not the entire Server runtime. """ from __future__ import annotations -import hashlib +import json import os import sqlite3 import threading from collections.abc import Iterator, Mapping -from contextlib import closing, contextmanager -from datetime import UTC, datetime +from contextlib import contextmanager from pathlib import Path -from typing import Any +from typing import Any, Literal, Self from uuid import uuid4 from alembic import command -from sqlalchemy import Connection, create_engine, text -from sqlalchemy.exc import OperationalError +from pydantic import BaseModel, ConfigDict, model_validator +from sqlalchemy import Connection, create_engine +from sqlalchemy.exc import DatabaseError, OperationalError +from .backup import BackupContext, BackupProvider, BackupRef, SQLiteBackupProvider from .bundle import MigrationBundle, sqlite_inventory from .locking import local_migration_lock from .models import MigrationError, MigrationPlan, MigrationResult, digest -_CONTROL_DDL = { - "pc_schema_revision": ( - "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " - "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num))" - ), - "pc_migration_runs": ( - "CREATE TABLE pc_migration_runs (run_id TEXT PRIMARY KEY NOT NULL, plan_json TEXT NOT NULL, " - "backup_path TEXT NOT NULL, backup_checksum TEXT NOT NULL, " - "state TEXT NOT NULL CHECK (state IN ('running', 'complete')), started_at TEXT NOT NULL)" - ), - "pc_migration_steps": ( - "CREATE TABLE pc_migration_steps (run_id TEXT NOT NULL, revision TEXT NOT NULL, checksum TEXT NOT NULL, " - "PRIMARY KEY (run_id, revision), FOREIGN KEY (run_id) REFERENCES pc_migration_runs(run_id))" - ), -} -_CONTROL_NAMES = frozenset(_CONTROL_DDL) -# Alembic's environment/op proxies are process-global, even for different DBs. +_VERSION_DDL = ( + "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " + "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num))" +) +_CONTROL_NAMES = frozenset({"pc_schema_revision"}) _ALEMBIC_CONTEXT_LOCK = threading.RLock() +BackupPolicy = Literal["auto", "manual", "skip"] + + +class _MaintenanceEvidence(BaseModel): + model_config = ConfigDict(extra="forbid") + + state: Literal["preparing", "active", "complete"] + database_id: str + source_revision: str | None + source_fingerprint: str + bundle_checksum: str + configuration_digest: str + backup_policy: BackupPolicy + maintenance_window_id: str + backup_state: Literal["pending", "completed", "user_confirmed", "skipped", "not_required"] + backup: BackupRef | None + manual_ref: str | None = None + release_checksums: dict[str, str] + retained_objects: tuple[str, ...] = () + + @model_validator(mode="after") + def consistent_recovery_point(self) -> Self: + if self.backup_state == "not_required": + if self.source_revision is not None or self.backup is not None: + raise ValueError("Only an originally empty database needs no recovery point") # noqa: TRY003 + return self + expected = {"auto": "completed", "manual": "user_confirmed", "skip": "skipped"}[self.backup_policy] + if self.backup_state != expected and not (self.state == "preparing" and self.backup_state == "pending"): + raise ValueError("The backup policy and recorded maintenance state disagree") # noqa: TRY003 + if self.backup_state == "completed": + ref = self.backup + if ref is None or ( + ref.database_id != self.database_id + or ref.source_revision != self.source_revision + or ref.bundle_checksum != self.bundle_checksum + or ref.maintenance_window_id != self.maintenance_window_id + ): + raise ValueError("The original backup reference belongs to another maintenance context") # noqa: TRY003 + elif self.backup is not None: + raise ValueError("An automatic recovery point is inconsistent with the declared backup state") # noqa: TRY003 + return self class SQLiteMigrationRunner: - """A synchronous maintenance executor. Call outside the business event loop. + """Keep business writers stopped throughout this synchronous executor. - File locks serialize cooperating migrators; BEGIN IMMEDIATE detects a current - SQLite writer. Neither proves that an idle old process has stopped, so the - operator's maintenance confirmation is required for every changing run. + The OS lock serializes cooperating migrators. BEGIN IMMEDIATE detects a + present writer, but cannot prove that idle old processes will remain stopped. """ def __init__( - self, database: Path, bundle: MigrationBundle, *, configuration: Mapping[str, Any] | None = None + self, + database: Path, + bundle: MigrationBundle, + *, + configuration: Mapping[str, Any] | None = None, + backup_provider: BackupProvider | None = None, ) -> None: self.database = database.expanduser().resolve() self.bundle = bundle self.configuration_digest = digest(configuration or {}) + self.backup_provider = backup_provider or SQLiteBackupProvider(self.database) + self.evidence_path = self.database.with_name(self.database.name + ".pc-migration-state") / "maintenance.json" - def plan(self) -> MigrationPlan: - """Inspect without creating the database, its parent, locks or version rows.""" - if MigrationBundle(self.bundle.directory).checksum != self.bundle.checksum: - raise MigrationError("plan_changed", "Migration resources changed after the bundle was loaded.") + def plan( + self, + *, + backup_policy: BackupPolicy = "auto", + service: dict[str, Any] | None = None, + shared_database: bool = False, + ) -> MigrationPlan: + """Inspect without creating a database, parent, lock or external evidence.""" + self.bundle.assert_unchanged() + if backup_policy not in {"auto", "manual", "skip"}: + raise MigrationError("invalid_backup_policy", "Choose auto, manual or skip.") if not self.database.exists(): - return self._plan(None, digest([]), initialized=False) + return self._plan( + None, + digest([]), + initialized=False, + backup_policy=backup_policy, + service=service, + shared_database=shared_database, + ) self._check_file() with self._connect(read_only=True) as connection: - return self._inspect(connection) + return self._inspect( + connection, backup_policy=backup_policy, service=service, shared_database=shared_database + ) def verify(self) -> MigrationResult: - plan = self.plan() + evidence = self._evidence() + policy = evidence["backup_policy"] if evidence and evidence["state"] != "complete" else "auto" + plan = self.plan(backup_policy=policy) if plan.state != "ready": - raise MigrationError(plan.state, "The bundle is not fully migrated; inspect its plan before applying.") + raise MigrationError(plan.state, "Inspect and explicitly apply the registered bundle before verification.") return MigrationResult(revision=self.bundle.head, changed=False) def apply( @@ -96,60 +151,98 @@ def apply( plan_id: str | None = None, maintenance_confirmed: bool = False, accepted: bool = False, - resume: str | None = None, + backup_policy: BackupPolicy = "auto", + backup_confirmed: bool = False, + accept_no_backup: bool = False, + backup_ref: str | None = None, + service: dict[str, Any] | None = None, + shared_database: bool = False, ) -> MigrationResult: - """Require an accepted plan; resume keeps its original plan and backup.""" - initial = self.plan() - if initial.state == "ready" and resume is None: + options: dict[str, Any] = { + "backup_policy": backup_policy, + "service": service, + "shared_database": shared_database, + } + initial = self.plan(**options) + if initial.state == "ready": return MigrationResult(revision=self.bundle.head, changed=False) - if not accepted or plan_id is None: - raise MigrationError("confirmation_required", "Accept the displayed plan ID explicitly.") - if not maintenance_confirmed: - raise MigrationError("maintenance_required", "Stop all application, SDK and background writers first.") - if initial.state == "recovery_required" and resume != initial.resume_run_id: - raise MigrationError("recovery_required", "Resume the recorded run; do not start a replacement run.") - if resume is not None and resume != initial.resume_run_id: - raise MigrationError("recovery_required", "The requested run is not the pending run.") - if initial.plan_id != plan_id: - raise MigrationError("plan_changed", "The reviewed plan no longer matches the database or bundle.") + self._require_acceptance(initial, plan_id, accepted, maintenance_confirmed, backup_confirmed, accept_no_backup) with local_migration_lock(self.database): - # Revalidate before opening in write mode, including a missing target. - if self.plan() != initial: + if self.plan(**options) != initial: raise MigrationError("plan_changed", "State changed while acquiring the migration lock.") with self._connect(read_only=False) as connection: connection.exec_driver_sql("PRAGMA foreign_keys=OFF") connection.commit() self._begin(connection) - locked = self._inspect(connection) - if locked != initial: - raise MigrationError("plan_changed", "State changed before acquiring the SQLite write transaction.") - if resume: - run_id, backup = self._resume(connection, resume, initial) - connection.commit() - else: - # A separate read connection can back up the committed WAL - # while this connection prevents writers with BEGIN IMMEDIATE. - backup, checksum = self._backup() - run_id = uuid4().hex - self._bootstrap(connection, initial, run_id, backup, checksum) - connection.commit() - self._upgrade(connection, run_id) + if self._inspect(connection, **options) != initial: + raise MigrationError("plan_changed", "State changed before acquiring the write transaction.") + evidence = self._prepare_evidence(initial, backup_ref) + self.bundle.assert_unchanged() + version_exists = ( + connection.exec_driver_sql("SELECT name FROM sqlite_schema WHERE name='pc_schema_revision'").first() + is not None + ) + if not version_exists: + connection.exec_driver_sql(_VERSION_DDL) + if initial.adopt_baseline and initial.source_revision is not None: + with _ALEMBIC_CONTEXT_LOCK: + command.stamp(self.bundle.config(connection), initial.source_revision) + connection.commit() + self._upgrade(connection) self._begin(connection) self._verify_schema(connection, self.bundle.head) - connection.execute( - text("UPDATE pc_migration_runs SET state='complete' WHERE run_id=:run"), {"run": run_id} - ) connection.commit() - return MigrationResult(revision=self.bundle.head, changed=True, run_id=run_id, backup_ref=str(backup)) + evidence["state"] = "complete" + evidence["release_checksums"] = self.bundle.checksums + inventory = sqlite_inventory(connection) + evidence["retained_objects"] = [ + name for name in self.bundle.retained_objects if any(row[1] == name for row in inventory) + ] + self._write_evidence(evidence) + ref = evidence.get("backup") + return MigrationResult( + revision=self.bundle.head, + changed=True, + backup_state=evidence["backup_state"], + backup_ref=ref["location"] if ref else evidence.get("manual_ref"), + ) + + @staticmethod + def _require_acceptance( + initial: MigrationPlan, + plan_id: str | None, + accepted: bool, + maintenance_confirmed: bool, + backup_confirmed: bool, + accept_no_backup: bool, + ) -> None: + if not accepted or plan_id is None: + raise MigrationError("confirmation_required", "Accept the displayed plan ID explicitly.") + if not maintenance_confirmed: + raise MigrationError( + "maintenance_required", "Stop every writer and coordinate all affected shared-db nodes." + ) + if initial.plan_id != plan_id: + raise MigrationError("plan_changed", "The reviewed target, schema, bundle or maintenance choices changed.") + if initial.state != "uninitialized": + if initial.backup_policy == "manual" and not backup_confirmed: + raise MigrationError( + "backup_confirmation_required", "Declare your manual backup; PC does not verify it." + ) + if initial.backup_policy == "skip" and not accept_no_backup: + raise MigrationError( + "backup_confirmation_required", "Accept that the original data may be unrecoverable." + ) + if initial.backup_policy == "auto" and not initial.backup_available: + raise MigrationError("backup_unsupported", "; ".join(initial.backup_reason)) def _check_file(self) -> None: if not self.database.is_file() or self.database.stat().st_nlink != 1: - raise MigrationError("unsupported_target", "Use a regular database file without hard-link aliases.") + raise MigrationError("unsupported_target", "Use a regular SQLite file without hard-link aliases.") @contextmanager def _connect(self, *, read_only: bool) -> Iterator[Connection]: - mode = "ro" if read_only else "rwc" - uri = self.database.as_uri() + "?mode=" + mode + uri = self.database.as_uri() + ("?mode=ro" if read_only else "?mode=rwc") engine = create_engine( "sqlite://", creator=lambda: sqlite3.connect(uri, uri=True, isolation_level=None, timeout=0) ) @@ -159,79 +252,84 @@ def _connect(self, *, read_only: bool) -> Iterator[Connection]: connection.exec_driver_sql("PRAGMA query_only=ON") connection.exec_driver_sql("BEGIN") yield connection + except DatabaseError as error: + if isinstance(error.orig, sqlite3.Error): + raise MigrationError( + "invalid_database", "SQLite could not read or execute the registered schema." + ) from error + raise finally: engine.dispose() - def _inspect(self, connection: Connection) -> MigrationPlan: + def _inspect(self, connection: Connection, **options: Any) -> MigrationPlan: inventory = sqlite_inventory(connection) controls = [row for row in inventory if row[1] in _CONTROL_NAMES or row[2] in _CONTROL_NAMES] initialized = bool(controls) + if initialized and controls != [["table", "pc_schema_revision", "pc_schema_revision", _VERSION_DDL]]: + raise MigrationError("recovery_required", "The Alembic version table has an unknown shape.") + fingerprint = digest( + sqlite_inventory(connection, exclude=_CONTROL_NAMES | frozenset(self.bundle.retained_objects)) + ) if initialized: - expected = sorted([["table", name, name, ddl] for name, ddl in _CONTROL_DDL.items()]) - if sorted(controls) != expected: + revisions = list(connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalars()) + if len(revisions) > 1: + raise MigrationError("incompatible_schema", "Multiple revision heads are unsupported.") + revision = revisions[0] if revisions else None + self._verify_schema(connection, revision) + if self.bundle.pending(revision) and self._evidence() is None: raise MigrationError( - "recovery_required", "Migration bookkeeping is incomplete or has an unknown shape." + "recovery_required", + "The original external maintenance evidence is missing; do not back up partial state.", ) - fingerprint = digest(sqlite_inventory(connection, exclude=_CONTROL_NAMES)) - if not initialized: - if not inventory: - return self._plan(None, fingerprint, initialized=False) + elif inventory: revision = self.bundle.baselines.get(fingerprint) if revision is None: - raise MigrationError("unknown_baseline", "Schema is not a recognized baseline; no changes were made.") + raise MigrationError("unknown_baseline", "Schema is not registered; no changes were made.") + self._verify_retained(inventory, revision) self._verify_integrity(connection) - return self._plan(revision, fingerprint, initialized=False) - revisions = list(connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalars()) - if len(revisions) > 1: - raise MigrationError("incompatible_schema", "Multiple database revision heads are unsupported.") - revision = revisions[0] if revisions else None - self._verify_schema(connection, revision) - self._verify_history(connection, revision) - recovery = self._pending_run(connection, revision) - if recovery is not None: - return recovery - return self._plan(revision, fingerprint, initialized=True) - - def _verify_history(self, connection: Connection, revision: str | None) -> None: - receipts = list(connection.exec_driver_sql("SELECT revision, checksum FROM pc_migration_steps")) - expected = self.bundle.revisions[: self.bundle.revisions.index(revision) + 1] if revision else () - if sorted(item[0] for item in receipts) != sorted(expected): - raise MigrationError("recovery_required", "Revision receipts do not match the database version.") - for recorded_revision, checksum in receipts: - if self.bundle.checksums.get(recorded_revision) != checksum: - raise MigrationError("checksum_conflict", "An applied revision is absent or its source has changed.") - - def _pending_run(self, connection: Connection, revision: str | None) -> MigrationPlan | None: - runs = list(connection.exec_driver_sql("SELECT run_id, plan_json FROM pc_migration_runs WHERE state='running'")) - if len(runs) > 1: - raise MigrationError("recovery_required", "Multiple unfinished runs require manual inspection.") - if runs: - run_id, plan_json = runs[0] - original = MigrationPlan.model_validate_json(plan_json) - if ( - original.bundle_checksum != self.bundle.checksum - or original.configuration_digest != self.configuration_digest - ): - raise MigrationError("plan_changed", "Resume requires the original migration bundle and configuration.") - if original.database_id != self._database_id(): - raise MigrationError("plan_changed", "The recovery record belongs to a different database target.") - completed_count = self.bundle.revisions.index(revision) + 1 if revision is not None else 0 - source_count = self.bundle.revisions.index(original.source_revision) + 1 if original.source_revision else 0 - if completed_count < source_count: - raise MigrationError("recovery_required", "Recorded progress precedes the original source revision.") - return original.model_copy(update={"state": "recovery_required", "resume_run_id": run_id}) - return None - - def _database_id(self) -> str: - # The deployment identity is the canonical path, never a DSN or payload. - return digest({"kind": "sqlite", "path": str(self.database)}) + else: + revision = None + return self._plan(revision, fingerprint, initialized=initialized, **options) - def _plan(self, revision: str | None, fingerprint: str, *, initialized: bool) -> MigrationPlan: + def _plan( + self, + revision: str | None, + fingerprint: str, + *, + initialized: bool, + backup_policy: BackupPolicy, + service: dict[str, Any] | None, + shared_database: bool, + ) -> MigrationPlan: pending = self.bundle.pending(revision) adopt = not initialized and revision is not None state = "migration_required" if pending or adopt else "ready" if not initialized and revision is None: state = "uninitialized" + evidence = self._evidence() + if evidence: + for recorded, checksum in evidence.get("release_checksums", {}).items(): + if self.bundle.checksums.get(recorded) != checksum: + raise MigrationError("checksum_conflict", "A recorded released migration resource changed.") + if evidence["state"] != "complete": + if ( + evidence["bundle_checksum"] != self.bundle.checksum + or evidence["configuration_digest"] != self.configuration_digest + ): + raise MigrationError( + "recovery_required", "Keep the original bundle, configuration and backup policy." + ) + pristine = self._pristine_preparation(evidence, revision, fingerprint) + if evidence["backup_policy"] != backup_policy and not pristine: + raise MigrationError("recovery_required", "Keep the original backup policy after migration begins.") + if not pristine: + state = "recovery_required" + context = self._backup_context(revision, "preview") + available = True + reasons: tuple[str, ...] = () + if backup_policy == "auto" and revision is not None and state != "ready": + capability = self.backup_provider.capabilities(context) + available, reasons = capability.available, capability.reasons fields = { "database_id": self._database_id(), "source_revision": revision, @@ -242,91 +340,146 @@ def _plan(self, revision: str | None, fingerprint: str, *, initialized: bool) -> "state": state, "revisions": pending, "adopt_baseline": adopt, + "backup_policy": backup_policy, + "backup_available": available, + "backup_reason": reasons, + "affected_objects": self.bundle.affected_objects, + "retained_objects": self.bundle.retained_objects, + "service": service, + "shared_database": shared_database, } return MigrationPlan.model_validate({"plan_id": digest(fields), **fields}) + def _database_id(self) -> str: + return digest({"kind": "sqlite", "path": str(self.database)}) + + def _backup_context(self, revision: str | None, window: str) -> BackupContext: + return BackupContext( + database_id=self._database_id(), + source_revision=revision, + bundle_checksum=self.bundle.checksum, + maintenance_window_id=window, + objects=self.bundle.affected_objects, + writers_stopped=True, + ) + + def _prepare_evidence(self, plan: MigrationPlan, manual_ref: str | None) -> dict[str, Any]: + previous = self._evidence() + if previous and previous["state"] != "complete": + recovered = self._reuse_evidence(previous, plan) + if recovered is not None: + return recovered + evidence: dict[str, Any] = ( + previous + if previous and previous["state"] != "complete" + else { + "state": "preparing", + "database_id": plan.database_id, + "source_revision": plan.source_revision, + "source_fingerprint": plan.schema_fingerprint, + "bundle_checksum": plan.bundle_checksum, + "configuration_digest": plan.configuration_digest, + "backup_policy": plan.backup_policy, + "maintenance_window_id": uuid4().hex, + "backup_state": "pending", + "backup": None, + "release_checksums": previous.get("release_checksums", {}) if previous else {}, + } + ) + self._write_evidence(evidence) + if plan.source_revision is None: + evidence["backup_state"] = "not_required" + elif plan.backup_policy == "auto": + ref = self.backup_provider.create_backup( + self._backup_context(plan.source_revision, evidence["maintenance_window_id"]) + ) + if self.backup_provider.inspect_backup(ref).state != "completed": + raise MigrationError("backup_failed", "The automatic recovery point did not pass provider inspection.") + evidence.update(backup=ref.model_dump(mode="json"), backup_state="completed") + elif plan.backup_policy == "manual": + evidence.update(backup_state="user_confirmed", manual_ref=manual_ref) + else: + evidence["backup_state"] = "skipped" + evidence["state"] = "active" + self._write_evidence(evidence) + return evidence + + def _reuse_evidence(self, previous: dict[str, Any], plan: MigrationPlan) -> dict[str, Any] | None: + if self._pristine_preparation(previous, plan.source_revision, plan.schema_fingerprint): + previous["backup_policy"] = plan.backup_policy + if previous.get("backup"): + ref = BackupRef.model_validate(previous["backup"]) + if self.backup_provider.inspect_backup(ref).state != "completed": + raise MigrationError( + "backup_required", "The original automatic recovery point or manifest is unavailable." + ) + return previous + if previous["backup_state"] in {"user_confirmed", "skipped", "not_required"}: + return previous + if previous["source_fingerprint"] != plan.schema_fingerprint: + raise MigrationError( + "backup_required", "Do not replace a missing original recovery point after schema changes." + ) + return None + @staticmethod - def _begin(connection: Connection) -> None: + def _pristine_preparation(evidence: dict[str, Any], revision: str | None, fingerprint: str) -> bool: + return ( + evidence["state"] == "preparing" + and evidence["backup_state"] == "pending" + and evidence["backup"] is None + and evidence["source_revision"] == revision + and evidence["source_fingerprint"] == fingerprint + ) + + def _evidence(self) -> dict[str, Any] | None: + if not self.evidence_path.exists(): + return None try: - connection.exec_driver_sql("BEGIN IMMEDIATE") - except OperationalError as error: - if isinstance(error.orig, sqlite3.OperationalError) and error.orig.sqlite_errorcode in { - sqlite3.SQLITE_BUSY, - sqlite3.SQLITE_LOCKED, - }: - raise MigrationError("active_writers", "SQLite is busy; stop writers before retrying.") from error - raise + value = _MaintenanceEvidence.model_validate_json(self.evidence_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise MigrationError( + "recovery_required", "The external maintenance evidence cannot be read safely." + ) from error + if value.database_id != self._database_id(): + raise MigrationError("recovery_required", "The external maintenance evidence belongs to another target.") + return value.model_dump(mode="json") - def _backup(self) -> tuple[Path, str]: - directory = self.database.with_name(self.database.name + ".pc-migration-backups") - backup = directory / f"{uuid4().hex}.sqlite3" + def _write_evidence(self, evidence: dict[str, Any]) -> None: + directory = self.evidence_path.parent + temporary = directory / f".{uuid4().hex}.json" try: - directory.mkdir(mode=0o700, exist_ok=True) - os.close(os.open(backup, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)) - with ( - closing(sqlite3.connect(self.database.as_uri() + "?mode=ro", uri=True)) as source, - closing(sqlite3.connect(backup)) as destination, - ): - source.backup(destination) - if destination.execute("PRAGMA integrity_check").fetchall() != [("ok",)]: - raise MigrationError("backup_required", "The consistent SQLite backup failed integrity checks.") - with backup.open("rb") as persisted: - os.fsync(persisted.fileno()) + directory.mkdir(parents=True, mode=0o700, exist_ok=True) + with temporary.open("x", encoding="utf-8") as output: + os.chmod(temporary, 0o600) + json.dump(evidence, output, ensure_ascii=False, sort_keys=True) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, self.evidence_path) if os.name != "nt": - directory_fd = os.open(directory, os.O_RDONLY) + descriptor = os.open(directory, os.O_RDONLY) try: - os.fsync(directory_fd) + os.fsync(descriptor) finally: - os.close(directory_fd) - return backup, _file_checksum(backup) - except (OSError, sqlite3.Error) as error: + os.close(descriptor) + except OSError as error: raise MigrationError( - "backup_required", "Could not create and verify a consistent SQLite backup." + "recovery_required", "Cannot persist external recovery evidence; migration is blocked." ) from error - def _bootstrap(self, connection: Connection, plan: MigrationPlan, run_id: str, backup: Path, checksum: str) -> None: - if plan.adopt_baseline or plan.state == "uninitialized": - for ddl in _CONTROL_DDL.values(): - connection.exec_driver_sql(ddl) - if plan.adopt_baseline and plan.source_revision is not None: - # Adoption follows exact baseline recognition and a verified backup. - with _ALEMBIC_CONTEXT_LOCK: - command.stamp(self.bundle.config(connection), plan.source_revision) - connection.execute( - text("INSERT INTO pc_migration_runs VALUES (:run, :plan, :backup, :checksum, 'running', :started)"), - { - "run": run_id, - "plan": plan.model_dump_json(), - "backup": str(backup), - "checksum": checksum, - "started": datetime.now(UTC).isoformat(), - }, - ) - if plan.adopt_baseline and plan.source_revision: - for revision in self.bundle.revisions[: self.bundle.revisions.index(plan.source_revision) + 1]: - connection.execute( - text("INSERT INTO pc_migration_steps VALUES (:run, :revision, :checksum)"), - {"run": run_id, "revision": revision, "checksum": self.bundle.checksums[revision]}, - ) - - def _resume(self, connection: Connection, run_id: str, plan: MigrationPlan) -> tuple[str, Path]: - path, checksum, plan_json = connection.execute( - text("SELECT backup_path, backup_checksum, plan_json FROM pc_migration_runs WHERE run_id=:run"), - {"run": run_id}, - ).one() - original = MigrationPlan.model_validate_json(plan_json) - if original.plan_id != plan.plan_id: - raise MigrationError("plan_changed", "The run does not match the accepted plan.") - backup = Path(path) + @staticmethod + def _begin(connection: Connection) -> None: try: - valid = backup.is_file() and _file_checksum(backup) == checksum - except OSError as error: - raise MigrationError("backup_required", "The original recovery point cannot be read.") from error - if not valid: - raise MigrationError("backup_required", "The original recovery point is missing or has changed.") - return run_id, backup + connection.exec_driver_sql("BEGIN IMMEDIATE") + except OperationalError as error: + if isinstance(error.orig, sqlite3.OperationalError) and error.orig.sqlite_errorcode in { + sqlite3.SQLITE_BUSY, + sqlite3.SQLITE_LOCKED, + }: + raise MigrationError("active_writers", "SQLite is busy; stop writers before retrying.") from error + raise - def _upgrade(self, connection: Connection, run_id: str) -> None: + def _upgrade(self, connection: Connection) -> None: current = connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one_or_none() connection.commit() for revision in self.bundle.pending(current): @@ -335,25 +488,33 @@ def _upgrade(self, connection: Connection, run_id: str) -> None: with _ALEMBIC_CONTEXT_LOCK: command.upgrade(self.bundle.config(connection), revision) self._verify_schema(connection, revision) - connection.execute( - text("INSERT INTO pc_migration_steps VALUES (:run, :revision, :checksum)"), - {"run": run_id, "revision": revision, "checksum": self.bundle.checksums[revision]}, - ) connection.commit() current = revision def _verify_schema(self, connection: Connection, revision: str | None) -> None: - self.bundle.verify(revision, digest(sqlite_inventory(connection, exclude=_CONTROL_NAMES))) + excluded = _CONTROL_NAMES | frozenset(self.bundle.retained_objects) + inventory = sqlite_inventory(connection) + self._verify_retained(inventory, revision) + self.bundle.verify(revision, digest(sqlite_inventory(connection, exclude=excluded))) self._verify_integrity(connection) - @staticmethod - def _verify_integrity(connection: Connection) -> None: - if connection.exec_driver_sql("PRAGMA foreign_key_check").first() is not None: + def _verify_retained(self, inventory: list[list[Any]], revision: str | None) -> None: + for name in self.bundle.retained_objects: + objects = [row for row in inventory if row[1] == name or row[2] == name] + if objects and self.bundle.retained_revisions[name] in self.bundle.pending(revision): + raise MigrationError("recovery_required", "A retained object exists before its introducing revision.") + if objects and digest(objects) not in self.bundle.retained_fingerprints.get(name, []): + raise MigrationError("incompatible_schema", "A retained historical object has an unregistered shape.") + evidence = self._evidence() + if evidence and any(not any(row[1] == name for row in inventory) for name in evidence["retained_objects"]): + raise MigrationError( + "recovery_required", "A registered retained table was removed without a cleanup revision." + ) + + def _verify_integrity(self, connection: Connection) -> None: + if any( + row[0] not in self.bundle.retained_objects for row in connection.exec_driver_sql("PRAGMA foreign_key_check") + ): raise MigrationError("verification_failed", "Foreign-key violations block migration readiness.") if connection.exec_driver_sql("PRAGMA quick_check").scalars().all() != ["ok"]: raise MigrationError("verification_failed", "SQLite integrity verification failed.") - - -def _file_checksum(path: Path) -> str: - with path.open("rb") as source: - return hashlib.file_digest(source, "sha256").hexdigest() diff --git a/src/powercontext/server/cli.py b/src/powercontext/server/cli.py index 36a850cf36..092db71227 100644 --- a/src/powercontext/server/cli.py +++ b/src/powercontext/server/cli.py @@ -45,6 +45,7 @@ resolve_server_environment_file, server_settings_context, ) +from powercontext.server.database_migration import app as database_migration_app from powercontext.server.factory import create_server_app from powercontext.server.logging import configure_server_logging from powercontext.server.processing_security import build_worker_security @@ -81,6 +82,7 @@ help="Run a configured PowerContext service.", no_args_is_help=True, ) +app.add_typer(database_migration_app, name="db-migrate") @app.callback() diff --git a/src/powercontext/server/database_migration.py b/src/powercontext/server/database_migration.py new file mode 100644 index 0000000000..64843391a6 --- /dev/null +++ b/src/powercontext/server/database_migration.py @@ -0,0 +1,303 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Explicit maintenance of the database selected by Server configuration.""" + +from __future__ import annotations + +import json +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from typing import Annotated, Literal + +import typer +from sqlalchemy.exc import SQLAlchemyError + +from powercontext.builtin.persistence.migrations import MigrationError, MigrationPlan, MigrationResult +from powercontext.builtin.persistence.migrations.deployment import deployment_runner +from powercontext.builtin.persistence.migrations.sqlite import SQLiteMigrationRunner +from powercontext.server.configuration import ( + ServerConfigurationError, + resolve_server_environment_file, + server_settings_context, +) +from powercontext.service.controller import ServiceController +from powercontext.service.model import ServiceError + +BackupPolicy = Literal["auto", "manual", "skip"] +EnvFile = Annotated[Path | None, typer.Option(help="Read the same environment file as the deployed Server.")] +ManageService = Annotated[ + bool, + typer.Option(help="Request local service maintenance; unavailable for the current partial acceptance bundle."), +] +SharedDatabase = Annotated[ + bool, + typer.Option(help="Declare other Servers share this database; coordinate all writers through deployment tooling."), +] + +app = typer.Typer( + name="db-migrate", + context_settings={"help_option_names": ("-h", "--help")}, + help="Inspect and explicitly migrate the configured persistent database.", + no_args_is_help=True, +) + + +@app.callback() +def main() -> None: + """Manage schema revisions independently of installing software.""" + + +@app.command() +def status(env_file: EnvFile = None) -> None: + """Inspect the schema version without creating a database or control tables.""" + _inspect(env_file, verify=False) + + +@app.command() +def verify(env_file: EnvFile = None) -> None: + """Verify database readiness; this does not assert Server or cluster readiness.""" + _inspect(env_file, verify=True) + + +@app.command() +def plan( + env_file: EnvFile = None, + backup: Annotated[BackupPolicy, typer.Option(help="Bind the selected backup policy to the plan.")] = "auto", + manage_service: ManageService = False, + shared_database: SharedDatabase = False, +) -> None: + """Preview the target, resource digests, backup policy, and maintenance scope.""" + with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: + runner = deployment_runner(settings) + service = _service_scope(_environment_file(env_file)) if manage_service else None + reviewed = runner.plan(backup_policy=backup, service=service, shared_database=shared_database) + _write_payload(reviewed) + + +@app.command() +def apply( + env_file: EnvFile = None, + backup: Annotated[ + BackupPolicy | None, + typer.Option(help="PC native backup, a user-declared manual backup, or explicit no-backup risk."), + ] = None, + plan_id: Annotated[str | None, typer.Option(help="The exact plan accepted by the operator.")] = None, + yes: Annotated[ + bool, typer.Option("--yes", help="Accept the plan; does not imply backup or maintenance consent.") + ] = False, + maintenance_confirmed: Annotated[ + bool, + typer.Option( + help="Declare all writers stopped and accept coordinating every affected shared-database node upgrade." + ), + ] = False, + backup_confirmed: Annotated[ + bool, typer.Option(help="Declare a manual backup exists; PC does not inspect or verify it.") + ] = False, + accept_no_backup: Annotated[ + bool, typer.Option(help="Explicitly accept that failure may make original data unrecoverable.") + ] = False, + backup_ref: Annotated[str | None, typer.Option(help="Optional reference for the operator's manual backup.")] = None, + manage_service: ManageService = False, + shared_database: SharedDatabase = False, +) -> None: + """Review once, then stop writes, back up, migrate, verify, and optionally switch a local service.""" + with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: + runner = deployment_runner(settings) + policy: BackupPolicy = backup or "auto" + initial = runner.plan(backup_policy=policy, shared_database=shared_database) + if initial.state == "ready": + _write_payload(runner.verify()) + return + service = _service_scope(_environment_file(env_file)) if manage_service else None + reviewed = runner.plan(backup_policy=policy, service=service, shared_database=shared_database) + if policy == "auto" and not reviewed.backup_available and reviewed.state != "uninitialized": + raise MigrationError( + "backup_unsupported", + "PC automatic backup is unavailable; select --backup manual or --backup skip and review a new plan.", + ) + if yes: + if backup is None or plan_id is None: + raise MigrationError("confirmation_required", "Automation requires explicit --backup and --plan-id.") + elif _interactive(): + _write_confirmation(reviewed, policy) + if not typer.confirm("Accept this plan and the declarations above?", default=False): + raise MigrationError("confirmation_required", "The plan was not accepted.") + plan_id = reviewed.plan_id + maintenance_confirmed = True + backup_confirmed = policy == "manual" + accept_no_backup = policy == "skip" + else: + raise MigrationError("confirmation_required", "Review a plan and provide --plan-id, --backup and --yes.") + if plan_id != reviewed.plan_id: + raise MigrationError("plan_changed", "The accepted plan does not match the current options or database.") + _check_declarations( + reviewed, + maintenance_confirmed=maintenance_confirmed, + backup_confirmed=backup_confirmed, + accept_no_backup=accept_no_backup, + ) + result = _apply_reviewed( + runner, + reviewed, + maintenance_confirmed=maintenance_confirmed, + backup_confirmed=backup_confirmed, + accept_no_backup=accept_no_backup, + backup_ref=backup_ref, + env_file=_environment_file(env_file), + ) + _write_payload(result) + + +def _inspect(env_file: Path | None, *, verify: bool) -> None: + with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: + runner = deployment_runner(settings) + result = runner.verify() if verify else runner.plan() + _write_payload(result) + + +def _environment_file(env_file: Path | None) -> Path | None: + return resolve_server_environment_file(env_file, discover=True) + + +def _interactive() -> bool: + return sys.stdin.isatty() + + +def _write_payload(value: MigrationPlan | MigrationResult, *, indent: int | None = None) -> None: + typer.echo( + json.dumps( + {**value.model_dump(mode="json"), "server_ready": False}, + indent=indent, + ) + ) + + +def _write_confirmation(reviewed: MigrationPlan, policy: BackupPolicy) -> None: + _write_payload(reviewed, indent=2) + typer.echo( + "This acceptance bundle does not verify complete Server schema, task formats, indexes or startup readiness." + ) + typer.echo("Maintenance pauses writes. Stop all other Servers, Workers, SDK clients and automatic restarts first.") + typer.echo("PC cannot automatically discover every node or client connected to this database.") + if reviewed.shared_database: + typer.echo("You are responsible for upgrading every affected shared-database node before restoring traffic.") + if reviewed.service is not None: + typer.echo( + "PC manages only the displayed local service; all other writers remain the operator's responsibility." + ) + if policy == "auto": + typer.echo("PC will create a native recovery point. Large databases may need considerable time and free space.") + elif policy == "manual": + typer.echo("By confirming, you declare that you have backed up the data. PC will not verify that backup.") + else: + typer.echo("By confirming, you accept that migration failure may make original data unrecoverable.") + typer.echo("Old tables and recovery points are retained.") + + +def _service_scope(env_file: Path | None) -> dict[str, object]: + del env_file + raise MigrationError( + "service_unsupported", + "This partial acceptance bundle cannot establish complete Server readiness or switch a business service.", + ) + + +def _apply_reviewed( + runner: SQLiteMigrationRunner, + reviewed: MigrationPlan, + *, + maintenance_confirmed: bool, + backup_confirmed: bool, + accept_no_backup: bool, + backup_ref: str | None, + env_file: Path | None, +) -> MigrationResult: + def migrate() -> MigrationResult: + return runner.apply( + plan_id=reviewed.plan_id, + accepted=True, + maintenance_confirmed=maintenance_confirmed, + backup_policy=reviewed.backup_policy, + backup_confirmed=backup_confirmed, + accept_no_backup=accept_no_backup, + backup_ref=backup_ref, + service=reviewed.service, + shared_database=reviewed.shared_database, + ) + + if reviewed.service is None: + return migrate() + fingerprint = reviewed.service.get("fingerprint") + if not isinstance(fingerprint, str): + raise MigrationError("service_unsupported", "The local service does not provide a stable maintenance identity.") + with ServiceController().maintenance(expected_fingerprint=fingerprint, env_file=env_file) as session: + result = migrate() + try: + session.complete() + except (ServiceError, OSError): + _fail( + "service_start_failed", + "The database is ready but the service switch or startup failed; inspect service status and logs.", + database_state="ready", + service_state="failed", + revision=result.revision, + ) + return result + + +def _check_declarations( + reviewed: MigrationPlan, + *, + maintenance_confirmed: bool, + backup_confirmed: bool, + accept_no_backup: bool, +) -> None: + if not maintenance_confirmed: + raise MigrationError("maintenance_required", "Stop all writers and provide --maintenance-confirmed.") + if reviewed.state == "uninitialized": + return + if reviewed.backup_policy == "manual" and not backup_confirmed: + raise MigrationError("confirmation_required", "Manual backup requires --backup-confirmed.") + if reviewed.backup_policy == "skip" and not accept_no_backup: + raise MigrationError("confirmation_required", "Skipping backup requires --accept-no-backup.") + + +@contextmanager +def _operator_errors() -> Iterator[None]: + try: + yield + except MigrationError as error: + _fail(error.code, str(error)) + except ServerConfigurationError: + _fail("configuration_invalid", "The Server configuration is invalid; check the environment file and settings.") + except SQLAlchemyError: + _fail("invalid_target", "The database cannot be inspected; check its format, access and availability.") + except OSError: + _fail("target_unavailable", "The maintenance target or its recovery storage cannot be accessed.") + except ServiceError: + _fail( + "service_maintenance_failed", "The local service cannot enter maintenance; inspect service status and logs." + ) + + +def _fail(code: str, message: str, **details: object) -> None: + typer.echo(json.dumps({"state": "blocked", "error": code, "message": message, **details})) + raise typer.Exit(code=1) + + +__all__ = ["app"] diff --git a/src/powercontext/service/adapters/base.py b/src/powercontext/service/adapters/base.py index 9624c5d478..f8a3b25617 100644 --- a/src/powercontext/service/adapters/base.py +++ b/src/powercontext/service/adapters/base.py @@ -68,6 +68,12 @@ def stop(self) -> None: ... def disable(self) -> None: ... + def suspend(self, marker: Path) -> None: ... + + def resume(self) -> None: ... + + def update_suspended(self) -> None: ... + def remove(self) -> None: ... def manager_state(self) -> ManagerState: ... @@ -121,6 +127,15 @@ def stop(self) -> None: def disable(self) -> None: raise ServiceError(self._detail) + def suspend(self, marker: Path) -> None: + raise ServiceError(self._detail) + + def resume(self) -> None: + raise ServiceError(self._detail) + + def update_suspended(self) -> None: + raise ServiceError(self._detail) + def remove(self) -> None: raise ServiceError(self._detail) diff --git a/src/powercontext/service/adapters/launchd.py b/src/powercontext/service/adapters/launchd.py index 92b4927035..36d342a318 100644 --- a/src/powercontext/service/adapters/launchd.py +++ b/src/powercontext/service/adapters/launchd.py @@ -231,6 +231,19 @@ def disable(self) -> None: if registration.definition is not None: _clear_retry_files(registration.definition) + def suspend(self, marker: Path) -> None: + # The persistent disabled override survives logout/reboot. bootout in + # stop() then removes the existing KeepAlive job as well. + self.disable() + + def resume(self) -> None: + self.enable() + + def update_suspended(self) -> None: + # A suspended LaunchAgent was booted out; its next bootstrap reads the + # replaced plist. The persistent disable override remains in force. + return None + def remove(self) -> None: self.artifact_path.unlink(missing_ok=True) diff --git a/src/powercontext/service/adapters/systemd.py b/src/powercontext/service/adapters/systemd.py index d25269f00c..a97ea43b7b 100644 --- a/src/powercontext/service/adapters/systemd.py +++ b/src/powercontext/service/adapters/systemd.py @@ -212,7 +212,53 @@ def disable(self) -> None: _require_owned_or_not_loaded(self.loaded_registration()) self._run("disable", self.identifier) + def suspend(self, marker: Path) -> None: + """Prevent every native activation while preserving the installed unit.""" + + _require_owned_or_not_loaded(self.loaded_registration()) + path = self.artifact_path.with_name(f"{self.identifier}.d") / "90-powercontext-maintenance.conf" + if path.parent.is_symlink(): + raise ServiceError("the service maintenance drop-in directory must not be a symlink") # noqa: TRY003 + content = self._maintenance_dropin(marker) + if path.exists() and (path.is_symlink() or path.read_bytes() != content): + raise ServiceError("the service maintenance drop-in is not owned by PowerContext") # noqa: TRY003 + atomic_write(path, content, mode=0o600) + self.reload() + + def resume(self) -> None: + _require_owned_or_not_loaded(self.loaded_registration()) + path = self.artifact_path.with_name(f"{self.identifier}.d") / "90-powercontext-maintenance.conf" + marker = self.lock_path.with_name(f"{self.lock_path.name}.maintenance.json") + if path.exists(): + if path.is_symlink() or path.parent.is_symlink() or path.read_bytes() != self._maintenance_dropin(marker): + raise ServiceError("the service maintenance drop-in is not owned by PowerContext") # noqa: TRY003 + path.unlink() + self.reload() + self.enable() + + def update_suspended(self) -> None: + self.reload() + + @staticmethod + def _maintenance_dropin(marker: Path) -> bytes: + path = str(marker) + if not marker.is_absolute() or any(character in path for character in "\x00\r\n"): + raise ServiceError("the maintenance marker must be an absolute path without control characters") # noqa: TRY003 + # ConditionPathExists parses ! before the path and does not use + # ExecStart's shell-like quoting or dollar expansion. A quoted !path + # would be ignored as a relative path, silently removing the guard. + condition = path.replace("%", "%%") + return ( + f"# Managed by PowerContext: maintenance activation guard\n[Unit]\nConditionPathExists=!{condition}\n" + ).encode() + def remove(self) -> None: + path = self.artifact_path.with_name(f"{self.identifier}.d") / "90-powercontext-maintenance.conf" + marker = self.lock_path.with_name(f"{self.lock_path.name}.maintenance.json") + if path.exists(): + if path.is_symlink() or path.parent.is_symlink() or path.read_bytes() != self._maintenance_dropin(marker): + raise ServiceError("the service maintenance drop-in is not owned by PowerContext") # noqa: TRY003 + path.unlink() self.artifact_path.unlink(missing_ok=True) def manager_state(self) -> ManagerState: diff --git a/src/powercontext/service/adapters/windows.py b/src/powercontext/service/adapters/windows.py index d331f1778b..01efe725dd 100644 --- a/src/powercontext/service/adapters/windows.py +++ b/src/powercontext/service/adapters/windows.py @@ -22,6 +22,7 @@ import shutil import subprocess import sys +import tempfile import time import xml.etree.ElementTree as ET from collections.abc import Sequence @@ -289,6 +290,37 @@ def disable(self) -> None: return self._run("/Change", "/TN", self.identifier, "/DISABLE", "/HRESULT") + def suspend(self, marker: Path) -> None: + # Disabling is separate from End: it prevents queued/login triggers + # and RestartOnFailure from recreating the old writer. + self.disable() + + def resume(self) -> None: + self.enable() + + def update_suspended(self) -> None: + """Replace the native task while keeping it disabled, including its login trigger.""" + + _require_owned_or_not_loaded(self.loaded_registration()) + registration = self.inspect() + if registration.content is None: + raise ServiceError("the Task Scheduler definition is not installed") # noqa: TRY003 + root = _parse_xml(registration.content) + settings = _child(root, "Settings") + if settings is None: + raise ServiceError("the Task Scheduler definition has no settings") # noqa: TRY003 + ET.SubElement(settings, _tag("Enabled")).text = "false" + disabled = ET.tostring(root, encoding="utf-16", xml_declaration=True) + descriptor, name = tempfile.mkstemp( + prefix=".powercontext-disabled-", suffix=".xml", dir=self.artifact_path.parent + ) + try: + with os.fdopen(descriptor, "wb") as output: + output.write(disabled) + self._run("/Create", "/TN", self.identifier, "/XML", name, "/F", "/HRESULT") + finally: + Path(name).unlink(missing_ok=True) + def remove(self) -> None: loaded = self.loaded_registration() _require_owned_or_not_loaded(loaded) @@ -370,7 +402,7 @@ def _wait_for_inactive(self) -> None: deadline = time.monotonic() + _TASK_TIMEOUT_SECONDS while time.monotonic() < deadline: state = self.manager_state() - if state is ManagerState.INACTIVE: + if state in {ManagerState.INACTIVE, ManagerState.FAILED}: return if state is ManagerState.UNKNOWN: raise ServiceError( # noqa: TRY003 diff --git a/src/powercontext/service/cli.py b/src/powercontext/service/cli.py index 25d3710a39..e5f2564f4f 100644 --- a/src/powercontext/service/cli.py +++ b/src/powercontext/service/cli.py @@ -36,7 +36,7 @@ app = typer.Typer( name="service", context_settings={"help_option_names": HELP_OPTION_NAMES}, - help="Install and inspect the current user's persistent PowerContext Server.", + help="Manage the current user's persistent PowerContext Server.", no_args_is_help=True, ) @@ -121,6 +121,40 @@ def status( raise typer.Exit(code=1) +@app.command() +def start() -> None: + """Start the owned registered Server and check business readiness.""" + + _run_lifecycle("start") + + +@app.command() +def stop() -> None: + """Stop the owned Server and suppress automatic activation, retaining configuration.""" + + _run_lifecycle("stop") + + +@app.command() +def restart() -> None: + """Stop then start the registered Server; run required migrations separately.""" + + _run_lifecycle("restart") + + +def _run_lifecycle(operation: str) -> None: + controller = _controller() + try: + service_status = getattr(controller, operation)() + except (OSError, ServiceError) as error: + typer.echo(f"PowerContext personal service {operation} failed: {error}", err=True) + if isinstance(error, ServiceError) and error.status is not None: + _write_status(error.status, json_output=False) + raise typer.Exit(code=error.exit_code if isinstance(error, ServiceError) else 1) from error + typer.echo(f"PowerContext personal service {operation} completed.") + _write_status(service_status, json_output=False) + + @app.command() def uninstall() -> None: """Stop and remove the PowerContext-owned personal service registration.""" diff --git a/src/powercontext/service/controller.py b/src/powercontext/service/controller.py index 8aa7462549..6650515efa 100644 --- a/src/powercontext/service/controller.py +++ b/src/powercontext/service/controller.py @@ -16,22 +16,25 @@ from __future__ import annotations +import hashlib +import json import os import shlex import sys import time from collections.abc import Callable, Generator from contextlib import contextmanager, nullcontext, suppress -from dataclasses import replace +from dataclasses import dataclass, replace from importlib.metadata import version from pathlib import Path from typing import cast +from urllib.parse import urlsplit from powercontext.cli.env_file import environment_context from powercontext.paths import POWERCONTEXT_HOME_ENV, powercontext_data_dir from powercontext.server.configuration import ServerConfigurationError, server_settings_context from powercontext.service.adapters import NativeServiceAdapter, native_service_adapter -from powercontext.service.adapters.base import definition_state, service_python_executable +from powercontext.service.adapters.base import atomic_write, definition_state, service_python_executable from powercontext.service.environment import ProtectedEnvironmentFileError, load_protected_environment_file from powercontext.service.model import ( DEFINITION_VERSION, @@ -63,16 +66,71 @@ _MANAGER_RECHECK_SECONDS = 5.0 +@dataclass(frozen=True) +class ServiceMaintenanceSummary: + """The exact local registration and new executable accepted in a migration plan.""" + + identifier: str + artifact_path: str + definition: ServiceDefinition + target_definition: ServiceDefinition + originally_running: bool + registration_checksum: str + + def as_dict(self) -> dict[str, object]: + return {**self._payload(), "fingerprint": self.fingerprint} + + def _payload(self) -> dict[str, object]: + return { + "identifier": self.identifier, + "artifact_path": self.artifact_path, + "definition": self.definition.as_dict(), + "target_definition": self.target_definition.as_dict(), + "originally_running": self.originally_running, + "registration_checksum": self.registration_checksum, + "scope": "current-user local registration only; other writers require external coordination", + } + + @property + def fingerprint(self) -> str: + return hashlib.sha256(json.dumps(self._payload(), sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +class ServiceMaintenanceSession: + """A stopped service, with an explicit database-ready completion boundary.""" + + def __init__(self, controller: ServiceController, summary: ServiceMaintenanceSummary) -> None: + self._controller = controller + self.summary = summary + self.completed = False + self._closed = False + + def complete(self) -> ServiceStatus: + """Switch to the confirmed new executable only after database verification.""" + + if self._closed or self.completed: + raise ServiceError("service maintenance completion requires an active, uncompleted session") # noqa: TRY003 + status = self._controller._complete_maintenance(self.summary) + self.completed = True + return status + + class ServiceController: def __init__( self, adapter: NativeServiceAdapter | None = None, *, probe: Callable[[str], ProbeResult] = probe_server, + readiness_probe: Callable[[str], ProbeResult] | None = None, sleep: Callable[[float], None] = time.sleep, ) -> None: self._adapter = native_service_adapter() if adapter is None else adapter self._probe = probe + if readiness_probe is None: + from powercontext.service.probe import probe_readiness + + readiness_probe = probe_readiness + self._readiness_probe = readiness_probe self._sleep = sleep def install(self, *, env_file: Path | None = None, start_on_login: bool = True) -> ServiceStatus: @@ -92,6 +150,10 @@ def install(self, *, env_file: Path | None = None, start_on_login: bool = True) ) with _service_lock(self._adapter.lock_path): + if self._maintenance_record() is not None: + raise ServiceError( # noqa: TRY003 + "the personal service is stopped for maintenance; finish migration or use service start first" + ) registration = self._adapter.inspect() self._require_mutable_registration(registration) loaded = self._adapter.loaded_registration() @@ -130,6 +192,228 @@ def install(self, *, env_file: Path | None = None, start_on_login: bool = True) ) return self.status() + @property + def maintenance_path(self) -> Path: + return self._adapter.lock_path.with_name(f"{self._adapter.lock_path.name}.maintenance.json") + + def start(self) -> ServiceStatus: + """Start the registered executable without reinstalling or changing configuration.""" + + with _service_lock(self._adapter.lock_path): + registration, running = self._owned_registration() + record = self._maintenance_record() + if record is not None and record.get("phase") not in {"manual_stopped", "database_ready", "start_failed"}: + raise ServiceError( # noqa: TRY003 + "the database migration has not been verified; complete it before starting the service" + ) + if running and record is None: + definition = registration.definition + if definition is None or self._readiness_probe(definition.endpoint).state is not ProbeState.LIVE: + raise ServiceError("the running service is not ready; inspect its diagnostics") # noqa: TRY003 + return self.status() + if record is None: + # A failed start must also leave a durable activation guard; + # systemd's condition alone is ineffective without this file. + self._write_maintenance_record({"phase": "manual_stopped"}) + self._start_registered(registration.definition) + self.maintenance_path.unlink(missing_ok=True) + return self.status() + + def stop(self) -> ServiceStatus: + """Stop the owned service and persistently suppress native automatic activation.""" + + with _service_lock(self._adapter.lock_path): + registration, _running = self._owned_registration() + record = self._maintenance_record() + if record is None: + self._write_maintenance_record({"phase": "manual_stopped"}) + self._adapter.suspend(self.maintenance_path) + self._adapter.stop() + self._require_stopped(registration.definition) + return self.status() + + def restart(self) -> ServiceStatus: + """Stop then start the registered executable; migration is a separate operation.""" + + self.stop() + return self.start() + + def maintenance_summary(self, *, env_file: Path | None = None) -> ServiceMaintenanceSummary: + registration, running = self._owned_registration() + definition = registration.definition + if definition is None: + raise ServiceError("the installed service has no definition") # noqa: TRY003 + try: + if env_file is not None: + expected = load_protected_environment_file(env_file).identity + if definition.env_file != expected: + raise ServiceError( # noqa: TRY003 + "the migration environment file does not match the registered service configuration", + exit_code=2, + ) + elif definition.env_file is not None: + load_protected_environment_file(Path(definition.env_file.path), expected=definition.env_file) + except ProtectedEnvironmentFileError as error: + raise ServiceError(f"the registered environment file is not usable: {error}", exit_code=2) from error # noqa: TRY003 + record = self._maintenance_record() + if record is not None and record.get("phase") not in {"manual_stopped", "database_ready", "start_failed"}: + previous = record.get("summary") + if not isinstance(previous, dict) or previous.get("definition") != definition.as_dict(): + raise ServiceError("the interrupted service maintenance record does not match the registration") # noqa: TRY003 + running = previous.get("originally_running") + if not isinstance(running, bool): + raise ServiceError("the interrupted service maintenance record is invalid") # noqa: TRY003 + target = replace( + definition, + definition_version=DEFINITION_VERSION, + package_version=version("powercontext"), + python_executable=service_python_executable(), + ) + return ServiceMaintenanceSummary( + self._adapter.identifier, + str(self._adapter.artifact_path), + definition, + target, + running, + hashlib.sha256(registration.content or b"").hexdigest(), + ) + + @contextmanager + def maintenance( + self, + *, + expected_fingerprint: str, + env_file: Path | None = None, + ) -> Generator[ServiceMaintenanceSession, None, None]: + """Keep the local service stopped across backup/migration; failures retain the guard.""" + + with _service_lock(self._adapter.lock_path): + summary = self.maintenance_summary(env_file=env_file) + if summary.fingerprint != expected_fingerprint: + raise ServiceError("the service configuration or running state changed; review a new plan") # noqa: TRY003 + self._write_maintenance_record({"phase": "stopping", "summary": summary.as_dict()}) + self._adapter.suspend(self.maintenance_path) + self._adapter.stop() + self._require_stopped(summary.definition) + self._write_maintenance_record({"phase": "stopped", "summary": summary.as_dict()}) + session = ServiceMaintenanceSession(self, summary) + try: + yield session + finally: + session._closed = True + + def _complete_maintenance(self, summary: ServiceMaintenanceSummary) -> ServiceStatus: + registration, running = self._owned_registration() + if ( + running + or registration.definition != summary.definition + or hashlib.sha256(registration.content or b"").hexdigest() != summary.registration_checksum + ): + raise ServiceError("the stopped service registration changed during migration") # noqa: TRY003 + self._adapter.write(self._adapter.render(summary.target_definition)) + self._adapter.update_suspended() + self._write_maintenance_record({"phase": "database_ready", "summary": summary.as_dict()}) + if summary.originally_running: + try: + self._start_registered(summary.target_definition) + except (OSError, ServiceError): + self._write_maintenance_record({"phase": "start_failed", "summary": summary.as_dict()}) + raise + self.maintenance_path.unlink(missing_ok=True) + return self.status() + + def _owned_registration(self) -> tuple[NativeRegistration, bool]: + support, detail = self._adapter.support() + if support is SupportState.UNSUPPORTED: + raise ServiceError(detail) + registration = self._adapter.inspect() + self._require_mutable_registration(registration) + if registration.state is not RegistrationState.INSTALLED or registration.definition is None: + raise ServiceError("no owned personal service is installed; use external service management") # noqa: TRY003 + try: + endpoint = urlsplit(registration.definition.endpoint) + if endpoint.scheme != "http" or endpoint.port is None or not is_loopback_host(endpoint.hostname or ""): + raise ValueError("a local HTTP endpoint is required") # noqa: TRY003, TRY301 + except ValueError as error: + raise ServiceError("the service registration has an invalid local endpoint", exit_code=2) from error # noqa: TRY003 + loaded = self._adapter.loaded_registration() + self._require_mutable_manager_registration(loaded) + if loaded.state is ManagerOwnershipState.OWNED and loaded.definition != registration.definition: + raise ServiceError("loaded and installed service definitions differ; resolve them before maintenance") # noqa: TRY003 + manager = ( + self._adapter.manager_state() if loaded.state is ManagerOwnershipState.OWNED else ManagerState.INACTIVE + ) + if manager is ManagerState.UNKNOWN: + raise ServiceError("cannot determine the personal service running state") # noqa: TRY003 + probe = self._probe(registration.definition.endpoint) + if probe.state is ProbeState.CONFLICT or ( + manager is not ManagerState.ACTIVE and probe.state is ProbeState.LIVE + ): + raise ServiceError("another listener may own the registered endpoint; external maintenance is required") # noqa: TRY003 + return registration, manager is ManagerState.ACTIVE + + def _require_stopped(self, definition: ServiceDefinition | None) -> None: + if definition is None: + raise ServiceError("the installed service has no definition") # noqa: TRY003 + deadline = time.monotonic() + 30.0 + while True: + state = self._adapter.manager_state() + probe = self._probe(definition.endpoint) + if state in {ManagerState.INACTIVE, ManagerState.FAILED} and probe.state is ProbeState.UNREACHABLE: + return + if state is ManagerState.UNKNOWN or probe.state is ProbeState.CONFLICT or time.monotonic() >= deadline: + raise ServiceError("cannot prove the personal service stopped; maintenance must not continue") # noqa: TRY003 + self._sleep(0.1) + + def _start_registered(self, definition: ServiceDefinition | None) -> None: + if definition is None: + raise ServiceError("the installed service has no definition") # noqa: TRY003 + if not Path(definition.python_executable).is_file(): + raise ServiceError("the registered service executable is unavailable") # noqa: TRY003 + try: + self._adapter.resume() + self._adapter.start(reload_definition=False) + result = self._wait_until_live(definition.endpoint) + if result.state is not ProbeState.LIVE: + raise ServiceError("the registered service did not become live") # noqa: TRY003, TRY301 + loaded = self._adapter.loaded_registration() + if loaded.state is not ManagerOwnershipState.OWNED or loaded.definition != definition: + raise ServiceError("the running service does not use the confirmed executable and configuration") # noqa: TRY003, TRY301 + deadline = time.monotonic() + _START_TIMEOUT_SECONDS + while True: + readiness = self._readiness_probe(definition.endpoint) + if readiness.state is ProbeState.LIVE: + return + if readiness.state is ProbeState.CONFLICT or time.monotonic() >= deadline: + raise ServiceError(f"the service failed readiness: {readiness.detail}") # noqa: TRY003, TRY301 + self._sleep(0.1) + except (OSError, ServiceError): + # Restore the guard even if startup partially succeeded. Never fall + # back to an old executable after the database changed. + self._adapter.suspend(self.maintenance_path) + self._adapter.stop() + raise + + def _maintenance_record(self) -> dict[str, object] | None: + path = self.maintenance_path + if not path.exists(): + if path.is_symlink(): + raise ServiceError("the service maintenance record must not be a symlink") # noqa: TRY003 + return None + if path.is_symlink() or not path.is_file(): + raise ServiceError("the service maintenance record is not a regular file") # noqa: TRY003 + try: + result = json.loads(path.read_text()) + except (OSError, ValueError) as error: + raise ServiceError("cannot read the service maintenance record") from error # noqa: TRY003 + if not isinstance(result, dict) or not isinstance(result.get("phase"), str): + raise ServiceError("the service maintenance record is invalid") # noqa: TRY003 + return result + + def _write_maintenance_record(self, value: dict[str, object]) -> None: + self._maintenance_record() + atomic_write(self.maintenance_path, json.dumps(value, sort_keys=True).encode(), mode=0o600) + def registration_status(self) -> ServiceStatus: """Inspect support, artifact, and definition without touching the manager or endpoint.""" @@ -245,6 +529,7 @@ def uninstall(self) -> ServiceStatus: self._run_uninstall_stage("disable", self._adapter.disable) self._run_uninstall_stage("remove", self._remove_owned_artifact) self._run_uninstall_stage("reload", self._adapter.reload) + self.maintenance_path.unlink(missing_ok=True) return self.status() def _build_definition(self, env_file: Path | None, *, start_on_login: bool) -> ServiceDefinition: @@ -491,4 +776,4 @@ def _recovery_action( return None -__all__ = ["ServiceController"] +__all__ = ["ServiceController", "ServiceMaintenanceSession", "ServiceMaintenanceSummary"] diff --git a/src/powercontext/service/probe.py b/src/powercontext/service/probe.py index 8b894289b1..b864462ecf 100644 --- a/src/powercontext/service/probe.py +++ b/src/powercontext/service/probe.py @@ -25,7 +25,7 @@ from pydantic import ValidationError -from powercontext.http import HealthResponse +from powercontext.http import HealthResponse, ReadinessResponse, ReadinessStatus from powercontext.service.model import ProbeResult, ProbeState _REQUEST_ID = re.compile(r"^[0-9a-f]{16}$") @@ -77,4 +77,33 @@ def probe_server(endpoint: str, *, timeout: float = 2.0) -> ProbeResult: return ProbeResult(ProbeState.LIVE, f"{endpoint} status=ok") -__all__ = ["probe_server"] +def probe_readiness(endpoint: str, *, timeout: float = 2.0) -> ProbeResult: + """Check business readiness, including the startup compatibility gate.""" + + if probe_server(endpoint, timeout=timeout).state is not ProbeState.LIVE: + return ProbeResult(ProbeState.UNREACHABLE, "the PowerContext service is not live") + request = Request( # noqa: S310 - probe_server validates the registered local endpoint first. + f"{endpoint.rstrip('/')}/health/ready", + headers={"Accept": "application/json", "User-Agent": "powercontext-service"}, + ) + try: + with build_opener(ProxyHandler({})).open(request, timeout=timeout) as response: + if response.getcode() != 200 or response.headers.get_content_type() != "application/json": + return ProbeResult(ProbeState.CONFLICT, "listener returned an invalid readiness response") + if _REQUEST_ID.fullmatch(response.headers.get("X-PowerContext-Request-ID", "")) is None: + return ProbeResult(ProbeState.CONFLICT, "listener returned an invalid readiness request ID") + readiness = ReadinessResponse.model_validate(json.load(response)) + except HTTPError as error: + error.close() + state = ProbeState.UNREACHABLE if error.code == 503 else ProbeState.CONFLICT + return ProbeResult(state, f"readiness returned HTTP {error.code}") + except (OSError, URLError): + return ProbeResult(ProbeState.UNREACHABLE, "cannot reach service readiness") + except (ValueError, ValidationError): + return ProbeResult(ProbeState.CONFLICT, "listener returned invalid readiness JSON") + if readiness.status is ReadinessStatus.NOT_READY: + return ProbeResult(ProbeState.UNREACHABLE, "required Server bindings are not ready") + return ProbeResult(ProbeState.LIVE, f"{endpoint} readiness={readiness.status.value}") + + +__all__ = ["probe_readiness", "probe_server"] diff --git a/tests/builtin/persistence/test_database_backup_providers.py b/tests/builtin/persistence/test_database_backup_providers.py new file mode 100644 index 0000000000..b00c48c6f4 --- /dev/null +++ b/tests/builtin/persistence/test_database_backup_providers.py @@ -0,0 +1,366 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Recovery point behavior; simulated feature versions do not certify a backend.""" + +from __future__ import annotations + +import asyncio +import os +import sqlite3 +from contextlib import closing +from pathlib import Path +from typing import Any, cast + +import pytest +from sqlalchemy import Connection + +from powercontext.builtin.persistence.migrations.backup import ( + BackupContext, + BackupRef, + ForkBackupProvider, + ForkWorkflowAcceptance, + SQLiteBackupProvider, + identify_fork_server, +) +from powercontext.builtin.persistence.migrations.models import MigrationError, digest + + +def context(*, writers_stopped: bool = True) -> BackupContext: + return BackupContext( + database_id="scratch-database", + source_revision="p0001", + bundle_checksum="package-checksum", + maintenance_window_id="window-1", + objects=("pc_artifacts",), + writers_stopped=writers_stopped, + ) + + +def test_sqlite_online_backup_restores_committed_wal_data_and_keeps_original_point(tmp_path: Path) -> None: + database = tmp_path / "source.db" + with closing(sqlite3.connect(database)) as writer: + assert writer.execute("PRAGMA journal_mode=WAL").fetchone() == ("wal",) + writer.execute("PRAGMA wal_autocheckpoint=0") + writer.execute("CREATE TABLE pc_artifacts (id INTEGER PRIMARY KEY, content TEXT)") + writer.execute("INSERT INTO pc_artifacts VALUES (1, 'original')") + writer.commit() + assert database.with_name(database.name + "-wal").stat().st_size > 0 + provider = SQLiteBackupProvider(database) + ref = provider.create_backup(context()) + assert provider.inspect_backup(ref).state == "completed" + assert provider.inspect_backup(ref).check_level == "integrity_checked" + assert not provider.inspect_backup(ref).recovery_verified + + writer.execute("UPDATE pc_artifacts SET content='changed'") + writer.execute("ALTER TABLE pc_artifacts ADD COLUMN changed INTEGER") + writer.commit() + restored = tmp_path / "restored.db" + with closing(sqlite3.connect(ref.location)) as source, closing(sqlite3.connect(restored)) as destination: + source.backup(destination) + assert destination.execute("SELECT content FROM pc_artifacts").fetchall() == [("original",)] + + assert provider.create_backup(context()) == ref + with pytest.raises(MigrationError) as changed_plan: + provider.create_backup(context().model_copy(update={"bundle_checksum": "different-package"})) + assert changed_plan.value.code == "backup_failed" + assert tuple(provider.directory.glob("*.sqlite3")) == (Path(ref.location),) + second = provider.create_backup(context().model_copy(update={"maintenance_window_id": "window-2"})) + assert second.location != ref.location + assert provider.inspect_backup(ref).state == "completed" + assert provider.restore_plan(ref).automatic is False + assert Path(ref.location).is_file() + + +def test_sqlite_rejects_unstopped_writers_before_creating_backup_files(tmp_path: Path) -> None: + database = tmp_path / "source.db" + with closing(sqlite3.connect(database)) as writer: + writer.execute("CREATE TABLE pc_artifacts (id INTEGER)") + provider = SQLiteBackupProvider(database) + with pytest.raises(MigrationError) as error: + provider.create_backup(context(writers_stopped=False)) + assert error.value.code == "active_writers" + assert not provider.directory.exists() + + +def test_sqlite_changed_original_recovery_point_blocks_inspection_and_restore(tmp_path: Path) -> None: + database = tmp_path / "source.db" + with closing(sqlite3.connect(database)) as writer: + writer.execute("CREATE TABLE pc_artifacts (id INTEGER)") + provider = SQLiteBackupProvider(database) + ref = provider.create_backup(context()) + with Path(ref.location).open("ab") as changed: + changed.write(b"modified") + assert provider.inspect_backup(ref).state == "failed" + with pytest.raises(MigrationError) as error: + provider.restore_plan(ref) + assert error.value.code == "backup_failed" + assert Path(ref.location).is_file() + + +def test_sqlite_missing_original_manifest_blocks_retry_without_replacing_snapshot(tmp_path: Path) -> None: + database = tmp_path / "source.db" + with closing(sqlite3.connect(database)) as writer: + writer.execute("CREATE TABLE pc_artifacts (id INTEGER)") + provider = SQLiteBackupProvider(database) + ref = provider.create_backup(context()) + original = Path(ref.location).read_bytes() + (provider.directory / f"{ref.ref_id}.json").unlink() + with pytest.raises(MigrationError) as error: + provider.create_backup(context()) + assert error.value.code == "backup_failed" + assert Path(ref.location).read_bytes() == original + + +def test_sqlite_invalid_target_reports_unsupported_without_creating_anything(tmp_path: Path) -> None: + database = tmp_path / "invalid.db" + database.write_text("not a database", encoding="utf-8") + provider = SQLiteBackupProvider(database) + assert not provider.capabilities(context()).available + with pytest.raises(MigrationError) as error: + provider.create_backup(context()) + assert error.value.code == "backup_unsupported" + assert not provider.directory.exists() + + +@pytest.mark.parametrize( + ("version", "comment", "product", "expected"), + [ + ("5.7.25-OceanBase seekdb-v1.4.0.0", "OceanBase 4.3.5.3 seekdb (r1.4.0.0)", "seekdb", (1, 4, 0)), + ("5.7.25", "OceanBase seekdb (r1.1.0.0)", "seekdb", (1, 1, 0)), + ("5.7.25-OceanBase_CE-v4.6.2.0", "OceanBase", "oceanbase_ai", (4, 6, 2)), + ("8.0.36", "MySQL Community Server", None, None), + ("5.7.25-OceanBase", "OceanBase seekdb", "seekdb", None), + ], +) +def test_server_identity_uses_product_version_instead_of_mysql_handshake_version( + version: str, comment: str, product: str | None, expected: tuple[int, int, int] | None +) -> None: + assert identify_fork_server(version, comment) == (product, expected) + + +class _ReadResult: + def __init__(self, rows: list[tuple[Any, ...]]) -> None: + self.rows = rows + + def all(self) -> list[tuple[Any, ...]]: + return self.rows + + def one(self) -> tuple[Any, ...]: + assert len(self.rows) == 1 + return self.rows[0] + + def scalar_one(self) -> Any: + return self.one()[0] + + +class _ReadOnlyServer: + """Simulated server responses exercise refusal, not successful Fork support.""" + + class dialect: + name = "oceanbase" + + def __init__(self, version: str, comment: str) -> None: + self.version = version + self.comment = comment + self.queries: list[str] = [] + + def exec_driver_sql(self, query: str) -> _ReadResult: + self.queries.append(query) + if query == "SELECT VERSION()": + return _ReadResult([(self.version,)]) + if query == "SELECT @@version_comment": + return _ReadResult([(self.comment,)]) + if query == "SELECT DATABASE()": + return _ReadResult([("test",)]) + if query == "SHOW VARIABLES LIKE 'ob_compatibility_mode'": + return _ReadResult([("ob_compatibility_mode", "MYSQL")]) + if query.startswith("SHOW CREATE TABLE"): + return _ReadResult([("pc_artifacts", "CREATE TABLE pc_artifacts (id INT PRIMARY KEY)")]) + raise AssertionError(query) + + def execute(self, query: object, parameters: object) -> _ReadResult: + sql = str(query) + self.queries.append(sql) + if "TABLE_NAME, TABLE_TYPE" in sql: + return _ReadResult([("pc_artifacts", "BASE TABLE")]) + if "COUNT(*)" in sql: + return _ReadResult([(0,)]) + raise AssertionError(sql) + + +@pytest.mark.parametrize( + ("product", "version", "table_available", "database_available"), + [ + ("seekdb", "1.0.0", False, False), + ("seekdb", "1.1.0", True, False), + ("seekdb", "1.2.0", True, True), + ("oceanbase_ai", "4.6.1", False, False), + ("oceanbase_ai", "4.6.2", True, True), + ], +) +def test_product_specific_fork_minimum_never_enables_unaccepted_recovery( + tmp_path: Path, product: str, version: str, table_available: bool, database_available: bool +) -> None: + marker = "seekdb" if product == "seekdb" else "OceanBase_CE" + server = _ReadOnlyServer(f"5.7.25-{marker}-v{version}", marker) + provider = ForkBackupProvider( + cast(Connection, server), + database_name="test", + expected_product=cast(Any, product), + directory=tmp_path / "manifests", + ) + capability = provider.capabilities(context()) + assert capability.table_fork_available == table_available + assert capability.database_fork_available == database_available + assert not capability.available + with pytest.raises(MigrationError) as error: + provider.create_backup(context()) + assert error.value.code == "backup_unsupported" + assert all(query.startswith(("SELECT", "SHOW")) for query in server.queries) + assert not provider.directory.exists() + + +@pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") +def test_real_seekdb_reports_engine_fork_version_without_claiming_recovery_acceptance(tmp_path: Path) -> None: + from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile + + async def scenario() -> None: + async with ( + SeekDBProfile.open(SeekDBConfig(path=tmp_path / "seekdb"), tables=()) as profile, + profile.database.engine.connect() as connection, + ): + await connection.exec_driver_sql("CREATE TABLE pc_artifacts (id INT PRIMARY KEY)") + await connection.commit() + + def inspect_provider(sync: Connection) -> None: + provider = ForkBackupProvider( + sync, + database_name="test", + expected_product="seekdb", + directory=tmp_path / "manifests", + ) + capability = provider.capabilities(context()) + assert capability.product == "seekdb" + assert capability.server_version is not None + assert not capability.available + with pytest.raises(MigrationError) as error: + provider.create_backup(context()) + assert error.value.code == "backup_unsupported" + assert not provider.directory.exists() + + await connection.run_sync(inspect_provider) + + asyncio.run(scenario()) + + +@pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") +def test_real_seekdb_fork_simple_table_survives_ddl_restart_and_original_name_restore(tmp_path: Path) -> None: + """Acceptance for one scratch-table shape, not the complete PC schema.""" + from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile + + async def scenario() -> None: + path = tmp_path / "seekdb" + config = SeekDBConfig(path=path) + async with ( + SeekDBProfile.open(config, tables=()) as profile, + profile.database.engine.connect() as connection, + ): + await connection.exec_driver_sql("CREATE TABLE pc_artifacts (id INT PRIMARY KEY, content VARCHAR(40))") + await connection.exec_driver_sql("INSERT INTO pc_artifacts VALUES (1, 'original')") + await connection.commit() + version = str((await connection.exec_driver_sql("SELECT VERSION()")).scalar_one()) + comment = str((await connection.exec_driver_sql("SELECT @@version_comment")).scalar_one()) + number = identify_fork_server(version, comment)[1] + assert number is not None + definition = str((await connection.exec_driver_sql("SHOW CREATE TABLE test.pc_artifacts")).one()[1]) + # This test performs every claimed DDL/restart/restore check below. + # It does not install this evidence in any production adapter. + acceptance = ForkWorkflowAcceptance( + product="seekdb", + server_version=".".join(str(part) for part in number), + server_identity_digest=digest([version, comment]), + method="fork_database", + schema_fingerprint=digest({"pc_artifacts": definition}), + bundle_checksum=context().bundle_checksum, + covered_objects=("pc_artifacts",), + evidence_reference="scratch-table acceptance executed by this integration test", + restore_steps=( + "While stopped, Fork the retained table into a new table in the original test database.", + "Rename the current table to a retained name, then atomically give the restored table its original name.", + "Verify original data and revision; preserve the original Fork database and failed table.", + ), + ddl_verified=True, + restart_verified=True, + restore_to_original_database=True, + events_not_supported=True, + ) + + def create_provider_backup(sync: Connection) -> BackupRef: + provider = ForkBackupProvider( + sync, + database_name="test", + expected_product="seekdb", + directory=tmp_path / "manifests", + acceptance=acceptance, + ) + assert provider.capabilities(context()).available + return provider.create_backup(context()) + + ref = await connection.run_sync(create_provider_backup) + await connection.exec_driver_sql("ALTER TABLE pc_artifacts ADD COLUMN changed INT") + await connection.exec_driver_sql("UPDATE pc_artifacts SET content='changed'") + await connection.commit() + + async with ( + SeekDBProfile.open(config, tables=()) as reopened, + reopened.database.engine.connect() as connection, + ): + + def inspect_provider_backup(sync: Connection) -> None: + provider = ForkBackupProvider( + sync, + database_name="test", + expected_product="seekdb", + directory=tmp_path / "manifests", + acceptance=acceptance, + ) + assert provider.inspect_backup(ref).state == "completed" + assert provider.inspect_backup(ref).check_level == "metadata_checked" + assert not provider.inspect_backup(ref).recovery_verified + assert not provider.restore_plan(ref).automatic + # Source DDL and process restart must not replace the original + # point if the executor lost its own post-backup receipt. + assert provider.create_backup(context()) == ref + + await connection.run_sync(inspect_provider_backup) + # The generated name is hexadecimal; it is not user SQL input. + await connection.exec_driver_sql(f"FORK TABLE `{ref.location}`.`pc_artifacts` TO `test`.`pc_restore_probe`") + await connection.exec_driver_sql( + "RENAME TABLE test.pc_artifacts TO test.pc_failed_retained,test.pc_restore_probe TO test.pc_artifacts" + ) + await connection.commit() + assert (await connection.exec_driver_sql("SELECT id,content FROM test.pc_artifacts")).all() == [ + (1, "original") + ] + assert (await connection.exec_driver_sql("SELECT content FROM test.pc_failed_retained")).all() == [ + ("changed",) + ] + assert ( + await connection.exec_driver_sql( + f"SELECT id,content FROM `{ref.location}`.`pc_artifacts`" # noqa: S608 -- generated hexadecimal name + ) + ).all() == [(1, "original")] + + asyncio.run(scenario()) diff --git a/tests/builtin/persistence/test_database_migrations.py b/tests/builtin/persistence/test_database_migrations.py index e0d8a26b7f..ca2c55e060 100644 --- a/tests/builtin/persistence/test_database_migrations.py +++ b/tests/builtin/persistence/test_database_migrations.py @@ -23,13 +23,14 @@ import sys from concurrent.futures import ThreadPoolExecutor from pathlib import Path -from typing import Any +from typing import Any, Literal import pytest from alembic import command from sqlalchemy import Connection, create_engine, inspect, text from powercontext.builtin.persistence.migrations import MigrationBundle, MigrationError, SQLiteMigrationRunner +from powercontext.builtin.persistence.migrations.backup import BackupCapabilities, SQLiteBackupProvider FIXTURE = Path(__file__).resolve().parents[2] / "fixtures/database_migrations" @@ -100,14 +101,14 @@ def test_empty_initialization_and_noop_apply(tmp_path: Path, bundle: MigrationBu database = tmp_path / "new.sqlite3" runner = SQLiteMigrationRunner(database, bundle) result = apply(runner) - assert result.changed and result.run_id and result.backup_ref + assert result.changed and result.backup_state == "not_required" and result.backup_ref is None assert runner.verify().revision == "p0003" before = database.read_bytes() files = sorted(str(path) for path in tmp_path.rglob("*")) assert not runner.apply().changed assert database.read_bytes() == before assert sorted(str(path) for path in tmp_path.rglob("*")) == files - assert rows(database, "SELECT COUNT(*) FROM pc_migration_runs") == [(1,)] + assert rows(database, "SELECT name FROM sqlite_schema WHERE name LIKE 'pc_migration_%'") == [] def test_distinct_databases_can_migrate_in_one_process_without_crossing_contexts( @@ -116,10 +117,10 @@ def test_distinct_databases_can_migrate_in_one_process_without_crossing_contexts runners = [SQLiteMigrationRunner(tmp_path / f"database-{index}.sqlite3", bundle) for index in range(4)] with ThreadPoolExecutor(max_workers=4) as executor: results = list(executor.map(apply, runners)) - assert len({result.run_id for result in results}) == 4 + assert all(result.changed and result.backup_state == "not_required" for result in results) for runner in runners: assert runner.verify().state == "ready" - assert rows(runner.database, "SELECT COUNT(*) FROM pc_migration_runs") == [(1,)] + assert rows(runner.database, "SELECT version_num FROM pc_schema_revision") == [("p0003",)] @pytest.mark.parametrize("source", ["pre_dream", "v1_1_0"]) @@ -222,7 +223,7 @@ def test_backup_failure_precedes_schema_writes(tmp_path: Path, bundle: Migration populate(database) database.with_name(database.name + ".pc-migration-backups").write_text("not a directory") before = database.read_bytes() - with pytest.raises(MigrationError, match="backup_required"): + with pytest.raises(MigrationError, match="backup_failed"): apply(SQLiteMigrationRunner(database, bundle)) assert database.read_bytes() == before assert not rows(database, "SELECT name FROM sqlite_schema WHERE name='pc_schema_revision'") @@ -264,12 +265,11 @@ def test_current_writer_blocks_even_with_confirmation(tmp_path: Path, bundle: Mi @pytest.mark.parametrize("revision", ["p0001", "p0002", "p0003"]) @pytest.mark.parametrize("after_commit", [False, True]) -def test_interruption_at_each_revision_boundary_resumes_one_run_and_backup( +def test_interruption_at_each_revision_boundary_retries_from_verified_revision( tmp_path: Path, bundle: MigrationBundle, monkeypatch: pytest.MonkeyPatch, revision: str, after_commit: bool ) -> None: database = tmp_path / "interrupted.sqlite3" runner = SQLiteMigrationRunner(database, bundle) - original_plan = runner.plan() real_upgrade = command.upgrade real_commit = Connection.commit interrupted = False @@ -277,45 +277,31 @@ def test_interruption_at_each_revision_boundary_resumes_one_run_and_backup( def upgrade(config, target, **kwargs): real_upgrade(config, target, **kwargs) if target == revision: - raise InterruptedError("Injected after real DDL, before transaction commit") # noqa: TRY003 + raise InterruptedError("Injected after real DDL, before commit") # noqa: TRY003 def commit(connection): nonlocal interrupted - present = connection.exec_driver_sql("SELECT name FROM sqlite_schema WHERE name='pc_migration_steps'").first() - completed = ( - present - and connection.execute( - text("SELECT revision FROM pc_migration_steps WHERE revision=:revision"), {"revision": revision} - ).first() - ) + present = connection.exec_driver_sql("SELECT name FROM sqlite_schema WHERE name='pc_schema_revision'").first() + current = connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar() if present else None real_commit(connection) - if completed and not interrupted: + if current == revision and not interrupted: interrupted = True - raise InterruptedError("Injected immediately after atomic commit") # noqa: TRY003 + raise InterruptedError("Injected after commit") # noqa: TRY003 with monkeypatch.context() as injection: - if after_commit: - injection.setattr(Connection, "commit", commit) - else: - injection.setattr(command, "upgrade", upgrade) + injection.setattr(Connection, "commit", commit) if after_commit else injection.setattr( + command, "upgrade", upgrade + ) with pytest.raises(InterruptedError): apply(runner) - recovery = runner.plan() - assert recovery.state == "recovery_required" - assert recovery.plan_id == original_plan.plan_id - assert recovery.resume_run_id - before_backups = list(tmp_path.glob("*.pc-migration-backups/*.sqlite3")) - assert len(before_backups) == 1 - with pytest.raises(MigrationError, match="recovery_required"): - apply(runner) + assert runner.plan().state == "recovery_required" with pytest.raises(MigrationError, match="recovery_required"): runner.verify() - result = apply(runner, resume=recovery.resume_run_id) - assert result.run_id == recovery.resume_run_id - assert Path(result.backup_ref) == before_backups[0] - assert list(tmp_path.glob("*.pc-migration-backups/*.sqlite3")) == before_backups - assert rows(database, "SELECT COUNT(*) FROM pc_migration_runs") == [(1,)] - assert rows(database, "SELECT COUNT(*) FROM pc_migration_steps") == [(3,)] + result = apply(runner) + assert result.backup_state == "not_required" + assert result.backup_ref is None + assert rows(database, "SELECT version_num FROM pc_schema_revision") == [("p0003",)] + assert rows(database, "SELECT name FROM sqlite_schema WHERE name LIKE 'pc_migration_%'") == [] assert runner.verify().state == "ready" @@ -331,10 +317,13 @@ def test_applied_revision_checksum_conflict_is_rejected(tmp_path: Path, resource SQLiteMigrationRunner(database, MigrationBundle(copied)).verify() -def test_resume_requires_the_original_backup( +def test_retry_requires_the_original_backup( tmp_path: Path, bundle: MigrationBundle, monkeypatch: pytest.MonkeyPatch ) -> None: - runner = SQLiteMigrationRunner(tmp_path / "pc.sqlite3", bundle) + database = tmp_path / "pc.sqlite3" + load_schema(database) + populate(database) + runner = SQLiteMigrationRunner(database, bundle) real_upgrade = command.upgrade def interrupted(config, target, **kwargs): @@ -345,17 +334,54 @@ def interrupted(config, target, **kwargs): injection.setattr(command, "upgrade", interrupted) with pytest.raises(InterruptedError): apply(runner) - plan = runner.plan() backups = list(tmp_path.glob("*.pc-migration-backups/*.sqlite3")) assert len(backups) == 1 backups[0].write_bytes(b"not the original recovery point") before = runner.database.read_bytes() with pytest.raises(MigrationError, match="backup_required"): - apply(runner, resume=plan.resume_run_id) + apply(runner) assert runner.database.read_bytes() == before assert list(tmp_path.glob("*.pc-migration-backups/*.sqlite3")) == backups +def test_missing_original_evidence_does_not_back_up_partial_state( + tmp_path: Path, bundle: MigrationBundle, monkeypatch: pytest.MonkeyPatch +) -> None: + database = tmp_path / "pc.sqlite3" + load_schema(database) + runner = SQLiteMigrationRunner(database, bundle) + upgrade = command.upgrade + + def interrupted(config, target, **kwargs): + upgrade(config, target, **kwargs) + if target == "p0003": + raise InterruptedError + + with monkeypatch.context() as injection: + injection.setattr(command, "upgrade", interrupted) + with pytest.raises(InterruptedError): + apply(runner) + assert rows(database, "SELECT version_num FROM pc_schema_revision") == [("p0002",)] + backups = list(tmp_path.glob("*.pc-migration-backups/*.sqlite3")) + assert len(backups) == 1 + runner.evidence_path.unlink() + before = database.read_bytes() + with pytest.raises(MigrationError, match="recovery_required"): + apply(SQLiteMigrationRunner(database, bundle)) + assert database.read_bytes() == before + assert list(tmp_path.glob("*.pc-migration-backups/*.sqlite3")) == backups + + +def test_incomplete_external_evidence_reports_recovery_error(tmp_path: Path, bundle: MigrationBundle) -> None: + runner = SQLiteMigrationRunner(tmp_path / "pc.sqlite3", bundle) + apply(runner) + evidence = json.loads(runner.evidence_path.read_text()) + del evidence["bundle_checksum"] + runner.evidence_path.write_text(json.dumps(evidence)) + with pytest.raises(MigrationError, match="recovery_required"): + runner.plan() + + def test_integrity_failure_blocks_readiness(tmp_path: Path, bundle: MigrationBundle) -> None: database = tmp_path / "corrupt.sqlite3" apply(SQLiteMigrationRunner(database, bundle)) @@ -415,3 +441,175 @@ def test_batch_rebuild_preserves_registered_triggers_and_foreign_keys(bundle: Mi } connection.commit() engine.dispose() + + +@pytest.mark.parametrize("policy", ["manual", "skip"]) +def test_operator_backup_choices_do_not_inspect_manual_backups( + tmp_path: Path, bundle: MigrationBundle, monkeypatch: pytest.MonkeyPatch, policy: Literal["manual", "skip"] +) -> None: + database = tmp_path / "pc.sqlite3" + load_schema(database) + provider = SQLiteBackupProvider(database) + + def forbidden(*args, **kwargs): + pytest.fail("An operator backup declaration must not call the automatic provider") + + monkeypatch.setattr(provider, "capabilities", forbidden) + monkeypatch.setattr(provider, "create_backup", forbidden) + monkeypatch.setattr(provider, "inspect_backup", forbidden) + runner = SQLiteMigrationRunner(database, bundle, backup_provider=provider) + plan = runner.plan(backup_policy=policy) + with pytest.raises(MigrationError, match="backup_confirmation_required"): + runner.apply(plan_id=plan.plan_id, backup_policy=policy, accepted=True, maintenance_confirmed=True) + result = runner.apply( + plan_id=plan.plan_id, + backup_policy=policy, + accepted=True, + maintenance_confirmed=True, + backup_confirmed=policy == "manual", + accept_no_backup=policy == "skip", + backup_ref="unreachable://user-backup", + ) + assert result.backup_state == ("user_confirmed" if policy == "manual" else "skipped") + assert not database.with_name(database.name + ".pc-migration-backups").exists() + assert runner.verify().state == "ready" + + +def test_unavailable_automatic_backup_blocks_before_writing( + tmp_path: Path, bundle: MigrationBundle, monkeypatch: pytest.MonkeyPatch +) -> None: + database = tmp_path / "pc.sqlite3" + load_schema(database) + provider = SQLiteBackupProvider(database) + monkeypatch.setattr( + provider, + "capabilities", + lambda _: BackupCapabilities( + available=False, + product="sqlite", + reasons=("Native backup is unavailable.",), + ), + ) + runner = SQLiteMigrationRunner(database, bundle, backup_provider=provider) + before = database.read_bytes() + assert not runner.plan().backup_available + with pytest.raises(MigrationError, match="backup_unsupported"): + apply(runner) + assert database.read_bytes() == before + assert not runner.evidence_path.exists() + + +@pytest.mark.parametrize("policy", ["manual", "skip"]) +def test_failed_backup_before_ddl_allows_explicit_new_choice( + tmp_path: Path, bundle: MigrationBundle, monkeypatch: pytest.MonkeyPatch, policy: Literal["manual", "skip"] +) -> None: + database = tmp_path / "pc.sqlite3" + load_schema(database) + provider = SQLiteBackupProvider(database) + + def unavailable(context): + raise MigrationError("backup_failed", "Insufficient backup storage.") + + monkeypatch.setattr(provider, "create_backup", unavailable) + runner = SQLiteMigrationRunner(database, bundle, backup_provider=provider) + before = database.read_bytes() + with pytest.raises(MigrationError, match="backup_failed"): + apply(runner) + assert database.read_bytes() == before + revised = runner.plan(backup_policy=policy) + result = runner.apply( + plan_id=revised.plan_id, + backup_policy=policy, + accepted=True, + maintenance_confirmed=True, + backup_confirmed=policy == "manual", + accept_no_backup=policy == "skip", + ) + assert result.backup_state == ("user_confirmed" if policy == "manual" else "skipped") + + +def test_changed_bundle_during_backup_is_not_executed(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + copied = tmp_path / "bundle" + shutil.copytree(FIXTURE, copied) + database = tmp_path / "pc.sqlite3" + load_schema(database) + provider = SQLiteBackupProvider(database) + backup = provider.create_backup + script = copied / "versions/p0002_citations.py" + + def change_source(context): + ref = backup(context) + script.write_text(script.read_text() + "\n# Changed during backup\n") + return ref + + monkeypatch.setattr(provider, "create_backup", change_source) + runner = SQLiteMigrationRunner(database, MigrationBundle(copied), backup_provider=provider) + with pytest.raises(MigrationError, match="plan_changed"): + apply(runner) + assert "memory_citations" not in {row[1] for row in rows(database, "PRAGMA table_info(pc_artifacts)")} + assert rows(database, "SELECT name FROM sqlite_schema WHERE name='pc_schema_revision'") == [] + assert len(list(tmp_path.glob("*.pc-migration-backups/*.sqlite3"))) == 1 + + +def test_alembic_executes_reviewed_bytes_when_sources_change_between_revisions( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + copied = tmp_path / "bundle" + shutil.copytree(FIXTURE, copied) + database = tmp_path / "pc.sqlite3" + runner = SQLiteMigrationRunner(database, MigrationBundle(copied)) + reviewed = runner.plan() + upgrade = command.upgrade + script = copied / "versions/p0002_citations.py" + + def alter_installed_script(config, target, **kwargs): + upgrade(config, target, **kwargs) + if target == "p0001": + script.write_text( + script.read_text().replace( + "def upgrade() -> None:", + "def upgrade() -> None:\n op.execute('CREATE TABLE unapproved (id INTEGER)')", + ) + ) + + monkeypatch.setattr(command, "upgrade", alter_installed_script) + result = runner.apply(plan_id=reviewed.plan_id, accepted=True, maintenance_confirmed=True) + assert result.state == "ready" + assert rows(database, "SELECT name FROM sqlite_schema WHERE name='unapproved'") == [] + assert rows(database, "SELECT version_num FROM pc_schema_revision") == [("p0003",)] + with pytest.raises(MigrationError, match="checksum_conflict"): + SQLiteMigrationRunner(database, MigrationBundle(copied)).verify() + + +def test_invalid_sqlite_file_reports_operator_error(tmp_path: Path, bundle: MigrationBundle) -> None: + database = tmp_path / "pc.sqlite3" + database.write_text("not a SQLite database") + with pytest.raises(MigrationError, match="invalid_database"): + SQLiteMigrationRunner(database, bundle).plan() + + +def test_replaced_table_is_retained_across_verification_and_retry(tmp_path: Path, bundle: MigrationBundle) -> None: + database = tmp_path / "pc.sqlite3" + load_schema(database) + populate(database) + runner = SQLiteMigrationRunner(database, bundle) + apply(runner) + expected = [("Keep",)] + assert rows(database, "SELECT tag FROM pc_retained_p0002_artifact_tags") == expected + assert runner.verify().state == "ready" + assert not runner.apply().changed + assert rows(database, "SELECT tag FROM pc_retained_p0002_artifact_tags") == expected + + +def test_retained_table_cannot_precede_its_revision(tmp_path: Path, bundle: MigrationBundle) -> None: + database = tmp_path / "pc.sqlite3" + load_schema(database) + with sqlite3.connect(database) as connection: + original = connection.execute("SELECT sql FROM sqlite_schema WHERE name='pc_artifact_tags'").fetchone()[0] + connection.execute( + original.replace("CREATE TABLE pc_artifact_tags", "CREATE TABLE pc_retained_p0002_artifact_tags") + ) + before = database.read_bytes() + with pytest.raises(MigrationError, match="recovery_required"): + apply(SQLiteMigrationRunner(database, bundle)) + assert database.read_bytes() == before diff --git a/tests/fixtures/database_migrations/README.md b/tests/fixtures/database_migrations/README.md index c814520648..a8b4494757 100644 --- a/tests/fixtures/database_migrations/README.md +++ b/tests/fixtures/database_migrations/README.md @@ -1,70 +1,59 @@ -# Database migration prototype +# Database migration acceptance bundle -Implementation of the Phase A migration primitives in [RFC #1771](https://github.com/oceanbase/powercontext/pull/1771), tracked by [#1756](https://github.com/oceanbase/powercontext/issues/1756). +The bundle validates Phase A of [RFC #1771](https://github.com/oceanbase/powercontext/pull/1771), tracked by [#1756](https://github.com/oceanbase/powercontext/issues/1756). Use disposable databases. Its `ready` result covers only four registered Artifact tables, not complete Server schemas, task formats, search projections or cluster readiness. -This is a repository-only prototype for **disposable databases**. Its `ready` result means that this bundle's four tables and migration receipts verify. It does not mean that a PowerContext Server, its data migrations, or its search projections are ready. The production `powercontext server` commands and startup behavior are unchanged. +The installed copy in `src/powercontext/builtin/persistence/migrations/resources/` is included in the wheel. The public `server db-migrate` command reads Server configuration without opening a business runtime; full unregistered databases are rejected. Ordinary Server startup and the existing `processing-migrate` flow are not yet transferred to this bundle. -## Fixed historical inputs +## Frozen historical inputs -The JSON resources contain the original compiled definitions of `pc_artifacts`, `pc_artifact_heads`, `pc_artifact_candidate_versions`, and `pc_artifact_tags`, including indexes, constraints, and MySQL collations. They do not import current application metadata. +The JSON resources contain original compiled definitions, indexes, constraints and MySQL collations for `pc_artifacts`, `pc_artifact_heads`, `pc_artifact_candidate_versions` and `pc_artifact_tags`. Revisions never import current application metadata. | Input | Source | Purpose | | --- | --- | --- | -| `schemas/pre_dream.json` | [495cc93d67387879382c98706f4975f638faec15](https://github.com/oceanbase/powercontext/commit/495cc93d67387879382c98706f4975f638faec15), before Dream citations | Upgrade existing rows through both change classes | -| `schemas/v1_1_0.json` | [powercontext-v1.1.0 / 86e687f13fec62151ee5657a90733586fd5158dc](https://github.com/oceanbase/powercontext/commit/86e687f13fec62151ee5657a90733586fd5158dc) | Recognize the already-upgraded released table shapes | +| `schemas/pre_dream.json` | [495cc93d67387879382c98706f4975f638faec15](https://github.com/oceanbase/powercontext/commit/495cc93d67387879382c98706f4975f638faec15) | Exercise both change classes with existing data | +| `schemas/v1_1_0.json` | [v1.1.0 / 86e687f13fec62151ee5657a90733586fd5158dc](https://github.com/oceanbase/powercontext/commit/86e687f13fec62151ee5657a90733586fd5158dc) | Recognize released table shapes | -Both resources record the source Git blob and the SQLAlchemy compiler version. They were captured from the original commit's `tables.py` **and `limits.py`**, selecting these four original tables and compiling `CreateTable` plus each named `CreateIndex` for SQLite and MySQL. The pre-Dream fixture was not synthesized by subtracting columns from today's models. These are table subsets, not complete released database images. +Each resource records source blobs and compiler versions. Definitions were captured from the original commit's `tables.py` and `limits.py`, compiling `CreateTable` and named `CreateIndex` for SQLite and MySQL. These are subsets, not full released database images. -The single Alembic chain is: +The single chain contains: 1. `p0001`: create the frozen pre-Dream structures. -2. `p0002`: add nullable `memory_citations` columns to immutable artifact and candidate revisions. -3. `p0003`: remove `ck_pc_artifact_tags_family`; use an Alembic batch rebuild on SQLite and explicit constraint DDL on MySQL dialects. Preserve the target constraint, foreign key, indexes, rows, and registered triggers. +2. `p0002`: add nullable `memory_citations` to Artifact and candidate revisions. +3. `p0003`: remove `ck_pc_artifact_tags_family`; SQLite rebuilds the table, preserves rows, constraints, indexes and registered triggers, and retains its prior layout and rows as `pc_retained_p0002_artifact_tags`. MySQL uses explicit constraint DDL. -`manifest.json` records exact SQLite schema fingerprints at each revision and the two supported unversioned baselines. Exact matching intentionally rejects unregistered tables, indexes, triggers, views, temporary-table residue, and alternate DDL spellings. The released table definitions and the migrated definitions have separate admitted fingerprints. No generic `stamp head` fallback exists. +The manifest registers exact business fingerprints, alternate released shapes, retained object shapes and their introduction revisions. Unregistered objects, premature retained objects, residue and unknown versions are rejected. There is no optimistic `stamp head` fallback. -Revision receipts bind each script, its declared historical resources, and the Alembic environment. A reviewed plan also binds the complete bundle, configuration digest, canonical database path, source fingerprint, and target. Editing a recorded revision or its declared resources prevents acceptance. +Plans bind the canonical target, actual schema, immutable resource snapshot, configuration digest and maintenance choices. Alembic executes the reviewed snapshot even if installed resources change; changes detected during backup stop before migration DDL. -## SQLite executor +## Explicit maintenance -Run from the repository after `uv sync --locked`: +Repository-only scratch commands: ```bash -uv run --locked python scripts/database_migration_probe.py plan --sqlite-path /tmp/pc-migration-probe.sqlite3 -uv run --locked python scripts/database_migration_probe.py apply --sqlite-path /tmp/pc-migration-probe.sqlite3 --maintenance-confirmed --yes --plan-id PLAN_ID -uv run --locked python scripts/database_migration_probe.py verify --sqlite-path /tmp/pc-migration-probe.sqlite3 +uv run --locked python scripts/database_migration_probe.py plan --sqlite-path /tmp/pc-probe.sqlite3 --backup auto +uv run --locked python scripts/database_migration_probe.py apply --sqlite-path /tmp/pc-probe.sqlite3 --backup auto --maintenance-confirmed --yes --plan-id PLAN_ID +uv run --locked python scripts/database_migration_probe.py verify --sqlite-path /tmp/pc-probe.sqlite3 ``` -Replace `PLAN_ID` with the returned ID. An interactive apply prints a fresh plan and asks once; non-interactive apply requires both the exact ID and `--yes`. The command never discovers `.env` files or opens the business runtime. `status`, `plan`, and `verify` open an existing target read-only and never create a missing target, parent, lock, or bookkeeping table. A no-op apply needs no confirmation, lock, backup, or new run. +Read-only commands create no missing target, parent, lock or control table. No-op apply performs no backup or lifecycle changes. Changing runs require explicit plan acceptance and stopped-writer confirmation. Canonical-path OS locking covers symlinks and process exit; hard links are rejected. `BEGIN IMMEDIATE` detects present writers, but cannot prove idle old clients will remain stopped. -Changing runs require the operator to stop all writers. A canonical-path OS lock excludes other migrators across processes, including symlink aliases; hard-linked SQLite files are rejected. Each schema transaction begins with `BEGIN IMMEDIATE`, which also rejects a currently active database writer. This check cannot prove that an idle legacy process has stopped. The maintenance confirmation remains an operator responsibility. +Only Alembic's standard `pc_schema_revision(version_num)` control table is created. There are no run, step or task control tables and no generic resume ID. Each SQLite revision, version update and verification commits atomically. Retry inspects actual structure and revision, then runs only pending revisions. Partial or missing evidence blocks execution rather than replacing an original recovery point. -Before bookkeeping or business DDL, the executor uses SQLite's online backup API from a separate read connection while holding the database write reservation. Backups include committed WAL data, are created with restrictive permissions, and pass integrity verification. They are retained beside the target in `.pc-migration-backups/`; migration does not automatically delete them or restore over the database. +Automatic SQLite backup uses Online Backup API while holding a write reservation and includes committed WAL. Its immutable provider manifest and the maintenance summary are stored outside the target database with restrictive permissions and durable writes. Retry reuses the original verified recovery point. Empty initialization reports `not_required`; manual backup is an unchecked declaration (`user_confirmed`); skip requires risk acceptance (`skipped`). Retained tables and recovery points are not automatically deleted or restored. -Bookkeeping bootstrap is atomic on SQLite. Each subsequent revision, version update, schema/foreign-key verification, and completion receipt commits together. Foreign-key enforcement is disabled only on the dedicated migration connection before its explicit transaction; `foreign_key_check` runs before commit. An interrupted committed run blocks readiness until explicitly resumed: +## Real backend probes ```bash -uv run --locked python scripts/database_migration_probe.py status --sqlite-path /tmp/pc-migration-probe.sqlite3 -uv run --locked python scripts/database_migration_probe.py apply --sqlite-path /tmp/pc-migration-probe.sqlite3 --maintenance-confirmed --yes --plan-id ORIGINAL_PLAN_ID --resume RUN_ID +uv run --locked pytest -q tests/builtin/persistence/test_database_migrations.py tests/test_database_migration_probe.py tests/test_database_migration_cli.py +POWERCONTEXT_TEST_MIGRATION_SEEKDB=1 uv run --locked --extra seekdb pytest -q tests/builtin/persistence/test_database_migration_backends.py tests/builtin/persistence/test_database_backup_providers.py ``` -Recovery requires the original bundle, configuration, target, and unmodified backup. It retains the run ID and recovery point, skipping only atomically completed revisions. A crash before bootstrap commits can leave an unreferenced backup file; it is preserved, while the rolled-back database can be planned again. +seekdb probes use a new temporary instance and a directory lock around engine startup and shutdown. The main workflow runs actual DDL probes on Linux Python 3.11 and 3.14. A simple-table Fork test additionally covers source DDL, restart and restoration to the original table name while retaining failed and Fork objects. This does not certify full PC full-text, vector, foreign-key or queue recovery. -## Backend probes and tests +OceanBase requires an empty dedicated database named `pc_migration_probe_` through `POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL`. The probe refuses other names and nonempty targets; it leaves the resulting schema and never provisions or drops databases. Its lock probe uses two physical sessions and verifies ownership after commit and DDL. Missing environments and skipped tests are not acceptance. -```bash -uv run --locked pytest -q tests/builtin/persistence/test_database_migrations.py tests/test_database_migration_probe.py -POWERCONTEXT_TEST_MIGRATION_SEEKDB=1 uv run --locked --extra seekdb pytest -q tests/builtin/persistence/test_database_migration_backends.py -k seekdb -``` - -The seekDB probe holds the directory lock before engine startup until after shutdown and runs actual Alembic DDL on a new temporary instance. The main workflow runs it on Linux with Python 3.11 and 3.14. The probe establishes additive and constraint-change behavior, data/index preservation, and repeated `upgrade head`; it does not establish backup or interrupted-DDL recovery support. - -For OceanBase, provision an **empty dedicated database** named `pc_migration_probe_` and supply its `mysql+aoceanbase` URL through `POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL`. Run the same backend test file with `-k oceanbase`. It refuses other database names and nonempty targets, leaves the resulting schema for inspection, and does not provision or drop databases. No deployment credentials are embedded in the fixtures or test output. - -The OceanBase lock primitive tests exclusion with two distinct physical sessions, ownership after commit, and the pinned session after DDL. It refuses unsupported locking rather than falling back to a local mutex. This is one capability probe; it does not prove exclusion across every node/proxy route. The probe has no production backup or nontransactional-DDL recovery executor. - -SQLite tests exercise empty initialization, both historical table subsets, payload preservation, new tag families, constraints/indexes/triggers, consistent WAL backup and restore, no-op behavior, read-only inspection, explicit confirmation, plan changes, active writers, concurrent processes, lock-owner exit, interruption before/after every revision commit, original-backup reuse, checksum conflicts, unknown/newer schemas, and integrity failures. Backend tests skipped for missing configuration are not counted as acceptance. +The independent `BackupProvider` reports Fork features separately from an accepted PC recovery workflow. OceanBase AI Database requires 4.6.2 for both table and database Fork; seekdb requires 1.1.0 and 1.2.0 respectively. Actual product/version, full object coverage, permissions, DDL restrictions, restart and restoration must pass release-owned acceptance before automatic Fork is enabled. It never silently falls back to physical backup or skipped backup. -## Before production enablement +## Production gates -The remaining RFC phases require complete released database fixtures and ownership inventory; full application revision chains; required data/projection task scheduling; seekDB consistent backup and OceanBase external recovery evidence; nontransactional-DDL postcondition recovery; all-profile startup/SDK/Worker gates; the public `server db-migrate` interface and `processing-migrate` compatibility; and mandatory schema-change CI checks. These primitives do not bypass those release gates. PR #1716 remains independent. +Full historical baselines, complete object ownership, data/projection and old-task validators, no-side-effect remote connections, nontransactional DDL recovery, startup/SDK/Worker gates and mandatory schema-change CI remain prerequisites for production enablement. The current limited bundle refuses `--manage-service`; its result must not start a full business runtime. Native service start/stop/restart and the separately tested maintenance lifecycle API do not establish those database gates. PR #1716 remains independent. diff --git a/tests/fixtures/database_migrations/manifest.json b/tests/fixtures/database_migrations/manifest.json index ce51a824ee..d3c36a44e4 100644 --- a/tests/fixtures/database_migrations/manifest.json +++ b/tests/fixtures/database_migrations/manifest.json @@ -21,5 +21,22 @@ "9958fa0413ed3c16442df27dc48057d438e2cf3f7e4312c5a914cccd9d3acaab", "fcee685ed307ee6d02fe5e950ed4e80be75f38d19de902ccacf443f21ee8c73d" ] + }, + "affected_objects": [ + "pc_artifacts", + "pc_artifact_heads", + "pc_artifact_candidate_versions", + "pc_artifact_tags" + ], + "retained_objects": [ + "pc_retained_p0002_artifact_tags" + ], + "retained_fingerprints": { + "pc_retained_p0002_artifact_tags": [ + "33052f9e6ae051c52e7ec13d53033b6c11e96d7f5b2c1df40047a7344e471eee" + ] + }, + "retained_revisions": { + "pc_retained_p0002_artifact_tags": "p0003" } } diff --git a/tests/fixtures/database_migrations/versions/p0003_tag_families.py b/tests/fixtures/database_migrations/versions/p0003_tag_families.py index 90e722eb4e..cc98f5e684 100644 --- a/tests/fixtures/database_migrations/versions/p0003_tag_families.py +++ b/tests/fixtures/database_migrations/versions/p0003_tag_families.py @@ -23,6 +23,14 @@ def upgrade() -> None: connection = op.get_bind() if connection.dialect.name == "sqlite": + # Keep a frozen copy of the replaced layout and rows. Cleanup belongs + # to a later, independently reviewed maintenance revision. + original = connection.exec_driver_sql( + "SELECT sql FROM sqlite_schema WHERE type='table' AND name='pc_artifact_tags'" + ).scalar_one() + retained = original.replace("CREATE TABLE pc_artifact_tags", "CREATE TABLE pc_retained_p0002_artifact_tags", 1) + connection.exec_driver_sql(retained) + connection.exec_driver_sql("INSERT INTO pc_retained_p0002_artifact_tags SELECT * FROM pc_artifact_tags") # Alembic reflects indexes and named CHECKs, but not application triggers. triggers = ( connection diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index e58d3b02bf..915a40ce83 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -99,6 +99,42 @@ def test_native_personal_service_lifecycle(tmp_path: Path) -> None: _cleanup(adapter) +def test_native_maintenance_preserves_registration_and_original_running_state(tmp_path: Path) -> None: + """The manager must not recreate an old writer while a migration owns maintenance.""" + + adapter = _native_adapter(suffix="maintenance") + controller = ServiceController(adapter) + environment = _environment_file(tmp_path) + try: + assert controller.install(env_file=environment).ok + original = adapter.inspect().definition + summary = controller.maintenance_summary(env_file=environment) + assert summary.originally_running + with controller.maintenance(expected_fingerprint=summary.fingerprint, env_file=environment) as session: + assert adapter.inspect().definition == original + assert controller.maintenance_path.exists() + # Cross the native RestartSec/ThrottleInterval boundary so a + # KeepAlive/retry policy cannot silently restart the old process. + time.sleep(6) + assert adapter.manager_state() in {ManagerState.INACTIVE, ManagerState.FAILED} + assert session.complete().ok + assert not controller.maintenance_path.exists() + + controller.stop() + stopped = controller.maintenance_summary(env_file=environment) + assert not stopped.originally_running + with controller.maintenance(expected_fingerprint=stopped.fingerprint, env_file=environment) as session: + session.complete() + assert adapter.inspect().definition == stopped.target_definition + assert adapter.manager_state() in {ManagerState.INACTIVE, ManagerState.FAILED} + assert controller.start().ok + finally: + _capture_native_failure(adapter, tmp_path) + with suppress(Exception): + controller.uninstall() + _cleanup(adapter) + + def test_failure_diagnostics_reports_an_absent_service(tmp_path: Path) -> None: """The report must survive the state it exists to describe: nothing there. diff --git a/tests/test_database_migration_cli.py b/tests/test_database_migration_cli.py new file mode 100644 index 0000000000..17ee1fc4a5 --- /dev/null +++ b/tests/test_database_migration_cli.py @@ -0,0 +1,290 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The configured maintenance CLI preserves one-plan consent and read-only inspection.""" + +import asyncio +import json +import os +import sqlite3 +from pathlib import Path + +from typer.testing import CliRunner + +from powercontext.builtin.persistence.migrations.deployment import production_bundle +from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile +from powercontext.builtin.persistence.tables import BUILTIN_TABLES +from powercontext.cli.app import create_cli +from powercontext.server import database_migration +from powercontext.server.cli import app + + +def _configure( + tmp_path: Path, monkeypatch, *, existing: bool = False, complete: bool = False +) -> tuple[Path, list[str]]: + for key in tuple(os.environ): + if key.startswith("POWERCONTEXT_SERVER_"): + monkeypatch.delenv(key) + database = tmp_path / "database" / "server.db" + environment = tmp_path / "deployment.env" + environment.write_text( + f"POWERCONTEXT_SERVER_DATABASE_KIND=sqlite\nPOWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite:///{database}\n", + encoding="utf-8", + ) + if existing: + database.parent.mkdir(parents=True) + bundle = production_bundle() + schema = json.loads((bundle.snapshot_directory / "schemas/pre_dream.json").read_text()) + with sqlite3.connect(database) as connection: + for statement in schema["sqlite"]: + connection.execute(statement) + if complete: + + async def initialize() -> None: + async with SQLiteProfile.open(SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), tables=BUILTIN_TABLES): + pass + + asyncio.run(initialize()) + return database, ["server", "db-migrate"] + + +def _invoke(command: list[str], action: str, tmp_path: Path, *options: str, prompt_input: str | None = None): + return CliRunner().invoke( + create_cli([app]), + [*command, action, "--env-file", str(tmp_path / "deployment.env"), *options], + input=prompt_input, + ) + + +def test_read_only_commands_never_create_missing_target(tmp_path, monkeypatch): + database, command = _configure(tmp_path, monkeypatch) + for action in ("status", "plan", "verify"): + result = _invoke(command, action, tmp_path) + payload = json.loads(result.output) + if action == "verify": + assert result.exit_code == 1 + assert payload["error"] in {"migration_required", "uninitialized"} + else: + assert result.exit_code == 0, result.output + assert payload["state"] == "uninitialized" + assert not database.parent.exists() + + +def test_apply_requires_exact_plan_and_explicit_noninteractive_consent(tmp_path, monkeypatch): + database, command = _configure(tmp_path, monkeypatch) + plan = json.loads(_invoke(command, "plan", tmp_path).output) + denied = _invoke(command, "apply", tmp_path, "--yes", "--backup", "auto") + assert denied.exit_code == 1 + assert json.loads(denied.output)["error"] == "confirmation_required" + changed = _invoke( + command, + "apply", + tmp_path, + "--yes", + "--backup", + "auto", + "--plan-id", + "wrong", + "--maintenance-confirmed", + ) + assert changed.exit_code == 1 + assert json.loads(changed.output)["error"] == "plan_changed" + assert not database.exists() + applied = _invoke( + command, + "apply", + tmp_path, + "--yes", + "--backup", + "auto", + "--plan-id", + plan["plan_id"], + "--maintenance-confirmed", + ) + assert applied.exit_code == 0, applied.output + payload = json.loads(applied.output) + assert payload["state"] == "ready" + assert payload["changed"] is True + assert payload["backup_state"] == "not_required" + with sqlite3.connect(database) as connection: + tables = {row[0] for row in connection.execute("SELECT name FROM sqlite_master WHERE type='table'")} + assert "pc_artifacts" in tables + assert "pc_schema_revision" in tables + assert "pc_migration_runs" not in tables + assert "pc_migration_steps" not in tables + assert _invoke(command, "verify", tmp_path).exit_code == 0 + no_op = _invoke(command, "apply", tmp_path, "--manage-service") + assert no_op.exit_code == 0, no_op.output + assert json.loads(no_op.output)["changed"] is False + assert not list(database.parent.glob("*backup*")) + + +def test_interactive_manual_backup_combines_declarations_in_one_confirmation(tmp_path, monkeypatch): + database, command = _configure(tmp_path, monkeypatch, existing=True) + monkeypatch.setattr(database_migration, "_interactive", lambda: True) + result = _invoke(command, "apply", tmp_path, "--backup", "manual", prompt_input="y\n") + assert result.exit_code == 0, result.output + assert result.output.count("Accept this plan and the declarations above?") == 1 + assert "PC will not verify that backup" in result.output + assert '"backup_state": "user_confirmed"' in result.output + assert database.exists() + + +def test_existing_database_manual_backup_is_only_an_explicit_declaration(tmp_path, monkeypatch): + database, command = _configure(tmp_path, monkeypatch, existing=True) + original = database.read_bytes() + planned = _invoke(command, "plan", tmp_path, "--backup", "manual") + assert planned.exit_code == 0, planned.output + plan = json.loads(planned.output) + missing = _invoke( + command, + "apply", + tmp_path, + "--yes", + "--backup", + "manual", + "--plan-id", + plan["plan_id"], + "--maintenance-confirmed", + ) + assert missing.exit_code == 1 + assert json.loads(missing.output)["error"] == "confirmation_required" + assert database.read_bytes() == original + result = _invoke( + command, + "apply", + tmp_path, + "--yes", + "--backup", + "manual", + "--plan-id", + plan["plan_id"], + "--maintenance-confirmed", + "--backup-confirmed", + "--backup-ref", + "operator-reference", + ) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["backup_state"] == "user_confirmed" + assert payload["backup_ref"] == "operator-reference" + + +def test_invalid_sqlite_file_reports_payload_free_json(tmp_path, monkeypatch): + database, command = _configure(tmp_path, monkeypatch) + database.parent.mkdir() + private_value = "private-database-content-do-not-print" + database.write_text(private_value, encoding="utf-8") + result = _invoke(command, "plan", tmp_path) + assert result.exit_code == 1 + assert json.loads(result.output)["error"] == "invalid_database" + assert private_value not in result.output + assert "Traceback" not in result.output + + +def test_incomplete_external_evidence_is_a_structured_recovery_error(tmp_path, monkeypatch): + database, command = _configure(tmp_path, monkeypatch, existing=True) + inspected = json.loads(_invoke(command, "status", tmp_path).output) + evidence = database.with_name(database.name + ".pc-migration-state") / "maintenance.json" + evidence.parent.mkdir() + evidence.write_text( + json.dumps({"database_id": inspected["database_id"], "state": "active", "backup_policy": "auto"}), + encoding="utf-8", + ) + before = database.read_bytes() + result = _invoke(command, "status", tmp_path) + assert result.exit_code == 1 + assert json.loads(result.output)["error"] == "recovery_required" + assert "Traceback" not in result.output + assert database.read_bytes() == before + + +def test_invalid_configuration_does_not_print_credentials(tmp_path, monkeypatch): + _database, command = _configure(tmp_path, monkeypatch) + private_value = "private-token-do-not-print" + (tmp_path / "deployment.env").write_text( + f"POWERCONTEXT_SERVER_DATABASE_KIND=unknown\nPOWERCONTEXT_SERVER_AUTH_TOKEN={private_value}\n", + encoding="utf-8", + ) + result = _invoke(command, "status", tmp_path) + assert result.exit_code == 1 + assert json.loads(result.output)["error"] == "configuration_invalid" + assert private_value not in result.output + assert "Traceback" not in result.output + + +def test_complete_server_database_is_not_silently_adopted_by_acceptance_bundle(tmp_path, monkeypatch): + database, command = _configure(tmp_path, monkeypatch, complete=True) + before = database.read_bytes() + result = _invoke(command, "plan", tmp_path) + assert result.exit_code == 1 + assert json.loads(result.output)["error"] == "unknown_baseline" + assert database.read_bytes() == before + + +def test_partial_bundle_rejects_business_service_management_before_writing(tmp_path, monkeypatch): + database, command = _configure(tmp_path, monkeypatch) + result = _invoke(command, "apply", tmp_path, "--manage-service") + assert result.exit_code == 1 + assert json.loads(result.output)["error"] == "service_unsupported" + assert not database.exists() + + +def test_remote_backend_is_not_reported_ready_without_acceptance(tmp_path, monkeypatch): + _database, command = _configure(tmp_path, monkeypatch) + directory = tmp_path / "seekdb" + (tmp_path / "deployment.env").write_text( + f"POWERCONTEXT_SERVER_DATABASE_KIND=seekdb\nPOWERCONTEXT_SERVER_DATABASE_PATH={directory}\n", + encoding="utf-8", + ) + result = _invoke(command, "status", tmp_path) + assert result.exit_code == 1 + assert json.loads(result.output)["error"] == "unsupported_backend" + assert not directory.exists() + + +def test_no_backup_requires_separate_risk_consent_and_policy_is_plan_bound(tmp_path, monkeypatch): + database, command = _configure(tmp_path, monkeypatch, existing=True) + before = database.read_bytes() + auto = json.loads(_invoke(command, "plan", tmp_path, "--backup", "auto").output) + skip = json.loads(_invoke(command, "plan", tmp_path, "--backup", "skip").output) + assert auto["plan_id"] != skip["plan_id"] + missing = _invoke( + command, + "apply", + tmp_path, + "--yes", + "--backup", + "skip", + "--plan-id", + skip["plan_id"], + "--maintenance-confirmed", + ) + assert missing.exit_code == 1 + assert json.loads(missing.output)["error"] == "confirmation_required" + assert database.read_bytes() == before + result = _invoke( + command, + "apply", + tmp_path, + "--yes", + "--backup", + "skip", + "--plan-id", + skip["plan_id"], + "--maintenance-confirmed", + "--accept-no-backup", + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["backup_state"] == "skipped" diff --git a/tests/test_service.py b/tests/test_service.py index 80c0b5ca15..ece742aaf1 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -161,6 +161,7 @@ def __init__(self, tmp_path: Path) -> None: self.fail_disable = False self.fail_remove = False self.fail_reload = False + self.suspended = False def support(self) -> tuple[SupportState, str]: return SupportState.SUPPORTED, "fake manager available" @@ -222,6 +223,18 @@ def disable(self) -> None: if self.fail_disable: raise ServiceError("disable failed") # noqa: TRY003 + def suspend(self, marker: Path) -> None: + self.events.append("suspend") + self.suspended = True + + def resume(self) -> None: + self.events.append("resume") + self.suspended = False + + def update_suspended(self) -> None: + self.events.append("update_suspended") + self.loaded_definition = self.definition + def remove(self) -> None: self.events.append("remove") if self.fail_remove: diff --git a/tests/test_service_maintenance.py b/tests/test_service_maintenance.py new file mode 100644 index 0000000000..0b72dc83ef --- /dev/null +++ b/tests/test_service_maintenance.py @@ -0,0 +1,286 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import os +import subprocess +import xml.etree.ElementTree as ET +from dataclasses import replace +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +import powercontext.service.cli as service_cli +from powercontext.service.adapters.base import service_python_executable +from powercontext.service.adapters.systemd import SystemdUserAdapter +from powercontext.service.adapters.windows import WindowsTaskSchedulerAdapter +from powercontext.service.controller import ServiceController +from powercontext.service.model import ( + ManagerOwnershipState, + ManagerRegistration, + ManagerState, + ProbeResult, + ProbeState, + ServiceError, +) +from powercontext.service.probe import probe_readiness +from tests.test_service import FakeAdapter, _definition, _LivenessHandler, _manager_probe, _serve + + +def _installed(tmp_path: Path, *, running: bool = True) -> tuple[ServiceController, FakeAdapter]: + adapter = FakeAdapter(tmp_path) + old_python = tmp_path / "old-python" + old_python.touch() + adapter.definition = _definition(tmp_path, python_executable=str(old_python), package_version="old") + adapter.content = adapter.render(adapter.definition) + adapter.loaded_definition = adapter.definition + adapter.manager = ManagerState.ACTIVE if running else ManagerState.INACTIVE + probe = _manager_probe(adapter) + return ServiceController(adapter, probe=probe, readiness_probe=probe), adapter + + +def test_service_lifecycle_commands_preserve_registration_and_configuration( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + controller, adapter = _installed(tmp_path) + definition = adapter.definition + monkeypatch.setattr(service_cli, "_controller", lambda: controller) + runner = CliRunner() + + assert runner.invoke(service_cli.app, ["stop"]).exit_code == 0 + assert adapter.definition == definition + assert adapter.suspended + assert adapter.manager is ManagerState.INACTIVE + assert runner.invoke(service_cli.app, ["start"]).exit_code == 0 + assert adapter.definition == definition + assert not adapter.suspended + assert adapter.manager is ManagerState.ACTIVE + assert runner.invoke(service_cli.app, ["restart"]).exit_code == 0 + assert adapter.definition == definition + assert adapter.manager is ManagerState.ACTIVE + + +def test_successful_maintenance_switches_to_confirmed_executable_and_preserves_configuration(tmp_path: Path) -> None: + controller, adapter = _installed(tmp_path) + original = adapter.definition + assert original is not None + summary = controller.maintenance_summary() + assert summary.as_dict()["fingerprint"] == summary.fingerprint + assert summary.target_definition.python_executable == service_python_executable() + + with controller.maintenance(expected_fingerprint=summary.fingerprint) as session: + assert adapter.suspended + assert adapter.manager is ManagerState.INACTIVE + assert adapter.definition == original + assert not session.completed + completed = session.complete() + + assert completed.ok + assert adapter.definition == summary.target_definition + assert adapter.definition is not None + assert adapter.definition.data_dir == original.data_dir + assert adapter.definition.env_file == original.env_file + assert not adapter.suspended + assert not controller.maintenance_path.exists() + + +def test_failed_migration_keeps_service_stopped_and_retry_preserves_original_running_state(tmp_path: Path) -> None: + controller, adapter = _installed(tmp_path) + summary = controller.maintenance_summary() + original = adapter.definition + + with ( + pytest.raises(RuntimeError, match="migration failed"), + controller.maintenance(expected_fingerprint=summary.fingerprint), + ): + raise RuntimeError("migration failed") # noqa: TRY003 + + assert adapter.suspended + assert adapter.manager is ManagerState.INACTIVE + assert adapter.definition == original + with pytest.raises(ServiceError, match="not been verified"): + controller.start() + assert controller.maintenance_summary().fingerprint == summary.fingerprint + with controller.maintenance(expected_fingerprint=summary.fingerprint) as session: + session.complete() + assert adapter.manager is ManagerState.ACTIVE + + +def test_maintenance_retains_an_originally_stopped_service_without_automatic_start(tmp_path: Path) -> None: + controller, adapter = _installed(tmp_path, running=False) + summary = controller.maintenance_summary() + with controller.maintenance(expected_fingerprint=summary.fingerprint) as session: + session.complete() + + assert adapter.manager is ManagerState.INACTIVE + assert adapter.suspended + assert adapter.definition == summary.target_definition + assert json.loads(controller.maintenance_path.read_text())["phase"] == "database_ready" + assert not any(event.startswith("start:") for event in adapter.events) + assert controller.start().ok + + +def test_changed_service_plan_is_rejected_before_stopping_the_service(tmp_path: Path) -> None: + controller, adapter = _installed(tmp_path) + summary = controller.maintenance_summary() + assert adapter.definition is not None + adapter.definition = replace(adapter.definition, endpoint="http://127.0.0.1:9000") + adapter.loaded_definition = adapter.definition + + with pytest.raises(ServiceError, match="changed"), controller.maintenance(expected_fingerprint=summary.fingerprint): + pytest.fail("a changed service plan must not enter maintenance") + assert not adapter.suspended + assert adapter.manager is ManagerState.ACTIVE + assert not controller.maintenance_path.exists() + + +def test_service_completion_requires_the_still_locked_maintenance_session(tmp_path: Path) -> None: + controller, adapter = _installed(tmp_path) + original = adapter.definition + summary = controller.maintenance_summary() + with controller.maintenance(expected_fingerprint=summary.fingerprint) as session: + pass + + with pytest.raises(ServiceError, match="active, uncompleted"): + session.complete() + assert adapter.definition == original + assert adapter.suspended + assert adapter.manager is ManagerState.INACTIVE + + +@pytest.mark.parametrize("operation", ["stop", "start", "restart", "maintenance_summary"]) +def test_foreign_native_registration_is_not_mutated(tmp_path: Path, operation: str) -> None: + controller, adapter = _installed(tmp_path) + adapter.manager_registration_override = ManagerRegistration(ManagerOwnershipState.FOREIGN, detail="foreign task") + + with pytest.raises(ServiceError, match="foreign task"): + getattr(controller, operation)() + assert adapter.events == [] + assert not controller.maintenance_path.exists() + + +def test_startup_readiness_failure_is_separate_from_database_ready_and_keeps_service_stopped(tmp_path: Path) -> None: + controller, adapter = _installed(tmp_path) + controller = ServiceController( + adapter, + probe=_manager_probe(adapter), + readiness_probe=lambda _: ProbeResult(ProbeState.CONFLICT, "schema is not ready"), + ) + summary = controller.maintenance_summary() + with ( + controller.maintenance(expected_fingerprint=summary.fingerprint) as session, + pytest.raises(ServiceError, match="failed readiness"), + ): + session.complete() + assert adapter.suspended + assert adapter.manager is ManagerState.INACTIVE + assert adapter.definition == summary.target_definition + assert json.loads(controller.maintenance_path.read_text())["phase"] == "start_failed" + + +@pytest.mark.skipif(os.name == "nt", reason="requires creating a symlink without elevation") +def test_foreign_maintenance_record_is_not_overwritten(tmp_path: Path) -> None: + controller, adapter = _installed(tmp_path) + external = tmp_path / "external.json" + external.write_text('{"phase":"manual_stopped"}') + controller.maintenance_path.symlink_to(external) + + with pytest.raises(ServiceError, match="regular file"): + controller.stop() + assert adapter.events == [] + assert external.read_text() == '{"phase":"manual_stopped"}' + + +@pytest.mark.parametrize( + ("status", "code", "expected"), + [("ready", 200, ProbeState.LIVE), ("degraded", 200, ProbeState.LIVE), ("not_ready", 503, ProbeState.UNREACHABLE)], +) +def test_service_readiness_uses_business_readiness_beyond_liveness( + status: str, code: int, expected: ProbeState +) -> None: + class Handler(_LivenessHandler): + def do_GET(self) -> None: + if self.path == "/health/live": + super().do_GET() + return + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("X-PowerContext-Request-ID", "0123456789abcdef") + self.end_headers() + self.wfile.write(json.dumps({"status": status, "checks": {"application": status}}).encode()) + + server, thread, endpoint = _serve(Handler) + try: + assert probe_readiness(endpoint).state is expected + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def test_windows_definition_switch_registers_a_disabled_task_without_changing_login_configuration( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + adapter = WindowsTaskSchedulerAdapter( + config_home=tmp_path, + identifier="PowerContext Maintenance Test", + user_account="test\\alice", + user_sid="S-1-5-21-1", + ) + definition = _definition(tmp_path) + adapter.write(adapter.render(definition)) + monkeypatch.setattr( + adapter, "loaded_registration", lambda: ManagerRegistration(ManagerOwnershipState.OWNED, definition=definition) + ) + registered: list[bytes] = [] + + def run(*arguments: str, **_kwargs: object) -> subprocess.CompletedProcess[str]: + if "/XML" in arguments: + registered.append(Path(arguments[arguments.index("/XML") + 1]).read_bytes()) + return subprocess.CompletedProcess(["schtasks.exe", *arguments], 0, "", "") + + monkeypatch.setattr(adapter, "_run", run) + adapter.update_suspended() + + assert registered + loaded = ET.fromstring(registered[-1]) # noqa: S314 + namespace = "{http://schemas.microsoft.com/windows/2004/02/mit/task}" + enabled = loaded.find(f"{namespace}Settings/{namespace}Enabled") + assert enabled is not None and enabled.text == "false" + assert loaded.find(f"{namespace}Triggers/{namespace}LogonTrigger") is not None + assert adapter.inspect().definition == definition + + +def test_systemd_maintenance_guard_uses_a_negated_absolute_path_and_preserves_the_unit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + adapter = SystemdUserAdapter(config_home=tmp_path / "config with $ and %") + definition = _definition(tmp_path) + unit = adapter.render(definition) + adapter.write(unit) + marker = adapter.lock_path.with_name(f"{adapter.lock_path.name}.maintenance.json") + marker.write_text('{"phase":"stopped"}') + monkeypatch.setattr( + adapter, "loaded_registration", lambda: ManagerRegistration(ManagerOwnershipState.OWNED, definition=definition) + ) + monkeypatch.setattr(adapter, "reload", lambda: None) + + adapter.suspend(marker) + + dropin = adapter.artifact_path.with_name(f"{adapter.identifier}.d") / "90-powercontext-maintenance.conf" + assert f"ConditionPathExists=!{str(marker).replace('%', '%%')}\n" in dropin.read_text() + assert adapter.artifact_path.read_bytes() == unit From d4288897043681bbe24507b48248cb2f22fe9072 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Wed, 30 Sep 2026 17:48:37 +0800 Subject: [PATCH 19/37] fix(migrations): preserve recovery evidence and resume service switches --- .../builtin/persistence/migrations/backup.py | 14 ++- .../migrations/resources/manifest.json | 2 +- .../resources/versions/p0003_tag_families.py | 10 ++- .../builtin/persistence/migrations/sqlite.py | 29 +++++- src/powercontext/service/controller.py | 67 ++++++++++++-- .../test_database_backup_providers.py | 31 +++++++ .../persistence/test_database_migrations.py | 55 ++++++++++++ .../database_migrations/manifest.json | 2 +- .../versions/p0003_tag_families.py | 10 ++- tests/test_service_maintenance.py | 88 +++++++++++++++++++ 10 files changed, 293 insertions(+), 15 deletions(-) diff --git a/src/powercontext/builtin/persistence/migrations/backup.py b/src/powercontext/builtin/persistence/migrations/backup.py index 86b933c333..c6e28fd01c 100644 --- a/src/powercontext/builtin/persistence/migrations/backup.py +++ b/src/powercontext/builtin/persistence/migrations/backup.py @@ -174,6 +174,7 @@ def create_backup(self, context: BackupContext) -> BackupRef: os.close(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)) with closing(self._read_database(self.database)) as source, closing(sqlite3.connect(target)) as destination: source.backup(destination) + destination.execute("PRAGMA journal_mode=DELETE") if destination.execute("PRAGMA integrity_check").fetchall() != [("ok",)]: raise MigrationError("backup_failed", "The SQLite recovery point failed integrity checks.") _fsync_file(target) @@ -228,9 +229,17 @@ def inspect_backup(self, ref: BackupRef) -> BackupInspection: return BackupInspection( state="failed", reasons=("The original recovery point or manifest has changed.",) ) - with closing(self._read_database(target)) as connection: + if any(Path(str(target) + suffix).exists() for suffix in ("-wal", "-journal")): + return BackupInspection( + state="failed", reasons=("The recovery point has unrecorded journal sidecars.",) + ) + with closing(sqlite3.connect(target.as_uri() + "?mode=ro&immutable=1", uri=True)) as connection: if connection.execute("PRAGMA integrity_check").fetchall() != [("ok",)]: return BackupInspection(state="failed", reasons=("The recovery point failed integrity checks.",)) + if ref.checksum != _file_checksum(target) or any( + Path(str(target) + suffix).exists() for suffix in ("-wal", "-journal") + ): + return BackupInspection(state="failed", reasons=("The recovery point changed during inspection.",)) except (OSError, sqlite3.Error, ValueError): return BackupInspection(state="failed", reasons=("The original recovery point cannot be inspected.",)) return BackupInspection(state="completed", check_level="integrity_checked") @@ -248,7 +257,8 @@ def restore_plan(self, ref: BackupRef) -> RestorePlan: ), steps=( "Preserve the current database and its WAL/SHM files as failure evidence.", - "Restore the checked backup into a separate SQLite file with the Online Backup API.", + "Recheck the recorded checksum and absence of WAL/journal sidecars; keep the backup unchanged during restore.", + "Open the recorded backup with mode=ro&immutable=1 and copy it to a separate file with the Online Backup API.", "Check integrity, schema revision, required extensions, queues, and business invariants.", "With all connections closed, explicitly replace the configured database and remove only stale WAL/SHM sidecars.", "Start a compatible application and verify readiness before restoring traffic; retain the original backup.", diff --git a/src/powercontext/builtin/persistence/migrations/resources/manifest.json b/src/powercontext/builtin/persistence/migrations/resources/manifest.json index d3c36a44e4..65c798ea8b 100644 --- a/src/powercontext/builtin/persistence/migrations/resources/manifest.json +++ b/src/powercontext/builtin/persistence/migrations/resources/manifest.json @@ -33,7 +33,7 @@ ], "retained_fingerprints": { "pc_retained_p0002_artifact_tags": [ - "33052f9e6ae051c52e7ec13d53033b6c11e96d7f5b2c1df40047a7344e471eee" + "e399e72af968ea0df073f3199dcc007ebb5063c9787c57badb4f0d06d2e884c2" ] }, "retained_revisions": { diff --git a/src/powercontext/builtin/persistence/migrations/resources/versions/p0003_tag_families.py b/src/powercontext/builtin/persistence/migrations/resources/versions/p0003_tag_families.py index cc98f5e684..a8fcd575db 100644 --- a/src/powercontext/builtin/persistence/migrations/resources/versions/p0003_tag_families.py +++ b/src/powercontext/builtin/persistence/migrations/resources/versions/p0003_tag_families.py @@ -29,7 +29,15 @@ def upgrade() -> None: "SELECT sql FROM sqlite_schema WHERE type='table' AND name='pc_artifact_tags'" ).scalar_one() retained = original.replace("CREATE TABLE pc_artifact_tags", "CREATE TABLE pc_retained_p0002_artifact_tags", 1) - connection.exec_driver_sql(retained) + # The registered source shape contains this exact FK. Preserve its other + # constraints without allowing live-head deletions to erase historical rows. + foreign_key = ( + "FOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads " + "(scope_id, family, artifact_id) ON DELETE CASCADE, " + ) + if retained.count(foreign_key) != 1: + raise ValueError("The retained tag source has an unexpected foreign key") # noqa: TRY003 + connection.exec_driver_sql(retained.replace(foreign_key, "", 1)) connection.exec_driver_sql("INSERT INTO pc_retained_p0002_artifact_tags SELECT * FROM pc_artifact_tags") # Alembic reflects indexes and named CHECKs, but not application triggers. triggers = ( diff --git a/src/powercontext/builtin/persistence/migrations/sqlite.py b/src/powercontext/builtin/persistence/migrations/sqlite.py index 9041d44080..541c5be518 100644 --- a/src/powercontext/builtin/persistence/migrations/sqlite.py +++ b/src/powercontext/builtin/persistence/migrations/sqlite.py @@ -66,6 +66,7 @@ class _MaintenanceEvidence(BaseModel): manual_ref: str | None = None release_checksums: dict[str, str] retained_objects: tuple[str, ...] = () + backup_history: list[dict[str, Any]] = [] @model_validator(mode="after") def consistent_recovery_point(self) -> Self: @@ -316,12 +317,12 @@ def _plan( evidence["bundle_checksum"] != self.bundle.checksum or evidence["configuration_digest"] != self.configuration_digest ): + raise MigrationError("recovery_required", "Keep the original bundle and configuration.") + pristine = self._pristine_preparation(evidence, revision, fingerprint) + if evidence["backup_policy"] != backup_policy and backup_policy == "auto" and not pristine: raise MigrationError( - "recovery_required", "Keep the original bundle, configuration and backup policy." + "recovery_required", "Do not create an automatic backup of partially migrated state." ) - pristine = self._pristine_preparation(evidence, revision, fingerprint) - if evidence["backup_policy"] != backup_policy and not pristine: - raise MigrationError("recovery_required", "Keep the original backup policy after migration begins.") if not pristine: state = "recovery_required" context = self._backup_context(revision, "preview") @@ -366,6 +367,26 @@ def _backup_context(self, revision: str | None, window: str) -> BackupContext: def _prepare_evidence(self, plan: MigrationPlan, manual_ref: str | None) -> dict[str, Any]: previous = self._evidence() if previous and previous["state"] != "complete": + if previous["backup_policy"] != plan.backup_policy: + # apply has already required the new plan ID and explicit backup consent. + previous.setdefault("backup_history", []).append({ + key: previous.get(key) for key in ("backup_policy", "backup_state", "backup", "manual_ref") + }) + previous.update( + state="preparing", + backup_policy=plan.backup_policy, + backup_state="pending", + backup=None, + manual_ref=None, + ) + if plan.backup_policy in {"manual", "skip"}: + previous.update( + state="active", + backup_state="user_confirmed" if plan.backup_policy == "manual" else "skipped", + manual_ref=manual_ref if plan.backup_policy == "manual" else None, + ) + self._write_evidence(previous) + return previous recovered = self._reuse_evidence(previous, plan) if recovered is not None: return recovered diff --git a/src/powercontext/service/controller.py b/src/powercontext/service/controller.py index 6650515efa..1ce5c0c4c8 100644 --- a/src/powercontext/service/controller.py +++ b/src/powercontext/service/controller.py @@ -200,6 +200,7 @@ def start(self) -> ServiceStatus: """Start the registered executable without reinstalling or changing configuration.""" with _service_lock(self._adapter.lock_path): + self._recover_definition_switch() registration, running = self._owned_registration() record = self._maintenance_record() if record is not None and record.get("phase") not in {"manual_stopped", "database_ready", "start_failed"}: @@ -310,9 +311,12 @@ def _complete_maintenance(self, summary: ServiceMaintenanceSummary) -> ServiceSt or hashlib.sha256(registration.content or b"").hexdigest() != summary.registration_checksum ): raise ServiceError("the stopped service registration changed during migration") # noqa: TRY003 - self._adapter.write(self._adapter.render(summary.target_definition)) - self._adapter.update_suspended() - self._write_maintenance_record({"phase": "database_ready", "summary": summary.as_dict()}) + self._write_maintenance_record({ + "phase": "switching", + "summary": summary.as_dict(), + "target_checksum": hashlib.sha256(self._adapter.render(summary.target_definition)).hexdigest(), + }) + self._recover_definition_switch() if summary.originally_running: try: self._start_registered(summary.target_definition) @@ -322,6 +326,56 @@ def _complete_maintenance(self, summary: ServiceMaintenanceSummary) -> ServiceSt self.maintenance_path.unlink(missing_ok=True) return self.status() + def _recover_definition_switch(self) -> None: + """Resume only a database-verified switch with intact original/target evidence.""" + record = self._maintenance_record() + if record is None or record.get("phase") != "switching": + return + try: + payload = record["summary"] + if not isinstance(payload, dict): + raise TypeError("invalid summary") # noqa: TRY003, TRY301 + payload = cast(dict[str, object], payload) + originally_running = payload["originally_running"] + if not isinstance(originally_running, bool): + raise TypeError("invalid original service state") # noqa: TRY003, TRY301 + summary = ServiceMaintenanceSummary( + str(payload["identifier"]), + str(payload["artifact_path"]), + ServiceDefinition.from_dict(payload["definition"]), + ServiceDefinition.from_dict(payload["target_definition"]), + originally_running, + str(payload["registration_checksum"]), + ) + target = self._adapter.render(summary.target_definition) + if ( + summary.identifier != self._adapter.identifier + or summary.artifact_path != str(self._adapter.artifact_path) + or summary.fingerprint != payload.get("fingerprint") + or hashlib.sha256(target).hexdigest() != record.get("target_checksum") + ): + raise ValueError("changed switch intent") # noqa: TRY003, TRY301 + except (KeyError, TypeError, ValueError) as error: + raise ServiceError("the pending service switch evidence is invalid") from error # noqa: TRY003 + registration = self._adapter.inspect() + self._require_mutable_registration(registration) + loaded = self._adapter.loaded_registration() + self._require_mutable_manager_registration(loaded) + if ( + registration.definition not in (summary.definition, summary.target_definition) + or hashlib.sha256(registration.content or b"").hexdigest() + not in (summary.registration_checksum, record["target_checksum"]) + or ( + loaded.state is ManagerOwnershipState.OWNED + and loaded.definition not in (summary.definition, summary.target_definition) + ) + ): + raise ServiceError("the service registration changed outside the pending switch") # noqa: TRY003 + self._require_stopped(summary.definition) + self._adapter.write(target) + self._adapter.update_suspended() + self._write_maintenance_record({"phase": "database_ready", "summary": summary.as_dict()}) + def _owned_registration(self) -> tuple[NativeRegistration, bool]: support, detail = self._adapter.support() if support is SupportState.UNSUPPORTED: @@ -361,8 +415,11 @@ def _require_stopped(self, definition: ServiceDefinition | None) -> None: probe = self._probe(definition.endpoint) if state in {ManagerState.INACTIVE, ManagerState.FAILED} and probe.state is ProbeState.UNREACHABLE: return - if state is ManagerState.UNKNOWN or probe.state is ProbeState.CONFLICT or time.monotonic() >= deadline: - raise ServiceError("cannot prove the personal service stopped; maintenance must not continue") # noqa: TRY003 + if time.monotonic() >= deadline: + raise ServiceError( # noqa: TRY003 + f"cannot prove the personal service stopped; maintenance must not continue: " + f"manager={state.value}, probe={probe.state.value}, detail={probe.detail}" + ) self._sleep(0.1) def _start_registered(self, definition: ServiceDefinition | None) -> None: diff --git a/tests/builtin/persistence/test_database_backup_providers.py b/tests/builtin/persistence/test_database_backup_providers.py index b00c48c6f4..b075d30095 100644 --- a/tests/builtin/persistence/test_database_backup_providers.py +++ b/tests/builtin/persistence/test_database_backup_providers.py @@ -364,3 +364,34 @@ def inspect_provider_backup(sync: Connection) -> None: ).all() == [(1, "original")] asyncio.run(scenario()) + + +def test_sqlite_backup_rejects_unrecorded_wal_even_when_main_checksum_matches(tmp_path: Path) -> None: + import hashlib + import subprocess + import sys + + database = tmp_path / "source.db" + with closing(sqlite3.connect(database)) as writer: + writer.execute("CREATE TABLE pc_artifacts (id INTEGER)") + provider = SQLiteBackupProvider(database) + ref = provider.create_backup(context()) + # Simulate an older WAL-mode backup and record its main-file checksum. + with closing(sqlite3.connect(ref.location)) as writer: + writer.execute("PRAGMA journal_mode=WAL") + ref = ref.model_copy(update={"checksum": hashlib.sha256(Path(ref.location).read_bytes()).hexdigest()}) + (provider.directory / f"{ref.ref_id}.json").write_text(ref.model_dump_json()) + subprocess.run( + [ + sys.executable, + "-c", + "import os,sqlite3,sys; c=sqlite3.connect(sys.argv[1]); " + "c.execute('INSERT INTO pc_artifacts VALUES (7)'); c.commit(); os._exit(0)", + ref.location, + ], + check=True, + ) + assert hashlib.sha256(Path(ref.location).read_bytes()).hexdigest() == ref.checksum + assert provider.inspect_backup(ref).state == "failed" + with pytest.raises(MigrationError, match="backup_failed"): + provider.restore_plan(ref) diff --git a/tests/builtin/persistence/test_database_migrations.py b/tests/builtin/persistence/test_database_migrations.py index ca2c55e060..198d4cc814 100644 --- a/tests/builtin/persistence/test_database_migrations.py +++ b/tests/builtin/persistence/test_database_migrations.py @@ -596,6 +596,10 @@ def test_replaced_table_is_retained_across_verification_and_retry(tmp_path: Path apply(runner) expected = [("Keep",)] assert rows(database, "SELECT tag FROM pc_retained_p0002_artifact_tags") == expected + with sqlite3.connect(database) as connection: + connection.execute("PRAGMA foreign_keys=ON") + connection.execute("DELETE FROM pc_artifact_heads") + assert rows(database, "SELECT tag FROM pc_artifact_tags") == [] assert runner.verify().state == "ready" assert not runner.apply().changed assert rows(database, "SELECT tag FROM pc_retained_p0002_artifact_tags") == expected @@ -613,3 +617,54 @@ def test_retained_table_cannot_precede_its_revision(tmp_path: Path, bundle: Migr with pytest.raises(MigrationError, match="recovery_required"): apply(SQLiteMigrationRunner(database, bundle)) assert database.read_bytes() == before + + +@pytest.mark.parametrize("policy", ["manual", "skip"]) +def test_partial_migration_can_reconfirm_backup_policy_without_replacing_original( + tmp_path: Path, + bundle: MigrationBundle, + monkeypatch: pytest.MonkeyPatch, + policy: Literal["manual", "skip"], +) -> None: + database = tmp_path / "pc.sqlite3" + load_schema(database) + populate(database) + runner = SQLiteMigrationRunner(database, bundle) + original_upgrade = command.upgrade + + def interrupt(config, target, **kwargs): + if target == "p0003": + raise InterruptedError + original_upgrade(config, target, **kwargs) + + with monkeypatch.context() as injection: + injection.setattr(command, "upgrade", interrupt) + with pytest.raises(InterruptedError): + apply(runner) + assert rows(database, "SELECT version_num FROM pc_schema_revision") == [("p0002",)] + evidence = json.loads(runner.evidence_path.read_text()) + backup = Path(evidence["backup"]["location"]) + backup.write_bytes(b"damaged recovery point") + old_plan = runner.plan() + plan = runner.plan(backup_policy=policy) + assert plan.plan_id != old_plan.plan_id + with pytest.raises(MigrationError, match="plan_changed"): + runner.apply(plan_id=old_plan.plan_id, accepted=True, maintenance_confirmed=True, backup_policy=policy) + with pytest.raises(MigrationError, match="backup_confirmation_required"): + runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True, backup_policy=policy) + result = runner.apply( + plan_id=plan.plan_id, + accepted=True, + maintenance_confirmed=True, + backup_policy=policy, + backup_confirmed=policy == "manual", + accept_no_backup=policy == "skip", + backup_ref="user declaration", + ) + assert runner.verify().state == "ready" + assert result.backup_state == ("user_confirmed" if policy == "manual" else "skipped") + final = json.loads(runner.evidence_path.read_text()) + assert final["backup_history"][0]["backup"] == evidence["backup"] + assert final["maintenance_window_id"] == evidence["maintenance_window_id"] + assert final["source_revision"] == evidence["source_revision"] + assert backup.read_bytes() == b"damaged recovery point" diff --git a/tests/fixtures/database_migrations/manifest.json b/tests/fixtures/database_migrations/manifest.json index d3c36a44e4..65c798ea8b 100644 --- a/tests/fixtures/database_migrations/manifest.json +++ b/tests/fixtures/database_migrations/manifest.json @@ -33,7 +33,7 @@ ], "retained_fingerprints": { "pc_retained_p0002_artifact_tags": [ - "33052f9e6ae051c52e7ec13d53033b6c11e96d7f5b2c1df40047a7344e471eee" + "e399e72af968ea0df073f3199dcc007ebb5063c9787c57badb4f0d06d2e884c2" ] }, "retained_revisions": { diff --git a/tests/fixtures/database_migrations/versions/p0003_tag_families.py b/tests/fixtures/database_migrations/versions/p0003_tag_families.py index cc98f5e684..a8fcd575db 100644 --- a/tests/fixtures/database_migrations/versions/p0003_tag_families.py +++ b/tests/fixtures/database_migrations/versions/p0003_tag_families.py @@ -29,7 +29,15 @@ def upgrade() -> None: "SELECT sql FROM sqlite_schema WHERE type='table' AND name='pc_artifact_tags'" ).scalar_one() retained = original.replace("CREATE TABLE pc_artifact_tags", "CREATE TABLE pc_retained_p0002_artifact_tags", 1) - connection.exec_driver_sql(retained) + # The registered source shape contains this exact FK. Preserve its other + # constraints without allowing live-head deletions to erase historical rows. + foreign_key = ( + "FOREIGN KEY(scope_id, family, artifact_id) REFERENCES pc_artifact_heads " + "(scope_id, family, artifact_id) ON DELETE CASCADE, " + ) + if retained.count(foreign_key) != 1: + raise ValueError("The retained tag source has an unexpected foreign key") # noqa: TRY003 + connection.exec_driver_sql(retained.replace(foreign_key, "", 1)) connection.exec_driver_sql("INSERT INTO pc_retained_p0002_artifact_tags SELECT * FROM pc_artifact_tags") # Alembic reflects indexes and named CHECKs, but not application triggers. triggers = ( diff --git a/tests/test_service_maintenance.py b/tests/test_service_maintenance.py index 0b72dc83ef..5bf193ed21 100644 --- a/tests/test_service_maintenance.py +++ b/tests/test_service_maintenance.py @@ -284,3 +284,91 @@ def test_systemd_maintenance_guard_uses_a_negated_absolute_path_and_preserves_th dropin = adapter.artifact_path.with_name(f"{adapter.identifier}.d") / "90-powercontext-maintenance.conf" assert f"ConditionPathExists=!{str(marker).replace('%', '%%')}\n" in dropin.read_text() assert adapter.artifact_path.read_bytes() == unit + + +@pytest.mark.parametrize("after_reload", [False, True]) +def test_interrupted_definition_switch_can_resume_with_start( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + after_reload: bool, +) -> None: + controller, adapter = _installed(tmp_path) + summary = controller.maintenance_summary() + reload = adapter.update_suspended + + def fail(): + assert json.loads(controller.maintenance_path.read_text())["phase"] == "switching" + if after_reload: + reload() + raise OSError("interrupted reload") # noqa: TRY003 + + with ( + controller.maintenance(expected_fingerprint=summary.fingerprint) as session, + monkeypatch.context() as injection, + ): + injection.setattr(adapter, "update_suspended", fail) + with pytest.raises(OSError, match="interrupted reload"): + session.complete() + assert adapter.suspended + assert adapter.manager is ManagerState.INACTIVE + assert controller.start().ok + assert adapter.definition == summary.target_definition + assert not controller.maintenance_path.exists() + + +def test_stop_waits_for_transient_connection_conflict(tmp_path: Path) -> None: + _, adapter = _installed(tmp_path) + outcomes = iter([ + ProbeResult(ProbeState.LIVE, "owned server"), + ProbeResult(ProbeState.CONFLICT, "connection reset during shutdown"), + ProbeResult(ProbeState.UNREACHABLE, "stopped"), + ]) + controller = ServiceController( + adapter, + probe=lambda _: next(outcomes, ProbeResult(ProbeState.UNREACHABLE, "stopped")), + sleep=lambda _: None, + ) + controller.stop() + assert adapter.manager is ManagerState.INACTIVE + assert adapter.suspended + + +def test_stop_rejects_a_persistent_listener_after_wait_budget(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import powercontext.service.controller as module + + _, adapter = _installed(tmp_path) + elapsed = iter([0.0, 0.1, 31.0]) + monkeypatch.setattr(module.time, "monotonic", lambda: next(elapsed)) + controller = ServiceController( + adapter, + probe=lambda _: ProbeResult( + ProbeState.LIVE if adapter.manager is ManagerState.ACTIVE else ProbeState.CONFLICT, "listener remains" + ), + sleep=lambda _: None, + ) + with pytest.raises(ServiceError, match=r"cannot prove.*stopped"): + controller.stop() + assert adapter.suspended + assert controller.maintenance_path.exists() + + +def test_pending_switch_refuses_a_replaced_registration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + controller, adapter = _installed(tmp_path) + summary = controller.maintenance_summary() + + def fail(): + raise OSError("reload failed") # noqa: TRY003 + + with ( + controller.maintenance(expected_fingerprint=summary.fingerprint) as session, + monkeypatch.context() as injection, + ): + injection.setattr(adapter, "update_suspended", fail) + with pytest.raises(OSError): + session.complete() + adapter.write(adapter.render(replace(summary.target_definition, endpoint="http://127.0.0.1:9010"))) + events = list(adapter.events) + with pytest.raises(ServiceError, match="changed outside"): + controller.start() + assert adapter.events == events + assert adapter.suspended From e97cbbc9b220d2a2bd14903f17751f54810b039f Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 8 Oct 2026 10:37:46 +0800 Subject: [PATCH 20/37] fix(service): stop services when startup is interrupted --- src/powercontext/service/controller.py | 9 ++-- tests/test_service_maintenance.py | 74 ++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/src/powercontext/service/controller.py b/src/powercontext/service/controller.py index 1ce5c0c4c8..d57280a3a6 100644 --- a/src/powercontext/service/controller.py +++ b/src/powercontext/service/controller.py @@ -320,7 +320,8 @@ def _complete_maintenance(self, summary: ServiceMaintenanceSummary) -> ServiceSt if summary.originally_running: try: self._start_registered(summary.target_definition) - except (OSError, ServiceError): + except BaseException: + # Catchable interruptions need the same explicit recovery state as startup errors. self._write_maintenance_record({"phase": "start_failed", "summary": summary.as_dict()}) raise self.maintenance_path.unlink(missing_ok=True) @@ -444,9 +445,9 @@ def _start_registered(self, definition: ServiceDefinition | None) -> None: if readiness.state is ProbeState.CONFLICT or time.monotonic() >= deadline: raise ServiceError(f"the service failed readiness: {readiness.detail}") # noqa: TRY003, TRY301 self._sleep(0.1) - except (OSError, ServiceError): - # Restore the guard even if startup partially succeeded. Never fall - # back to an old executable after the database changed. + except BaseException: + # Restore the guard even on Ctrl+C during startup verification, then + # propagate the interruption. Never fall back to an old executable. self._adapter.suspend(self.maintenance_path) self._adapter.stop() raise diff --git a/tests/test_service_maintenance.py b/tests/test_service_maintenance.py index 5bf193ed21..a9a4b8eab3 100644 --- a/tests/test_service_maintenance.py +++ b/tests/test_service_maintenance.py @@ -30,6 +30,7 @@ from powercontext.service.adapters.windows import WindowsTaskSchedulerAdapter from powercontext.service.controller import ServiceController from powercontext.service.model import ( + LivenessState, ManagerOwnershipState, ManagerRegistration, ManagerState, @@ -192,6 +193,79 @@ def test_startup_readiness_failure_is_separate_from_database_ready_and_keeps_ser assert json.loads(controller.maintenance_path.read_text())["phase"] == "start_failed" +@pytest.mark.parametrize("startup_check", ["liveness", "readiness"]) +def test_cancelled_maintenance_startup_stays_stopped_until_explicit_recovery( + tmp_path: Path, startup_check: str +) -> None: + _, adapter = _installed(tmp_path) + manager_probe = _manager_probe(adapter) + interrupt_pending = False + + def probe(endpoint: str) -> ProbeResult: + nonlocal interrupt_pending + result = manager_probe(endpoint) + if startup_check == "liveness" and interrupt_pending and result.state is ProbeState.LIVE: + interrupt_pending = False + raise KeyboardInterrupt + return result + + def readiness_probe(endpoint: str) -> ProbeResult: + nonlocal interrupt_pending + if startup_check == "readiness" and interrupt_pending: + interrupt_pending = False + raise KeyboardInterrupt + return manager_probe(endpoint) + + controller = ServiceController(adapter, probe=probe, readiness_probe=readiness_probe) + summary = controller.maintenance_summary() + with controller.maintenance(expected_fingerprint=summary.fingerprint) as session: + # Only the new service's startup check is interrupted, after migration + # has stopped the old service and the caller has verified the database. + interrupt_pending = True + with pytest.raises(KeyboardInterrupt): + session.complete() + assert not session.completed + + assert adapter.manager is ManagerState.INACTIVE + assert adapter.suspended + assert adapter.definition == summary.target_definition + assert json.loads(controller.maintenance_path.read_text())["phase"] == "start_failed" + assert controller.start().ok + assert adapter.definition == summary.target_definition + assert not adapter.suspended + assert not controller.maintenance_path.exists() + + +def test_cancelled_public_start_restores_stopped_service_and_allows_explicit_retry(tmp_path: Path) -> None: + controller, adapter = _installed(tmp_path) + controller.stop() + original = adapter.definition + manager_probe = _manager_probe(adapter) + interrupt_pending = True + + def readiness_probe(endpoint: str) -> ProbeResult: + nonlocal interrupt_pending + if interrupt_pending: + interrupt_pending = False + raise KeyboardInterrupt + return manager_probe(endpoint) + + controller = ServiceController(adapter, probe=manager_probe, readiness_probe=readiness_probe) + with pytest.raises(KeyboardInterrupt): + controller.start() + + assert adapter.manager is ManagerState.INACTIVE + assert adapter.suspended + assert adapter.definition == original + assert json.loads(controller.maintenance_path.read_text())["phase"] == "manual_stopped" + recovered = controller.start() + assert recovered.manager is ManagerState.ACTIVE + assert recovered.server_liveness is LivenessState.LIVE + assert adapter.definition == original + assert not adapter.suspended + assert not controller.maintenance_path.exists() + + @pytest.mark.skipif(os.name == "nt", reason="requires creating a symlink without elevation") def test_foreign_maintenance_record_is_not_overwritten(tmp_path: Path) -> None: controller, adapter = _installed(tmp_path) From 719af5413d88cb1a039343d2a976abf71cffff30 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 8 Oct 2026 12:44:33 +0800 Subject: [PATCH 21/37] fix(db): validate retained tables during recovery --- .../builtin/persistence/migrations/sqlite.py | 13 ++- .../persistence/test_database_migrations.py | 96 +++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/src/powercontext/builtin/persistence/migrations/sqlite.py b/src/powercontext/builtin/persistence/migrations/sqlite.py index 541c5be518..cf9bec0517 100644 --- a/src/powercontext/builtin/persistence/migrations/sqlite.py +++ b/src/powercontext/builtin/persistence/migrations/sqlite.py @@ -405,6 +405,7 @@ def _prepare_evidence(self, plan: MigrationPlan, manual_ref: str | None) -> dict "backup_state": "pending", "backup": None, "release_checksums": previous.get("release_checksums", {}) if previous else {}, + "retained_objects": previous.get("retained_objects", []) if previous else [], } ) self._write_evidence(evidence) @@ -527,9 +528,17 @@ def _verify_retained(self, inventory: list[list[Any]], revision: str | None) -> if objects and digest(objects) not in self.bundle.retained_fingerprints.get(name, []): raise MigrationError("incompatible_schema", "A retained historical object has an unregistered shape.") evidence = self._evidence() - if evidence and any(not any(row[1] == name for row in inventory) for name in evidence["retained_objects"]): + required = set(evidence["retained_objects"]) if evidence else set() + if evidence: + # A revision can commit before its retained inventory is recorded. + # Adoption at the current revision did not execute that revision. + committed = set(self.bundle.pending(evidence["source_revision"])) - set(self.bundle.pending(revision)) + required.update( + name for name, introduced in self.bundle.retained_revisions.items() if introduced in committed + ) + if any(not any(row[1] == name for row in inventory) for name in required): raise MigrationError( - "recovery_required", "A registered retained table was removed without a cleanup revision." + "recovery_required", "A required retained table was removed without a cleanup revision." ) def _verify_integrity(self, connection: Connection) -> None: diff --git a/tests/builtin/persistence/test_database_migrations.py b/tests/builtin/persistence/test_database_migrations.py index 198d4cc814..e2a82551d0 100644 --- a/tests/builtin/persistence/test_database_migrations.py +++ b/tests/builtin/persistence/test_database_migrations.py @@ -605,6 +605,102 @@ def test_replaced_table_is_retained_across_verification_and_retry(tmp_path: Path assert rows(database, "SELECT tag FROM pc_retained_p0002_artifact_tags") == expected +@pytest.mark.parametrize("remove_retained", [False, True]) +def test_committed_revision_retains_old_table_before_completion_evidence( + tmp_path: Path, bundle: MigrationBundle, remove_retained: bool +) -> None: + database = tmp_path / "pc.sqlite3" + load_schema(database) + populate(database) + code = """ +import os +import sys +from pathlib import Path +from sqlalchemy import Connection +from powercontext.builtin.persistence.migrations import MigrationBundle, SQLiteMigrationRunner + +commit = Connection.commit +def crash_after_commit(connection): + present = connection.exec_driver_sql("SELECT name FROM sqlite_schema WHERE name='pc_schema_revision'").first() + current = connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar() if present else None + commit(connection) + if current == 'p0003': + os._exit(73) +Connection.commit = crash_after_commit +runner = SQLiteMigrationRunner(Path(sys.argv[1]), MigrationBundle(Path(sys.argv[2]))) +runner.apply(plan_id=runner.plan().plan_id, accepted=True, maintenance_confirmed=True) +""" + process = subprocess.run([sys.executable, "-c", code, str(database), str(FIXTURE)], check=False, timeout=30) + assert process.returncode == 73 + assert rows(database, "SELECT version_num FROM pc_schema_revision") == [("p0003",)] + assert rows(database, "SELECT tag FROM pc_retained_p0002_artifact_tags") == [("Keep",)] + runner = SQLiteMigrationRunner(database, bundle) + evidence = runner.evidence_path.read_bytes() + + if remove_retained: + with sqlite3.connect(database) as connection: + connection.execute("DROP TABLE pc_retained_p0002_artifact_tags") + for operation in (runner.plan, lambda: apply(runner), runner.verify): + with pytest.raises(MigrationError, match="recovery_required"): + operation() + assert runner.evidence_path.read_bytes() == evidence + assert rows(database, "SELECT tag FROM pc_artifact_tags") == [("Keep",)] + else: + assert runner.plan().state == "recovery_required" + assert apply(runner).state == "ready" + assert runner.verify().state == "ready" + assert rows(database, "SELECT tag FROM pc_retained_p0002_artifact_tags") == [("Keep",)] + + +def test_adopting_current_baseline_does_not_require_an_unexecuted_retention_revision( + tmp_path: Path, bundle: MigrationBundle +) -> None: + database = tmp_path / "pc.sqlite3" + load_schema(database, "v1_1_0") + populate(database) + runner = SQLiteMigrationRunner(database, bundle) + assert runner.plan().source_revision == "p0003" + assert apply(runner).state == "ready" + assert runner.verify().state == "ready" + assert rows(database, "SELECT name FROM sqlite_schema WHERE name='pc_retained_p0002_artifact_tags'") == [] + assert rows(database, "SELECT tag FROM pc_artifact_tags") == [("Keep",)] + + +def test_a_new_maintenance_window_preserves_existing_retention_requirements( + tmp_path: Path, bundle: MigrationBundle, monkeypatch: pytest.MonkeyPatch +) -> None: + database = tmp_path / "pc.sqlite3" + load_schema(database) + populate(database) + apply(SQLiteMigrationRunner(database, bundle)) + copied = tmp_path / "bundle" + shutil.copytree(FIXTURE, copied) + (copied / "versions/p0004.py").write_text( + 'revision = "p0004"\ndown_revision = "p0003"\ndef upgrade():\n pass\n', encoding="utf-8" + ) + manifest = json.loads((copied / "manifest.json").read_text()) + manifest["revision_resources"]["p0004"] = [] + manifest["sqlite_fingerprints"]["p0004"] = manifest["sqlite_fingerprints"]["p0003"] + (copied / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + runner = SQLiteMigrationRunner(database, MigrationBundle(copied)) + + def interrupt(config, target, **kwargs): + raise InterruptedError + + with monkeypatch.context() as injection: + injection.setattr(command, "upgrade", interrupt) + with pytest.raises(InterruptedError): + apply(runner) + assert rows(database, "SELECT version_num FROM pc_schema_revision") == [("p0003",)] + evidence = runner.evidence_path.read_bytes() + with sqlite3.connect(database) as connection: + connection.execute("DROP TABLE pc_retained_p0002_artifact_tags") + for operation in (runner.plan, lambda: apply(runner), runner.verify): + with pytest.raises(MigrationError, match="recovery_required"): + operation() + assert runner.evidence_path.read_bytes() == evidence + + def test_retained_table_cannot_precede_its_revision(tmp_path: Path, bundle: MigrationBundle) -> None: database = tmp_path / "pc.sqlite3" load_schema(database) From 2050234927352e4ecdee3e4403df1f394fb19c48 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 8 Oct 2026 12:44:36 +0800 Subject: [PATCH 22/37] fix(service): update manually stopped registrations --- docs/en/docs/operate/database-migrations.md | 4 + docs/en/docs/operate/deploy-server.md | 7 +- docs/zh/docs/operate/database-migrations.md | 4 + docs/zh/docs/operate/deploy-server.md | 6 +- src/powercontext/service/cli.py | 22 ++-- src/powercontext/service/controller.py | 70 +++++++++-- .../native/test_personal_service_lifecycle.py | 52 ++++++++ tests/test_service_maintenance.py | 114 ++++++++++++++++-- 8 files changed, 255 insertions(+), 24 deletions(-) diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index ea35548ed4..ba0c974ebf 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -110,6 +110,10 @@ powercontext service start powercontext service restart ``` +After a manual `service stop`, `service install` from the new environment can update the stopped registration without +starting the old executable. Pass the existing `--env-file` when configured. The service remains stopped with automatic +activation suppressed until an explicit `service start`; an unverified migration continues to block registration updates. + They manage only the current user's PC-owned local service, not a cluster or other clients. `stop` suppresses automatic restart during maintenance; `start` restores startup. One `restart` cannot replace stopped-write migration. diff --git a/docs/en/docs/operate/deploy-server.md b/docs/en/docs/operate/deploy-server.md index 311f30f895..dcb2e8c864 100644 --- a/docs/en/docs/operate/deploy-server.md +++ b/docs/en/docs/operate/deploy-server.md @@ -59,7 +59,12 @@ owner first. The native definition stores only the absolute file path and non-content file identity metadata. On Windows this includes the current user's owner SID, which is revalidated whenever the launcher starts. It does not copy credentials or the caller's shell environment. Re-run `service install` after upgrading PowerContext or changing the -environment file. Remove the registration without deleting Server data or logs with: +environment file. After an explicit `service stop`, run `service install` from the new environment, passing the same +`--env-file` when one is configured. This updates the registration even if the old Python executable has been removed, +while keeping the service stopped and automatic activation suppressed. Run `service start` when it is safe to resume. +An unverified database migration continues to block registration updates. + +Remove the registration without deleting Server data or logs with: ```bash powercontext service uninstall diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index e423512741..05742ab402 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -77,6 +77,10 @@ powercontext service start powercontext service restart ``` +手动执行 `service stop` 后，可以在新版环境中通过 `service install` 更新已停止的服务注册，无需先启动旧程序。 +如果原服务使用环境文件，请继续传入同一个 `--env-file`。更新注册后仍保持停服并禁止自动启动，直到显式执行 +`service start`；尚未验证完成的迁移仍会阻止更新注册。 + 这些命令只管理当前用户归 PC 所有的服务，不管理集群或其他客户端。`stop` 禁止维护期间自动拉起旧服务；`start` 恢复启动。涉及迁移时不能用一次 `restart` 替代停写维护。 完整升级的托管设计为：排空请求、停服并禁止自动重启 → 取迁移锁并复核 → 按策略备份 → 迁移 → 验证 → 将启动定义切换到已确认新版程序 → 启动原本运行的服务 → readiness。迁移失败保持停服；原本停止的服务保持停止。数据库验证成功而服务启动失败时分别报告，处理启动问题而不重跑迁移。 diff --git a/docs/zh/docs/operate/deploy-server.md b/docs/zh/docs/operate/deploy-server.md index a4aa911a11..3893403051 100644 --- a/docs/zh/docs/operate/deploy-server.md +++ b/docs/zh/docs/operate/deploy-server.md @@ -53,7 +53,11 @@ icacls $env:USERPROFILE\powercontext.env /inheritance:r /grant:r "${env:USERNAME 原生定义只记录环境文件的绝对路径和不含内容的文件 identity metadata；在 Windows 上还记录当前用户的 owner SID， launcher 每次启动都会重新校验它。不复制 credential 或调用者的 shell environment。 -升级 PowerContext 或修改环境文件后应重新执行 `service install`。以下命令会删除注册，但保留 Server 数据和日志： +升级 PowerContext 或修改环境文件后应重新执行 `service install`。显式执行 `service stop` 后，可以在新版环境中运行 +`service install`；如果原服务使用环境文件，请继续传入同一个 `--env-file`。即使旧 Python 程序已删除，也可以更新注册， +服务仍保持停止并禁止自动启动。确认可以恢复服务后，再执行 `service start`。尚未验证完成的数据库迁移仍会阻止更新注册。 + +以下命令会删除注册，但保留 Server 数据和日志： ```bash powercontext service uninstall diff --git a/src/powercontext/service/cli.py b/src/powercontext/service/cli.py index e5f2564f4f..5508feff05 100644 --- a/src/powercontext/service/cli.py +++ b/src/powercontext/service/cli.py @@ -27,7 +27,7 @@ from powercontext.server import cli as _server_role_dependency from powercontext.server.configuration import ServerConfigurationError, server_settings_context from powercontext.service.controller import ServiceController -from powercontext.service.model import ServiceError, ServiceStatus +from powercontext.service.model import ManagerState, ServiceError, ServiceStatus del _server_role_dependency @@ -60,7 +60,7 @@ def install( ), ] = None, ) -> None: - """Install the personal Server service and optionally start it at user login.""" + """Install or update the personal Server service, preserving an explicit stop.""" if start_on_login is None: start_on_login = ( @@ -88,11 +88,19 @@ def install( else: exit_code = 1 raise typer.Exit(code=exit_code) from error - message = ( - "PowerContext personal service installed with login auto-start." - if start_on_login - else "PowerContext personal service installed without login auto-start." - ) + if status.manager in {ManagerState.INACTIVE, ManagerState.FAILED}: + message = ( + "PowerContext personal service registration updated " + f"{'with login auto-start configured' if start_on_login else 'without login auto-start'}. " + "The service remains stopped with automatic activation suppressed. " + "Run `powercontext service start` when ready." + ) + else: + message = ( + "PowerContext personal service installed with login auto-start." + if start_on_login + else "PowerContext personal service installed without login auto-start." + ) typer.echo(message) _write_environment_guidance(expanded_env_file) write_inference_capability_notice( diff --git a/src/powercontext/service/controller.py b/src/powercontext/service/controller.py index d57280a3a6..0e2b339510 100644 --- a/src/powercontext/service/controller.py +++ b/src/powercontext/service/controller.py @@ -150,10 +150,15 @@ def install(self, *, env_file: Path | None = None, start_on_login: bool = True) ) with _service_lock(self._adapter.lock_path): - if self._maintenance_record() is not None: - raise ServiceError( # noqa: TRY003 - "the personal service is stopped for maintenance; finish migration or use service start first" - ) + record = self._maintenance_record() + if record is not None: + if record.get("phase") not in {"manual_stopped", "manual_switching"}: + raise ServiceError( # noqa: TRY003 + "the personal service is stopped for maintenance; finish migration or use service start first" + ) + self._recover_definition_switch() + self._update_manually_stopped_definition(definition) + return self.status() registration = self._adapter.inspect() self._require_mutable_registration(registration) loaded = self._adapter.loaded_registration() @@ -327,11 +332,41 @@ def _complete_maintenance(self, summary: ServiceMaintenanceSummary) -> ServiceSt self.maintenance_path.unlink(missing_ok=True) return self.status() + def _update_manually_stopped_definition(self, definition: ServiceDefinition) -> None: + """Update an owned registration without undoing the user's explicit stop.""" + + registration, _running = self._owned_registration() + self._adapter.suspend(self.maintenance_path) + self._adapter.stop() + self._require_stopped(registration.definition) + if registration.definition is None: + raise ServiceError("the installed service has no definition") # noqa: TRY003 + if definition.endpoint != registration.definition.endpoint: + self._require_stopped(definition) + target = self._adapter.render(definition) + if registration.definition == definition and registration.content == target: + return + summary = ServiceMaintenanceSummary( + self._adapter.identifier, + str(self._adapter.artifact_path), + registration.definition, + definition, + False, + hashlib.sha256(registration.content or b"").hexdigest(), + ) + self._write_maintenance_record({ + "phase": "manual_switching", + "summary": summary.as_dict(), + "target_checksum": hashlib.sha256(target).hexdigest(), + }) + self._recover_definition_switch() + def _recover_definition_switch(self) -> None: - """Resume only a database-verified switch with intact original/target evidence.""" + """Resume a verified migration or explicit stopped-registration update.""" record = self._maintenance_record() - if record is None or record.get("phase") != "switching": + if record is None or record.get("phase") not in {"switching", "manual_switching"}: return + manual_update = record.get("phase") == "manual_switching" try: payload = record["summary"] if not isinstance(payload, dict): @@ -340,6 +375,8 @@ def _recover_definition_switch(self) -> None: originally_running = payload["originally_running"] if not isinstance(originally_running, bool): raise TypeError("invalid original service state") # noqa: TRY003, TRY301 + if manual_update and originally_running: + raise ValueError("a manual registration update must preserve the stopped state") # noqa: TRY003, TRY301 summary = ServiceMaintenanceSummary( str(payload["identifier"]), str(payload["artifact_path"]), @@ -372,10 +409,18 @@ def _recover_definition_switch(self) -> None: ) ): raise ServiceError("the service registration changed outside the pending switch") # noqa: TRY003 + self._adapter.suspend(self.maintenance_path) + self._adapter.stop() self._require_stopped(summary.definition) + if summary.target_definition.endpoint != summary.definition.endpoint: + self._require_stopped(summary.target_definition) self._adapter.write(target) self._adapter.update_suspended() - self._write_maintenance_record({"phase": "database_ready", "summary": summary.as_dict()}) + self._require_stopped(summary.target_definition) + if manual_update: + self._write_maintenance_record({"phase": "manual_stopped"}) + else: + self._write_maintenance_record({"phase": "database_ready", "summary": summary.as_dict()}) def _owned_registration(self) -> tuple[NativeRegistration, bool]: support, detail = self._adapter.support() @@ -559,6 +604,17 @@ def status(self) -> ServiceStatus: probe = self._probe(registration.endpoint or "") liveness = LivenessState.LIVE if probe.state is ProbeState.LIVE else LivenessState.UNREACHABLE recovery = _recovery_action(registration.definition, manager, probe, loaded) + record = self._maintenance_record() + if ( + loaded.state in {ManagerOwnershipState.OWNED, ManagerOwnershipState.NOT_LOADED} + and manager in {ManagerState.INACTIVE, ManagerState.FAILED} + and probe.state is ProbeState.UNREACHABLE + and record is not None + ): + if record.get("phase") == "manual_switching": + recovery = "run `powercontext service install` to finish the interrupted stopped-registration update" + elif record.get("phase") == "manual_stopped" and registration.definition is DefinitionState.CURRENT: + recovery = "run `powercontext service start` when ready to resume the manually stopped service" details = [detail for detail in (loaded.detail, probe.detail) if detail] return replace( registration, diff --git a/tests/native/test_personal_service_lifecycle.py b/tests/native/test_personal_service_lifecycle.py index 915a40ce83..9d2c807c10 100644 --- a/tests/native/test_personal_service_lifecycle.py +++ b/tests/native/test_personal_service_lifecycle.py @@ -135,6 +135,58 @@ def test_native_maintenance_preserves_registration_and_original_running_state(tm _cleanup(adapter) +def test_native_stopped_service_registration_updates_without_starting(tmp_path: Path) -> None: + """Updating a stopped registration must leave activation to an explicit start.""" + + adapter = _native_adapter(suffix="stopped-upgrade") + controller = ServiceController(adapter) + environment = _environment_file(tmp_path) + try: + assert controller.install(env_file=environment).ok + original = adapter.inspect().definition + assert original is not None and original.env_file is not None + controller.stop() + assert adapter.manager_state() is ManagerState.INACTIVE + assert controller.maintenance_path.exists() + + # Keep the same valid configuration and permissions while changing its + # protected identity, as an upgrade may update the deployment file. + environment.write_text(environment.read_text(encoding="utf-8") + "# upgraded registration\n", encoding="utf-8") + controller.install(env_file=environment) + updated = adapter.inspect().definition + assert updated is not None and updated.env_file is not None + assert updated.env_file.path == original.env_file.path + assert updated.env_file != original.env_file + assert adapter.manager_state() is ManagerState.INACTIVE + assert controller.maintenance_path.exists() + + controller.install(env_file=environment) + assert adapter.inspect().definition == updated + # Cross the native restart-policy boundary to catch accidental + # enablement during either registration update or its idempotent retry. + time.sleep(6) + assert adapter.manager_state() is ManagerState.INACTIVE + assert controller.maintenance_path.exists() + loaded = adapter.loaded_registration() + if isinstance(adapter, LaunchdUserAdapter): + assert loaded.state is ManagerOwnershipState.NOT_LOADED + else: + assert loaded.state is ManagerOwnershipState.OWNED + assert loaded.definition == updated + + assert controller.start().ok + assert not controller.maintenance_path.exists() + loaded = adapter.loaded_registration() + assert loaded.state is ManagerOwnershipState.OWNED + assert loaded.definition == updated + assert adapter.manager_state() is ManagerState.ACTIVE + finally: + _capture_native_failure(adapter, tmp_path) + with suppress(Exception): + controller.uninstall() + _cleanup(adapter) + + def test_failure_diagnostics_reports_an_absent_service(tmp_path: Path) -> None: """The report must survive the state it exists to describe: nothing there. diff --git a/tests/test_service_maintenance.py b/tests/test_service_maintenance.py index a9a4b8eab3..6e021bb11f 100644 --- a/tests/test_service_maintenance.py +++ b/tests/test_service_maintenance.py @@ -75,6 +75,93 @@ def test_service_lifecycle_commands_preserve_registration_and_configuration( assert adapter.manager is ManagerState.ACTIVE +@pytest.mark.parametrize("manager_loads_definition", [False, True]) +def test_install_updates_a_manually_stopped_service_without_starting_the_removed_old_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, manager_loads_definition: bool +) -> None: + controller, adapter = _installed(tmp_path) + original = adapter.definition + assert original is not None + controller.stop() + Path(original.python_executable).unlink() + if not manager_loads_definition: + # An unloaded manager, like launchd after bootout, reads the new + # artifact only when the user explicitly starts the service. + monkeypatch.setattr(adapter, "update_suspended", lambda: None) + monkeypatch.setattr(service_cli, "_controller", lambda: controller) + runner = CliRunner() + + updated = runner.invoke(service_cli.app, ["install", "--start-on-login"]) + + assert updated.exit_code == 0, updated.output + assert "remains stopped" in updated.output + assert "automatic activation suppressed" in updated.output + assert "powercontext service start" in updated.output + assert adapter.definition is not None + assert adapter.definition.python_executable == service_python_executable() + assert adapter.definition != original + assert adapter.manager is ManagerState.INACTIVE + assert adapter.suspended + assert json.loads(controller.maintenance_path.read_text())["phase"] == "manual_stopped" + assert "powercontext service start" in (controller.status().recovery_action or "") + assert runner.invoke(service_cli.app, ["start"]).exit_code == 0 + assert adapter.manager is ManagerState.ACTIVE + assert not adapter.suspended + assert not controller.maintenance_path.exists() + + +@pytest.mark.parametrize("error_type", [ServiceError, KeyboardInterrupt]) +@pytest.mark.parametrize("recovery_operation", ["install", "start"]) +def test_interrupted_manual_registration_update_remains_stopped_and_can_be_retried( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, error_type: type[BaseException], recovery_operation: str +) -> None: + controller, adapter = _installed(tmp_path) + original = adapter.definition + assert original is not None + controller.stop() + Path(original.python_executable).unlink() + + def fail_update() -> None: + # A failed native replacement can retain the stopped old definition + # after the artifact already contains the accepted new definition. + adapter.loaded_definition = original + raise error_type("interrupted native registration update") # noqa: TRY003 + + with monkeypatch.context() as injection: + injection.setattr(adapter, "update_suspended", fail_update) + with pytest.raises(error_type, match="interrupted native registration update"): + controller.install() + + assert adapter.manager is ManagerState.INACTIVE + assert adapter.suspended + assert json.loads(controller.maintenance_path.read_text())["phase"] == "manual_switching" + if recovery_operation == "install": + assert controller.install().manager is ManagerState.INACTIVE + assert adapter.suspended + assert json.loads(controller.maintenance_path.read_text())["phase"] == "manual_stopped" + assert controller.start().ok + assert adapter.definition is not None + assert adapter.definition.python_executable == service_python_executable() + assert not adapter.suspended + assert not controller.maintenance_path.exists() + + +def test_install_cannot_replace_a_service_with_an_unverified_migration(tmp_path: Path) -> None: + controller, adapter = _installed(tmp_path) + original = adapter.definition + summary = controller.maintenance_summary() + with controller.maintenance(expected_fingerprint=summary.fingerprint): + pass + + with pytest.raises(ServiceError, match="finish migration"): + controller.install() + + assert adapter.definition == original + assert adapter.manager is ManagerState.INACTIVE + assert adapter.suspended + assert json.loads(controller.maintenance_path.read_text())["phase"] == "stopped" + + def test_successful_maintenance_switches_to_confirmed_executable_and_preserves_configuration(tmp_path: Path) -> None: controller, adapter = _installed(tmp_path) original = adapter.definition @@ -426,21 +513,32 @@ def test_stop_rejects_a_persistent_listener_after_wait_budget(tmp_path: Path, mo assert controller.maintenance_path.exists() -def test_pending_switch_refuses_a_replaced_registration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize("manual_update", [False, True], ids=["migration", "manual-update"]) +@pytest.mark.parametrize("changed_registration", ["artifact", "loaded"]) +def test_pending_switch_refuses_a_replaced_registration( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, manual_update: bool, changed_registration: str +) -> None: controller, adapter = _installed(tmp_path) summary = controller.maintenance_summary() def fail(): raise OSError("reload failed") # noqa: TRY003 - with ( - controller.maintenance(expected_fingerprint=summary.fingerprint) as session, - monkeypatch.context() as injection, - ): + with monkeypatch.context() as injection: injection.setattr(adapter, "update_suspended", fail) - with pytest.raises(OSError): - session.complete() - adapter.write(adapter.render(replace(summary.target_definition, endpoint="http://127.0.0.1:9010"))) + if manual_update: + controller.stop() + with pytest.raises(OSError): + controller.install() + else: + with controller.maintenance(expected_fingerprint=summary.fingerprint) as session, pytest.raises(OSError): + session.complete() + assert adapter.definition is not None + replacement = replace(adapter.definition, endpoint="http://127.0.0.1:9010") + if changed_registration == "artifact": + adapter.write(adapter.render(replacement)) + else: + adapter.loaded_definition = replacement events = list(adapter.events) with pytest.raises(ServiceError, match="changed outside"): controller.start() From 63303b9fdacbcce76733e707c78e6600b2adeb84 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 8 Oct 2026 15:45:52 +0800 Subject: [PATCH 23/37] fix: verify migration targets and service startup --- .../persistence/migrations/deployment.py | 9 +- src/powercontext/service/cli.py | 6 + src/powercontext/service/controller.py | 41 ++++-- tests/test_database_migration_cli.py | 99 ++++++++++++++ tests/test_service_maintenance.py | 121 ++++++++++++++++++ 5 files changed, 264 insertions(+), 12 deletions(-) diff --git a/src/powercontext/builtin/persistence/migrations/deployment.py b/src/powercontext/builtin/persistence/migrations/deployment.py index f9da0c4a57..afb721fa12 100644 --- a/src/powercontext/builtin/persistence/migrations/deployment.py +++ b/src/powercontext/builtin/persistence/migrations/deployment.py @@ -20,6 +20,7 @@ from __future__ import annotations +import os from pathlib import Path from typing import TYPE_CHECKING @@ -48,10 +49,12 @@ def deployment_runner(settings: ServerSettings) -> SQLiteMigrationRunner: "unsupported_backend", "Remote and seekdb deployment migration awaits full-backend acceptance." ) url = make_url(database.url) - if database.is_in_memory or url.query or not url.database: + if database.is_in_memory or url.query or not url.database or url.username or url.password or url.host or url.port: raise MigrationError( - "unsupported_target", "Use an explicitly configured regular SQLite file without URI options." + "unsupported_target", "Use a regular SQLite file URL without credentials, host, port or URI options." ) + # Match SQLite's URL normalization before the runner expands standalone Path inputs. + target = Path(os.path.abspath(url.database)) return SQLiteMigrationRunner( - Path(url.database), production_bundle(), configuration=database.model_dump(exclude={"url", "echo"}) + target, production_bundle(), configuration=database.model_dump(exclude={"url", "echo"}) ) diff --git a/src/powercontext/service/cli.py b/src/powercontext/service/cli.py index 5508feff05..2c512750e0 100644 --- a/src/powercontext/service/cli.py +++ b/src/powercontext/service/cli.py @@ -156,9 +156,15 @@ def _run_lifecycle(operation: str) -> None: service_status = getattr(controller, operation)() except (OSError, ServiceError) as error: typer.echo(f"PowerContext personal service {operation} failed: {error}", err=True) + for note in getattr(error, "__notes__", ()): + typer.echo(note, err=True) if isinstance(error, ServiceError) and error.status is not None: _write_status(error.status, json_output=False) raise typer.Exit(code=error.exit_code if isinstance(error, ServiceError) else 1) from error + except KeyboardInterrupt as error: + for note in getattr(error, "__notes__", ()): + typer.echo(note, err=True) + raise typer.echo(f"PowerContext personal service {operation} completed.") _write_status(service_status, json_output=False) diff --git a/src/powercontext/service/controller.py b/src/powercontext/service/controller.py index 0e2b339510..c23fd3da1b 100644 --- a/src/powercontext/service/controller.py +++ b/src/powercontext/service/controller.py @@ -216,6 +216,7 @@ def start(self) -> ServiceStatus: definition = registration.definition if definition is None or self._readiness_probe(definition.endpoint).state is not ProbeState.LIVE: raise ServiceError("the running service is not ready; inspect its diagnostics") # noqa: TRY003 + self._require_running_definition(definition) return self.status() if record is None: # A failed start must also leave a durable activation guard; @@ -325,9 +326,14 @@ def _complete_maintenance(self, summary: ServiceMaintenanceSummary) -> ServiceSt if summary.originally_running: try: self._start_registered(summary.target_definition) - except BaseException: + except BaseException as error: # Catchable interruptions need the same explicit recovery state as startup errors. - self._write_maintenance_record({"phase": "start_failed", "summary": summary.as_dict()}) + try: + self._write_maintenance_record({"phase": "start_failed", "summary": summary.as_dict()}) + except BaseException as record_error: + error.add_note( + f"could not record the failed startup: {type(record_error).__name__}: {record_error}" + ) raise self.maintenance_path.unlink(missing_ok=True) return self.status() @@ -468,6 +474,13 @@ def _require_stopped(self, definition: ServiceDefinition | None) -> None: ) self._sleep(0.1) + def _require_running_definition(self, definition: ServiceDefinition) -> None: + loaded = self._adapter.loaded_registration() + if loaded.state is not ManagerOwnershipState.OWNED or loaded.definition != definition: + raise ServiceError("the running service does not use the confirmed executable and configuration") # noqa: TRY003 + if self._adapter.manager_state() is not ManagerState.ACTIVE: + raise ServiceError("the managed service is not running; another listener may own the endpoint") # noqa: TRY003 + def _start_registered(self, definition: ServiceDefinition | None) -> None: if definition is None: raise ServiceError("the installed service has no definition") # noqa: TRY003 @@ -479,22 +492,32 @@ def _start_registered(self, definition: ServiceDefinition | None) -> None: result = self._wait_until_live(definition.endpoint) if result.state is not ProbeState.LIVE: raise ServiceError("the registered service did not become live") # noqa: TRY003, TRY301 - loaded = self._adapter.loaded_registration() - if loaded.state is not ManagerOwnershipState.OWNED or loaded.definition != definition: - raise ServiceError("the running service does not use the confirmed executable and configuration") # noqa: TRY003, TRY301 deadline = time.monotonic() + _START_TIMEOUT_SECONDS while True: readiness = self._readiness_probe(definition.endpoint) if readiness.state is ProbeState.LIVE: + self._require_running_definition(definition) return if readiness.state is ProbeState.CONFLICT or time.monotonic() >= deadline: raise ServiceError(f"the service failed readiness: {readiness.detail}") # noqa: TRY003, TRY301 self._sleep(0.1) - except BaseException: + except BaseException as error: # Restore the guard even on Ctrl+C during startup verification, then - # propagate the interruption. Never fall back to an old executable. - self._adapter.suspend(self.maintenance_path) - self._adapter.stop() + # stop independently: failure to restore protection must not skip + # stopping the process. Preserve the original failure and diagnostics. + try: + self._adapter.suspend(self.maintenance_path) + except BaseException as suspend_error: + error.add_note( + "automatic activation suppression could not be confirmed: " + f"{type(suspend_error).__name__}: {suspend_error}" + ) + try: + self._adapter.stop() + except BaseException as stop_error: + error.add_note( + f"stopping the managed service could not be confirmed: {type(stop_error).__name__}: {stop_error}" + ) raise def _maintenance_record(self) -> dict[str, object] | None: diff --git a/tests/test_database_migration_cli.py b/tests/test_database_migration_cli.py index 17ee1fc4a5..3d9b0cd7ea 100644 --- a/tests/test_database_migration_cli.py +++ b/tests/test_database_migration_cli.py @@ -19,7 +19,10 @@ import os import sqlite3 from pathlib import Path +from uuid import uuid4 +import pytest +from sqlalchemy.ext.asyncio import create_async_engine from typer.testing import CliRunner from powercontext.builtin.persistence.migrations.deployment import production_bundle @@ -81,6 +84,102 @@ def test_read_only_commands_never_create_missing_target(tmp_path, monkeypatch): assert not database.parent.exists() +def test_sqlite_url_authority_is_rejected_without_creating_a_local_target(tmp_path, monkeypatch): + _database, command = _configure(tmp_path, monkeypatch) + monkeypatch.chdir(tmp_path) + filename = f"pc-migration-{uuid4().hex}.db" + private_host = "private-database-host.example" + (tmp_path / "deployment.env").write_text( + "POWERCONTEXT_SERVER_DATABASE_KIND=sqlite\n" + f"POWERCONTEXT_SERVER_DATABASE_URL=sqlite+aiosqlite://{private_host}/{filename}\n", + encoding="utf-8", + ) + for action in ("plan", "apply"): + options = ( + ["--yes", "--backup", "auto", "--plan-id", "unaccepted-plan", "--maintenance-confirmed"] + if action == "apply" + else [] + ) + result = _invoke(command, action, tmp_path, *options) + assert result.exit_code == 1, result.output + assert json.loads(result.output)["error"] == "unsupported_target" + assert private_host not in result.output + assert filename not in result.output + assert not list(tmp_path.glob(f"{filename}*")) + + +@pytest.mark.parametrize( + ("directory", "url_directory"), + [ + pytest.param("~", "~", id="literal-tilde"), + pytest.param("relative", "relative", id="relative-path"), + pytest.param( + "relative", + "linked/../relative", + id="lexical-parent-through-symlink", + marks=pytest.mark.skipif( + os.name == "nt", reason="creating symlinks may require elevated Windows privileges" + ), + ), + ], +) +def test_migration_commands_use_the_configured_sqlite_url_target(tmp_path, monkeypatch, directory, url_directory): + original_database, command = _configure(tmp_path, monkeypatch, existing=True) + monkeypatch.chdir(tmp_path) + filename = f"pc-migration-{uuid4().hex}.db" + home_database = Path.home() / filename + assert not home_database.exists() + database = tmp_path / directory / filename + database.parent.mkdir() + original_database.rename(database) + if url_directory != directory: + link_target = tmp_path / "other" / "link-target" + link_target.mkdir(parents=True) + (tmp_path / "linked").symlink_to(link_target, target_is_directory=True) + url = f"sqlite+aiosqlite:///{url_directory}/{filename}" + (tmp_path / "deployment.env").write_text( + f"POWERCONTEXT_SERVER_DATABASE_KIND=sqlite\nPOWERCONTEXT_SERVER_DATABASE_URL={url}\n", + encoding="utf-8", + ) + + planned = _invoke(command, "plan", tmp_path) + assert planned.exit_code == 0, planned.output + plan = json.loads(planned.output) + assert plan["state"] == "migration_required" + assert plan["source_revision"] == "p0001" + applied = _invoke( + command, + "apply", + tmp_path, + "--yes", + "--backup", + "auto", + "--plan-id", + plan["plan_id"], + "--maintenance-confirmed", + ) + assert applied.exit_code == 0, applied.output + assert json.loads(applied.output)["backup_state"] == "completed" + + async def inspect_configured_database() -> tuple[Path, str]: + engine = create_async_engine(url) + try: + async with engine.connect() as connection: + target = (await connection.exec_driver_sql("PRAGMA database_list")).all()[0][2] + revision = (await connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision")).scalar_one() + return Path(target), revision + finally: + await engine.dispose() + + target, revision = asyncio.run(inspect_configured_database()) + assert target == database + assert revision == production_bundle().head + verified = _invoke(command, "verify", tmp_path) + assert verified.exit_code == 0, verified.output + assert json.loads(verified.output)["state"] == "ready" + assert not home_database.exists() + + def test_apply_requires_exact_plan_and_explicit_noninteractive_consent(tmp_path, monkeypatch): database, command = _configure(tmp_path, monkeypatch) plan = json.loads(_invoke(command, "plan", tmp_path).output) diff --git a/tests/test_service_maintenance.py b/tests/test_service_maintenance.py index 6e021bb11f..d48070884f 100644 --- a/tests/test_service_maintenance.py +++ b/tests/test_service_maintenance.py @@ -14,6 +14,7 @@ from __future__ import annotations +import errno import json import os import subprocess @@ -280,6 +281,126 @@ def test_startup_readiness_failure_is_separate_from_database_ready_and_keeps_ser assert json.loads(controller.maintenance_path.read_text())["phase"] == "start_failed" +def test_start_rejects_an_external_listener_after_the_managed_launcher_exits( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, adapter = _installed(tmp_path, running=False) + adapter.definition = _definition(tmp_path, python_executable=service_python_executable()) + adapter.content = adapter.render(adapter.definition) + adapter.loaded_definition = adapter.definition + external_listener = False + + def probe(endpoint: str) -> ProbeResult: + state = ProbeState.LIVE if external_listener else ProbeState.UNREACHABLE + return ProbeResult(state, endpoint) + + def start(*, reload_definition: bool) -> None: + nonlocal external_listener + # The launcher exits successfully when a foreground PC wins the port. + # systemd and Task Scheduler still retain the owned job definition. + external_listener = True + adapter.manager = ManagerState.INACTIVE + adapter.loaded_definition = adapter.definition + + monkeypatch.setattr(adapter, "start", start) + controller = ServiceController(adapter, probe=probe, readiness_probe=probe) + monkeypatch.setattr(service_cli, "_controller", lambda: controller) + + result = CliRunner().invoke(service_cli.app, ["start"]) + + assert result.exit_code != 0 + assert "start completed" not in result.output + assert "not running" in result.output + assert adapter.manager is ManagerState.INACTIVE + assert adapter.suspended + assert controller.maintenance_path.exists() + assert external_listener + + +def test_start_stops_the_service_when_restoring_automatic_activation_protection_fails( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + controller, adapter = _installed(tmp_path) + controller.stop() + + def fail_suspend(_marker: Path) -> None: + raise OSError(errno.ENOSPC, "No space left on device") + + monkeypatch.setattr(adapter, "suspend", fail_suspend) + controller = ServiceController( + adapter, + probe=_manager_probe(adapter), + readiness_probe=lambda _: ProbeResult(ProbeState.CONFLICT, "schema is not ready"), + ) + monkeypatch.setattr(service_cli, "_controller", lambda: controller) + + result = CliRunner().invoke(service_cli.app, ["start"]) + + assert result.exit_code != 0 + assert "failed readiness" in result.output + assert "automatic activation suppression could not be confirmed" in result.output + assert "No space left on device" in result.output + assert "start completed" not in result.output + assert adapter.manager is ManagerState.INACTIVE + assert not adapter.suspended + assert json.loads(controller.maintenance_path.read_text())["phase"] == "manual_stopped" + + +def test_interrupted_start_reports_both_cleanup_failures(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + controller, adapter = _installed(tmp_path) + controller.stop() + interruption = KeyboardInterrupt() + + def fail_suspend(_marker: Path) -> None: + raise OSError(errno.ENOSPC, "No space left on device") + + def readiness(_endpoint: str) -> ProbeResult: + raise interruption + + monkeypatch.setattr(adapter, "suspend", fail_suspend) + adapter.fail_stop = True + controller = ServiceController(adapter, probe=_manager_probe(adapter), readiness_probe=readiness) + monkeypatch.setattr(service_cli, "_controller", lambda: controller) + + result = CliRunner().invoke(service_cli.app, ["start"]) + + assert result.exit_code == 130 + assert "automatic activation suppression could not be confirmed" in result.output + assert "stopping the managed service could not be confirmed" in result.output + assert "stop failed" in result.output + assert "start completed" not in result.output + assert adapter.manager is ManagerState.ACTIVE + assert not adapter.suspended + assert json.loads(controller.maintenance_path.read_text())["phase"] == "manual_stopped" + + +def test_maintenance_startup_preserves_the_failure_when_its_recovery_record_cannot_be_written( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, adapter = _installed(tmp_path) + interruption = KeyboardInterrupt() + + def fail_write(_record: dict[str, object]) -> None: + raise OSError(errno.ENOSPC, "No space left on device") + + def readiness(_endpoint: str) -> ProbeResult: + monkeypatch.setattr(controller, "_write_maintenance_record", fail_write) + raise interruption + + controller = ServiceController(adapter, probe=_manager_probe(adapter), readiness_probe=readiness) + summary = controller.maintenance_summary() + with controller.maintenance(expected_fingerprint=summary.fingerprint) as session: + with pytest.raises(KeyboardInterrupt) as caught: + session.complete() + assert caught.value is interruption + assert "could not record the failed startup" in "\n".join(interruption.__notes__) + assert not session.completed + + assert adapter.manager is ManagerState.INACTIVE + assert adapter.suspended + assert json.loads(controller.maintenance_path.read_text())["phase"] == "database_ready" + + @pytest.mark.parametrize("startup_check", ["liveness", "readiness"]) def test_cancelled_maintenance_startup_stays_stopped_until_explicit_recovery( tmp_path: Path, startup_check: str From 279df94ef272ef12579c188e468c4a9ec3cedf65 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 8 Oct 2026 17:28:48 +0800 Subject: [PATCH 24/37] feat(db): add seekdb and OceanBase migration executors --- .github/workflows/master.yml | 7 +- docs/en/docs/operate/database-migrations.md | 119 ++- docs/zh/docs/operate/database-migrations.md | 62 +- .../builtin/persistence/migrations/bundle.py | 3 + .../persistence/migrations/connections.py | 254 +++++++ .../persistence/migrations/deployment.py | 39 +- .../builtin/persistence/migrations/mysql.py | 659 +++++++++++++++++ .../migrations/mysql_resources/env.py | 26 + .../versions/p0001_baseline.py | 24 + .../versions/p0002_citations.py | 24 + .../versions/p0003_tag_families.py | 24 + .../persistence/migrations/mysql_schema.py | 699 ++++++++++++++++++ .../builtin/persistence/migrations/sqlite.py | 12 +- src/powercontext/server/database_migration.py | 27 +- .../test_database_migration_connections.py | 255 +++++++ .../test_mysql_migration_runner.py | 309 ++++++++ .../test_mysql_migration_schema.py | 347 +++++++++ tests/test_database_migration_cli.py | 119 ++- 18 files changed, 2950 insertions(+), 59 deletions(-) create mode 100644 src/powercontext/builtin/persistence/migrations/connections.py create mode 100644 src/powercontext/builtin/persistence/migrations/mysql.py create mode 100644 src/powercontext/builtin/persistence/migrations/mysql_resources/env.py create mode 100644 src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0001_baseline.py create mode 100644 src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0002_citations.py create mode 100644 src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0003_tag_families.py create mode 100644 src/powercontext/builtin/persistence/migrations/mysql_schema.py create mode 100644 tests/builtin/persistence/test_database_migration_connections.py create mode 100644 tests/builtin/persistence/test_mysql_migration_runner.py create mode 100644 tests/builtin/persistence/test_mysql_migration_schema.py diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 385515218d..cb27bdf829 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -154,13 +154,16 @@ jobs: - name: Test code indexing with embedded seekdb run: make code-seekdb-test - - name: Probe Alembic migrations with embedded seekdb + - name: Test unified migrations with embedded seekdb env: POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" run: >- uv run --locked --extra seekdb pytest -q tests/builtin/persistence/test_database_migration_backends.py - tests/builtin/persistence/test_database_backup_providers.py -k seekdb + tests/builtin/persistence/test_database_backup_providers.py + tests/builtin/persistence/test_database_migration_connections.py + tests/builtin/persistence/test_mysql_migration_schema.py + tests/builtin/persistence/test_mysql_migration_runner.py -k seekdb windows-unit-portability: timeout-minutes: 20 diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index ba0c974ebf..fd7168b2d3 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -12,16 +12,19 @@ The design follows [RFC #1771](https://github.com/oceanbase/powercontext/pull/17 ## Current availability -This branch provides `powercontext server db-migrate status/plan/apply/verify`. They read the same deployment settings as -the Server and do not accept arbitrary test-bundle paths. Frozen resources ship in the wheel for Phase A acceptance of a -four-table Artifact migration. Only persistent SQLite databases with registered schema shapes are supported. A complete -Server database containing unmanaged objects returns `unknown_baseline`; unified seekdb and OceanBase migration returns -`unsupported_backend`. +`powercontext server db-migrate status/plan/apply/verify` use the configured persistent SQLite, embedded seekdb, or +OceanBase MySQL-mode database. They do not accept arbitrary test-bundle paths. Explicit environment files are authoritative +for maintenance commands; use the deployment's environment file rather than relying on temporary shell overrides. +Frozen resources ship in the wheel for Phase A acceptance of four Artifact tables: `pc_artifacts`, `pc_artifact_heads`, +`pc_artifact_candidate_versions`, and `pc_artifact_tags`. Only empty targets and registered historical layouts are accepted. +A complete Server database containing unmanaged objects is rejected. `ready` means this migration bundle passed its schema and data checks. Output explicitly includes `readiness_scope=registered_bundle` and `server_ready=false`: it does not establish complete Server, index, legacy-task, or cluster readiness. Do not substitute it for a production upgrade or initialization of the complete business database. -Full startup gates and historical baselines require acceptance across all three backends before integration. +Full startup gates and complete historical baselines remain outside this bundle. OceanBase execution and full Fork +recovery acceptance require a dedicated real test environment; local tests and an implemented adapter do not establish +production readiness. The existing `server processing-migrate` remains available. Follow [Migrate Artifact processing state](artifact-processing-migration.md); it does not yet forward to the unified entry point. @@ -29,7 +32,9 @@ See [Deploy the Server](deploy-server.md) for service deployment. ## Read-only inspection and one confirmation -Inspect the target or preview a plan. These commands never create a missing database, parent directory, or control table: +The `deployment.env` examples below use SQLite. OceanBase commands also require the evidence-directory option shown +in the next section. Inspect the target or preview a plan; these commands never create a missing database, parent +directory, or control table: ```bash powercontext server db-migrate status --env-file deployment.env @@ -67,6 +72,58 @@ powercontext server db-migrate apply --env-file deployment.env --backup manual \ When the bundle is already verified and no persistent change is needed, `apply` returns `changed=false` without backup or service lifecycle actions. Switching the application version remains a separate deployment operation. +## Target configuration and external evidence + +Select the database using the existing [Server configuration](configuration.md). The maintenance connection does not +initialize business tables from current application metadata. OceanBase must already have a user database in a +MySQL-mode user tenant; these commands do not create a cluster, tenant, or database. + +OceanBase maintenance must be able to read the tenant ID, cluster ID, and tenant creation metadata through +`effective_tenant_id()`, `oceanbase.GV$OB_PARAMETERS`, and `oceanbase.DBA_OB_TENANTS`. Unavailable permissions or incomplete +identity return `target_identity_unavailable`; a proxy hostname is not used as a substitute database identity. + +seekdb inspection does not start an engine for a missing path or empty directory. A nonempty directory without a +recognized engine layout returns `unsupported_target`. Opening a valid existing engine may write native logs and other +engine housekeeping files; read-only migration commands do not perform business DDL or initialize tables. + +`--evidence-dir PATH` selects persistent storage outside the target database for maintenance progress and recovery +references. Use the same directory for `status`, `plan`, `apply`, `verify`, and retries. Keep it available after a process, +container, or migration Job exits. Give each database its own directory and restrict access to the migration operator. + +| Backend | Evidence directory | +| --- | --- | +| SQLite | Defaults to `.pc-migration-state` beside the canonical target file; override with `--evidence-dir` | +| Embedded seekdb | Defaults to `.pc-migration-state` beside the canonical engine directory; override with `--evidence-dir` | +| OceanBase | Explicit `--evidence-dir` is required by all four commands; every migration Job for the same database must share this durable directory | + +For seekdb, configure its path in `seekdb.env` and use the same commands. Before this interactive manual-backup example, +complete your backup and stop every writer: + +```bash +powercontext server db-migrate status --env-file seekdb.env +powercontext server db-migrate apply --env-file seekdb.env --backup manual +powercontext server db-migrate verify --env-file seekdb.env +``` + +For multiple Servers sharing OceanBase, make a manual backup and stop all writers through deployment tooling. The +following automation uses one persistent shared directory and the same scope when planning and applying: + +```bash +powercontext server db-migrate status --env-file oceanbase.env \ + --evidence-dir /mnt/shared/pc-migration-state +powercontext server db-migrate plan --env-file oceanbase.env --backup manual \ + --shared-database --evidence-dir /mnt/shared/pc-migration-state +powercontext server db-migrate apply --env-file oceanbase.env --backup manual \ + --shared-database --evidence-dir /mnt/shared/pc-migration-state \ + --plan-id PLAN_ID --backup-confirmed --maintenance-confirmed --yes +powercontext server db-migrate verify --env-file oceanbase.env \ + --evidence-dir /mnt/shared/pc-migration-state +``` + +Replace `PLAN_ID` with the returned plan ID. Do not move evidence to an empty directory to bypass a recovery error. +Missing or inconsistent evidence for an interrupted migration returns `recovery_required`; selecting manual backup or +accepting no-backup risk does not authorize an unknown schema or an unproven DDL transition. + ## Backup choices | Policy | Behavior | Options | @@ -81,10 +138,14 @@ It returns `backup_unsupported` or a backup failure category and never silently confirmation. A separate `BackupProvider` reuses backend settings and maintenance connections. SQLite uses Online Backup API to create -an independent file, including committed WAL, and checks integrity. The seekdb/OceanBase cluster design prefers native -`FORK DATABASE`. `FORK TABLE` is eligible only when every affected object and dependency, cross-table consistency, and -restoration path have passed adapter acceptance. PC does not automatically replace unavailable remote Fork with physical -backup. +an independent file, including committed WAL, and checks integrity. seekdb and OceanBase executors reuse +`ForkBackupProvider`, preferring native `FORK DATABASE`. The current executor rejects automatic `FORK TABLE`: enabling +it requires tracking same-database backup objects separately and accepting every affected dependency, cross-table +consistency, and restoration path. Until the actual engine and +schema have accepted recovery evidence, an upgrade requiring backup reports `backup_unsupported` for `--backup auto` +before migration writes. Manual backup and explicitly accepted no-backup risk remain available without Fork support. +PC neither declares acceptance on the operator's behalf nor automatically replaces unavailable Fork with physical backup +or another policy. | Product | First `FORK TABLE` version | First `FORK DATABASE` version | | --- | --- | --- | @@ -93,13 +154,19 @@ backup. Version eligibility is only the start: capability checks also consider actual product, tenant mode, permissions, objects, subsequent DDL limits, and restoration. Several table Forks do not automatically constitute one database snapshot. Fork -shares underlying storage and does not protect against disk failure. Remote unified migration has not completed full -acceptance; an implemented provider is not a production-upgrade guarantee. +shares underlying storage and does not protect against disk failure. A simple-table Fork probe does not establish +recovery of this four-table bundle. OceanBase has not completed real environment acceptance; an implemented provider +is not a production-upgrade guarantee. Retain old business tables, Fork recovery points, and SQLite backups during this upgrade. Migration success, normal startup, and retry do not delete them. A later independent revision removes old tables; explicit maintenance removes recovery points after checking dependencies, retention windows, and recovery obligations. +In this bundle, SQLite revision `p0003` recreates the tag table and retains its old layout and data in +`pc_retained_p0002_artifact_tags`. seekdb/OceanBase `p0003` changes the CHECK constraint in place without replacing or +deleting the business table, so it does not create the SQLite history copy. Neither path automatically deletes backup +recovery points. + ## Stopping, starting, and upgrading services Personal services provide these commands while retaining registration and settings: @@ -139,8 +206,9 @@ powercontext server run --env-file deployment.env --role all For multiple Servers sharing one database, run one migration Job per database. Deployment tooling stops traffic and task production, drains requests and tasks requiring old handlers, stops all writers, and suspends scaling and automatic -restarts. Coordinate subsequent node upgrades; each node passes its schema, task, and capability checks before traffic -returns. A migration lock excludes other migrators, not unknown external clients. +restarts. Replacement migration Jobs use the same persistent `--evidence-dir`; the original recovery reference must +survive Job replacement. Coordinate subsequent node upgrades; each node passes its schema, task, and capability checks +before traffic returns. A migration lock excludes other migrators, not unknown external clients. Release authors provide affected objects, execution mode, binary/schema compatibility, API changes, and task formats. Breaking public API replacements require deprecation, alternatives, and removal plans. Same-package internal entry points @@ -155,13 +223,26 @@ receipts. The Phase A bundle does not yet cover these complete business checks. ## Storage layers and recovery boundaries Maintenance reuses configuration and keeps Alembic and locking on a dedicated connection. Frozen historical SQL does not -depend on current Repositories or application metadata. Repositories run after schema checks; versioned resources and -controlled rebuilds manage full-text/vector indexes. seekdb/OceanBase DDL may commit implicitly, so one Python transaction -cannot make the entire upgrade atomic. +depend on current Repositories or application metadata. seekdb holds its canonical engine-directory lock before startup +until engine shutdown. OceanBase holds a named lock on a pinned connection, checks that a second connection is excluded +across commits, and verifies ownership before and after each DDL. Lock loss blocks further writes; it does not reconnect +and continue. These locks do not stop business writers. + +seekdb/OceanBase DDL may commit implicitly, so one Python transaction cannot make the entire upgrade atomic. Recovery +recognizes only the registered sequence: `p0001` creates the four frozen tables and two indexes; `p0002` adds the two +nullable `MEDIUMBLOB` citation columns; `p0003` removes the tag-family CHECK. Actual schema, data checks, and persisted +maintenance evidence must establish the known completed prefix before continuing. Alembic advances the version only +after the revision's postconditions pass. Unknown objects, layouts, or partial transitions block execution instead of +being optimistically stamped. + +Repositories run after schema checks. Versioned resources and controlled rebuilds for complete business full-text/vector +indexes, plus legacy-task readiness, remain outside this acceptance bundle. One version table offers no generic run-ID resume. After interruption, inspect actual schema/data through read-only commands. Ambiguous state returns `recovery_required`; never blindly stamp or replace the original recovery point with a -partial-state backup. External summaries record execution and backup information. +partial-state backup. External summaries record execution and backup information before DDL. A retry preserves the +original maintenance window and recovery point. Manual/skip policies can be explicitly reconfirmed when recovery +conditions allow, but they do not bypass structural verification or replace the required progress evidence. Restoration is separate, explicit, and performed with writes stopped. Restore the database, required files, and matching program before verification. Application rollback is not database rollback. Manual mode does not claim verified recovery; diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index 05742ab402..b797f69b23 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -10,15 +10,15 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 ## 当前可用范围 -本分支提供 `powercontext server db-migrate status/plan/apply/verify`，读取与 Server 相同的部署配置，不接受任意测试 bundle 路径。随 wheel 发布的冻结资源目前用于阶段 A 的 Artifact 四表迁移验收；只有已登记结构的持久 SQLite 数据库可以执行。完整 Server 数据库含有尚未纳管的对象时返回 `unknown_baseline`，seekdb 和 OceanBase 统一迁移返回 `unsupported_backend`。 +`powercontext server db-migrate status/plan/apply/verify` 使用部署配置中的持久 SQLite、嵌入式 seekdb 或 OceanBase MySQL 模式数据库，不接受任意测试 bundle 路径。维护命令以显式环境文件为准，应使用部署的环境文件，不依赖临时 shell 环境覆盖。随 wheel 发布的冻结资源用于阶段 A 的 Artifact 四表迁移：`pc_artifacts`、`pc_artifact_heads`、`pc_artifact_candidate_versions`、`pc_artifact_tags`。仅接受空目标和已登记的历史结构；完整 Server 数据库含有尚未纳管的对象时拒绝执行。 -输出中的 `ready` 仅表示当前迁移 bundle 已通过结构和数据验证；`readiness_scope=registered_bundle`、`server_ready=false` 明确区分它与完整 Server、索引、旧任务及集群可用。不要以此替代生产升级或完整业务库初始化。普通业务启动的全面版本门禁和完整历史基线仍须在三后端验收后接入。 +输出中的 `ready` 仅表示当前迁移 bundle 已通过结构和数据验证；`readiness_scope=registered_bundle`、`server_ready=false` 明确区分它与完整 Server、索引、旧任务及集群可用。不要以此替代生产升级或完整业务库初始化。普通业务启动的全面版本门禁和完整历史基线尚未纳管。OceanBase 实际执行和完整 Fork 恢复仍需专用真实环境验收；本地测试与已有适配器不能证明生产可用。 原有 `server processing-migrate` 保留，请按[迁移 Artifact 处理状态](artifact-processing-migration.md)使用。它尚未自动转发到统一入口。服务部署参考[部署 Server](deploy-server.md)。 ## 只读检查与一次确认 -先检查目标或预览计划；这些命令不会创建缺失数据库、父目录或控制表： +下面的 `deployment.env` 示例使用 SQLite；OceanBase 命令还须传入下一节的证据目录参数。先检查目标或预览计划，这些命令不会创建缺失数据库、父目录或控制表： ```bash powercontext server db-migrate status --env-file deployment.env @@ -46,6 +46,46 @@ powercontext server db-migrate apply --env-file deployment.env --backup manual \ `--yes` 仅接受计划，不隐含手动备份声明、无备份风险或所有节点已停写。`--shared-database` 将共享库维护责任加入计划，不使 SQLite 获得跨主机部署能力。数据库已达到 bundle 目标且无需持久化变更时，`apply` 返回 `changed=false`，不备份、不启停服务；程序版本切换仍须单独安排。 +## 目标配置与库外证据 + +数据库通过现有 [Server 配置](configuration.md)选择。维护连接不使用当前应用 metadata 初始化业务表。OceanBase 须已有 MySQL 模式用户租户和用户数据库；迁移命令不创建集群、租户或数据库。 + +OceanBase 维护须能通过 `effective_tenant_id()`、`oceanbase.GV$OB_PARAMETERS`、`oceanbase.DBA_OB_TENANTS` 读取租户 ID、集群 ID 和租户创建信息。权限不足或身份信息不完整时返回 `target_identity_unavailable`，不以代理主机名代替数据库身份。 + +seekdb 只读检查遇到缺失路径或空目录时不启动引擎；非空目录缺少可识别的引擎结构时返回 `unsupported_target`。打开有效的已有引擎可能写入原生日志和引擎维护文件；只读迁移命令不执行业务 DDL、不初始化表。 + +`--evidence-dir PATH` 指定目标库之外的持久目录，保存维护进度及恢复引用。`status`、`plan`、`apply`、`verify` 和重试必须使用同一目录；进程、容器或迁移 Job 退出后仍须保留。每个数据库使用独立目录，并限制为迁移操作者可访问。 + +| 后端 | 证据目录 | +| --- | --- | +| SQLite | 默认在规范化后的数据库文件旁使用 `<数据库文件名>.pc-migration-state`，可用 `--evidence-dir` 覆盖 | +| 嵌入式 seekdb | 默认在规范化后的引擎目录旁使用 `<目录名>.pc-migration-state`，可用 `--evidence-dir` 覆盖 | +| OceanBase | 四个命令均须显式传入 `--evidence-dir`；同一数据库的所有迁移 Job 共享该持久目录 | + +seekdb 在 `seekdb.env` 中配置路径后使用同一组命令。下面的手动备份交互示例要求先完成备份并停止所有写入者： + +```bash +powercontext server db-migrate status --env-file seekdb.env +powercontext server db-migrate apply --env-file seekdb.env --backup manual +powercontext server db-migrate verify --env-file seekdb.env +``` + +多个 Server 共用 OceanBase 时，先由用户手动备份，部署系统停止所有写入者。下面的自动化示例使用一个持久共享目录，并在计划和执行时使用相同范围： + +```bash +powercontext server db-migrate status --env-file oceanbase.env \ + --evidence-dir /mnt/shared/pc-migration-state +powercontext server db-migrate plan --env-file oceanbase.env --backup manual \ + --shared-database --evidence-dir /mnt/shared/pc-migration-state +powercontext server db-migrate apply --env-file oceanbase.env --backup manual \ + --shared-database --evidence-dir /mnt/shared/pc-migration-state \ + --plan-id PLAN_ID --backup-confirmed --maintenance-confirmed --yes +powercontext server db-migrate verify --env-file oceanbase.env \ + --evidence-dir /mnt/shared/pc-migration-state +``` + +将 `PLAN_ID` 替换为返回的计划 ID。不能换到空目录绕过恢复错误：中断迁移的证据缺失或不一致时返回 `recovery_required`；选择手动备份或接受不备份风险，也不能授权未知结构或无法证明的 DDL 进度。 + ## 备份选择 | 策略 | 行为 | 参数 | @@ -56,17 +96,19 @@ powercontext server db-migrate apply --env-file deployment.env --backup manual \ 自动备份报告 `completed`，手动备份报告 `user_confirmed`，跳过报告 `skipped`；真正空库初始化报告 `not_required`。自动备份不支持、失败或未完成时停在迁移写入之前，返回 `backup_unsupported` 或备份失败类别，不能自动改成跳过。选择其他策略后需要重新确认计划。 -独立 `BackupProvider` 复用后端配置和维护连接。SQLite 使用 Online Backup API 创建独立文件，包含已提交 WAL，并检查文件完整性。seekdb 和 OceanBase 集群的设计采用原生 `FORK DATABASE`，在所有受影响表、依赖和跨表一致性均能覆盖且恢复路径经过验收时可选 `FORK TABLE`。PC 不将不支持 Fork 的远端引擎自动切换到物理备份。 +独立 `BackupProvider` 复用后端配置和维护连接。SQLite 使用 Online Backup API 创建独立文件，包含已提交 WAL，并检查文件完整性。seekdb 和 OceanBase 执行器复用 `ForkBackupProvider`，优先采用原生 `FORK DATABASE`。当前执行器拒绝自动 `FORK TABLE`；启用前须单独登记同库备份对象，并完成受影响依赖、跨表一致性和恢复路径的验收。对于需要备份的升级，实际引擎和结构缺少通过验收的恢复证据时，`--backup auto` 在迁移写入前返回 `backup_unsupported`。用户手动备份与明确接受风险的不备份路径不依赖 Fork 支持。PC 不替用户声明已完成验收，也不自动改为物理备份或其他策略。 | 产品 | `FORK TABLE` 起始版本 | `FORK DATABASE` 起始版本 | | --- | --- | --- | | OceanBase AI 数据库 | V4.6.2 | V4.6.2 | | seekdb | V1.1.0 | V1.2.0 | -版本满足只是能力检查的起点，还需检查实际产品、租户模式、权限、对象覆盖、后续 DDL 限制和恢复路径。多个表 Fork 不自动代表整库共同快照；Fork 共享底层存储，不提供磁盘损坏保护。当前远端统一迁移尚未通过完整验收，存在 provider 不等于可用生产升级。 +版本满足只是能力检查的起点，还需检查实际产品、租户模式、权限、对象覆盖、后续 DDL 限制和恢复路径。多个表 Fork 不自动代表整库共同快照；Fork 共享底层存储，不提供磁盘损坏保护。简单表的 Fork 探针不代表四表 bundle 的完整恢复能力；OceanBase 尚未完成真实环境验收，存在 provider 不等于可用生产升级。 旧业务表、Fork 恢复点和 SQLite 备份本次保留，不在迁移成功、普通启动或重试时删除。旧表删除由后续版本单独 revision 实现；恢复点由显式维护操作清理，均需验证依赖、保留窗口及恢复责任。 +本 bundle 的 SQLite `p0003` 会重建标签表，将旧结构和数据保存在 `pc_retained_p0002_artifact_tags`。seekdb／OceanBase 的 `p0003` 原地修改 CHECK，不替换或删除业务表，因此不创建 SQLite 的历史副本。两条路径均不自动删除备份恢复点。 + ## 服务停止、启动与升级 本机个人服务提供以下命令，保留注册和配置： @@ -95,7 +137,7 @@ powercontext server run --env-file deployment.env --role all ## 共享数据库、接口与旧任务 -多个 Server 共用一个数据库时，每库只执行一个迁移 Job。部署系统停止流量和任务生产、排空在途请求及需由旧处理器消费的任务、停止全部写入者，并暂停扩容和自动重启。迁移后协调全部相关节点升级，各节点通过自身 schema、任务和能力检查后才恢复流量。迁移锁只排斥迁移进程，不阻止未知外部客户端。 +多个 Server 共用一个数据库时，每库只执行一个迁移 Job。部署系统停止流量和任务生产、排空在途请求及需由旧处理器消费的任务、停止全部写入者，并暂停扩容和自动重启。替换的迁移 Job 继续使用同一个持久 `--evidence-dir`，保留原始恢复引用。迁移后协调全部相关节点升级，各节点通过自身 schema、任务和能力检查后才恢复流量。迁移锁只排斥迁移进程，不阻止未知外部客户端。 发布作者应提供受影响对象、执行模式、程序／schema 兼容范围、API 变化和任务格式声明。公共 API 不兼容替换需标记弃用并说明替代入口和移除计划；同包内部入口可与调用方一起替换。保留的 API 可转换请求／响应后共用当前实现，不要求每个 handler 支持两套表结构。 @@ -103,8 +145,12 @@ powercontext server run --env-file deployment.env --role all ## 存储层与恢复边界 -维护执行器复用配置，使用专用连接承载 Alembic 和锁；冻结历史 SQL 不依赖当前 Repository 或应用 metadata。Repository 仅在结构满足后运行；全文／向量索引由版本化资源和受控重建管理。seekdb／OceanBase DDL 可能隐式提交，不能用 Python 事务承诺整个迁移原子化。 +维护执行器复用配置，使用专用连接承载 Alembic 和锁；冻结历史 SQL 不依赖当前 Repository 或应用 metadata。seekdb 从启动引擎前到关闭引擎后持有规范化目录的锁。OceanBase 在固定连接上持有命名锁，检查另一连接在提交前后仍被排斥，并在每条 DDL 前后验证持有权；失锁后阻止后续写入，不重连继续。这些锁不阻止业务写入者。 + +seekdb／OceanBase DDL 可能隐式提交，不能用 Python 事务承诺整个迁移原子化。恢复只识别已登记顺序：`p0001` 创建冻结四表及两个索引；`p0002` 添加两个可空的 `MEDIUMBLOB` 引用列；`p0003` 删除标签 family 的 CHECK。继续执行前须用实际结构、数据检查和持久维护证据证明已完成的已知前缀；revision 的后置条件通过后才推进 Alembic 版本。未知对象、结构或中间状态拒绝执行，不乐观 stamp。 + +Repository 仅在结构满足后运行。完整业务的全文／向量索引版本管理及受控重建、旧任务 readiness 尚未包含在本验收 bundle 中。 -单版本表不提供通用 run ID 续跑。中断后先用只读命令检查真实结构和数据；含混状态返回 `recovery_required`，不盲目 stamp、不用部分状态的备份替换原恢复点。库外摘要记录执行结果及备份信息。 +单版本表不提供通用 run ID 续跑。中断后先用只读命令检查真实结构和数据；含混状态返回 `recovery_required`，不盲目 stamp、不用部分状态的备份替换原恢复点。库外摘要在 DDL 前记录执行及备份信息；重试保留原维护窗口和恢复点。在恢复条件允许时可显式重新确认 manual／skip 策略，但不绕过结构验证，也不替代必需的进度证据。 恢复独立、显式执行，并保持停写。恢复数据库、必要文件和匹配程序后重新验证；应用回退不等于数据库回滚。manual 不承诺已验证可恢复，skip 不承诺恢复原数据。 diff --git a/src/powercontext/builtin/persistence/migrations/bundle.py b/src/powercontext/builtin/persistence/migrations/bundle.py index d957935229..c38bc2671e 100644 --- a/src/powercontext/builtin/persistence/migrations/bundle.py +++ b/src/powercontext/builtin/persistence/migrations/bundle.py @@ -18,6 +18,7 @@ import hashlib import json +import threading from pathlib import Path from tempfile import TemporaryDirectory from typing import Any @@ -28,6 +29,8 @@ from .models import MigrationError, digest +ALEMBIC_CONTEXT_LOCK = threading.RLock() + class MigrationBundle: """Load a trusted local bundle. No revision imports current application metadata. diff --git a/src/powercontext/builtin/persistence/migrations/connections.py b/src/powercontext/builtin/persistence/migrations/connections.py new file mode 100644 index 0000000000..fcb638a613 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/connections.py @@ -0,0 +1,254 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dedicated backend connections without business schema initialization.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Literal, TypeVar + +from sqlalchemy import Connection, text +from sqlalchemy.exc import DBAPIError, SQLAlchemyError +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine +from sqlalchemy.pool import NullPool + +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.persistence.oceanbase import profile as oceanbase_profile +from powercontext.builtin.persistence.seekdb import SeekDBConfig +from powercontext.builtin.persistence.seekdb import profile as seekdb_profile + +from .locking import local_migration_lock +from .models import MigrationError, digest +from .oceanbase import PinnedOceanBaseLock, oceanbase_migration_lock + +_T = TypeVar("_T") + + +@dataclass(frozen=True) +class BackendIdentity: + """Stable maintenance identity, independent of credentials and proxy routes.""" + + product: Literal["seekdb", "oceanbase"] + database_id: str + database_name: str + + +class MaintenanceConnections: + """Run synchronous migration code inside the official async driver bridge. + + The connection and lock verifier are valid only during the supplied callback. + A callback must commit its completed work explicitly; connection shutdown + rolls back unfinished transactions. A missing seekdb inspection supplies no + connection and does not start an engine or create any directory. + """ + + def __init__(self, config: SeekDBConfig | OceanBaseConfig) -> None: + self.config = config + + def run( + self, + operation: Callable[[Connection | None, BackendIdentity, Callable[[], None]], _T], + *, + writable: bool = False, + ) -> _T: + if isinstance(self.config, SeekDBConfig): + path = self.config.path.expanduser().resolve() + identity = BackendIdentity( + "seekdb", + digest({"backend": "seekdb", "path": str(path), "database": self.config.database}), + self.config.database, + ) + if not path.exists() and not writable: + return operation(None, identity, lambda: None) + with local_migration_lock(path): + if path.exists() and not path.is_dir(): + raise MigrationError("unsupported_target", "The configured seekdb target must be a directory.") + nonempty = path.exists() and any(path.iterdir()) + if not writable and not nonempty: + return operation(None, identity, lambda: None) + if nonempty and not _seekdb_storage_exists(path): + raise MigrationError( + "unsupported_target", "A nonempty seekdb directory must contain recognized native storage." + ) + initial_inode = _directory_inode(path) if path.exists() else None + return asyncio.run(self._seekdb(operation, self.config, path, identity, initial_inode)) + return asyncio.run(self._oceanbase(operation, self.config, writable=writable)) + + async def _seekdb( + self, + operation: Callable[[Connection | None, BackendIdentity, Callable[[], None]], _T], + config: SeekDBConfig, + path: Path, + identity: BackendIdentity, + initial_inode: tuple[int, int] | None, + ) -> _T: + try: + module = seekdb_profile._load_binding() + except (seekdb_profile.SeekDBUnavailableError, ImportError) as error: + raise MigrationError("backend_unavailable", "The embedded seekdb binding is unavailable.") from error + try: + instance = await seekdb_profile._open_instance(module, path) + except RuntimeError as error: + raise MigrationError("backend_open_failed", "The embedded seekdb instance could not be opened.") from error + try: + engine = seekdb_profile._create_engine( + config.model_copy(update={"echo": False}), instance.connection_options() + ) + try: + inode = _directory_inode(path) + if initial_inode is not None and initial_inode != inode: + raise MigrationError("migration_lock_lost", "The seekdb directory changed during engine startup.") + async with engine.connect() as connection: + + def execute(sync: Connection) -> _T: + def verify() -> None: + if sync.closed or sync.invalidated or _directory_inode(path) != inode: + raise MigrationError("migration_lock_lost", "The seekdb maintenance target was lost.") + + verify() + if sync.exec_driver_sql("SELECT DATABASE()").scalar_one() != identity.database_name: + raise MigrationError("target_identity_unavailable", "The seekdb database identity changed.") + result = operation(sync, identity, verify) + verify() + return result + + return await connection.run_sync(execute) + finally: + await _dispose_engine(engine) + finally: + instance.close() + + async def _oceanbase( + self, + operation: Callable[[Connection | None, BackendIdentity, Callable[[], None]], _T], + config: OceanBaseConfig, + *, + writable: bool, + ) -> _T: + oceanbase_profile._register_official_dialect() + engine = create_async_engine( + config.url.get_secret_value(), + connect_args={"init_command": "SET autocommit = 0"}, + echo=False, + hide_parameters=True, + pool_pre_ping=config.pool_pre_ping, + poolclass=NullPool, + ) + try: + async with engine.connect() as connection: + identity = await connection.run_sync(_oceanbase_identity) + if not writable: + return await connection.run_sync(lambda sync: operation(sync, identity, lambda: None)) + async with engine.connect() as contender: + other_identity = await contender.run_sync(_oceanbase_identity) + if other_identity != identity: + raise MigrationError( + "target_identity_unavailable", "Maintenance sessions reached different database identities." + ) + async with oceanbase_migration_lock( + connection, contender, database_id=identity.database_id + ) as lock: + + def execute(sync: Connection) -> _T: + def verify() -> None: + _verify_remote_lock(sync, lock) + + verify() + result = operation(sync, identity, verify) + verify() + return result + + return await connection.run_sync(execute) + finally: + await _dispose_engine(engine) + + +def _seekdb_storage_exists(path: Path) -> bool: + # These native markers are verified by the isolated seekdb reopen test. + return (path / "etc" / "seekdb.data_version.bin").is_file() and (path / "store" / "sstable" / "meta.db").is_file() + + +def _directory_inode(path: Path) -> tuple[int, int]: + try: + stat = path.stat() + except OSError as error: + raise MigrationError("migration_lock_lost", "The seekdb maintenance directory is unavailable.") from error + return stat.st_dev, stat.st_ino + + +def _oceanbase_identity(connection: Connection) -> BackendIdentity: + try: + mode = connection.exec_driver_sql("SHOW VARIABLES LIKE 'ob_compatibility_mode'").first() + if mode is None or len(mode) < 2 or str(mode[1]).upper() != "MYSQL": + raise MigrationError("unsupported_target", "OceanBase maintenance requires a MySQL user tenant.") + database_name, tenant_id = connection.exec_driver_sql("SELECT DATABASE(), effective_tenant_id()").one() + cluster_ids = ( + connection + .exec_driver_sql("SELECT DISTINCT VALUE FROM oceanbase.GV$OB_PARAMETERS WHERE NAME = 'cluster_id'") + .scalars() + .all() + ) + tenant_created, tenant_type = connection.exec_driver_sql( + "SELECT CREATE_TIME, TENANT_TYPE FROM oceanbase.DBA_OB_TENANTS WHERE TENANT_ID = effective_tenant_id()" + ).one() + if ( + not isinstance(database_name, str) + or not database_name + or not isinstance(tenant_id, int) + or tenant_id <= 0 + or len(cluster_ids) != 1 + or int(cluster_ids[0]) <= 0 + or not isinstance(tenant_created, datetime) + ): + raise MigrationError("target_identity_unavailable", "The database did not return a stable identity.") + if tenant_type != "USER": + raise MigrationError("unsupported_target", "OceanBase maintenance requires a MySQL user tenant.") + database_id = digest({ + "backend": "oceanbase", + "cluster_id": int(cluster_ids[0]), + "tenant_id": tenant_id, + "tenant_created": tenant_created.isoformat(), + "database_name": database_name, + }) + return BackendIdentity("oceanbase", database_id, database_name) + except (SQLAlchemyError, ValueError, TypeError) as error: + raise MigrationError( + "target_identity_unavailable", "The connected OceanBase identity could not be verified." + ) from error + + +def _verify_remote_lock(connection: Connection, lock: PinnedOceanBaseLock) -> None: + try: + if connection.invalidated or connection.closed: + raise MigrationError("migration_lock_lost", "The lock connection was lost; inspect DDL before recovery.") + session = connection.scalar(text("SELECT CONNECTION_ID()")) + owner = connection.scalar(text("SELECT IS_USED_LOCK(:name)"), {"name": lock.name}) + if session != lock.session_id or owner != lock.session_id: + raise MigrationError("migration_lock_lost", "The pinned session no longer owns the migration lock.") + except DBAPIError as error: + raise MigrationError("migration_lock_lost", "The database migration lock could not be verified.") from error + + +async def _dispose_engine(engine: AsyncEngine) -> None: + task = asyncio.create_task(engine.dispose()) + try: + await asyncio.shield(task) + except asyncio.CancelledError: + await seekdb_profile._finish_task(task) + raise diff --git a/src/powercontext/builtin/persistence/migrations/deployment.py b/src/powercontext/builtin/persistence/migrations/deployment.py index afb721fa12..2652623218 100644 --- a/src/powercontext/builtin/persistence/migrations/deployment.py +++ b/src/powercontext/builtin/persistence/migrations/deployment.py @@ -12,11 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Packaged acceptance bundle, isolated from side-effectful runtime factories. - -This release does not register full Server baselines or remote executors. A -complete deployment must not be optimistically stamped from a four-table test. -""" +"""Select maintenance adapters without initializing the business runtime.""" from __future__ import annotations @@ -26,10 +22,14 @@ from sqlalchemy.engine import make_url +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.persistence.seekdb import SeekDBConfig from powercontext.builtin.persistence.sqlite import SQLiteConfig from .bundle import MigrationBundle +from .connections import MaintenanceConnections from .models import MigrationError +from .mysql import MySQLMigrationRunner from .sqlite import SQLiteMigrationRunner if TYPE_CHECKING: @@ -41,13 +41,29 @@ def production_bundle() -> MigrationBundle: return MigrationBundle(Path(__file__).parent / "resources") -def deployment_runner(settings: ServerSettings) -> SQLiteMigrationRunner: +MigrationRunner = SQLiteMigrationRunner | MySQLMigrationRunner + + +def deployment_runner(settings: ServerSettings, *, evidence_dir: Path | None = None) -> MigrationRunner: """Use the real configured identity without opening a business profile.""" database = settings.database - if not isinstance(database, SQLiteConfig): - raise MigrationError( - "unsupported_backend", "Remote and seekdb deployment migration awaits full-backend acceptance." + if isinstance(database, (SeekDBConfig, OceanBaseConfig)): + if evidence_dir is None: + if isinstance(database, OceanBaseConfig): + raise MigrationError( + "evidence_required", + "OceanBase maintenance requires --evidence-dir on durable storage shared by migration Jobs.", + ) + target = database.path.expanduser().resolve() + evidence_dir = target.with_name(target.name + ".pc-migration-state") + return MySQLMigrationRunner( + MaintenanceConnections(database), + production_bundle(), + evidence_directory=evidence_dir, + configuration=database.model_dump(mode="json", exclude={"url", "echo"}), ) + if not isinstance(database, SQLiteConfig): + raise MigrationError("unsupported_backend", "The database has no registered migration adapter.") url = make_url(database.url) if database.is_in_memory or url.query or not url.database or url.username or url.password or url.host or url.port: raise MigrationError( @@ -56,5 +72,8 @@ def deployment_runner(settings: ServerSettings) -> SQLiteMigrationRunner: # Match SQLite's URL normalization before the runner expands standalone Path inputs. target = Path(os.path.abspath(url.database)) return SQLiteMigrationRunner( - target, production_bundle(), configuration=database.model_dump(exclude={"url", "echo"}) + target, + production_bundle(), + configuration=database.model_dump(exclude={"url", "echo"}), + evidence_directory=evidence_dir, ) diff --git a/src/powercontext/builtin/persistence/migrations/mysql.py b/src/powercontext/builtin/persistence/migrations/mysql.py new file mode 100644 index 0000000000..801107bab3 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/mysql.py @@ -0,0 +1,659 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline seekdb/OceanBase migrations with explicit nontransactional DDL recovery.""" + +from __future__ import annotations + +import hashlib +import os +from collections.abc import Callable, Mapping +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Literal +from uuid import uuid4 + +from alembic import command +from alembic.config import Config +from pydantic import BaseModel, ConfigDict, model_validator +from sqlalchemy import Connection + +from .backup import BackupContext, BackupProvider, BackupRef, ForkBackupProvider +from .bundle import ALEMBIC_CONTEXT_LOCK, MigrationBundle +from .connections import BackendIdentity, MaintenanceConnections +from .models import MigrationError, MigrationPlan, MigrationResult, digest +from .mysql_schema import MySQLSchema, mysql_inventory, read_revision + +BackupPolicy = Literal["auto", "manual", "skip"] +BackupFactory = Callable[[Connection, BackendIdentity, Path], BackupProvider] + + +class _Evidence(BaseModel): + model_config = ConfigDict(extra="forbid") + + state: Literal["preparing", "active", "complete"] + database_id: str + source_revision: str | None + source_fingerprint: str + bundle_checksum: str + configuration_digest: str + last_revision: str | None + pending_revision: str | None = None + adopting: bool = False + maintenance_window_id: str + backup_policy: BackupPolicy + backup_state: Literal["pending", "completed", "user_confirmed", "skipped", "not_required"] + backup: BackupRef | None = None + manual_ref: str | None = None + backup_history: list[dict[str, Any]] = [] + release_checksums: dict[str, str] = {} + + @model_validator(mode="after") + def validate_backup(self) -> _Evidence: + if self.backup_state == "not_required": + if self.source_revision is not None or self.backup is not None: + raise ValueError("Only an empty source needs no recovery point") # noqa: TRY003 + elif self.backup_state == "pending": + if self.state != "preparing" or self.backup is not None: + raise ValueError("Pending backup is only valid before migration") # noqa: TRY003 + elif ( + self.backup_state + != {"auto": "completed", "manual": "user_confirmed", "skip": "skipped"}[self.backup_policy] + ): + raise ValueError("Backup declaration and policy disagree") # noqa: TRY003 + if self.backup_state == "completed": + ref = self.backup + if ref is None or ( + ref.database_id != self.database_id + or ref.source_revision != self.source_revision + or ref.bundle_checksum != self.bundle_checksum + or ref.maintenance_window_id != self.maintenance_window_id + ): + raise ValueError("Recovery point belongs to another maintenance context") # noqa: TRY003 + elif self.backup is not None: + raise ValueError("Unexpected automatic recovery point") # noqa: TRY003 + return self + + +class _Journal: + def __init__(self, directory: Path, identity: BackendIdentity) -> None: + self.database_id = identity.database_id + self.path = directory / identity.database_id / "maintenance.json" + + def read(self) -> _Evidence | None: + if not self.path.exists(): + return None + try: + evidence = _Evidence.model_validate_json(self.path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise MigrationError( + "recovery_required", "The external maintenance evidence cannot be read safely." + ) from error + if evidence.database_id != self.database_id: + raise MigrationError("recovery_required", "The maintenance evidence belongs to another database.") + return evidence + + def write(self, evidence: _Evidence) -> None: + # Revalidate mutations before replacing the durable recovery record. + value = _Evidence.model_validate(evidence.model_dump()) + directory = self.path.parent + temporary = directory / f".{uuid4().hex}.json" + try: + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + with temporary.open("x", encoding="utf-8") as output: + os.chmod(temporary, 0o600) + output.write(value.model_dump_json()) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, self.path) + if os.name != "nt": + descriptor = os.open(directory, os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except OSError as error: + raise MigrationError( + "recovery_required", "Cannot persist maintenance evidence; migration is blocked." + ) from error + + +class MySQLMigrationRunner: + """Use one version table, a pinned backend lock, and persistent external intent. + + The supported historical layouts are intentionally finite. A partial DDL + state is resumable only when both the original intent and its exact declared + schema prefix agree. Each Alembic upgrade callback verifies the full target + before Alembic advances its standard version_num row. + """ + + def __init__( + self, + connections: MaintenanceConnections, + bundle: MigrationBundle, + *, + evidence_directory: Path, + configuration: Mapping[str, Any] | None = None, + backup_factory: BackupFactory | None = None, + ) -> None: + self.connections = connections + self.bundle = bundle + self.schema = MySQLSchema(bundle) + self.evidence_directory = evidence_directory.expanduser().resolve() + self.configuration_digest = digest(configuration or {}) + self.backup_factory = backup_factory or _fork_provider + self._history_directory = Path(__file__).parent / "mysql_resources" + resources = { + path.relative_to(self._history_directory).as_posix(): path.read_bytes() + for path in sorted(self._history_directory.rglob("*.py")) + } + self._history_bytes = resources + self._snapshot = TemporaryDirectory(prefix="pc-mysql-migrations-") + for name, content in resources.items(): + target = Path(self._snapshot.name) / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + hashes = {name: hashlib.sha256(content).hexdigest() for name, content in resources.items()} + self.checksum = digest({"bundle": bundle.checksum, "mysql_history": hashes}) + self.checksums = { + revision: digest({ + "revision": checksum, + "env": hashes["env.py"], + "wrapper": {name: value for name, value in hashes.items() if name.startswith(f"versions/{revision}_")}, + }) + for revision, checksum in bundle.checksums.items() + } + + def _assert_unchanged(self) -> None: + self.bundle.assert_unchanged() + resources = { + path.relative_to(self._history_directory).as_posix(): path.read_bytes() + for path in sorted(self._history_directory.rglob("*.py")) + } + if resources != self._history_bytes: + raise MigrationError("plan_changed", "MySQL migration resources changed after planning.") + + def plan( + self, + *, + backup_policy: BackupPolicy = "auto", + service: dict[str, Any] | None = None, + shared_database: bool = False, + ) -> MigrationPlan: + self._assert_unchanged() + return self.connections.run( + lambda connection, identity, _guard: self._inspect( + connection, identity, backup_policy=backup_policy, service=service, shared_database=shared_database + ) + ) + + def verify(self) -> MigrationResult: + def verify( + connection: Connection | None, identity: BackendIdentity, _guard: Callable[[], None] + ) -> MigrationResult: + evidence = _Journal(self.evidence_directory, identity).read() + policy = evidence.backup_policy if evidence and evidence.state != "complete" else "auto" + plan = self._inspect(connection, identity, backup_policy=policy) + if plan.state != "ready": + raise MigrationError( + plan.state, "Inspect and explicitly apply the registered bundle before verification." + ) + return MigrationResult(revision=self.bundle.head, changed=False) + + self._assert_unchanged() + return self.connections.run(verify) + + def apply( + self, + *, + plan_id: str | None = None, + maintenance_confirmed: bool = False, + accepted: bool = False, + backup_policy: BackupPolicy = "auto", + backup_confirmed: bool = False, + accept_no_backup: bool = False, + backup_ref: str | None = None, + service: dict[str, Any] | None = None, + shared_database: bool = False, + ) -> MigrationResult: + options: dict[str, Any] = { + "backup_policy": backup_policy, + "service": service, + "shared_database": shared_database, + } + initial = self.plan(**options) + if initial.state == "ready": + return MigrationResult(revision=self.bundle.head, changed=False) + self._check_consent( + initial, + plan_id=plan_id, + accepted=accepted, + maintenance_confirmed=maintenance_confirmed, + backup_confirmed=backup_confirmed, + accept_no_backup=accept_no_backup, + ) + return self.connections.run( + lambda connection, identity, guard: self._apply_locked( + connection, identity, guard, initial=initial, manual_ref=backup_ref + ), + writable=True, + ) + + @staticmethod + def _check_consent( + initial: MigrationPlan, + *, + plan_id: str | None, + accepted: bool, + maintenance_confirmed: bool, + backup_confirmed: bool, + accept_no_backup: bool, + ) -> None: + if not accepted or plan_id is None: + raise MigrationError("confirmation_required", "Accept the displayed plan ID explicitly.") + if initial.plan_id != plan_id: + raise MigrationError( + "plan_changed", "The reviewed target, schema, resources or maintenance choices changed." + ) + if not maintenance_confirmed: + raise MigrationError("maintenance_required", "Stop every writer and coordinate all shared-database nodes.") + if initial.state != "uninitialized": + if initial.backup_policy == "manual" and not backup_confirmed: + raise MigrationError( + "backup_confirmation_required", "Declare your manual backup; PC does not verify it." + ) + if initial.backup_policy == "skip" and not accept_no_backup: + raise MigrationError("backup_confirmation_required", "Accept that original data may be unrecoverable.") + if initial.backup_policy == "auto" and not initial.backup_available: + raise MigrationError("backup_unsupported", "; ".join(initial.backup_reason)) + + def _apply_locked( + self, + connection: Connection | None, + identity: BackendIdentity, + guard: Callable[[], None], + *, + initial: MigrationPlan, + manual_ref: str | None, + ) -> MigrationResult: + if connection is None: + raise MigrationError("target_unavailable", "The maintenance connection is unavailable.") + guard() + self._assert_unchanged() + if ( + self._inspect( + connection, + identity, + backup_policy=initial.backup_policy, + service=initial.service, + shared_database=initial.shared_database, + ) + != initial + ): + raise MigrationError("plan_changed", "The database changed before the maintenance lock was acquired.") + journal = _Journal(self.evidence_directory, identity) + provider = self.backup_factory(connection, identity, journal.path.parent / "backups") + evidence = self._prepare_evidence(initial, journal, provider, manual_ref=manual_ref) + guard() + self._assert_unchanged() + if self.schema.fingerprint(mysql_inventory(connection)) != initial.schema_fingerprint: + raise MigrationError("plan_changed", "The source schema changed during backup.") + config = self._config(connection) + if initial.adopt_baseline: + self._adopt(connection, config, journal, evidence, initial.source_revision, guard) + current = initial.source_revision + for target in self.bundle.pending(current): + # An interrupted prior run can have committed Alembic's version row + # before saving the journal. Normalize that observed head first. + evidence.last_revision = current + evidence.pending_revision = target + journal.write(evidence) + self._upgrade(connection, config, current, target, guard) + evidence.last_revision = target + evidence.pending_revision = None + evidence.release_checksums[target] = self.checksums[target] + journal.write(evidence) + current = target + self.schema.verify(mysql_inventory(connection), self.bundle.head) + self.schema.validate_data(connection, self.bundle.head) + guard() + evidence.state = "complete" + evidence.last_revision = self.bundle.head + evidence.pending_revision = None + evidence.adopting = False + evidence.release_checksums = self.checksums.copy() + journal.write(evidence) + if evidence.backup_state == "pending": + raise MigrationError("backup_required", "The backup decision has not completed.") + return MigrationResult( + revision=self.bundle.head, + changed=True, + backup_state=evidence.backup_state, + backup_ref=evidence.backup.location if evidence.backup else evidence.manual_ref, + ) + + def _adopt( + self, + connection: Connection, + config: Config, + journal: _Journal, + evidence: _Evidence, + revision: str | None, + guard: Callable[[], None], + ) -> None: + if revision is None: + raise MigrationError("unknown_baseline", "Cannot stamp an unidentified baseline.") + self.schema.verify(mysql_inventory(connection), revision) + evidence.adopting = True + journal.write(evidence) + connection.commit() + guard() + with ALEMBIC_CONTEXT_LOCK: + command.stamp(config, revision) + connection.commit() + guard() + evidence.adopting = False + journal.write(evidence) + + def _upgrade( + self, + connection: Connection, + config: Config, + source: str | None, + target: str, + guard: Callable[[], None], + ) -> None: + def upgrade(revision: str) -> None: + if revision != target: + raise MigrationError("invalid_bundle", "Alembic selected an unexpected migration revision.") + guard() + recovery = self.schema.recovery(mysql_inventory(connection), source) + if recovery.revision != revision: + raise MigrationError("recovery_required", "The stored intent does not match the schema transition.") + for statement in recovery.pending_statements: + guard() + connection.exec_driver_sql(statement) + # DDL may already have committed; rollback cannot undo it. + guard() + self.schema.recovery(mysql_inventory(connection), source) + self.schema.verify(mysql_inventory(connection), revision) + self.schema.validate_data(connection, revision) + guard() + + config.attributes["execute_revision"] = upgrade + connection.commit() + guard() + with ALEMBIC_CONTEXT_LOCK: + command.upgrade(config, target) + connection.commit() + guard() + self.schema.verify(mysql_inventory(connection), target) + + def _config(self, connection: Connection) -> Config: + config = Config() + config.set_main_option("script_location", self._snapshot.name.replace("%", "%%")) + config.attributes["connection"] = connection + return config + + def _inspect( + self, + connection: Connection | None, + identity: BackendIdentity, + *, + backup_policy: BackupPolicy, + service: dict[str, Any] | None = None, + shared_database: bool = False, + ) -> MigrationPlan: + if backup_policy not in {"auto", "manual", "skip"}: + raise MigrationError("invalid_backup_policy", "Choose auto, manual or skip.") + journal = _Journal(self.evidence_directory, identity) + evidence = journal.read() + inventory = ( + mysql_inventory(connection) if connection is not None else {"tables": {}, "views": [], "triggers": []} + ) + initialized, current = read_revision(connection, inventory) if connection is not None else (False, None) + self._validate_evidence(evidence) + current, adopt = self._recognize_source(inventory, initialized, current, evidence) + if initialized and self.bundle.pending(current) and evidence is None: + raise MigrationError("recovery_required", "The original external maintenance evidence is missing.") + self._validate_position(evidence, current) + if connection is not None and current is not None: + self.schema.validate_data(connection, current) + fingerprint = self.schema.fingerprint(inventory) + state = "migration_required" if self.bundle.pending(current) or adopt else "ready" + if not initialized and current is None: + state = "uninitialized" + if evidence and evidence.state != "complete": + state = "recovery_required" + if ( + evidence.backup_policy != backup_policy + and backup_policy == "auto" + and not ( + evidence.state == "preparing" + and evidence.backup_state == "pending" + and evidence.source_fingerprint == fingerprint + ) + ): + raise MigrationError( + "recovery_required", "Do not create an automatic backup of partially migrated state." + ) + available, reasons = self._backup_availability( + connection, identity, journal, evidence, current, backup_policy=backup_policy, ready=state == "ready" + ) + fields = { + "database_id": identity.database_id, + "source_revision": current, + "target_revision": self.bundle.head, + "schema_fingerprint": fingerprint, + "bundle_checksum": self.checksum, + "configuration_digest": self.configuration_digest, + "state": state, + "revisions": self.bundle.pending(current), + "adopt_baseline": adopt, + "backup_policy": backup_policy, + "backup_available": available, + "backup_reason": reasons, + "affected_objects": self.bundle.affected_objects, + "retained_objects": (), + "service": service, + "shared_database": shared_database, + } + return MigrationPlan.model_validate({"plan_id": digest(fields), **fields}) + + def _validate_evidence(self, evidence: _Evidence | None) -> None: + if evidence is None: + return + for revision, checksum in evidence.release_checksums.items(): + if self.checksums.get(revision) != checksum: + raise MigrationError("checksum_conflict", "A recorded released migration resource changed.") + if evidence.state != "complete" and ( + evidence.bundle_checksum != self.checksum or evidence.configuration_digest != self.configuration_digest + ): + raise MigrationError("recovery_required", "Keep the original migration resources and configuration.") + for revision in (evidence.source_revision, evidence.last_revision, evidence.pending_revision): + if revision is not None and revision not in self.checksums: + raise MigrationError("recovery_required", "The maintenance evidence contains an unknown revision.") + if evidence.pending_revision is not None and ( + self.bundle.pending(evidence.last_revision)[:1] != (evidence.pending_revision,) + ): + raise MigrationError("recovery_required", "The pending revision is not the next registered migration.") + if evidence.state == "complete" and (evidence.pending_revision is not None or evidence.adopting): + raise MigrationError("recovery_required", "Completed maintenance evidence still records unfinished work.") + + def _validate_position(self, evidence: _Evidence | None, current: str | None) -> None: + if evidence is None: + return + if evidence.state == "complete": + if current != evidence.last_revision: + raise MigrationError("recovery_required", "The database differs from its completed maintenance record.") + elif current not in {evidence.last_revision, evidence.pending_revision}: + raise MigrationError("recovery_required", "The database head is outside the stored migration intent.") + + def _recognize_source( + self, + inventory: dict[str, Any], + initialized: bool, + current: str | None, + evidence: _Evidence | None, + ) -> tuple[str | None, bool]: + if not initialized: + try: + current = self.schema.identify(inventory) + except MigrationError as error: + raise MigrationError( + "unknown_baseline", "Schema is not a registered baseline; no changes were made." + ) from error + if current is not None and current not in set(self.bundle.baselines.values()): + raise MigrationError( + "unknown_baseline", "This unversioned layout is not a registered adoption baseline." + ) + return current, current is not None + if evidence and evidence.adopting and current is None: + current = evidence.source_revision + if current is None: + raise MigrationError("recovery_required", "The adoption intent has no source revision.") + self.schema.verify(inventory, current) + return current, True + try: + self.schema.verify(inventory, current) + except MigrationError: + if not evidence or evidence.state != "active" or evidence.pending_revision is None: + raise MigrationError("recovery_required", "Partial DDL requires the original durable intent.") from None + recovery = self.schema.recovery(inventory, current) + if recovery.revision != evidence.pending_revision: + raise MigrationError( + "recovery_required", "Partial DDL does not match the stored migration intent." + ) from None + return current, False + + def _backup_availability( + self, + connection: Connection | None, + identity: BackendIdentity, + journal: _Journal, + evidence: _Evidence | None, + current: str | None, + *, + backup_policy: BackupPolicy, + ready: bool, + ) -> tuple[bool, tuple[str, ...]]: + if backup_policy != "auto" or ready or connection is None or current is None: + return True, () + active = evidence if evidence and evidence.state != "complete" else None + if active and active.backup_state == "not_required": + return True, () + provider = self.backup_factory(connection, identity, journal.path.parent / "backups") + if active and active.backup: + available = provider.inspect_backup(active.backup).state == "completed" + return available, () if available else ("The original automatic recovery point is unavailable.",) + capability = provider.capabilities(self._backup_context(identity.database_id, current, "preview")) + if capability.available and capability.method == "fork_table": + return False, ( + "This executor has no accepted inventory and recovery workflow for same-database Fork tables.", + ) + return capability.available, capability.reasons + + def _backup_context(self, database_id: str, revision: str | None, window: str) -> BackupContext: + return BackupContext( + database_id=database_id, + source_revision=revision, + bundle_checksum=self.checksum, + maintenance_window_id=window, + objects=self.bundle.affected_objects, + writers_stopped=True, + ) + + def _prepare_evidence( + self, plan: MigrationPlan, journal: _Journal, provider: BackupProvider, *, manual_ref: str | None + ) -> _Evidence: + previous = journal.read() + if previous and previous.state != "complete": + if self._reuse_evidence(previous, plan, journal, provider, manual_ref=manual_ref): + return previous + if previous.source_fingerprint != plan.schema_fingerprint: + raise MigrationError("backup_required", "Do not replace the original backup after schema changes.") + evidence = previous + else: + evidence = _Evidence( + state="preparing", + database_id=plan.database_id, + source_revision=plan.source_revision, + source_fingerprint=plan.schema_fingerprint, + bundle_checksum=self.checksum, + configuration_digest=self.configuration_digest, + last_revision=plan.source_revision, + maintenance_window_id=uuid4().hex, + backup_policy=plan.backup_policy, + backup_state="pending", + release_checksums=previous.release_checksums.copy() if previous else {}, + ) + journal.write(evidence) + if evidence.source_revision is None: + evidence.backup_state = "not_required" + elif plan.backup_policy == "auto": + ref = provider.create_backup( + self._backup_context(plan.database_id, evidence.source_revision, evidence.maintenance_window_id) + ) + if provider.inspect_backup(ref).state != "completed": + raise MigrationError("backup_failed", "The native recovery point did not pass provider inspection.") + evidence.backup = ref + evidence.backup_state = "completed" + elif plan.backup_policy == "manual": + evidence.backup_state = "user_confirmed" + evidence.manual_ref = manual_ref + else: + evidence.backup_state = "skipped" + evidence.state = "active" + journal.write(evidence) + return evidence + + @staticmethod + def _reuse_evidence( + previous: _Evidence, + plan: MigrationPlan, + journal: _Journal, + provider: BackupProvider, + *, + manual_ref: str | None, + ) -> bool: + if previous.backup_policy != plan.backup_policy: + previous.backup_history.append({ + key: previous.model_dump(mode="json")[key] + for key in ("backup_policy", "backup_state", "backup", "manual_ref") + }) + previous.backup_policy = plan.backup_policy + previous.backup = None + previous.manual_ref = manual_ref if plan.backup_policy == "manual" else None + if plan.backup_policy == "auto": + return False + previous.backup_state = "user_confirmed" if plan.backup_policy == "manual" else "skipped" + previous.state = "active" + journal.write(previous) + return True + if previous.backup_state == "completed": + if previous.backup is None or provider.inspect_backup(previous.backup).state != "completed": + raise MigrationError("backup_required", "The original automatic recovery point is unavailable.") + return True + return previous.backup_state in {"user_confirmed", "skipped", "not_required"} + + +def _fork_provider(connection: Connection, identity: BackendIdentity, directory: Path) -> BackupProvider: + # Engine feature versions are insufficient to claim a verified restoration + # workflow. An adapter without release acceptance reports unavailable. + return ForkBackupProvider( + connection, + database_name=identity.database_name, + expected_product="seekdb" if identity.product == "seekdb" else "oceanbase_ai", + directory=directory, + ) diff --git a/src/powercontext/builtin/persistence/migrations/mysql_resources/env.py b/src/powercontext/builtin/persistence/migrations/mysql_resources/env.py new file mode 100644 index 0000000000..2db6a78803 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/mysql_resources/env.py @@ -0,0 +1,26 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Advance the standard Alembic head only after a registered DDL postcondition.""" + +from alembic import context + +context.configure( + connection=context.config.attributes["connection"], + version_table="pc_schema_revision", + transactional_ddl=False, + transaction_per_migration=True, +) +with context.begin_transaction(): + context.run_migrations() diff --git a/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0001_baseline.py b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0001_baseline.py new file mode 100644 index 0000000000..5e0cfa44e7 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0001_baseline.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Initialize the frozen MySQL baseline with declared partial-DDL recovery.""" + +from alembic import context + +revision = "p0001" +down_revision = None + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) diff --git a/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0002_citations.py b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0002_citations.py new file mode 100644 index 0000000000..960a1f2615 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0002_citations.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Add both citation columns, recognizing only registered intermediate states.""" + +from alembic import context + +revision = "p0002" +down_revision = "p0001" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) diff --git a/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0003_tag_families.py b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0003_tag_families.py new file mode 100644 index 0000000000..6b2c25edf1 --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/mysql_resources/versions/p0003_tag_families.py @@ -0,0 +1,24 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Relax the family check in place without deleting historical business tables.""" + +from alembic import context + +revision = "p0003" +down_revision = "p0002" + + +def upgrade() -> None: + context.config.attributes["execute_revision"](revision) diff --git a/src/powercontext/builtin/persistence/migrations/mysql_schema.py b/src/powercontext/builtin/persistence/migrations/mysql_schema.py new file mode 100644 index 0000000000..b3502db1bc --- /dev/null +++ b/src/powercontext/builtin/persistence/migrations/mysql_schema.py @@ -0,0 +1,699 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Recognize the frozen MySQL layouts without importing application metadata. + +Only the reviewed linear p0001--p0003 history is supported. DDL recovery accepts +an exact prefix of the next revision, never an arbitrary collection of objects. +The caller owns maintenance evidence, backup, the pinned database lock and the +version row; this module never writes or stamps a revision. +""" + +from __future__ import annotations + +import copy +import json +import re +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import Connection +from sqlalchemy.exc import DBAPIError + +from .bundle import MigrationBundle +from .models import MigrationError, digest + +_LEXEME = re.compile(r"\s+|`(?:``|[^`])*`|'(?:''|\\.|[^'\\])*'|[A-Za-z_][A-Za-z_0-9$]*|[0-9]+|>=|<=|<>|!=|[(),.=<>+-]") +_REVISIONS = ("p0001", "p0002", "p0003") +_CITATION_TABLES = ("pc_artifacts", "pc_artifact_candidate_versions") +_FAMILY_CHECK = "ck_pc_artifact_tags_family" +_CONTROL_TABLE = "pc_schema_revision" +_PHYSICAL_VALUES = frozenset({ + "row_format", + "compression", + "replica_num", + "block_size", + "key_block_size", + "use_bloom_filter", + "enable_macro_block_bloom_filter", + "tablet_size", + "pctfree", + "auto_increment", +}) + + +@dataclass(frozen=True) +class MySQLRecovery: + revision: str + pending_statements: tuple[str, ...] + + @property + def target_revision(self) -> str: + return self.revision + + @property + def completed_revision(self) -> str | None: + """A version may advance only after the entire postcondition is present.""" + return self.revision if not self.pending_statements else None + + +def mysql_inventory(connection: Connection, *, exclude: frozenset[str] = frozenset()) -> dict[str, Any]: + """Reflect tables, views and triggers on the caller's existing connection. + + SHOW CREATE preserves constraints, index prefixes and enforcement that a + partial column reflection can lose. Unknown syntax is rejected. Physical + placement/compression options do not define the application's schema. + """ + if connection.dialect.name != "mysql": + raise MigrationError("unsupported_backend", "MySQL schema reflection needs a MySQL-compatible connection.") + try: + database = connection.exec_driver_sql("SELECT DATABASE()").scalar_one() + if not isinstance(database, str) or not database: + raise MigrationError("incompatible_schema", "A current database must be selected for schema reflection.") + inventory = _empty_inventory() + _reflect_tables(connection, database, exclude, inventory) + # Do not hide a trigger merely because it writes an excluded version + # table: it would still change the migration's authoritative state. + inventory["triggers"] = sorted( + [ + {field: row[field] for field in ("Trigger", "Table", "Timing", "Event", "Statement")} + for row in connection.exec_driver_sql("SHOW TRIGGERS").mappings() + ], + key=lambda row: str(row.get("Trigger", "")), + ) + inventory["views"].sort() + except DBAPIError as error: + raise MigrationError( + "schema_reflection_failed", "Complete database schema reflection is unavailable." + ) from error + except (IndexError, KeyError, TypeError, ValueError) as error: + raise MigrationError( + "incompatible_schema", "A schema definition could not be completely interpreted." + ) from error + else: + return inventory + + +def _reflect_tables(connection: Connection, database: str, exclude: frozenset[str], inventory: dict[str, Any]) -> None: + for name, kind in connection.exec_driver_sql("SHOW FULL TABLES"): + if kind == "VIEW": + inventory["views"].append(name) + elif kind != "BASE TABLE": + raise MigrationError("incompatible_schema", "An unsupported database object was discovered.") + elif name not in exclude: + row = connection.exec_driver_sql(f"SHOW CREATE TABLE {_quote(name)}").first() + if row is None or not isinstance(row[1], str): + raise MigrationError("incompatible_schema", "A table definition could not be reflected.") + reflected_name, table = _parse_table(row[1], database=database) + if reflected_name != name: + raise MigrationError("incompatible_schema", "A reflected table identity is inconsistent.") + inventory["tables"][name] = table + + +def read_revision(connection: Connection, inventory: dict[str, Any]) -> tuple[bool, str | None]: + """Validate Alembic's only control table before reading its zero/one head.""" + _business_inventory(inventory) + if _CONTROL_TABLE not in inventory["tables"]: + return False, None + rows = connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision LIMIT 2").all() + if len(rows) > 1 or (rows and rows[0][0] not in _REVISIONS): + raise MigrationError("incompatible_schema", "The database has multiple or unknown migration heads.") + return True, rows[0][0] if rows else None + + +class MySQLSchema: + """Validate source, intermediate and post-DDL shapes from frozen resources.""" + + def __init__(self, bundle: MigrationBundle) -> None: + if bundle.revisions != _REVISIONS: + raise MigrationError("invalid_bundle", "MySQL schema recognition does not cover this revision chain.") + self.bundle = bundle + baseline = json.loads((bundle.snapshot_directory / "schemas/pre_dream.json").read_text())["mysql"] + current = json.loads((bundle.snapshot_directory / "schemas/v1_1_0.json").read_text())["mysql"] + self._statements = { + "p0001": tuple(baseline), + "p0002": tuple(f"ALTER TABLE {table} ADD COLUMN memory_citations MEDIUMBLOB" for table in _CITATION_TABLES), + "p0003": (f"ALTER TABLE pc_artifact_tags DROP CHECK {_FAMILY_CHECK}",), + } + inventory = _empty_inventory() + self._prefixes: dict[str, list[dict[str, Any]]] = {} + self._schemas: dict[str, dict[str, Any]] = {} + for revision in _REVISIONS: + prefixes = [copy.deepcopy(inventory)] + for statement in self._statements[revision]: + _apply_statement(inventory, statement) + prefixes.append(copy.deepcopy(inventory)) + self._prefixes[revision] = prefixes + self._schemas[revision] = copy.deepcopy(inventory) + frozen_current = _empty_inventory() + for statement in current: + _apply_statement(frozen_current, statement) + if frozen_current != self._schemas["p0003"]: + raise MigrationError("invalid_bundle", "Frozen MySQL layouts disagree with the registered revision steps.") + + @staticmethod + def fingerprint(inventory: dict[str, Any]) -> str: + return digest(inventory) + + def identify(self, inventory: dict[str, Any]) -> str | None: + """Only complete known shapes may be adopted as an unversioned source.""" + inventory = _business_inventory(inventory) + if inventory == _empty_inventory(): + return None + for revision, expected in self._schemas.items(): + if _matches(inventory, expected): + return revision + raise MigrationError("incompatible_schema", "Actual MySQL schema is not a registered complete layout.") + + def verify(self, inventory: dict[str, Any], revision: str | None) -> None: + inventory = _business_inventory(inventory) + expected = _empty_inventory() if revision is None else self._schemas.get(revision) + if expected is None or not _matches(inventory, expected): + raise MigrationError("incompatible_schema", "Actual MySQL schema does not match its recorded revision.") + + def recovery(self, inventory: dict[str, Any], revision: str | None) -> MySQLRecovery: + """Return only missing DDL in an exact prefix of the next revision. + + A runner must require matching active evidence before accepting a + partially executed revision. A recognized postcondition is necessary + for advancing the version row, and is not itself recovery authorization. + """ + inventory = _business_inventory(inventory) + if revision is None: + target = _REVISIONS[0] + elif revision not in _REVISIONS or revision == _REVISIONS[-1]: + raise MigrationError("incompatible_schema", "There is no registered next MySQL revision to recover.") + else: + target = _REVISIONS[_REVISIONS.index(revision) + 1] + for position, expected in enumerate(self._prefixes[target]): + if _matches(inventory, expected): + return MySQLRecovery(target, self._statements[target][position:]) + raise MigrationError("incompatible_schema", "Actual MySQL schema is not a controlled next-revision DDL state.") + + def validate_data(self, connection: Connection, revision: str) -> None: + """Check stored CHECK/FK invariants before adoption or version advance. + + Enabling foreign_key_checks does not validate existing rows. The queries + return at most one existence marker, never application payloads. They + can scan tables and belong inside the offline maintenance window. + """ + expected = self._schemas.get(revision) + if expected is None: + raise MigrationError("incompatible_schema", "Cannot validate data for an unknown MySQL revision.") + for name, table in expected["tables"].items(): + for check in table["checks"].values(): + expression = _expression_sql(check["expression"]) + violation = connection.exec_driver_sql( + f"SELECT 1 FROM {_quote(name)} WHERE NOT ({expression}) LIMIT 1" # noqa: S608 -- frozen expressions + ).first() + if violation is not None: + raise MigrationError("data_integrity_error", "Existing rows violate a registered CHECK constraint.") + for foreign_key in table["foreign_keys"]: + columns = foreign_key["columns"] + referred = foreign_key["referred_columns"] + nonnull = " AND ".join(f"s.{_quote(column)} IS NOT NULL" for column in columns) + equality = " AND ".join( + f"s.{_quote(left)} = r.{_quote(right)}" for left, right in zip(columns, referred, strict=True) + ) + # A FK-backed outer join can hide historical orphans on seekdb + # after enforcement is re-enabled; use a correlated existence + # check rather than trusting that already-declared constraint. + violation = connection.exec_driver_sql( + f"SELECT 1 FROM {_quote(name)} AS s WHERE {nonnull} AND NOT EXISTS " # noqa: S608 -- frozen, quoted identifiers + f"(SELECT 1 FROM {_quote(foreign_key['referred_table'])} AS r WHERE {equality}) LIMIT 1" + ).first() + if violation is not None: + raise MigrationError("data_integrity_error", "Existing rows violate a registered foreign key.") + + +def _empty_inventory() -> dict[str, Any]: + return {"tables": {}, "views": [], "triggers": []} + + +def _business_inventory(inventory: dict[str, Any]) -> dict[str, Any]: + table = inventory["tables"].get(_CONTROL_TABLE) + if table is None: + return inventory + _, expected = _parse_table( + "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, PRIMARY KEY (version_num))" + ) + if any(trigger.get("Table") == _CONTROL_TABLE for trigger in inventory["triggers"]) or not _matches( + {"tables": {_CONTROL_TABLE: table}, "views": [], "triggers": []}, + {"tables": {_CONTROL_TABLE: expected}, "views": [], "triggers": []}, + ): + raise MigrationError("incompatible_schema", "The migration version table has an unsupported definition.") + business = copy.deepcopy(inventory) + del business["tables"][_CONTROL_TABLE] + return business + + +def _quote(identifier: str) -> str: + return "`" + identifier.replace("`", "``") + "`" + + +def _tokens(statement: str) -> list[str]: + tokens: list[str] = [] + position = 0 + for match in _LEXEME.finditer(statement): + if match.start() != position: + raise MigrationError("incompatible_schema", "An unsupported schema definition lexeme was discovered.") + position = match.end() + value = match.group() + if value.isspace(): + continue + if value.startswith("`"): + value = value[1:-1].replace("``", "`") + tokens.append(value) + else: + tokens.append(value if value.startswith("'") else value.lower()) + if position != len(statement): + raise MigrationError("incompatible_schema", "An unsupported schema definition lexeme was discovered.") + return tokens + + +def _group(tokens: list[str], position: int) -> tuple[list[str], int]: + if position >= len(tokens) or tokens[position] != "(": + raise MigrationError("incompatible_schema", "A schema definition is missing a parenthesized group.") + depth = 0 + for end in range(position, len(tokens)): + lexeme = tokens[end] + if lexeme == "(": + depth += 1 + elif lexeme == ")": + depth -= 1 + if depth == 0: + return tokens[position + 1 : end], end + 1 + raise MigrationError("incompatible_schema", "A schema definition has unbalanced parentheses.") + + +def _parts(tokens: list[str]) -> list[list[str]]: + parts: list[list[str]] = [] + current: list[str] = [] + depth = 0 + for lexeme in tokens: + if lexeme == "," and depth == 0: + parts.append(current) + current = [] + else: + current.append(lexeme) + depth += (lexeme == "(") - (lexeme == ")") + parts.append(current) + if any(not part for part in parts): + raise MigrationError("incompatible_schema", "A schema definition has an empty clause.") + return parts + + +def _physical(tokens: list[str]) -> dict[str, str | None]: + options: dict[str, str | None] = {"charset": None, "collation": None} + position = 0 + while position < len(tokens): + lexeme = tokens[position] + if lexeme in {"default", "global", "local", "visible"}: + position += 1 + continue + if lexeme == "character" and tokens[position + 1 : position + 2] == ["set"]: + lexeme = "charset" + position += 1 + if lexeme in {"charset", "collate", "engine", "using", "organization", *_PHYSICAL_VALUES}: + position += 1 + if tokens[position : position + 1] == ["="]: + position += 1 + if position >= len(tokens): + raise MigrationError("incompatible_schema", "A schema option is missing its value.") + value = tokens[position] + _validate_physical_value(lexeme, value) + if lexeme in {"charset", "collate"}: + options["charset" if lexeme == "charset" else "collation"] = value + position += 1 + continue + raise MigrationError("incompatible_schema", "An unsupported schema option was discovered.") + return options + + +def _validate_physical_value(option: str, value: str) -> None: + supported = {"engine": {"innodb"}, "using": {"btree"}, "organization": {"index", "heap"}} + if option in supported and value not in supported[option]: + raise MigrationError("incompatible_schema", "An unsupported table engine or index organization was discovered.") + + +def _column(tokens: list[str], defaults: dict[str, str | None]) -> dict[str, Any]: + if len(tokens) < 2: + raise MigrationError("incompatible_schema", "A column definition is incomplete.") + kind = {"integer": "int"}.get(tokens[1], tokens[1]) + position = 2 + arguments: list[str] = [] + if tokens[position : position + 1] == ["("]: + arguments, position = _group(tokens, position) + if kind in {"int", "bigint"}: + if arguments and (len(arguments) != 1 or not arguments[0].isdigit()): + raise MigrationError("incompatible_schema", "An integer column has unsupported arguments.") + arguments = [] + if kind == "datetime" and arguments == ["0"]: + arguments = [] + column: dict[str, Any] = { + "type": [kind, arguments], + "nullable": True, + "default": None, + "charset": None, + "collation": None, + "extra": [], + } + if kind in {"varchar", "char", "mediumtext", "text"}: + column["charset"] = defaults["charset"] + column["collation"] = defaults["collation"] + _column_modifiers(tokens, position, column) + if column["collation"] and not column["charset"]: + column["charset"] = column["collation"].split("_", 1)[0] + return column + + +def _column_character_option(tokens: list[str], position: int) -> tuple[str, str, int]: + option = tokens[position] + if option == "character": + if tokens[position + 1 : position + 2] != ["set"]: + raise MigrationError("incompatible_schema", "An unsupported character column option was discovered.") + position += 1 + position += 1 + if position >= len(tokens): + raise MigrationError("incompatible_schema", "A character column option has no value.") + return ("collation" if option == "collate" else "charset"), tokens[position], position + 1 + + +def _column_default(tokens: list[str], position: int, kind: str) -> tuple[list[Any] | None, int]: + position += 1 + if tokens[position : position + 1] == ["_utf8mb4"]: + position += 1 + if position >= len(tokens): + raise MigrationError("incompatible_schema", "A column default has no value.") + value = tokens[position] + if value == "null": + default = None + elif value.startswith("'"): + literal = _literal(value) + default = ["number", int(literal)] if kind in {"int", "bigint"} and literal.isdigit() else ["literal", literal] + elif value.isdigit(): + default = ["number", int(value)] + else: + raise MigrationError("incompatible_schema", "An unsupported column default was discovered.") + return default, position + 1 + + +def _column_modifiers(tokens: list[str], position: int, column: dict[str, Any]) -> None: + while position < len(tokens): + lexeme = tokens[position] + if lexeme == "not" and tokens[position + 1 : position + 2] == ["null"]: + column["nullable"] = False + position += 2 + elif lexeme == "null": + position += 1 + elif lexeme in {"character", "charset", "collate"}: + option, value, position = _column_character_option(tokens, position) + column[option] = value + elif lexeme == "default": + column["default"], position = _column_default(tokens, position, column["type"][0]) + else: + column["extra"].append(lexeme) + position += 1 + + +def _key_columns(tokens: list[str]) -> list[list[Any]]: + columns: list[list[Any]] = [] + for part in _parts(tokens): + position = 1 + prefix = None + if part[position : position + 1] == ["("]: + length, position = _group(part, position) + if len(length) != 1 or not length[0].isdigit(): + raise MigrationError("incompatible_schema", "An unsupported index prefix was discovered.") + prefix = int(length[0]) + order = "asc" + if part[position : position + 1] in (["asc"], ["desc"]): + order = part[position] + position += 1 + if position != len(part) or not re.fullmatch(r"[a-z_][a-z_0-9$]*", part[0]): + raise MigrationError("incompatible_schema", "An unsupported index key was discovered.") + columns.append([part[0], prefix, order]) + return columns + + +def _qualified(tokens: list[str], position: int, database: str | None) -> tuple[str | None, str, int]: + name = tokens[position] + position += 1 + schema = None + if tokens[position : position + 1] == ["."]: + schema = name + name = tokens[position + 1] + position += 2 + return (None if schema == database else schema), name, position + + +def _foreign_key(part: list[str], database: str | None) -> dict[str, Any]: + columns, end = _group(part, 2) + if part[end : end + 1] != ["references"]: + raise MigrationError("incompatible_schema", "A foreign key has no referenced table.") + schema, referred, end = _qualified(part, end + 1, database) + referred_columns, end = _group(part, end) + local_keys = _key_columns(columns) + referred_keys = _key_columns(referred_columns) + if len(local_keys) != len(referred_keys) or any( + prefix is not None or order != "asc" for _, prefix, order in [*local_keys, *referred_keys] + ): + raise MigrationError("incompatible_schema", "A foreign key has unsupported key columns.") + actions = {"delete": "restrict", "update": "restrict"} + while end < len(part): + if part[end] != "on" or part[end + 1] not in actions: + raise MigrationError("incompatible_schema", "An unsupported foreign key option was discovered.") + action = part[end + 1] + end += 2 + value = part[end] + end += 1 + if value in {"no", "set"}: + value += " " + part[end] + end += 1 + actions[action] = "restrict" if value == "no action" else value + return { + "columns": [column[0] for column in local_keys], + "referred_schema": schema, + "referred_table": referred, + "referred_columns": [column[0] for column in referred_keys], + "ondelete": actions["delete"], + "onupdate": actions["update"], + } + + +def _parse_table(statement: str, *, database: str | None = None) -> tuple[str, dict[str, Any]]: + tokens = _tokens(statement) + if tokens[:2] != ["create", "table"]: + raise MigrationError("incompatible_schema", "An unsupported table definition was discovered.") + schema, name, position = _qualified(tokens, 2, database) + if schema is not None: + raise MigrationError("incompatible_schema", "A table belongs to an unexpected database.") + body, position = _group(tokens, position) + defaults = _physical(tokens[position:]) + table: dict[str, Any] = {"columns": {}, "primary_key": [], "foreign_keys": [], "checks": {}, "indexes": {}} + for part in _parts(body): + _table_clause(table, part, defaults, database) + table["foreign_keys"].sort(key=digest) + return name, table + + +def _put_named(objects: dict[str, Any], name: str, definition: dict[str, Any]) -> None: + if name in objects: + raise MigrationError("incompatible_schema", "Duplicate schema object names were discovered.") + objects[name] = definition + + +def _check_definition(part: list[str], constraint: str | None) -> tuple[str, dict[str, Any]]: + expression, end = _group(part, 1) + if constraint is None or part[end:] not in ([], ["enforced"], ["not", "enforced"]): + raise MigrationError("incompatible_schema", "An unsupported CHECK definition was discovered.") + return constraint, {"expression": _Expression(expression).parse(), "enforced": part[end:] != ["not", "enforced"]} + + +def _index_definition(part: list[str]) -> tuple[str, dict[str, Any]]: + unique = part[0] == "unique" + offset = 2 if unique and part[1] in {"key", "index"} else 1 + index_name = part[offset] + columns, end = _group(part, offset + 1) + _physical(part[end:]) + return index_name, {"columns": _key_columns(columns), "unique": unique} + + +def _table_clause( + table: dict[str, Any], part: list[str], defaults: dict[str, str | None], database: str | None +) -> None: + constraint = None + if part[0] == "constraint": + constraint = part[1] + part = part[2:] + if part[:2] == ["primary", "key"]: + columns, end = _group(part, 2) + _physical(part[end:]) + if table["primary_key"]: + raise MigrationError("incompatible_schema", "Duplicate primary keys were discovered.") + table["primary_key"] = _key_columns(columns) + elif part[:2] == ["foreign", "key"]: + table["foreign_keys"].append(_foreign_key(part, database)) + elif part[0] == "check": + constraint, definition = _check_definition(part, constraint) + _put_named(table["checks"], constraint, definition) + elif part[0] in {"key", "index", "unique"}: + index_name, definition = _index_definition(part) + _put_named(table["indexes"], index_name, definition) + elif constraint is None: + _put_named(table["columns"], part[0], _column(part, defaults)) + else: + raise MigrationError("incompatible_schema", "An unsupported constraint definition was discovered.") + + +def _apply_statement(inventory: dict[str, Any], statement: str) -> None: + tokens = _tokens(statement) + tables = inventory["tables"] + if tokens[:2] == ["create", "table"]: + name, table = _parse_table(statement) + if name in tables: + raise MigrationError("invalid_bundle", "A frozen table is declared more than once.") + tables[name] = table + elif tokens[:2] == ["create", "index"] and tokens[3:4] == ["on"]: + columns, end = _group(tokens, 5) + if end != len(tokens) or tokens[2] in tables[tokens[4]]["indexes"]: + raise MigrationError("invalid_bundle", "An unsupported frozen index was discovered.") + tables[tokens[4]]["indexes"][tokens[2]] = {"columns": _key_columns(columns), "unique": False} + elif tokens[:2] == ["alter", "table"] and tokens[3:5] == ["add", "column"]: + table = tables[tokens[2]] + if tokens[5] in table["columns"]: + raise MigrationError("invalid_bundle", "A frozen column is declared more than once.") + table["columns"][tokens[5]] = _column(tokens[5:], {"charset": None, "collation": None}) + elif tokens[:2] == ["alter", "table"] and tokens[3:5] == ["drop", "check"] and len(tokens) == 6: + del tables[tokens[2]]["checks"][tokens[5]] + else: + raise MigrationError("invalid_bundle", "A frozen MySQL revision operation is not recognized.") + + +def _matches(actual: dict[str, Any], expected: dict[str, Any]) -> bool: + normalized = copy.deepcopy(actual) + if set(normalized) != set(expected) or set(normalized["tables"]) != set(expected["tables"]): + return False + for name, table in normalized["tables"].items(): + expected_table = expected["tables"][name] + if set(table["columns"]) != set(expected_table["columns"]): + return False + for column, specification in table["columns"].items(): + # The frozen SQL inherits backend defaults for display text only. + # Identity columns explicitly require utf8mb4_bin and remain exact. + for option in ("charset", "collation"): + if expected_table["columns"][column][option] is None: + specification[option] = None + automatic_keys: set[tuple[str, ...]] = set() + for index_name, index in tuple(table["indexes"].items()): + if index_name in expected_table["indexes"] or index["unique"]: + continue + columns = tuple(key[0] for key in index["columns"]) + foreign_columns = {tuple(key["columns"]) for key in table["foreign_keys"]} + if ( + columns in foreign_columns + and columns not in automatic_keys + and all(prefix is None and order == "asc" for _, prefix, order in index["columns"]) + ): + # MySQL can generate one child-key index; OceanBase need not. + # Ignore its generated name, not arbitrary prefixes/unique keys. + del table["indexes"][index_name] + automatic_keys.add(columns) + return normalized == expected + + +def _literal(lexeme: str) -> str: + value = lexeme[1:-1].replace("''", "'") + escapes = {"0": "\0", "b": "\b", "n": "\n", "r": "\r", "t": "\t", "Z": "\x1a"} + return re.sub(r"\\(.)", lambda match: escapes.get(match[1], match[1]), value) + + +class _Expression: + def __init__(self, tokens: list[str]) -> None: + self.tokens = tokens + self.position = 0 + + def parse(self) -> list[Any]: + value = self._expression(0) + if self.position != len(self.tokens): + raise MigrationError("incompatible_schema", "An unsupported CHECK expression was discovered.") + return value + + def _expression(self, precedence: int) -> list[Any]: + left = self._value() + priorities = {"or": 1, "and": 2, "=": 3, ">": 3, "<": 3, ">=": 3, "<=": 3, "!=": 3, "<>": 3, "is": 3, "in": 3} + while self.position < len(self.tokens): + operator = self.tokens[self.position] + priority = priorities.get(operator, 0) + if priority <= precedence: + break + self.position += 1 + if operator == "is": + negate = self.tokens[self.position : self.position + 1] == ["not"] + self.position += int(negate) + if self.tokens[self.position : self.position + 1] != ["null"]: + raise MigrationError("incompatible_schema", "An unsupported CHECK predicate was discovered.") + self.position += 1 + left = ["is_not_null" if negate else "is_null", left] + elif operator == "in": + values, self.position = _group(self.tokens, self.position) + left = ["in", left, [_Expression(part).parse() for part in _parts(values)]] + else: + left = ["binary", operator, left, self._expression(priority)] + return left + + def _value(self) -> list[Any]: + if self.position >= len(self.tokens): + raise MigrationError("incompatible_schema", "A CHECK expression is incomplete.") + lexeme = self.tokens[self.position] + self.position += 1 + if lexeme == "(": + value = self._expression(0) + if self.tokens[self.position : self.position + 1] != [")"]: + raise MigrationError("incompatible_schema", "A CHECK expression has unbalanced parentheses.") + self.position += 1 + return value + if lexeme == "_utf8mb4": + if self.position >= len(self.tokens) or not self.tokens[self.position].startswith("'"): + raise MigrationError("incompatible_schema", "A CHECK expression has an unsupported literal introducer.") + lexeme = self.tokens[self.position] + self.position += 1 + if lexeme.startswith("'"): + return ["literal", _literal(lexeme)] + if lexeme.isdigit(): + return ["number", int(lexeme)] + if lexeme == "null": + return ["null"] + if not re.fullmatch(r"[a-z_][a-z_0-9$]*", lexeme): + raise MigrationError("incompatible_schema", "An unsupported CHECK value was discovered.") + return ["column", lexeme] + + +def _expression_sql(expression: list[Any]) -> str: + kind = expression[0] + if kind == "column": + return _quote(expression[1]) + if kind == "literal": + return "'" + expression[1].replace("'", "''") + "'" + if kind == "number": + return str(expression[1]) + if kind == "null": + return "NULL" + if kind in {"is_null", "is_not_null"}: + return f"({_expression_sql(expression[1])} IS {'NOT ' if kind == 'is_not_null' else ''}NULL)" + if kind == "in": + return f"({_expression_sql(expression[1])} IN ({', '.join(_expression_sql(value) for value in expression[2])}))" + if kind == "binary": + return f"({_expression_sql(expression[2])} {expression[1].upper()} {_expression_sql(expression[3])})" + raise MigrationError("invalid_bundle", "An unsupported frozen CHECK expression was discovered.") diff --git a/src/powercontext/builtin/persistence/migrations/sqlite.py b/src/powercontext/builtin/persistence/migrations/sqlite.py index cf9bec0517..2664f2f440 100644 --- a/src/powercontext/builtin/persistence/migrations/sqlite.py +++ b/src/powercontext/builtin/persistence/migrations/sqlite.py @@ -24,7 +24,6 @@ import json import os import sqlite3 -import threading from collections.abc import Iterator, Mapping from contextlib import contextmanager from pathlib import Path @@ -37,7 +36,7 @@ from sqlalchemy.exc import DatabaseError, OperationalError from .backup import BackupContext, BackupProvider, BackupRef, SQLiteBackupProvider -from .bundle import MigrationBundle, sqlite_inventory +from .bundle import ALEMBIC_CONTEXT_LOCK, MigrationBundle, sqlite_inventory from .locking import local_migration_lock from .models import MigrationError, MigrationPlan, MigrationResult, digest @@ -46,7 +45,6 @@ "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num))" ) _CONTROL_NAMES = frozenset({"pc_schema_revision"}) -_ALEMBIC_CONTEXT_LOCK = threading.RLock() BackupPolicy = Literal["auto", "manual", "skip"] @@ -105,12 +103,14 @@ def __init__( *, configuration: Mapping[str, Any] | None = None, backup_provider: BackupProvider | None = None, + evidence_directory: Path | None = None, ) -> None: self.database = database.expanduser().resolve() self.bundle = bundle self.configuration_digest = digest(configuration or {}) self.backup_provider = backup_provider or SQLiteBackupProvider(self.database) - self.evidence_path = self.database.with_name(self.database.name + ".pc-migration-state") / "maintenance.json" + directory = evidence_directory or self.database.with_name(self.database.name + ".pc-migration-state") + self.evidence_path = directory.expanduser().resolve() / "maintenance.json" def plan( self, @@ -186,7 +186,7 @@ def apply( if not version_exists: connection.exec_driver_sql(_VERSION_DDL) if initial.adopt_baseline and initial.source_revision is not None: - with _ALEMBIC_CONTEXT_LOCK: + with ALEMBIC_CONTEXT_LOCK: command.stamp(self.bundle.config(connection), initial.source_revision) connection.commit() self._upgrade(connection) @@ -507,7 +507,7 @@ def _upgrade(self, connection: Connection) -> None: for revision in self.bundle.pending(current): self._begin(connection) self._verify_schema(connection, current) - with _ALEMBIC_CONTEXT_LOCK: + with ALEMBIC_CONTEXT_LOCK: command.upgrade(self.bundle.config(connection), revision) self._verify_schema(connection, revision) connection.commit() diff --git a/src/powercontext/server/database_migration.py b/src/powercontext/server/database_migration.py index 64843391a6..37fac0e684 100644 --- a/src/powercontext/server/database_migration.py +++ b/src/powercontext/server/database_migration.py @@ -27,8 +27,7 @@ from sqlalchemy.exc import SQLAlchemyError from powercontext.builtin.persistence.migrations import MigrationError, MigrationPlan, MigrationResult -from powercontext.builtin.persistence.migrations.deployment import deployment_runner -from powercontext.builtin.persistence.migrations.sqlite import SQLiteMigrationRunner +from powercontext.builtin.persistence.migrations.deployment import MigrationRunner, deployment_runner from powercontext.server.configuration import ( ServerConfigurationError, resolve_server_environment_file, @@ -39,6 +38,10 @@ BackupPolicy = Literal["auto", "manual", "skip"] EnvFile = Annotated[Path | None, typer.Option(help="Read the same environment file as the deployed Server.")] +EvidenceDir = Annotated[ + Path | None, + typer.Option(help="Persistent maintenance evidence directory; required and shared across Jobs for OceanBase."), +] ManageService = Annotated[ bool, typer.Option(help="Request local service maintenance; unavailable for the current partial acceptance bundle."), @@ -62,27 +65,28 @@ def main() -> None: @app.command() -def status(env_file: EnvFile = None) -> None: +def status(env_file: EnvFile = None, evidence_dir: EvidenceDir = None) -> None: """Inspect the schema version without creating a database or control tables.""" - _inspect(env_file, verify=False) + _inspect(env_file, evidence_dir=evidence_dir, verify=False) @app.command() -def verify(env_file: EnvFile = None) -> None: +def verify(env_file: EnvFile = None, evidence_dir: EvidenceDir = None) -> None: """Verify database readiness; this does not assert Server or cluster readiness.""" - _inspect(env_file, verify=True) + _inspect(env_file, evidence_dir=evidence_dir, verify=True) @app.command() def plan( env_file: EnvFile = None, + evidence_dir: EvidenceDir = None, backup: Annotated[BackupPolicy, typer.Option(help="Bind the selected backup policy to the plan.")] = "auto", manage_service: ManageService = False, shared_database: SharedDatabase = False, ) -> None: """Preview the target, resource digests, backup policy, and maintenance scope.""" with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: - runner = deployment_runner(settings) + runner = deployment_runner(settings, evidence_dir=evidence_dir) service = _service_scope(_environment_file(env_file)) if manage_service else None reviewed = runner.plan(backup_policy=backup, service=service, shared_database=shared_database) _write_payload(reviewed) @@ -91,6 +95,7 @@ def plan( @app.command() def apply( env_file: EnvFile = None, + evidence_dir: EvidenceDir = None, backup: Annotated[ BackupPolicy | None, typer.Option(help="PC native backup, a user-declared manual backup, or explicit no-backup risk."), @@ -117,7 +122,7 @@ def apply( ) -> None: """Review once, then stop writes, back up, migrate, verify, and optionally switch a local service.""" with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: - runner = deployment_runner(settings) + runner = deployment_runner(settings, evidence_dir=evidence_dir) policy: BackupPolicy = backup or "auto" initial = runner.plan(backup_policy=policy, shared_database=shared_database) if initial.state == "ready": @@ -163,9 +168,9 @@ def apply( _write_payload(result) -def _inspect(env_file: Path | None, *, verify: bool) -> None: +def _inspect(env_file: Path | None, *, evidence_dir: Path | None, verify: bool) -> None: with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: - runner = deployment_runner(settings) + runner = deployment_runner(settings, evidence_dir=evidence_dir) result = runner.verify() if verify else runner.plan() _write_payload(result) @@ -218,7 +223,7 @@ def _service_scope(env_file: Path | None) -> dict[str, object]: def _apply_reviewed( - runner: SQLiteMigrationRunner, + runner: MigrationRunner, reviewed: MigrationPlan, *, maintenance_confirmed: bool, diff --git a/tests/builtin/persistence/test_database_migration_connections.py b/tests/builtin/persistence/test_database_migration_connections.py new file mode 100644 index 0000000000..bc5a806b4a --- /dev/null +++ b/tests/builtin/persistence/test_database_migration_connections.py @@ -0,0 +1,255 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Maintenance owns its backend lifetime without creating business schema.""" + +from __future__ import annotations + +import os +from collections.abc import Callable +from datetime import UTC, datetime +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from sqlalchemy import Connection, event +from sqlalchemy.ext.asyncio import create_async_engine + +from powercontext.builtin.persistence.migrations import connections +from powercontext.builtin.persistence.migrations.connections import BackendIdentity, MaintenanceConnections +from powercontext.builtin.persistence.migrations.locking import local_migration_lock +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.persistence.seekdb import SeekDBConfig + + +def test_missing_seekdb_inspection_creates_no_directory_or_database(tmp_path: Path) -> None: + path = tmp_path / "missing-parent" / "database" + + def inspect(connection: Connection | None, identity: BackendIdentity, verify: Callable[[], None]): + assert connection is None + assert identity.product == "seekdb" + assert identity.database_name == "test" + verify() + return identity + + identity = MaintenanceConnections(SeekDBConfig(path=path)).run(inspect) + assert len(identity.database_id) == 64 + assert not path.parent.exists() + + +def test_empty_seekdb_inspection_does_not_initialize_the_directory(tmp_path: Path) -> None: + path = tmp_path / "empty-database" + path.mkdir() + assert ( + MaintenanceConnections(SeekDBConfig(path=path)).run(lambda connection, _identity, _verify: connection) is None + ) + assert list(path.iterdir()) == [] + + +@pytest.mark.parametrize("writable", [False, True], ids=["inspect", "apply"]) +def test_unknown_nonempty_seekdb_directory_is_not_initialized(tmp_path: Path, writable: bool) -> None: + path = tmp_path / "unrelated-directory" + path.mkdir() + existing = path / "user-content.txt" + existing.write_text("preserve", encoding="utf-8") + operation = Mock() + with pytest.raises(MigrationError) as error: + MaintenanceConnections(SeekDBConfig(path=path)).run(operation, writable=writable) + assert error.value.code == "unsupported_target" + operation.assert_not_called() + assert list(path.iterdir()) == [existing] + assert existing.read_text(encoding="utf-8") == "preserve" + + +def test_seekdb_startup_failure_is_safe_to_report_and_releases_the_lock(tmp_path: Path, monkeypatch) -> None: + path = tmp_path / "seekdb" + private_diagnostic = "private engine diagnostic" + + async def fail_open(_path: str): + raise RuntimeError(private_diagnostic) + + monkeypatch.setattr(connections.seekdb_profile, "_load_binding", lambda: SimpleNamespace(aopen=fail_open)) + operation = Mock() + with pytest.raises(MigrationError) as error: + MaintenanceConnections(SeekDBConfig(path=path)).run(operation, writable=True) + assert error.value.code == "backend_open_failed" + assert "private" not in str(error.value) + operation.assert_not_called() + with local_migration_lock(path): + pass + + +def _fake_seekdb(tmp_path: Path, monkeypatch): + path = tmp_path / "seekdb" + engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path / 'maintenance.db'}") + disposed = False + + def assert_locked() -> None: + with pytest.raises(MigrationError) as error, local_migration_lock(path): + pytest.fail("A second maintenance operation acquired the engine directory lock") + assert error.value.code == "migration_locked" + + @event.listens_for(engine.sync_engine, "connect") + def configure_database(connection, _record) -> None: + connection.create_function("DATABASE", 0, lambda: "test") + + class Instance: + def connection_options(self): + return {} + + def close(self) -> None: + assert disposed + assert_locked() + + async def open_instance(_path: str): + assert_locked() + path.mkdir(exist_ok=True) + return Instance() + + dispose = engine.dispose + + async def dispose_engine() -> None: + nonlocal disposed + assert_locked() + await dispose() + disposed = True + + maintained_engine = SimpleNamespace(connect=engine.connect, dispose=dispose_engine) + monkeypatch.setattr(connections.seekdb_profile, "_load_binding", lambda: SimpleNamespace(aopen=open_instance)) + monkeypatch.setattr(connections.seekdb_profile, "_create_engine", lambda _config, _options: maintained_engine) + return path, assert_locked + + +def test_seekdb_lock_outlives_callback_failure_and_engine_shutdown(tmp_path: Path, monkeypatch) -> None: + path, assert_locked = _fake_seekdb(tmp_path, monkeypatch) + + def fail(connection: Connection | None, _identity: BackendIdentity, verify: Callable[[], None]): + assert connection is not None + assert_locked() + verify() + raise KeyboardInterrupt + + with pytest.raises(KeyboardInterrupt): + MaintenanceConnections(SeekDBConfig(path=path)).run(fail, writable=True) + with local_migration_lock(path): + pass + + +def test_seekdb_directory_replacement_is_rejected_before_success(tmp_path: Path, monkeypatch) -> None: + path, _assert_locked = _fake_seekdb(tmp_path, monkeypatch) + + def replace(connection: Connection | None, _identity: BackendIdentity, verify: Callable[[], None]): + assert connection is not None + verify() + path.rename(tmp_path / "original-directory") + path.mkdir() + + with pytest.raises(MigrationError) as error: + MaintenanceConnections(SeekDBConfig(path=path)).run(replace, writable=True) + assert error.value.code == "migration_lock_lost" + with local_migration_lock(path): + pass + + +class _RemoteSession: + def __init__(self, cluster_id: int) -> None: + self.connection = Mock(spec=Connection) + + def query(statement: str): + result = Mock() + if "ob_compatibility_mode" in statement: + result.first.return_value = ("ob_compatibility_mode", "MYSQL") + elif "SELECT DATABASE()" in statement: + result.one.return_value = ("scratch", 1002) + elif "GV$OB_PARAMETERS" in statement: + result.scalars.return_value.all.return_value = [str(cluster_id)] + elif "DBA_OB_TENANTS" in statement: + result.one.return_value = (datetime(2026, 1, 1, tzinfo=UTC), "USER") + else: + pytest.fail(f"Unexpected identity query: {statement}") + return result + + self.connection.exec_driver_sql.side_effect = query + + async def __aenter__(self): + return self + + async def __aexit__(self, *_args): + pass + + async def run_sync(self, operation): + return operation(self.connection) + + +def test_oceanbase_routes_to_different_database_identity_before_write_are_rejected(monkeypatch) -> None: + sessions = iter((_RemoteSession(1), _RemoteSession(2))) + engine = Mock() + engine.connect.side_effect = lambda: next(sessions) + + async def dispose() -> None: + pass + + engine.dispose = dispose + monkeypatch.setattr(connections, "create_async_engine", lambda *_args, **_options: engine) + operation = Mock() + config = OceanBaseConfig.model_validate({ + "url": "mysql+aoceanbase://private-user:private-password@private-route:2881/scratch?charset=utf8mb4" + }) + with pytest.raises(MigrationError) as error: + MaintenanceConnections(config).run(operation, writable=True) + assert error.value.code == "target_identity_unavailable" + assert "private" not in str(error.value) + operation.assert_not_called() + + +@pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") +def test_real_seekdb_maintenance_preserves_committed_data_without_initializing_business_tables( + tmp_path: Path, caplog +) -> None: + pytest.importorskip("pylibseekdb") + path = tmp_path / "seekdb" + adapter = MaintenanceConnections(SeekDBConfig(path=path, echo=True)) + missing = adapter.run(lambda connection, identity, _verify: (connection, identity)) + assert missing[0] is None + assert not path.exists() + + def write(connection: Connection | None, identity: BackendIdentity, verify: Callable[[], None]): + assert connection is not None + assert connection.exec_driver_sql("SHOW TABLES").all() == [] + with pytest.raises(MigrationError) as locked, local_migration_lock(path): + pytest.fail("Another maintenance operation entered the live seekdb directory") + assert locked.value.code == "migration_locked" + verify() + connection.exec_driver_sql("CREATE TABLE pc_maintenance_probe (id INT PRIMARY KEY, value VARCHAR(32))") + connection.exec_driver_sql("INSERT INTO pc_maintenance_probe VALUES (1, 'preserved')") + connection.commit() + verify() + return identity + + written = adapter.run(write, writable=True) + + def read(connection: Connection | None, identity: BackendIdentity, verify: Callable[[], None]): + assert connection is not None + verify() + assert connection.exec_driver_sql("SHOW TABLES").all() == [("pc_maintenance_probe",)] + assert ( + connection.exec_driver_sql("SELECT value FROM pc_maintenance_probe WHERE id = 1").scalar_one() + == "preserved" + ) + return identity + + assert adapter.run(read) == written == missing[1] + assert "preserved" not in caplog.text diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py new file mode 100644 index 0000000000..bc61dc2b2d --- /dev/null +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -0,0 +1,309 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Real seekdb maintenance acceptance on independent temporary instances.""" + +from __future__ import annotations + +import json +import os +from collections.abc import Callable +from pathlib import Path +from typing import Literal +from unittest.mock import Mock + +import pytest +from sqlalchemy import Connection, Engine, event, text + +from powercontext.builtin.persistence.migrations import mysql as mysql_migrations +from powercontext.builtin.persistence.migrations.backup import BackupCapabilities, BackupProvider +from powercontext.builtin.persistence.migrations.connections import BackendIdentity, MaintenanceConnections +from powercontext.builtin.persistence.migrations.deployment import production_bundle +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.mysql import MySQLMigrationRunner +from powercontext.builtin.persistence.seekdb import SeekDBConfig + +pytestmark = pytest.mark.skipif( + os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled" +) + + +def _connections(tmp_path: Path) -> MaintenanceConnections: + pytest.importorskip("pylibseekdb") + return MaintenanceConnections(SeekDBConfig(path=tmp_path / "seekdb")) + + +def _runner(tmp_path: Path, *, backup_factory=None) -> MySQLMigrationRunner: + return MySQLMigrationRunner( + _connections(tmp_path), + production_bundle(), + evidence_directory=tmp_path / "evidence", + backup_factory=backup_factory, + ) + + +def _forbidden_automatic_backup(_connection, _identity, _directory): + provider = Mock(spec=BackupProvider) + for name in ("capabilities", "create_backup", "inspect_backup", "restore_plan"): + getattr(provider, name).side_effect = AssertionError("This source must not invoke automatic backup") + return provider + + +def _load_baseline(tmp_path: Path) -> None: + bundle = production_bundle() + schema = json.loads((bundle.snapshot_directory / "schemas/pre_dream.json").read_text()) + + def load(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + for statement in schema["mysql"]: + connection.exec_driver_sql(statement) + connection.execute( + text( + "INSERT INTO pc_artifacts (scope_id,family,artifact_id,revision,content) " + "VALUES ('s','memory','m',1,:data)" + ), + {"data": b'{"content":"preserved"}'}, + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_heads (scope_id,family,artifact_id,revision) VALUES ('s','memory','m',1)" + ) + connection.exec_driver_sql( + "INSERT INTO pc_artifact_tags (scope_id,family,artifact_id,target_type,target_id,tag_key_hash,tag_key,tag,assigned_at) " + "VALUES ('s','memory','m','artifact','m',UNHEX(REPEAT('00',32)),'keep','Keep','2026-09-28 12:00:00')" + ) + connection.commit() + + _connections(tmp_path).run(load, writable=True) + + +def _apply(runner: MySQLMigrationRunner, *, policy: Literal["auto", "manual", "skip"] = "auto"): + plan = runner.plan(backup_policy=policy) + return runner.apply( + plan_id=plan.plan_id, + accepted=True, + maintenance_confirmed=True, + backup_policy=policy, + backup_confirmed=policy == "manual", + accept_no_backup=policy == "skip", + backup_ref="operator://manual-backup" if policy == "manual" else None, + ) + + +def _read_state(tmp_path: Path): + def inspect(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]): + assert connection is not None + tables = {name for (name,) in connection.exec_driver_sql("SHOW TABLES")} + revision = ( + connection.exec_driver_sql("SELECT version_num FROM pc_schema_revision").scalar_one_or_none() + if "pc_schema_revision" in tables + else None + ) + columns = { + name: {row[0] for row in connection.exec_driver_sql(f"SHOW COLUMNS FROM {name}")} + for name in ("pc_artifacts", "pc_artifact_candidate_versions") + if name in tables + } + content = ( + connection.exec_driver_sql("SELECT content FROM pc_artifacts").all() if "pc_artifacts" in tables else [] + ) + tags = ( + connection.exec_driver_sql("SELECT tag FROM pc_artifact_tags").all() if "pc_artifact_tags" in tables else [] + ) + return {"tables": tables, "revision": revision, "columns": columns, "content": content, "tags": tags} + + return _connections(tmp_path).run(inspect) + + +def test_seekdb_initialization_and_shared_evidence_noop(tmp_path: Path) -> None: + runner = _runner(tmp_path) + plan = runner.plan() + assert plan.state == "uninitialized" + assert not (tmp_path / "seekdb").exists() + result = _apply(runner) + assert result.state == "ready" + assert result.changed + assert result.backup_state == "not_required" + assert result.revision == "p0003" + replacement = _runner(tmp_path) + assert replacement.plan().state == "ready" + assert replacement.verify().revision == "p0003" + assert not replacement.apply().changed + + +@pytest.mark.parametrize("policy", ["manual", "skip"]) +def test_seekdb_operator_backup_choices_preserve_baseline_data_without_fork( + tmp_path: Path, policy: Literal["manual", "skip"] +) -> None: + _load_baseline(tmp_path) + runner = _runner(tmp_path, backup_factory=_forbidden_automatic_backup) + plan = runner.plan(backup_policy=policy) + assert plan.source_revision == "p0001" + assert plan.adopt_baseline + with pytest.raises(MigrationError) as error: + runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True, backup_policy=policy) + assert error.value.code == "backup_confirmation_required" + assert _read_state(tmp_path)["revision"] is None + result = _apply(runner, policy=policy) + assert result.backup_state == ("user_confirmed" if policy == "manual" else "skipped") + state = _read_state(tmp_path) + assert state["revision"] == "p0003" + assert state["content"] == [(b'{"content":"preserved"}',)] + assert state["tags"] == [("Keep",)] + assert "memory_citations" in state["columns"]["pc_artifacts"] + assert "memory_citations" in state["columns"]["pc_artifact_candidate_versions"] + assert _runner(tmp_path).verify().state == "ready" + + +@pytest.mark.parametrize("table_fork", [False, True]) +def test_seekdb_unsupported_backup_does_not_write_source_schema(tmp_path: Path, table_fork: bool) -> None: + _load_baseline(tmp_path) + before = _read_state(tmp_path) + + def advertised_table_fork(connection, identity, directory): + provider = _forbidden_automatic_backup(connection, identity, directory) + provider.capabilities.side_effect = None + provider.capabilities.return_value = BackupCapabilities( + available=True, method="fork_table", product="seekdb", recovery_workflow_accepted=True + ) + return provider + + runner = _runner(tmp_path, backup_factory=advertised_table_fork if table_fork else None) + plan = runner.plan() + assert not plan.backup_available + with pytest.raises(MigrationError) as error: + _apply(runner) + assert error.value.code == "backup_unsupported" + assert _read_state(tmp_path) == before + assert not (tmp_path / "evidence").exists() + + +class _InterruptedDDL(BaseException): + pass + + +def _interrupt_after_ddl( + runner: MySQLMigrationRunner, prefix: str, *, policy: Literal["auto", "manual", "skip"] = "manual" +) -> None: + def interrupt(_connection, _cursor, statement: str, _parameters, _context, _executemany) -> None: + if statement.lstrip().upper().startswith(prefix.upper()): + raise _InterruptedDDL + + event.listen(Engine, "after_cursor_execute", interrupt) + try: + with pytest.raises(_InterruptedDDL): + _apply(runner, policy=policy) + finally: + event.remove(Engine, "after_cursor_execute", interrupt) + + +def _interrupt_after_committed_citation_ddl(tmp_path: Path) -> None: + _load_baseline(tmp_path) + _interrupt_after_ddl(_runner(tmp_path), "ALTER TABLE pc_artifacts ADD COLUMN memory_citations") + state = _read_state(tmp_path) + assert state["revision"] == "p0001" + assert "memory_citations" in state["columns"]["pc_artifacts"] + assert "memory_citations" not in state["columns"]["pc_artifact_candidate_versions"] + assert state["content"] == [(b'{"content":"preserved"}',)] + + +def test_seekdb_committed_partial_ddl_resumes_from_shared_original_evidence(tmp_path: Path) -> None: + _interrupt_after_committed_citation_ddl(tmp_path) + replacement = _runner(tmp_path) + assert replacement.plan(backup_policy="manual").state == "recovery_required" + assert _apply(replacement, policy="manual").state == "ready" + state = _read_state(tmp_path) + assert state["revision"] == "p0003" + assert state["content"] == [(b'{"content":"preserved"}',)] + assert state["tags"] == [("Keep",)] + assert replacement.verify().state == "ready" + + +def test_seekdb_missing_original_evidence_blocks_partial_ddl_recovery(tmp_path: Path) -> None: + _interrupt_after_committed_citation_ddl(tmp_path) + before = _read_state(tmp_path) + evidence_files = list((tmp_path / "evidence").rglob("maintenance.json")) + assert len(evidence_files) == 1 + evidence_files[0].unlink() + replacement = _runner(tmp_path) + for operation in (lambda: replacement.plan(backup_policy="manual"), replacement.verify, replacement.apply): + with pytest.raises(MigrationError) as error: + operation() + assert error.value.code == "recovery_required" + assert _read_state(tmp_path) == before + + +def test_seekdb_empty_source_recovers_baseline_and_citation_ddl_without_probing_backup(tmp_path: Path) -> None: + _interrupt_after_ddl( + _runner(tmp_path, backup_factory=_forbidden_automatic_backup), "CREATE TABLE pc_artifacts", policy="auto" + ) + first = _read_state(tmp_path) + assert first["revision"] is None + assert "pc_schema_revision" in first["tables"] + assert "pc_artifacts" in first["tables"] + assert "pc_artifact_heads" not in first["tables"] + assert first["content"] == [] + _interrupt_after_ddl( + _runner(tmp_path, backup_factory=_forbidden_automatic_backup), + "ALTER TABLE pc_artifacts ADD COLUMN memory_citations", + policy="auto", + ) + second = _read_state(tmp_path) + assert second["revision"] == "p0001" + assert "memory_citations" in second["columns"]["pc_artifacts"] + assert "memory_citations" not in second["columns"]["pc_artifact_candidate_versions"] + replacement = _runner(tmp_path, backup_factory=_forbidden_automatic_backup) + result = _apply(replacement) + assert result.backup_state == "not_required" + assert replacement.verify().revision == "p0003" + assert _read_state(tmp_path)["content"] == [] + + +def test_seekdb_baseline_adoption_recovers_an_empty_committed_version_table(tmp_path: Path) -> None: + _load_baseline(tmp_path) + _interrupt_after_ddl(_runner(tmp_path), "CREATE TABLE pc_schema_revision") + state = _read_state(tmp_path) + assert "pc_schema_revision" in state["tables"] + assert state["revision"] is None + assert state["content"] == [(b'{"content":"preserved"}',)] + replacement = _runner(tmp_path) + recovered = replacement.plan(backup_policy="manual") + assert recovered.state == "recovery_required" + assert recovered.adopt_baseline + assert _apply(replacement, policy="manual").state == "ready" + assert replacement.verify().revision == "p0003" + assert _read_state(tmp_path)["tags"] == [("Keep",)] + + +def test_seekdb_committed_revision_recovers_before_evidence_advance(tmp_path: Path, monkeypatch) -> None: + _load_baseline(tmp_path) + write = mysql_migrations._Journal.write + + def interrupt(journal, evidence) -> None: + if evidence.state == "active" and evidence.last_revision == "p0002" and evidence.pending_revision is None: + raise _InterruptedDDL + write(journal, evidence) + + with monkeypatch.context() as patch: + patch.setattr(mysql_migrations._Journal, "write", interrupt) + with pytest.raises(_InterruptedDDL): + _apply(_runner(tmp_path), policy="manual") + state = _read_state(tmp_path) + assert state["revision"] == "p0002" + assert "memory_citations" in state["columns"]["pc_artifacts"] + assert "memory_citations" in state["columns"]["pc_artifact_candidate_versions"] + replacement = _runner(tmp_path) + assert replacement.plan(backup_policy="manual").source_revision == "p0002" + assert _apply(replacement, policy="manual").state == "ready" + assert replacement.verify().revision == "p0003" + assert _read_state(tmp_path)["content"] == [(b'{"content":"preserved"}',)] diff --git a/tests/builtin/persistence/test_mysql_migration_schema.py b/tests/builtin/persistence/test_mysql_migration_schema.py new file mode 100644 index 0000000000..f04773bd59 --- /dev/null +++ b/tests/builtin/persistence/test_mysql_migration_schema.py @@ -0,0 +1,347 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Frozen schema recognition; mocks alone do not establish backend acceptance.""" + +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from types import SimpleNamespace +from typing import Any, cast + +import pytest +from sqlalchemy import Connection, create_engine + +from powercontext.builtin.persistence.migrations.bundle import MigrationBundle +from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.mysql_schema import MySQLSchema, mysql_inventory, read_revision +from powercontext.builtin.persistence.seekdb import SeekDBConfig + +_RESOURCES = Path(__file__).parents[3] / "src/powercontext/builtin/persistence/migrations/resources" + + +class _Rows: + def __init__(self, rows: list[Any]) -> None: + self.rows = rows + + def __iter__(self): + return iter(self.rows) + + def first(self): + return self.rows[0] if self.rows else None + + def all(self): + return self.rows + + def scalar_one(self): + assert len(self.rows) == 1 + return self.rows[0][0] + + def mappings(self): + return iter(self.rows) + + +class _ReflectedDatabase: + """Scripted SHOW results, not a MySQL engine or backend acceptance test.""" + + dialect = SimpleNamespace(name="mysql") + + def __init__(self, tables: dict[str, str]) -> None: + self.tables = tables + self.views: list[str] = [] + self.triggers: list[dict[str, str]] = [] + self.heads: list[str] = [] + + def exec_driver_sql(self, statement: str) -> _Rows: + if statement == "SELECT DATABASE()": + return _Rows([("TestDB",)]) + if statement == "SHOW FULL TABLES": + return _Rows([(name, "BASE TABLE") for name in self.tables] + [(name, "VIEW") for name in self.views]) + if statement == "SHOW TRIGGERS": + return _Rows(self.triggers) + if statement.startswith("SHOW CREATE TABLE "): + name = statement.removeprefix("SHOW CREATE TABLE ").strip("`") + return _Rows([(name, self.tables[name])]) + if statement == "SELECT version_num FROM pc_schema_revision LIMIT 2": + return _Rows([(head,) for head in self.heads[:2]]) + pytest.fail(f"Unexpected query: {statement}") + + +@pytest.fixture +def schema() -> MySQLSchema: + return MySQLSchema(MigrationBundle(_RESOURCES)) + + +def _append_clause(statement: str, clause: str) -> str: + return statement.replace("\n)", ",\n\t" + clause + "\n)") + + +def _source_statements() -> list[str]: + return json.loads((_RESOURCES / "schemas/pre_dream.json").read_text())["mysql"] + + +def _show_tables(*, baseline_prefix: int = 6, citations: int = 0, drop_family_check: bool = False) -> dict[str, str]: + """Build scripted native output from frozen DDL, including inline indexes.""" + tables: dict[str, str] = {} + for statement in _source_statements()[:baseline_prefix]: + if statement.startswith("CREATE TABLE "): + name = statement.split()[2] + tables[name] = statement + else: + match = re.fullmatch(r"CREATE INDEX (\w+) ON (\w+) (\(.+\))", statement) + assert match is not None + index, table, columns = match.groups() + tables[table] = _append_clause(tables[table], f"KEY {index} {columns}") + for table in ("pc_artifacts", "pc_artifact_candidate_versions")[:citations]: + tables[table] = _append_clause(tables[table], "memory_citations MEDIUMBLOB DEFAULT NULL") + if drop_family_check: + tables["pc_artifact_tags"] = re.sub( + r",\s*CONSTRAINT ck_pc_artifact_tags_family CHECK \(family IN \([^)]*\)\)", + "", + tables["pc_artifact_tags"], + ) + return tables + + +def _inventory(database: _ReflectedDatabase) -> dict[str, Any]: + return mysql_inventory(cast(Connection, database)) + + +@pytest.mark.parametrize( + ("citations", "drop_family", "revision"), [(0, False, "p0001"), (2, False, "p0002"), (2, True, "p0003")] +) +def test_complete_frozen_layouts_are_identified_and_verified( + schema: MySQLSchema, citations: int, drop_family: bool, revision: str +) -> None: + inventory = _inventory(_ReflectedDatabase(_show_tables(citations=citations, drop_family_check=drop_family))) + + assert schema.identify(inventory) == revision + schema.verify(inventory, revision) + with pytest.raises(MigrationError, match="recorded revision"): + schema.verify(inventory, None) + + +def test_native_formatting_fk_names_and_optional_generated_indexes_preserve_the_layout(schema: MySQLSchema) -> None: + tables = _show_tables(citations=2, drop_family_check=True) + for name, statement in tables.items(): + statement = statement.replace(f"CREATE TABLE {name}", f"CREATE TABLE `{name}`") + statement = statement.replace("INTEGER", "int(11)").replace("BIGINT", "bigint(20)") + statement = re.sub(r"FOREIGN KEY\(", "CONSTRAINT `generated_OBFK_123` FOREIGN KEY (", statement) + statement = re.sub(r"REFERENCES (\w+) ", r"REFERENCES `TestDB`.\1 ", statement) + statement = statement.replace("ON DELETE RESTRICT", "ON UPDATE NO ACTION ON DELETE NO ACTION") + statement = statement.replace("CHECK (revision > 0)", "CHECK (((`revision` > 0)))") + statement += " ORGANIZATION INDEX DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC BLOCK_SIZE=16384" + tables[name] = statement + tables["pc_artifact_heads"] = _append_clause( + tables["pc_artifact_heads"], + "KEY generated_foreign_key_index (scope_id, family, artifact_id, revision) BLOCK_SIZE 16384 GLOBAL", + ) + inventory = _inventory(_ReflectedDatabase(tables)) + + assert schema.identify(inventory) == "p0003" + schema.verify(inventory, "p0003") + assert schema.fingerprint(inventory) == schema.fingerprint(_inventory(_ReflectedDatabase(tables))) + + +@pytest.mark.parametrize("applied", range(7)) +def test_baseline_recovery_accepts_only_the_completed_statement_prefix(schema: MySQLSchema, applied: int) -> None: + inventory = _inventory(_ReflectedDatabase(_show_tables(baseline_prefix=applied))) + + recovery = schema.recovery(inventory, None) + + assert recovery.revision == "p0001" + assert recovery.pending_statements == tuple(_source_statements()[applied:]) + assert recovery.completed_revision == ("p0001" if applied == 6 else None) + if 0 < applied < 6: + with pytest.raises(MigrationError, match="complete layout"): + schema.identify(inventory) + + +@pytest.mark.parametrize("applied", range(3)) +def test_citation_recovery_does_not_repeat_committed_add_columns(schema: MySQLSchema, applied: int) -> None: + inventory = _inventory(_ReflectedDatabase(_show_tables(citations=applied))) + + recovery = schema.recovery(inventory, "p0001") + + assert recovery.revision == "p0002" + expected = tuple( + f"ALTER TABLE {table} ADD COLUMN memory_citations MEDIUMBLOB" + for table in ("pc_artifacts", "pc_artifact_candidate_versions")[applied:] + ) + assert recovery.pending_statements == expected + assert recovery.completed_revision == ("p0002" if applied == 2 else None) + with pytest.raises(MigrationError, match="recorded revision"): + schema.verify(inventory, "p0002" if applied < 2 else "p0001") + + +@pytest.mark.parametrize("dropped", [False, True]) +def test_check_drop_recovery_requires_the_entire_predecessor_layout(schema: MySQLSchema, dropped: bool) -> None: + inventory = _inventory(_ReflectedDatabase(_show_tables(citations=2, drop_family_check=dropped))) + + recovery = schema.recovery(inventory, "p0002") + + assert recovery.revision == "p0003" + assert recovery.pending_statements == ( + () if dropped else ("ALTER TABLE pc_artifact_tags DROP CHECK ck_pc_artifact_tags_family",) + ) + assert recovery.completed_revision == ("p0003" if dropped else None) + + +def test_unordered_or_incompatible_partial_ddl_cannot_be_recovered(schema: MySQLSchema) -> None: + tables = _show_tables() + tables["pc_artifact_candidate_versions"] = _append_clause( + tables["pc_artifact_candidate_versions"], "memory_citations MEDIUMBLOB" + ) + with pytest.raises(MigrationError, match="controlled"): + schema.recovery(_inventory(_ReflectedDatabase(tables)), "p0001") + tables = _show_tables(citations=1) + tables["pc_artifacts"] = tables["pc_artifacts"].replace("memory_citations MEDIUMBLOB", "memory_citations BLOB") + with pytest.raises(MigrationError, match="controlled"): + schema.recovery(_inventory(_ReflectedDatabase(tables)), "p0001") + + +@pytest.mark.parametrize("change", ["table", "column", "index", "foreign_key", "disabled_check", "trigger", "view"]) +def test_unknown_objects_or_changed_integrity_constraints_cannot_be_adopted(schema: MySQLSchema, change: str) -> None: + database = _ReflectedDatabase(_show_tables()) + if change == "table": + database.tables["unknown_table"] = "CREATE TABLE unknown_table (id INTEGER NOT NULL, PRIMARY KEY (id))" + elif change == "column": + database.tables["pc_artifacts"] = _append_clause(database.tables["pc_artifacts"], "unexpected INTEGER") + elif change == "index": + database.tables["pc_artifact_tags"] = _append_clause( + database.tables["pc_artifact_tags"], "KEY unexpected (tag)" + ) + elif change == "foreign_key": + database.tables["pc_artifact_heads"] = database.tables["pc_artifact_heads"].replace( + "ON DELETE RESTRICT", "ON DELETE CASCADE" + ) + elif change == "disabled_check": + database.tables["pc_artifact_heads"] = database.tables["pc_artifact_heads"].replace( + "CHECK (revision > 0)", "CHECK (revision > 0) NOT ENFORCED" + ) + elif change == "trigger": + database.triggers = [ + { + "Trigger": "writer", + "Table": "pc_artifacts", + "Timing": "AFTER", + "Event": "INSERT", + "Statement": "DELETE FROM pc_artifact_heads", + } + ] + else: + database.views = ["unknown_projection"] + + with pytest.raises(MigrationError, match="complete layout"): + schema.identify(_inventory(database)) + + +def test_version_table_is_validated_before_reading_and_is_not_a_business_table(schema: MySQLSchema) -> None: + database = _ReflectedDatabase(_show_tables()) + assert read_revision(cast(Connection, database), _inventory(database)) == (False, None) + database.tables["pc_schema_revision"] = ( + "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " + "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num)\n)" + ) + assert read_revision(cast(Connection, database), _inventory(database)) == (True, None) + database.heads = ["p0001"] + inventory = _inventory(database) + assert read_revision(cast(Connection, database), inventory) == (True, "p0001") + assert schema.identify(inventory) == "p0001" + schema.verify(inventory, "p0001") + database.heads = ["p0001", "p0002"] + with pytest.raises(MigrationError, match="multiple or unknown"): + read_revision(cast(Connection, database), inventory) + database.heads = ["unknown"] + with pytest.raises(MigrationError, match="multiple or unknown"): + read_revision(cast(Connection, database), inventory) + database.tables["pc_schema_revision"] = _append_clause( + database.tables["pc_schema_revision"], "KEY unexpected (version_num)" + ) + with pytest.raises(MigrationError, match="version table"): + read_revision(cast(Connection, database), _inventory(database)) + + +@pytest.mark.parametrize("violation", ["foreign_key", "check"]) +def test_portable_integrity_queries_reject_preexisting_invalid_rows(schema: MySQLSchema, violation: str) -> None: + # Execute the portable existence queries on real SQLite, without claiming + # MySQL/seekdb acceptance from this regression's storage engine. + statements = json.loads((_RESOURCES / "schemas/v1_1_0.json").read_text())["sqlite"] + with create_engine("sqlite://").connect() as connection: + for statement in statements: + connection.exec_driver_sql(statement) + connection.exec_driver_sql("PRAGMA ignore_check_constraints=ON") + revision = 1 if violation == "foreign_key" else 0 + connection.exec_driver_sql( + "INSERT INTO pc_artifact_heads (scope_id, family, artifact_id, revision) VALUES ('scope', 'memory', 'missing', ?)", + (revision,), + ) + with pytest.raises(MigrationError, match="foreign key" if violation == "foreign_key" else "CHECK"): + schema.validate_data(connection, "p0003") + + +@pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") +def test_real_seekdb_frozen_schema_reflection_and_revision_recovery(tmp_path: Path) -> None: + pytest.importorskip("pylibseekdb") + schema = MySQLSchema(MigrationBundle(_RESOURCES)) + + def migrate(connection, _identity, guard) -> None: + assert connection is not None + assert read_revision(connection, mysql_inventory(connection)) == (False, None) + connection.exec_driver_sql( + "CREATE TABLE pc_schema_revision (version_num VARCHAR(32) NOT NULL, " + "CONSTRAINT pc_schema_revision_pkc PRIMARY KEY (version_num))" + ) + connection.commit() + assert read_revision(connection, mysql_inventory(connection)) == (True, None) + source = None + for target in ("p0001", "p0002", "p0003"): + while True: + guard() + inventory = mysql_inventory(connection) + recovery = schema.recovery(inventory, source) + assert recovery.revision == target + if recovery.completed_revision is not None: + break + connection.exec_driver_sql(recovery.pending_statements[0]) + connection.commit() + guard() + schema.verify(mysql_inventory(connection), target) + schema.validate_data(connection, target) + assert schema.identify(mysql_inventory(connection)) == target + source = target + if target == "p0001": + connection.exec_driver_sql( + "INSERT INTO pc_artifacts (scope_id, family, artifact_id, revision, content) " + "VALUES (%s, %s, %s, %s, %s)", + ("scope", "memory", "artifact", 1, b"payload"), + ) + connection.commit() + assert connection.exec_driver_sql("SELECT content FROM pc_artifacts").scalar_one() == b"payload" + connection.exec_driver_sql("SET foreign_key_checks=0") + connection.exec_driver_sql( + "INSERT INTO pc_artifact_heads (scope_id, family, artifact_id, revision) " + "VALUES ('scope', 'memory', 'orphan', 1)" + ) + connection.commit() + # Re-enabling enforcement does not repair or validate historical rows. + connection.exec_driver_sql("SET foreign_key_checks=1") + with pytest.raises(MigrationError, match="foreign key"): + schema.validate_data(connection, "p0003") + + MaintenanceConnections(SeekDBConfig(path=tmp_path / "seekdb")).run(migrate, writable=True) diff --git a/tests/test_database_migration_cli.py b/tests/test_database_migration_cli.py index 3d9b0cd7ea..f2fb1c41d1 100644 --- a/tests/test_database_migration_cli.py +++ b/tests/test_database_migration_cli.py @@ -19,12 +19,14 @@ import os import sqlite3 from pathlib import Path +from typing import Literal from uuid import uuid4 import pytest from sqlalchemy.ext.asyncio import create_async_engine from typer.testing import CliRunner +from powercontext.builtin.persistence.migrations.connections import BackendIdentity, MaintenanceConnections from powercontext.builtin.persistence.migrations.deployment import production_bundle from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import BUILTIN_TABLES @@ -340,7 +342,7 @@ def test_partial_bundle_rejects_business_service_management_before_writing(tmp_p assert not database.exists() -def test_remote_backend_is_not_reported_ready_without_acceptance(tmp_path, monkeypatch): +def test_missing_seekdb_is_inspected_without_initializing_the_engine(tmp_path, monkeypatch): _database, command = _configure(tmp_path, monkeypatch) directory = tmp_path / "seekdb" (tmp_path / "deployment.env").write_text( @@ -348,9 +350,120 @@ def test_remote_backend_is_not_reported_ready_without_acceptance(tmp_path, monke encoding="utf-8", ) result = _invoke(command, "status", tmp_path) - assert result.exit_code == 1 - assert json.loads(result.output)["error"] == "unsupported_backend" + assert result.exit_code == 0, result.output + assert json.loads(result.output)["state"] == "uninitialized" + assert json.loads(result.output)["server_ready"] is False assert not directory.exists() + assert not directory.with_name(directory.name + ".pc-migration.lock").exists() + + +@pytest.mark.parametrize("action", ["status", "plan", "apply", "verify"]) +def test_oceanbase_requires_persistent_evidence_before_connecting(tmp_path, monkeypatch, action): + _database, command = _configure(tmp_path, monkeypatch) + private_value = "private-password-do-not-print" + private_host = "private-database-host.invalid" + (tmp_path / "deployment.env").write_text( + "POWERCONTEXT_SERVER_DATABASE_KIND=oceanbase\n" + f"POWERCONTEXT_SERVER_DATABASE_URL=mysql+aoceanbase://operator:{private_value}@{private_host}:2881/pc_probe" + "?charset=utf8mb4\n", + encoding="utf-8", + ) + + def refuse_connection(*_args, **_kwargs): + pytest.fail("Missing persistent evidence must be rejected before contacting a database") + + monkeypatch.setattr(MaintenanceConnections, "run", refuse_connection) + result = _invoke(command, action, tmp_path) + assert result.exit_code == 1, result.output + assert json.loads(result.output)["error"] == "evidence_required" + assert "--evidence-dir" in result.output + assert private_value not in result.output + assert private_host not in result.output + assert "Traceback" not in result.output + + +@pytest.mark.parametrize("kind", ["seekdb", "oceanbase"]) +def test_remote_cli_uses_the_configured_maintenance_identity( + tmp_path, monkeypatch, kind: Literal["seekdb", "oceanbase"] +): + _database, command = _configure(tmp_path, monkeypatch) + database_name = "test" if kind == "seekdb" else "pc_probe" + environment = f"POWERCONTEXT_SERVER_DATABASE_KIND={kind}\n" + if kind == "seekdb": + environment += f"POWERCONTEXT_SERVER_DATABASE_PATH={tmp_path / 'seekdb'}\n" + else: + environment += ( + "POWERCONTEXT_SERVER_DATABASE_URL=mysql+aoceanbase://operator:secret@database.invalid:2881/pc_probe" + "?charset=utf8mb4\n" + ) + (tmp_path / "deployment.env").write_text(environment, encoding="utf-8") + identity = BackendIdentity(kind, f"simulated-{kind}-database-identity", database_name) + + def inspect_empty_database(_connections, operation, *, writable=False): + assert not writable + return operation(None, identity, lambda: None) + + monkeypatch.setattr(MaintenanceConnections, "run", inspect_empty_database) + evidence = tmp_path / "persistent-evidence" + for action in ("status", "plan"): + options = ["--evidence-dir", str(evidence)] + if action == "plan": + options.extend(["--backup", "manual"]) + result = _invoke(command, action, tmp_path, *options) + assert result.exit_code == 0, result.output + payload = json.loads(result.output) + assert payload["database_id"] == identity.database_id + assert payload["state"] == "uninitialized" + assert payload["readiness_scope"] == "registered_bundle" + assert payload["server_ready"] is False + assert not evidence.exists() + + +def test_sqlite_commands_use_the_explicit_external_evidence_directory(tmp_path, monkeypatch): + database, command = _configure(tmp_path, monkeypatch, existing=True) + evidence = tmp_path / "persistent-evidence" + options = ["--evidence-dir", str(evidence)] + before = database.read_bytes() + inspected = _invoke(command, "status", tmp_path, *options) + assert inspected.exit_code == 0, inspected.output + assert json.loads(inspected.output)["state"] == "migration_required" + planned = _invoke(command, "plan", tmp_path, *options, "--backup", "manual") + assert planned.exit_code == 0, planned.output + plan = json.loads(planned.output) + assert not evidence.exists() + assert database.read_bytes() == before + + applied = _invoke( + command, + "apply", + tmp_path, + *options, + "--yes", + "--backup", + "manual", + "--plan-id", + plan["plan_id"], + "--maintenance-confirmed", + "--backup-confirmed", + ) + assert applied.exit_code == 0, applied.output + assert json.loads(applied.output)["backup_state"] == "user_confirmed" + assert (evidence / "maintenance.json").is_file() + assert not database.with_name(database.name + ".pc-migration-state").exists() + verified = _invoke(command, "verify", tmp_path, *options) + assert verified.exit_code == 0, verified.output + assert json.loads(verified.output)["revision"] == production_bundle().head + + # All read paths must use the selected journal, rather than silently + # verifying against an empty default directory after the migration. + (evidence / "maintenance.json").write_text('{"state":"active"}', encoding="utf-8") + migrated = database.read_bytes() + for action in ("status", "plan", "verify"): + result = _invoke(command, action, tmp_path, *options) + assert result.exit_code == 1, result.output + assert json.loads(result.output)["error"] == "recovery_required" + assert "Traceback" not in result.output + assert database.read_bytes() == migrated def test_no_backup_requires_separate_risk_consent_and_policy_is_plan_bound(tmp_path, monkeypatch): From 0c524c0a944f9bcf150830c78279d53be99ba67c Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Thu, 8 Oct 2026 19:24:09 +0800 Subject: [PATCH 25/37] fix(db): coordinate OceanBase migrations safely through ODP --- docs/en/docs/operate/database-migrations.md | 100 ++++- docs/zh/docs/operate/database-migrations.md | 51 ++- .../persistence/migrations/connections.py | 59 ++- .../persistence/migrations/deployment.py | 26 +- .../builtin/persistence/migrations/locking.py | 21 +- .../builtin/persistence/migrations/models.py | 1 + .../builtin/persistence/migrations/mysql.py | 36 ++ .../persistence/migrations/oceanbase.py | 423 +++++++++++++++--- src/powercontext/server/database_migration.py | 68 ++- .../test_database_migration_backends.py | 29 +- .../test_database_migration_connections.py | 138 +++++- .../persistence/test_migration_locking.py | 110 +++++ .../test_mysql_migration_runner.py | 305 ++++++++++--- .../test_oceanbase_migration_lock.py | 412 +++++++++++++++++ tests/test_database_migration_cli.py | 191 +++++++- 15 files changed, 1744 insertions(+), 226 deletions(-) create mode 100644 tests/builtin/persistence/test_migration_locking.py create mode 100644 tests/builtin/persistence/test_oceanbase_migration_lock.py diff --git a/docs/en/docs/operate/database-migrations.md b/docs/en/docs/operate/database-migrations.md index fd7168b2d3..aa89e309a7 100644 --- a/docs/en/docs/operate/database-migrations.md +++ b/docs/en/docs/operate/database-migrations.md @@ -22,9 +22,10 @@ A complete Server database containing unmanaged objects is rejected. `ready` means this migration bundle passed its schema and data checks. Output explicitly includes `readiness_scope=registered_bundle` and `server_ready=false`: it does not establish complete Server, index, legacy-task, or cluster readiness. Do not substitute it for a production upgrade or initialization of the complete business database. -Full startup gates and complete historical baselines remain outside this bundle. OceanBase execution and full Fork -recovery acceptance require a dedicated real test environment; local tests and an implemented adapter do not establish -production readiness. +Full startup gates and complete historical baselines remain outside this bundle. Isolated four-table acceptance passed +through an ODP endpoint on OceanBase AI Database 4.6.3, covering coordinator exclusion, acknowledged recovery after +committed DDL, data preservation, and repeat execution. This does not establish complete Server readiness, automatic +detection of unfinished remote DDL after a crash, or full Fork restoration acceptance. The existing `server processing-migrate` remains available. Follow [Migrate Artifact processing state](artifact-processing-migration.md); it does not yet forward to the unified entry point. @@ -32,8 +33,8 @@ See [Deploy the Server](deploy-server.md) for service deployment. ## Read-only inspection and one confirmation -The `deployment.env` examples below use SQLite. OceanBase commands also require the evidence-directory option shown -in the next section. Inspect the target or preview a plan; these commands never create a missing database, parent +The `deployment.env` examples below use SQLite. OceanBase commands also require the evidence-directory and fixed-host +coordination options shown in the next section. Inspect the target or preview a plan; these commands never create a missing database, parent directory, or control table: ```bash @@ -43,7 +44,7 @@ powercontext server db-migrate verify --env-file deployment.env ``` Plans bind target identity, source/target revisions, actual schema, frozen resources, configuration digests, backup policy, -and service scope. Diagnostics do not print configuration or credential values. Expected failures return stable JSON +service scope, and migration coordination. Diagnostics do not print configuration or credential values. Expected failures return stable JSON categories and a nonzero exit status. Within the supported acceptance scope, interactive users can invoke `apply` directly. PC-managed backup is the default. @@ -94,7 +95,40 @@ container, or migration Job exits. Give each database its own directory and rest | --- | --- | | SQLite | Defaults to `.pc-migration-state` beside the canonical target file; override with `--evidence-dir` | | Embedded seekdb | Defaults to `.pc-migration-state` beside the canonical engine directory; override with `--evidence-dir` | -| OceanBase | Explicit `--evidence-dir` is required by all four commands; every migration Job for the same database must share this durable directory | +| OceanBase | All four commands require explicit `--evidence-dir` and `--lock-coordination single-host`; every migration Job for the same database uses this durable directory on one fixed maintenance host | + +OceanBase uses an OS-backed file lock in `/`, rather than a database named lock. Pin all +migration Jobs, retries, and inspections to one fixed maintenance host and the same canonical persistent directory. +Keep the hostname stable across replacement containers or Jobs as well; a different hostname conflicts with the durable host binding. +Several business Servers may share the database; migration commands remain on this one host. A shared filesystem, +replicated evidence directory, or identical directory name on another host does not establish supported cross-host +locking. Do not run migration Jobs concurrently on different hosts. + +The plan's `coordination` records `mode=single-host`, the host name, and the canonical database-specific directory. +Changing host or directory changes the plan and requires review again. The explicit option selects this deployment +contract; it does not replace stopped-write responsibilities. Interactive `apply` includes it in the existing single +confirmation. Missing evidence returns `evidence_required`; with evidence supplied but no coordination option, +`coordination_required` is returned before contacting OceanBase. + +OceanBase coordination works with the configured direct or ODP connection without using physical session IDs, +`PROCESS`, or process-list visibility. A durable run receipt stays outside the database alongside recovery evidence; +no extra control table or special Alembic revision row is added. Acquiring a released local lock does not prove previous +remote execution or background DDL has ended. PC does not automatically verify remote quiescence after interruption. + +Keep `/coordinator.json`, which binds the maintenance host, its `coordinator.lock`, and the database-specific `migration.lock`, +`active-run.json`, and recovery summaries. Unsupported OS locking returns `migration_lock_unsupported`; contention +returns `migration_locked`. An unresolved previous run, damaged run evidence, or conflicting host binding returns +`recovery_required`; lock-file or maintenance-connection loss returns `migration_lock_lost`. None of these errors +authorizes deleting the coordination files or continuing on another host. + +Only a normal return followed by successful primary-connection closure clears `active-run.json`. Execution interruption +or connection-close failure, including Ctrl+C after a known DDL completed, retains it. A read-only plan exposes its token as +`coordination.pending_run_id`, binds it to `plan_id`, and reports `state=recovery_required`. A DBA must first confirm +all execution and background DDL from that run have ended. Then review a new plan and explicitly supply its exact token +to `apply --acknowledge-previous-run TOKEN`. This is a manual declaration, not a PC verification, automatic approval, +schema bypass, or generic run-ID resume. `--yes` and the interactive confirmation never supply this token for you. +Missing or mismatched acknowledgement returns `recovery_required`; an acknowledgement without a pending run returns +`stale_ack`. The option is supported only by OceanBase `apply`; SQLite and seekdb reject it. For seekdb, configure its path in `seekdb.env` and use the same commands. Before this interactive manual-backup example, complete your backup and stop every writer: @@ -105,25 +139,43 @@ powercontext server db-migrate apply --env-file seekdb.env --backup manual powercontext server db-migrate verify --env-file seekdb.env ``` -For multiple Servers sharing OceanBase, make a manual backup and stop all writers through deployment tooling. The -following automation uses one persistent shared directory and the same scope when planning and applying: +For multiple Servers sharing OceanBase, make a manual backup and stop all writers through deployment tooling. Run every +command below on the fixed maintenance host, using its one persistent directory and the same scope when planning and applying: ```bash powercontext server db-migrate status --env-file oceanbase.env \ - --evidence-dir /mnt/shared/pc-migration-state + --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host powercontext server db-migrate plan --env-file oceanbase.env --backup manual \ - --shared-database --evidence-dir /mnt/shared/pc-migration-state + --shared-database --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host powercontext server db-migrate apply --env-file oceanbase.env --backup manual \ - --shared-database --evidence-dir /mnt/shared/pc-migration-state \ + --shared-database --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host \ --plan-id PLAN_ID --backup-confirmed --maintenance-confirmed --yes powercontext server db-migrate verify --env-file oceanbase.env \ - --evidence-dir /mnt/shared/pc-migration-state + --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host ``` Replace `PLAN_ID` with the returned plan ID. Do not move evidence to an empty directory to bypass a recovery error. Missing or inconsistent evidence for an interrupted migration returns `recovery_required`; selecting manual backup or accepting no-backup risk does not authorize an unknown schema or an unproven DDL transition. +After an interrupted OceanBase run, keep every writer stopped and retain the same host and directory. Have a DBA confirm +the previous remote execution and background DDL have ended, then inspect and review the recovery plan: + +```bash +powercontext server db-migrate plan --env-file oceanbase.env --backup manual \ + --shared-database --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host +powercontext server db-migrate apply --env-file oceanbase.env --backup manual \ + --shared-database --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host \ + --plan-id NEW_PLAN_ID --acknowledge-previous-run PENDING_RUN_ID \ + --backup-confirmed --maintenance-confirmed --yes +powercontext server db-migrate verify --env-file oceanbase.env \ + --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host +``` + +Use the latest plan's `plan_id` and `coordination.pending_run_id`. Reconfirm the existing manual backup; do not replace +the original recovery point with a backup of partial state. Normal upgrades still use one confirmation. Recovery requires +the additional explicit DBA declaration; PC never invents it when showing the plan. + ## Backup choices | Policy | Behavior | Options | @@ -155,8 +207,8 @@ or another policy. Version eligibility is only the start: capability checks also consider actual product, tenant mode, permissions, objects, subsequent DDL limits, and restoration. Several table Forks do not automatically constitute one database snapshot. Fork shares underlying storage and does not protect against disk failure. A simple-table Fork probe does not establish -recovery of this four-table bundle. OceanBase has not completed real environment acceptance; an implemented provider -is not a production-upgrade guarantee. +recovery of this four-table bundle. Full Fork restoration acceptance remains outstanding for OceanBase; an implemented +provider is not a production-upgrade guarantee. Retain old business tables, Fork recovery points, and SQLite backups during this upgrade. Migration success, normal startup, and retry do not delete them. A later independent revision removes old tables; explicit maintenance removes @@ -206,7 +258,8 @@ powercontext server run --env-file deployment.env --role all For multiple Servers sharing one database, run one migration Job per database. Deployment tooling stops traffic and task production, drains requests and tasks requiring old handlers, stops all writers, and suspends scaling and automatic -restarts. Replacement migration Jobs use the same persistent `--evidence-dir`; the original recovery reference must +restarts. OceanBase replacement migration Jobs stay on the fixed maintenance host and use the same persistent +`--evidence-dir` and `--lock-coordination single-host`; the original recovery reference and active-run evidence must survive Job replacement. Coordinate subsequent node upgrades; each node passes its schema, task, and capability checks before traffic returns. A migration lock excludes other migrators, not unknown external clients. @@ -222,11 +275,16 @@ receipts. The Phase A bundle does not yet cover these complete business checks. ## Storage layers and recovery boundaries -Maintenance reuses configuration and keeps Alembic and locking on a dedicated connection. Frozen historical SQL does not -depend on current Repositories or application metadata. seekdb holds its canonical engine-directory lock before startup -until engine shutdown. OceanBase holds a named lock on a pinned connection, checks that a second connection is excluded -across commits, and verifies ownership before and after each DDL. Lock loss blocks further writes; it does not reconnect -and continue. These locks do not stop business writers. +Maintenance reuses configuration and uses a dedicated connection for Alembic. Frozen historical SQL does not depend on +current Repositories or application metadata. seekdb holds its canonical engine-directory lock before startup until +engine shutdown. OceanBase verifies the OS lock, host binding, durable run receipt, and maintenance connection before +and after each DDL. On normal completion it closes the primary maintenance connection before clearing its receipt and +releasing the file lock; the engine closes afterward. Execution interruption or connection-close failure retains the receipt, including a connection-close +failure. Later execution requires the exact previous-run token and a DBA's declaration that remote work has ended. +PC does not infer completion from local process exit or lock availability. Replacing the lock file, losing the connection, +or finding an unacknowledged previous run blocks continuation. +Unavailable native file locking is rejected, including runtime fallback to a soft lock; it does not reconnect or silently +weaken coordination and continue. These locks do not stop business writers. seekdb/OceanBase DDL may commit implicitly, so one Python transaction cannot make the entire upgrade atomic. Recovery recognizes only the registered sequence: `p0001` creates the four frozen tables and two indexes; `p0002` adds the two diff --git a/docs/zh/docs/operate/database-migrations.md b/docs/zh/docs/operate/database-migrations.md index b797f69b23..b14dff53da 100644 --- a/docs/zh/docs/operate/database-migrations.md +++ b/docs/zh/docs/operate/database-migrations.md @@ -12,13 +12,13 @@ description: 了解数据库迁移命令、备份策略、服务启停及当前 `powercontext server db-migrate status/plan/apply/verify` 使用部署配置中的持久 SQLite、嵌入式 seekdb 或 OceanBase MySQL 模式数据库，不接受任意测试 bundle 路径。维护命令以显式环境文件为准，应使用部署的环境文件，不依赖临时 shell 环境覆盖。随 wheel 发布的冻结资源用于阶段 A 的 Artifact 四表迁移：`pc_artifacts`、`pc_artifact_heads`、`pc_artifact_candidate_versions`、`pc_artifact_tags`。仅接受空目标和已登记的历史结构；完整 Server 数据库含有尚未纳管的对象时拒绝执行。 -输出中的 `ready` 仅表示当前迁移 bundle 已通过结构和数据验证；`readiness_scope=registered_bundle`、`server_ready=false` 明确区分它与完整 Server、索引、旧任务及集群可用。不要以此替代生产升级或完整业务库初始化。普通业务启动的全面版本门禁和完整历史基线尚未纳管。OceanBase 实际执行和完整 Fork 恢复仍需专用真实环境验收；本地测试与已有适配器不能证明生产可用。 +输出中的 `ready` 仅表示当前迁移 bundle 已通过结构和数据验证；`readiness_scope=registered_bundle`、`server_ready=false` 明确区分它与完整 Server、索引、旧任务及集群可用。不要以此替代生产升级或完整业务库初始化。普通业务启动的全面版本门禁和完整历史基线尚未纳管。已通过 OceanBase AI 4.6.3 的 ODP 入口完成隔离四表验收，覆盖固定维护主机互斥、DDL 提交后中断的显式确认恢复、数据保留与重复执行。这不代表完整 Server 升级、崩溃后远端 DDL 自动判定或完整 Fork 恢复已通过验收。 原有 `server processing-migrate` 保留，请按[迁移 Artifact 处理状态](artifact-processing-migration.md)使用。它尚未自动转发到统一入口。服务部署参考[部署 Server](deploy-server.md)。 ## 只读检查与一次确认 -下面的 `deployment.env` 示例使用 SQLite；OceanBase 命令还须传入下一节的证据目录参数。先检查目标或预览计划，这些命令不会创建缺失数据库、父目录或控制表： +下面的 `deployment.env` 示例使用 SQLite；OceanBase 命令还须传入下一节的证据目录和固定主机协调参数。先检查目标或预览计划，这些命令不会创建缺失数据库、父目录或控制表： ```bash powercontext server db-migrate status --env-file deployment.env @@ -26,7 +26,7 @@ powercontext server db-migrate plan --env-file deployment.env --backup auto powercontext server db-migrate verify --env-file deployment.env ``` -计划绑定目标身份、源／目标 revision、实际结构摘要、冻结脚本摘要、配置摘要、备份策略和服务范围。配置与凭据值不输出到诊断中；预期错误返回稳定 JSON 类别和非零退出码。 +计划绑定目标身份、源／目标 revision、实际结构摘要、冻结脚本摘要、配置摘要、备份策略、服务范围和迁移协调范围。配置与凭据值不输出到诊断中；预期错误返回稳定 JSON 类别和非零退出码。 在支持范围内，终端用户可直接执行 `apply`，默认 PC 自动备份。命令展示计划、备份方法、维护影响及停写责任，确认一次后执行，不逐条 SQL 询问： @@ -60,7 +60,19 @@ seekdb 只读检查遇到缺失路径或空目录时不启动引擎；非空目 | --- | --- | | SQLite | 默认在规范化后的数据库文件旁使用 `<数据库文件名>.pc-migration-state`，可用 `--evidence-dir` 覆盖 | | 嵌入式 seekdb | 默认在规范化后的引擎目录旁使用 `<目录名>.pc-migration-state`，可用 `--evidence-dir` 覆盖 | -| OceanBase | 四个命令均须显式传入 `--evidence-dir`；同一数据库的所有迁移 Job 共享该持久目录 | +| OceanBase | 四个命令均须显式传入 `--evidence-dir` 和 `--lock-coordination single-host`；同一数据库的所有迁移 Job 使用固定维护主机上的同一持久目录 | + +OceanBase 在 `/` 中使用操作系统文件锁，不使用数据库命名锁。所有迁移 Job、重试和检查命令均固定在同一维护主机，并使用规范化后的同一持久目录。业务 Server 可以分布在多节点共用数据库，迁移命令仍只在这一台主机执行。共享文件系统、复制证据目录或其他主机上的同名目录，不代表已支持跨主机互斥；不要在不同主机并发运行迁移 Job。 + +替换容器或 Job 时也应保持主机名稳定；主机名改变会与持久主机绑定冲突，即使任务仍调度在同一物理节点上。 + +计划中的 `coordination` 记录 `mode=single-host`、主机名和规范化后的数据库专属目录。更换主机或目录会改变计划，需要重新审阅。显式参数表示选择这一部署约束，不替代停写声明；交互式 `apply` 将它纳入原有的一次确认，不新增询问步骤。缺少证据目录先返回 `evidence_required`；已有目录参数但未选择协调模式时，在连接 OceanBase 前返回 `coordination_required`。 + +OceanBase 协调使用配置中的直连或 ODP 连接，不依赖物理会话 ID、`PROCESS` 权限或进程列表观察。持久运行记录与恢复证据一起保存在库外，不新增控制表，也不向 Alembic 版本行写特殊锁标识。获得已释放的本地锁不代表旧远端执行或后台 DDL 已结束；PC 不自动核验中断后的远端执行是否结束。 + +应保留绑定维护主机的 `/coordinator.json`、同目录的 `coordinator.lock`，以及数据库专属目录中的 `migration.lock`、`active-run.json` 和恢复摘要。平台不支持原生锁返回 `migration_lock_unsupported`；其他进程持锁返回 `migration_locked`。旧运行尚未确认结束、运行证据损坏或绑定主机不一致返回 `recovery_required`；锁文件或维护连接丢失返回 `migration_lock_lost`。这些错误均不授权删除协调文件或换主机继续。 + +只有正常返回且迁移主连接成功关闭后，才清理 `active-run.json`。执行中断或主连接关闭失败会保留记录，包括已知 DDL 完成后按 Ctrl+C。只读计划将其 token 展示为 `coordination.pending_run_id`，纳入 `plan_id`，并报告 `state=recovery_required`。用户须先由 DBA 确认该次运行的全部远端执行和后台 DDL 已结束，再重新审阅计划，执行 `apply --acknowledge-previous-run TOKEN` 并提供准确 token。该参数仅作人工声明，不表示 PC 已核验，不自动审批、不绕过结构检查，也不是通用 run ID 续跑。`--yes` 与交互确认均不会替用户填入 token。缺少或不匹配的声明返回 `recovery_required`；没有遗留运行却提供 token 返回 `stale_ack`。参数仅支持 OceanBase 的 `apply`，SQLite 和 seekdb 拒绝该参数。 seekdb 在 `seekdb.env` 中配置路径后使用同一组命令。下面的手动备份交互示例要求先完成备份并停止所有写入者： @@ -70,22 +82,37 @@ powercontext server db-migrate apply --env-file seekdb.env --backup manual powercontext server db-migrate verify --env-file seekdb.env ``` -多个 Server 共用 OceanBase 时，先由用户手动备份，部署系统停止所有写入者。下面的自动化示例使用一个持久共享目录，并在计划和执行时使用相同范围： +多个 Server 共用 OceanBase 时，先由用户手动备份，部署系统停止所有写入者。以下所有命令都在固定维护主机执行，使用该主机上的同一持久目录，并在计划和执行时使用相同范围： ```bash powercontext server db-migrate status --env-file oceanbase.env \ - --evidence-dir /mnt/shared/pc-migration-state + --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host powercontext server db-migrate plan --env-file oceanbase.env --backup manual \ - --shared-database --evidence-dir /mnt/shared/pc-migration-state + --shared-database --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host powercontext server db-migrate apply --env-file oceanbase.env --backup manual \ - --shared-database --evidence-dir /mnt/shared/pc-migration-state \ + --shared-database --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host \ --plan-id PLAN_ID --backup-confirmed --maintenance-confirmed --yes powercontext server db-migrate verify --env-file oceanbase.env \ - --evidence-dir /mnt/shared/pc-migration-state + --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host ``` 将 `PLAN_ID` 替换为返回的计划 ID。不能换到空目录绕过恢复错误：中断迁移的证据缺失或不一致时返回 `recovery_required`；选择手动备份或接受不备份风险，也不能授权未知结构或无法证明的 DDL 进度。 +OceanBase 执行中断后，保持所有写入者停止，保留同一主机与目录。先由 DBA 确认旧远端执行和后台 DDL 已结束，再检查并审阅恢复计划： + +```bash +powercontext server db-migrate plan --env-file oceanbase.env --backup manual \ + --shared-database --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host +powercontext server db-migrate apply --env-file oceanbase.env --backup manual \ + --shared-database --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host \ + --plan-id NEW_PLAN_ID --acknowledge-previous-run PENDING_RUN_ID \ + --backup-confirmed --maintenance-confirmed --yes +powercontext server db-migrate verify --env-file oceanbase.env \ + --evidence-dir /var/lib/powercontext/migration-state --lock-coordination single-host +``` + +使用最新计划中的 `plan_id` 和 `coordination.pending_run_id`。重新声明既有手动备份，不用部分状态的新备份替换原恢复点。正常升级仍只确认一次；中断恢复另需用户显式作出 DBA 声明，PC 展示计划时不会自动替用户确认。 + ## 备份选择 | 策略 | 行为 | 参数 | @@ -103,7 +130,7 @@ powercontext server db-migrate verify --env-file oceanbase.env \ | OceanBase AI 数据库 | V4.6.2 | V4.6.2 | | seekdb | V1.1.0 | V1.2.0 | -版本满足只是能力检查的起点，还需检查实际产品、租户模式、权限、对象覆盖、后续 DDL 限制和恢复路径。多个表 Fork 不自动代表整库共同快照；Fork 共享底层存储，不提供磁盘损坏保护。简单表的 Fork 探针不代表四表 bundle 的完整恢复能力；OceanBase 尚未完成真实环境验收，存在 provider 不等于可用生产升级。 +版本满足只是能力检查的起点，还需检查实际产品、租户模式、权限、对象覆盖、后续 DDL 限制和恢复路径。多个表 Fork 不自动代表整库共同快照；Fork 共享底层存储，不提供磁盘损坏保护。简单表的 Fork 探针不代表四表 bundle 的完整恢复能力；OceanBase 完整 Fork 恢复仍待验收，存在 provider 不等于可用生产升级。 旧业务表、Fork 恢复点和 SQLite 备份本次保留，不在迁移成功、普通启动或重试时删除。旧表删除由后续版本单独 revision 实现；恢复点由显式维护操作清理，均需验证依赖、保留窗口及恢复责任。 @@ -137,7 +164,7 @@ powercontext server run --env-file deployment.env --role all ## 共享数据库、接口与旧任务 -多个 Server 共用一个数据库时，每库只执行一个迁移 Job。部署系统停止流量和任务生产、排空在途请求及需由旧处理器消费的任务、停止全部写入者，并暂停扩容和自动重启。替换的迁移 Job 继续使用同一个持久 `--evidence-dir`，保留原始恢复引用。迁移后协调全部相关节点升级，各节点通过自身 schema、任务和能力检查后才恢复流量。迁移锁只排斥迁移进程，不阻止未知外部客户端。 +多个 Server 共用一个数据库时，每库只执行一个迁移 Job。部署系统停止流量和任务生产、排空在途请求及需由旧处理器消费的任务、停止全部写入者，并暂停扩容和自动重启。OceanBase 替换的迁移 Job 仍在固定维护主机使用同一个持久 `--evidence-dir` 和 `--lock-coordination single-host`，保留原始恢复引用及运行证据。迁移后协调全部相关节点升级，各节点通过自身 schema、任务和能力检查后才恢复流量。迁移锁只排斥迁移进程，不阻止未知外部客户端。 发布作者应提供受影响对象、执行模式、程序／schema 兼容范围、API 变化和任务格式声明。公共 API 不兼容替换需标记弃用并说明替代入口和移除计划；同包内部入口可与调用方一起替换。保留的 API 可转换请求／响应后共用当前实现，不要求每个 handler 支持两套表结构。 @@ -145,7 +172,7 @@ powercontext server run --env-file deployment.env --role all ## 存储层与恢复边界 -维护执行器复用配置，使用专用连接承载 Alembic 和锁；冻结历史 SQL 不依赖当前 Repository 或应用 metadata。seekdb 从启动引擎前到关闭引擎后持有规范化目录的锁。OceanBase 在固定连接上持有命名锁，检查另一连接在提交前后仍被排斥，并在每条 DDL 前后验证持有权；失锁后阻止后续写入，不重连继续。这些锁不阻止业务写入者。 +维护执行器复用配置，使用专用连接承载 Alembic；冻结历史 SQL 不依赖当前 Repository 或应用 metadata。seekdb 从启动引擎前到关闭引擎后持有规范化目录的锁。OceanBase 在每条 DDL 前后验证操作系统锁、主机绑定、持久运行记录及维护连接。正常结束时先关闭迁移主连接，才清理运行记录并释放文件锁；引擎随后关闭。执行中断或关闭连接失败均保留记录；后续执行需要准确的旧运行 token，以及 DBA 确认远端执行已结束的人工声明。PC 不从本机进程退出或本地锁可获取推断远端执行结束。锁文件被替换、连接丢失或旧运行未确认时阻止继续。平台不支持原生文件锁时拒绝执行，包括运行时降级为软锁的情况，不重连或静默弱化互斥继续迁移。这些锁不阻止业务写入者。 seekdb／OceanBase DDL 可能隐式提交，不能用 Python 事务承诺整个迁移原子化。恢复只识别已登记顺序：`p0001` 创建冻结四表及两个索引；`p0002` 添加两个可空的 `MEDIUMBLOB` 引用列；`p0003` 删除标签 family 的 CHECK。继续执行前须用实际结构、数据检查和持久维护证据证明已完成的已知前缀；revision 的后置条件通过后才推进 Alembic 版本。未知对象、结构或中间状态拒绝执行，不乐观 stamp。 diff --git a/src/powercontext/builtin/persistence/migrations/connections.py b/src/powercontext/builtin/persistence/migrations/connections.py index fcb638a613..bc2ba8edfa 100644 --- a/src/powercontext/builtin/persistence/migrations/connections.py +++ b/src/powercontext/builtin/persistence/migrations/connections.py @@ -17,14 +17,15 @@ from __future__ import annotations import asyncio +import socket from collections.abc import Callable from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import Literal, TypeVar -from sqlalchemy import Connection, text -from sqlalchemy.exc import DBAPIError, SQLAlchemyError +from sqlalchemy import Connection +from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine from sqlalchemy.pool import NullPool @@ -35,7 +36,7 @@ from .locking import local_migration_lock from .models import MigrationError, digest -from .oceanbase import PinnedOceanBaseLock, oceanbase_migration_lock +from .oceanbase import oceanbase_migration_lock, pending_run _T = TypeVar("_T") @@ -58,8 +59,32 @@ class MaintenanceConnections: connection and does not start an engine or create any directory. """ - def __init__(self, config: SeekDBConfig | OceanBaseConfig) -> None: + def __init__( + self, + config: SeekDBConfig | OceanBaseConfig, + *, + evidence_directory: Path | None = None, + lock_coordination: Literal["single-host"] | None = None, + acknowledge_previous_run: str | None = None, + ) -> None: self.config = config + self.evidence_directory = evidence_directory.expanduser().resolve() if evidence_directory is not None else None + self.lock_coordination = lock_coordination + self.acknowledge_previous_run = acknowledge_previous_run + + def coordination(self, identity: BackendIdentity) -> dict[str, str]: + if isinstance(self.config, SeekDBConfig): + return {"mode": "engine-directory", "directory": str(self.config.path.expanduser().resolve())} + coordination = { + "mode": self.lock_coordination or "unselected", + "host": socket.gethostname(), + "directory": str(self.evidence_directory / identity.database_id) if self.evidence_directory else "", + } + if self.evidence_directory is not None: + previous = pending_run(self.evidence_directory, identity.database_id) + if previous is not None: + coordination["pending_run_id"] = previous + return coordination def run( self, @@ -141,6 +166,11 @@ async def _oceanbase( *, writable: bool, ) -> _T: + evidence_directory = self.evidence_directory + if writable and (self.lock_coordination != "single-host" or evidence_directory is None): + raise MigrationError( + "coordination_required", "Select single-host coordination and one persistent evidence directory." + ) oceanbase_profile._register_official_dialect() engine = create_async_engine( config.url.get_secret_value(), @@ -155,6 +185,8 @@ async def _oceanbase( identity = await connection.run_sync(_oceanbase_identity) if not writable: return await connection.run_sync(lambda sync: operation(sync, identity, lambda: None)) + if evidence_directory is None: + raise MigrationError("coordination_required", "Select one persistent evidence directory.") async with engine.connect() as contender: other_identity = await contender.run_sync(_oceanbase_identity) if other_identity != identity: @@ -162,12 +194,15 @@ async def _oceanbase( "target_identity_unavailable", "Maintenance sessions reached different database identities." ) async with oceanbase_migration_lock( - connection, contender, database_id=identity.database_id + connection, + database_id=identity.database_id, + directory=evidence_directory, + acknowledged_run=self.acknowledge_previous_run, ) as lock: def execute(sync: Connection) -> _T: def verify() -> None: - _verify_remote_lock(sync, lock) + lock.verify_sync(sync) verify() result = operation(sync, identity, verify) @@ -233,18 +268,6 @@ def _oceanbase_identity(connection: Connection) -> BackendIdentity: ) from error -def _verify_remote_lock(connection: Connection, lock: PinnedOceanBaseLock) -> None: - try: - if connection.invalidated or connection.closed: - raise MigrationError("migration_lock_lost", "The lock connection was lost; inspect DDL before recovery.") - session = connection.scalar(text("SELECT CONNECTION_ID()")) - owner = connection.scalar(text("SELECT IS_USED_LOCK(:name)"), {"name": lock.name}) - if session != lock.session_id or owner != lock.session_id: - raise MigrationError("migration_lock_lost", "The pinned session no longer owns the migration lock.") - except DBAPIError as error: - raise MigrationError("migration_lock_lost", "The database migration lock could not be verified.") from error - - async def _dispose_engine(engine: AsyncEngine) -> None: task = asyncio.create_task(engine.dispose()) try: diff --git a/src/powercontext/builtin/persistence/migrations/deployment.py b/src/powercontext/builtin/persistence/migrations/deployment.py index 2652623218..45a12a1ab8 100644 --- a/src/powercontext/builtin/persistence/migrations/deployment.py +++ b/src/powercontext/builtin/persistence/migrations/deployment.py @@ -18,7 +18,7 @@ import os from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal from sqlalchemy.engine import make_url @@ -44,20 +44,38 @@ def production_bundle() -> MigrationBundle: MigrationRunner = SQLiteMigrationRunner | MySQLMigrationRunner -def deployment_runner(settings: ServerSettings, *, evidence_dir: Path | None = None) -> MigrationRunner: +def deployment_runner( + settings: ServerSettings, + *, + evidence_dir: Path | None = None, + lock_coordination: Literal["single-host"] | None = None, + acknowledge_previous_run: str | None = None, +) -> MigrationRunner: """Use the real configured identity without opening a business profile.""" database = settings.database + if acknowledge_previous_run is not None and not isinstance(database, OceanBaseConfig): + raise MigrationError("unsupported_option", "--acknowledge-previous-run is supported only for OceanBase.") if isinstance(database, (SeekDBConfig, OceanBaseConfig)): if evidence_dir is None: if isinstance(database, OceanBaseConfig): raise MigrationError( "evidence_required", - "OceanBase maintenance requires --evidence-dir on durable storage shared by migration Jobs.", + "OceanBase maintenance requires --evidence-dir on one fixed host, shared by all migration Jobs.", ) target = database.path.expanduser().resolve() evidence_dir = target.with_name(target.name + ".pc-migration-state") + if isinstance(database, OceanBaseConfig) and lock_coordination != "single-host": + raise MigrationError( + "coordination_required", + "OceanBase requires --lock-coordination single-host; run all migrators on one fixed host and evidence directory.", + ) return MySQLMigrationRunner( - MaintenanceConnections(database), + MaintenanceConnections( + database, + evidence_directory=evidence_dir, + lock_coordination=lock_coordination, + acknowledge_previous_run=acknowledge_previous_run, + ), production_bundle(), evidence_directory=evidence_dir, configuration=database.model_dump(mode="json", exclude={"url", "echo"}), diff --git a/src/powercontext/builtin/persistence/migrations/locking.py b/src/powercontext/builtin/persistence/migrations/locking.py index 0c793808d2..1e3f62bb99 100644 --- a/src/powercontext/builtin/persistence/migrations/locking.py +++ b/src/powercontext/builtin/persistence/migrations/locking.py @@ -17,14 +17,19 @@ from __future__ import annotations from collections.abc import Iterator -from contextlib import contextmanager +from contextlib import contextmanager, suppress from pathlib import Path -from filelock import FileLock, SoftFileLock, Timeout +from filelock import BaseFileLock, FileLock, SoftFileLock, Timeout from .models import MigrationError +def _require_os_backed_lock(lock: BaseFileLock) -> None: + if isinstance(lock, SoftFileLock): + raise MigrationError("migration_lock_unsupported", "This platform does not provide an OS-backed file lock.") + + @contextmanager def local_migration_lock(target: Path) -> Iterator[None]: """Lock a canonical database file or embedded-engine directory before opening it. @@ -35,13 +40,19 @@ def local_migration_lock(target: Path) -> Iterator[None]: target = target.expanduser().resolve() target.parent.mkdir(parents=True, exist_ok=True) lock = FileLock(str(target.with_name(target.name + ".pc-migration.lock")), timeout=0, mode=0o600) - if isinstance(lock, SoftFileLock): - raise MigrationError("migration_lock_unsupported", "This platform does not provide an OS-backed file lock.") + _require_os_backed_lock(lock) try: lock.acquire() except Timeout as error: raise MigrationError("migration_locked", "Another process holds the migration lock.") from error try: + # filelock can switch to SoftFileLock during acquire when native locking is unsupported. + _require_os_backed_lock(lock) yield - finally: + except BaseException: + # Cleanup must not replace an unsupported-lock, migration, or interruption error. + with suppress(BaseException): + lock.release() + raise + else: lock.release() diff --git a/src/powercontext/builtin/persistence/migrations/models.py b/src/powercontext/builtin/persistence/migrations/models.py index 5e682b46bb..abbbd7af57 100644 --- a/src/powercontext/builtin/persistence/migrations/models.py +++ b/src/powercontext/builtin/persistence/migrations/models.py @@ -55,6 +55,7 @@ class MigrationPlan(BaseModel): affected_objects: tuple[str, ...] = () retained_objects: tuple[str, ...] = () shared_database: bool = False + coordination: dict[str, str] | None = None service: dict[str, Any] | None = None execution_mode: Literal["offline"] = "offline" readiness_scope: Literal["registered_bundle"] = "registered_bundle" diff --git a/src/powercontext/builtin/persistence/migrations/mysql.py b/src/powercontext/builtin/persistence/migrations/mysql.py index 801107bab3..9ef0708008 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql.py +++ b/src/powercontext/builtin/persistence/migrations/mysql.py @@ -18,6 +18,7 @@ import hashlib import os +import socket from collections.abc import Callable, Mapping from pathlib import Path from tempfile import TemporaryDirectory @@ -152,6 +153,21 @@ def __init__( self.schema = MySQLSchema(bundle) self.evidence_directory = evidence_directory.expanduser().resolve() self.configuration_digest = digest(configuration or {}) + if connections.config.kind == "oceanbase": + if ( + connections.lock_coordination != "single-host" + or connections.evidence_directory != self.evidence_directory + ): + raise MigrationError( + "coordination_required", + "Use single-host coordination and the same evidence directory for this runner.", + ) + self.configuration_digest = digest({ + "configuration": configuration or {}, + "coordination": "single-host", + "host": socket.gethostname(), + "evidence_directory": str(self.evidence_directory), + }) self.backup_factory = backup_factory or _fork_provider self._history_directory = Path(__file__).parent / "mysql_resources" resources = { @@ -233,6 +249,15 @@ def apply( "shared_database": shared_database, } initial = self.plan(**options) + previous_run = (initial.coordination or {}).get("pending_run_id") + acknowledgement = self.connections.acknowledge_previous_run + if previous_run is not None and acknowledgement != previous_run: + raise MigrationError( + "recovery_required", + "Confirm that the previous remote operation has ended, then acknowledge its exact pending run ID.", + ) + if previous_run is None and acknowledgement is not None: + raise MigrationError("stale_ack", "There is no pending run matching this acknowledgement.") if initial.state == "ready": return MigrationResult(revision=self.bundle.head, changed=False) self._check_consent( @@ -303,6 +328,13 @@ def _apply_locked( ): raise MigrationError("plan_changed", "The database changed before the maintenance lock was acquired.") journal = _Journal(self.evidence_directory, identity) + previous = journal.read() + if not initial.revisions and not initial.adopt_baseline and (previous is None or previous.state == "complete"): + # A completed migration can retain its run receipt after an + # interrupted connection close. Acknowledgement still requires + # the lock and full schema/data reinspection, but no new backup. + guard() + return MigrationResult(revision=self.bundle.head, changed=False) provider = self.backup_factory(connection, identity, journal.path.parent / "backups") evidence = self._prepare_evidence(initial, journal, provider, manual_ref=manual_ref) guard() @@ -451,6 +483,9 @@ def _inspect( available, reasons = self._backup_availability( connection, identity, journal, evidence, current, backup_policy=backup_policy, ready=state == "ready" ) + coordination = self.connections.coordination(identity) + if coordination.get("pending_run_id"): + state = "recovery_required" fields = { "database_id": identity.database_id, "source_revision": current, @@ -468,6 +503,7 @@ def _inspect( "retained_objects": (), "service": service, "shared_database": shared_database, + "coordination": coordination, } return MigrationPlan.model_validate({"plan_id": digest(fields), **fields}) diff --git a/src/powercontext/builtin/persistence/migrations/oceanbase.py b/src/powercontext/builtin/persistence/migrations/oceanbase.py index 374bf25e41..9ea6588881 100644 --- a/src/powercontext/builtin/persistence/migrations/oceanbase.py +++ b/src/powercontext/builtin/persistence/migrations/oceanbase.py @@ -12,87 +12,394 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Probe and hold a session lock on a pinned OceanBase connection. +"""Serialize remote maintenance on an explicitly selected coordinator host. -This is one prerequisite for a nontransactional-DDL executor, not that executor. -Deployment acceptance must additionally test its actual node/proxy routing. +This is an OS lock, not a database-wide or cross-host advisory lock. All +migration commands for the deployment must use the same host and evidence +directory. A durable run receipt fences recovery after any unclean exit. +Releasing a local OS lock does not establish that a request sent through ODP +has finished; only an explicit operator acknowledgment permits a retry. """ from __future__ import annotations -from collections.abc import AsyncIterator +import asyncio +import errno +import json +import os +import re +import socket +import stat +import tempfile +import uuid +from collections.abc import AsyncIterator, Awaitable from contextlib import asynccontextmanager +from contextvars import ContextVar from dataclasses import dataclass +from pathlib import Path +from typing import Any -from sqlalchemy import text -from sqlalchemy.exc import DBAPIError +from sqlalchemy import Connection from sqlalchemy.ext.asyncio import AsyncConnection -from .models import MigrationError, digest +from .models import MigrationError + + +class _OSLock: + def __init__(self, path: Path) -> None: + self.path = path + self.fd: int | None = None + self.parent_identity: tuple[int, int] | None = None + + def acquire(self) -> None: + if self.path.parent.is_symlink(): + raise MigrationError("migration_lock_unsupported", "The lock directory must not be a symbolic link.") + self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + self.parent_identity = _inode(self.path.parent) + try: + fd = os.open(self.path, os.O_CREAT | os.O_RDWR | getattr(os, "O_NOFOLLOW", 0), 0o600) + self.fd = fd + self._lock_descriptor(fd) + self.verify() + except BaseException as error: + try: + self.release() + except OSError: + error.add_note("The failed coordinator lock descriptor could not be closed.") + if isinstance(error, OSError): + if error.errno in {errno.EACCES, errno.EAGAIN, errno.EDEADLK}: + failure = MigrationError( + "migration_locked", "Another coordinator process holds the migration lock." + ) + else: + failure = MigrationError("migration_lock_unsupported", "The coordinator OS lock is unavailable.") + for note in getattr(error, "__notes__", []): + failure.add_note(note) + raise failure from error + raise + + def _lock_descriptor(self, fd: int) -> None: + if not stat.S_ISREG(os.fstat(fd).st_mode) or self.path.is_symlink(): + raise MigrationError("migration_lock_unsupported", "The lock must be a regular file.") + if os.name == "posix": + import fcntl + + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + elif os.name == "nt": + import msvcrt + + if os.fstat(fd).st_size == 0: + os.write(fd, b"\0") + os.fsync(fd) + os.lseek(fd, 0, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + else: + raise MigrationError("migration_lock_unsupported", "This platform has no supported OS lock.") + + def verify(self) -> None: + if self.fd is None: + raise MigrationError("migration_lock_lost", "The coordinator lock was released.") + try: + opened = os.fstat(self.fd) + visible = self.path.lstat() + if ( + not stat.S_ISREG(visible.st_mode) + or (opened.st_dev, opened.st_ino) != (visible.st_dev, visible.st_ino) + or _inode(self.path.parent) != self.parent_identity + ): + raise MigrationError("migration_lock_lost", "The coordinator lock file or directory changed.") + except OSError as error: + raise MigrationError("migration_lock_lost", "The coordinator lock file is unavailable.") from error + + def release(self) -> None: + fd, self.fd = self.fd, None + if fd is not None: + # Closing the only descriptor releases the OS lock. Keep its inode + # so that existing waiters and new processes lock the same file. + os.close(fd) + + +def _inode(path: Path) -> tuple[int, int]: + information = path.stat() + return information.st_dev, information.st_ino + + +def _read_record(path: Path) -> dict[str, Any] | None: + try: + if path.is_symlink(): + raise MigrationError("recovery_required", "A coordinator record must not be a symbolic link.") + with path.open(encoding="utf-8") as source: + record = json.load(source) + except FileNotFoundError: + return None + except (OSError, ValueError, UnicodeError) as error: + raise MigrationError("recovery_required", "A durable coordinator record could not be read.") from error + if not isinstance(record, dict): + raise MigrationError("recovery_required", "A durable coordinator record has an unknown format.") + return record + + +def _sync_directory(directory: Path) -> None: + if os.name == "posix": + fd = os.open(directory, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def _write_record(path: Path, record: dict[str, Any]) -> None: + temporary: Path | None = None + try: + if path.is_symlink(): + raise MigrationError("recovery_required", "A coordinator record must not be a symbolic link.") + with tempfile.NamedTemporaryFile("w", dir=path.parent, encoding="utf-8", delete=False) as output: + temporary = Path(output.name) + json.dump(record, output, ensure_ascii=False, sort_keys=True) + output.flush() + os.fsync(output.fileno()) + os.replace(temporary, path) + temporary = None + _sync_directory(path.parent) + except OSError as error: + raise MigrationError("recovery_required", "The coordinator record could not be saved durably.") from error + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) @dataclass(frozen=True) -class PinnedOceanBaseLock: +class _HostBinding: + directory: Path + hostname: str + directory_identity: tuple[int, int] + + def verify(self) -> None: + try: + if _inode(self.directory) != self.directory_identity: + raise MigrationError("migration_lock_lost", "The evidence directory changed.") + except OSError as error: + raise MigrationError("migration_lock_lost", "The evidence directory is unavailable.") from error + if _read_record(self.directory / "coordinator.json") != {"version": 1, "hostname": self.hostname}: + raise MigrationError("recovery_required", "The evidence directory belongs to a different coordinator host.") + + +def _bind_host(directory: Path) -> _HostBinding: + directory = directory.expanduser().resolve() + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + host = socket.gethostname().casefold() + if not host: + raise MigrationError("migration_lock_unsupported", "The coordinator hostname is unavailable.") + binding = _HostBinding(directory, host, _inode(directory)) + lock = _OSLock(directory / "coordinator.lock") + lock.acquire() + try: + path = directory / "coordinator.json" + if _read_record(path) is None: + _write_record(path, {"version": 1, "hostname": host}) + binding.verify() + finally: + lock.release() + return binding + + +def _validate_database_id(database_id: str) -> None: + if re.fullmatch(r"[A-Za-z0-9_-]{1,128}", database_id) is None: + raise MigrationError("unsupported_target", "The maintenance database identity is not a safe directory name.") + + +def _run_receipt(path: Path, *, database_id: str, hostname: str) -> dict[str, Any] | None: + receipt = _read_record(path) + if receipt is not None and ( + set(receipt) != {"version", "database_id", "hostname", "token", "previous_run"} + or type(receipt["version"]) is not int + or receipt["version"] != 1 + or receipt["database_id"] != database_id + or receipt["hostname"] != hostname + or not isinstance(receipt["token"], str) + or re.fullmatch(r"[0-9a-f]{32}", receipt["token"]) is None + or ( + receipt["previous_run"] is not None + and ( + not isinstance(receipt["previous_run"], str) + or re.fullmatch(r"[0-9a-f]{32}", receipt["previous_run"]) is None + or receipt["previous_run"] == receipt["token"] + ) + ) + ): + raise MigrationError("recovery_required", "The previous run receipt has an unknown identity or format.") + return receipt + + +@dataclass(frozen=True) +class _ActiveRun: + directory: Path + database_id: str + own_token: str + prior_token: str | None + + +_active_run: ContextVar[_ActiveRun | None] = ContextVar("oceanbase_active_migration_run", default=None) + + +def pending_run(directory: Path, database_id: str) -> str | None: + """Read an unresolved run token without creating files or querying the DB. + + Within an admitted operation this returns its predecessor's token, so that + rechecking the reviewed plan does not treat its own receipt as a new crash. + Independent callers continue to see the current operation's durable token. + """ + _validate_database_id(database_id) + directory = directory.expanduser().resolve() + hostname = socket.gethostname().casefold() + binding = _read_record(directory / "coordinator.json") + if binding is not None and binding != {"version": 1, "hostname": hostname}: + raise MigrationError("recovery_required", "The evidence directory belongs to a different coordinator host.") + receipt = _run_receipt(directory / database_id / "active-run.json", database_id=database_id, hostname=hostname) + active = _active_run.get() + if active is not None and active.directory == directory and active.database_id == database_id: + if receipt is None or receipt["token"] != active.own_token or receipt["previous_run"] != active.prior_token: + raise MigrationError("recovery_required", "The admitted run receipt changed; inspect it first.") + return active.prior_token + return receipt["token"] if receipt is not None else None + + +@dataclass(frozen=True) +class CoordinatorLock: connection: AsyncConnection name: str - session_id: int + _os_lock: _OSLock + _host: _HostBinding + _receipt_path: Path + _receipt: dict[str, Any] + + def _verify_local(self) -> None: + self._os_lock.verify() + self._host.verify() + if _read_record(self._receipt_path) != self._receipt: + raise MigrationError("recovery_required", "The active run receipt changed; inspect it first.") + + def verify_sync(self, connection: Connection) -> None: + """Check local coordination without relying on a physical ODP session.""" + self._verify_local() + if connection.closed or connection.invalidated: + raise MigrationError("migration_lock_lost", "The dedicated maintenance connection was lost.") async def verify(self) -> None: - """Call before and after each DDL; a reconnected session is not trusted.""" - if self.connection.invalidated or self.connection.closed: - raise MigrationError("migration_lock_lost", "The lock connection was lost; inspect DDL before recovery.") - current = await self.connection.scalar(text("SELECT CONNECTION_ID()")) - owner = await self.connection.scalar(text("SELECT IS_USED_LOCK(:name)"), {"name": self.name}) - if current != self.session_id or owner != self.session_id: - raise MigrationError("migration_lock_lost", "The pinned session no longer owns the migration lock.") + self._verify_local() + if self.connection.closed or self.connection.invalidated: + raise MigrationError("migration_lock_lost", "The dedicated maintenance connection was lost.") + + +def _clear_receipt(path: Path, receipt: dict[str, Any], lock: _OSLock, host: _HostBinding) -> None: + lock.verify() + host.verify() + if _read_record(path) != receipt: + raise MigrationError("recovery_required", "The active run receipt changed; retain it for inspection.") + path.unlink() + _sync_directory(path.parent) + + +async def _finish_cleanup(operation: Awaitable[None]) -> None: + task = asyncio.ensure_future(operation) + cancellation: asyncio.CancelledError | None = None + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError as error: + cancellation = cancellation or error + task.result() + if cancellation is not None: + raise cancellation + + +def _start_run(path: Path, database_id: str, hostname: str, acknowledgment: str | None) -> dict[str, Any]: + previous = _run_receipt(path, database_id=database_id, hostname=hostname) + prior_token = previous["token"] if previous is not None else None + if prior_token is not None and acknowledgment != prior_token: + raise MigrationError( + "recovery_required", "Confirm the previous remote operation has ended and acknowledge its run token." + ) + if prior_token is None and acknowledgment is not None: + raise MigrationError("stale_ack", "There is no previous run matching the supplied acknowledgment.") + receipt = { + "version": 1, + "database_id": database_id, + "hostname": hostname, + "token": uuid.uuid4().hex, + "previous_run": prior_token, + } + _write_record(path, receipt) + return receipt @asynccontextmanager async def oceanbase_migration_lock( - connection: AsyncConnection, contender: AsyncConnection, *, database_id: str -) -> AsyncIterator[PinnedOceanBaseLock]: - """Test exclusion and commit survival using two independently routed sessions. + connection: AsyncConnection, + *, + database_id: str, + directory: Path, + acknowledged_run: str | None = None, +) -> AsyncIterator[CoordinatorLock]: + """Hold one fixed host's lock and receipt through primary connection close. - Both connections must be dedicated to maintenance. The caller supplies a - stable tenant/schema identity, not a hostname that changes through a proxy. - A rejected capability never falls back to a process-local lock. + A normal callback and successful close clear the receipt. Every unclean run + keeps it, even when local close succeeds. A matching acknowledgment declares + that the operator verified the previous remote operation has ended; the + coordinator cannot establish that fact from ODP physical session IDs. """ - name = "pc-migrate-" + digest(database_id)[:48] - owned = False + _validate_database_id(database_id) + host = _bind_host(directory) + lock = _OSLock(host.directory / database_id / "migration.lock") + lock.acquire() + path = lock.path.parent / "active-run.json" + original: BaseException | None = None + completed = False + context_token = None + receipt: dict[str, Any] | None = None try: + receipt = _start_run(path, database_id, host.hostname, acknowledged_run) + context_token = _active_run.set( + _ActiveRun(host.directory, database_id, receipt["token"], receipt["previous_run"]) + ) + held = CoordinatorLock(connection, str(lock.path), lock, host, path, receipt) + await held.verify() + yield held + await held.verify() + completed = True + except BaseException as error: + original = error + raise + finally: + cleanup_error: BaseException | None = None try: - owner_id = await connection.scalar(text("SELECT CONNECTION_ID()")) - other_id = await contender.scalar(text("SELECT CONNECTION_ID()")) - if owner_id == other_id: - raise MigrationError("migration_lock_unsupported", "Lock probing needs two distinct physical sessions.") - acquired = await connection.scalar(text("SELECT GET_LOCK(:name, 0)"), {"name": name}) - if acquired == 0: - raise MigrationError("migration_locked", "Another session holds the database migration lock.") - if acquired != 1: - raise MigrationError("migration_lock_unsupported", "Named-lock acquisition is unavailable.") - owned = True - await connection.commit() - competing = await contender.scalar(text("SELECT GET_LOCK(:name, 0)"), {"name": name}) - if competing == 1: - await contender.scalar(text("SELECT RELEASE_LOCK(:name)"), {"name": name}) - await contender.commit() - if competing != 0: - raise MigrationError( - "migration_lock_unsupported", "The lock does not exclude a second session across commit." + # Close in a separate shielded task, but clear only after the caller + # has also returned normally from waiting for that task. Cancellation + # during close must retain evidence even if close eventually succeeds. + await _finish_cleanup(connection.close()) + if completed and receipt is not None: + _clear_receipt(path, receipt, lock, host) + except BaseException as error: + if original is None: + cleanup_error = error + else: + original.add_note( + f"Coordinator cleanup did not complete safely ({type(error).__name__}); retain the run receipt." ) - if not isinstance(owner_id, int): - raise MigrationError("migration_lock_unsupported", "The backend did not return a physical session ID.") - lock = PinnedOceanBaseLock(connection, name, owner_id) - await lock.verify() - except DBAPIError as error: - raise MigrationError( - "migration_lock_unsupported", "Named-lock capability could not be verified." - ) from error - yield lock - finally: - if owned and not connection.invalidated and not connection.closed: - # Never reconnect simply to release: a replacement session owns no lock. - # Releasing a lock must not commit unfinished work in the caller. - await connection.rollback() - await connection.scalar(text("SELECT RELEASE_LOCK(:name)"), {"name": name}) - await connection.commit() + finally: + if context_token is not None: + _active_run.reset(context_token) + try: + lock.release() + except OSError as error: + if original is not None: + original.add_note("The coordinator lock descriptor could not be closed.") + elif cleanup_error is not None: + cleanup_error.add_note("The coordinator lock descriptor could not be closed.") + else: + cleanup_error = MigrationError( + "recovery_required", "The coordinator lock descriptor could not be closed." + ) + cleanup_error.__cause__ = error + if cleanup_error is not None: + raise cleanup_error diff --git a/src/powercontext/server/database_migration.py b/src/powercontext/server/database_migration.py index 37fac0e684..d813cff2f9 100644 --- a/src/powercontext/server/database_migration.py +++ b/src/powercontext/server/database_migration.py @@ -40,7 +40,13 @@ EnvFile = Annotated[Path | None, typer.Option(help="Read the same environment file as the deployed Server.")] EvidenceDir = Annotated[ Path | None, - typer.Option(help="Persistent maintenance evidence directory; required and shared across Jobs for OceanBase."), + typer.Option( + help="Persistent maintenance evidence directory; OceanBase Jobs must use the same directory on one fixed host." + ), +] +LockCoordination = Annotated[ + Literal["single-host"] | None, + typer.Option(help="OceanBase: run all migrators on one fixed host using the same evidence directory."), ] ManageService = Annotated[ bool, @@ -65,28 +71,33 @@ def main() -> None: @app.command() -def status(env_file: EnvFile = None, evidence_dir: EvidenceDir = None) -> None: +def status( + env_file: EnvFile = None, evidence_dir: EvidenceDir = None, lock_coordination: LockCoordination = None +) -> None: """Inspect the schema version without creating a database or control tables.""" - _inspect(env_file, evidence_dir=evidence_dir, verify=False) + _inspect(env_file, evidence_dir=evidence_dir, lock_coordination=lock_coordination, verify=False) @app.command() -def verify(env_file: EnvFile = None, evidence_dir: EvidenceDir = None) -> None: +def verify( + env_file: EnvFile = None, evidence_dir: EvidenceDir = None, lock_coordination: LockCoordination = None +) -> None: """Verify database readiness; this does not assert Server or cluster readiness.""" - _inspect(env_file, evidence_dir=evidence_dir, verify=True) + _inspect(env_file, evidence_dir=evidence_dir, lock_coordination=lock_coordination, verify=True) @app.command() def plan( env_file: EnvFile = None, evidence_dir: EvidenceDir = None, + lock_coordination: LockCoordination = None, backup: Annotated[BackupPolicy, typer.Option(help="Bind the selected backup policy to the plan.")] = "auto", manage_service: ManageService = False, shared_database: SharedDatabase = False, ) -> None: """Preview the target, resource digests, backup policy, and maintenance scope.""" with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: - runner = deployment_runner(settings, evidence_dir=evidence_dir) + runner = deployment_runner(settings, evidence_dir=evidence_dir, lock_coordination=lock_coordination) service = _service_scope(_environment_file(env_file)) if manage_service else None reviewed = runner.plan(backup_policy=backup, service=service, shared_database=shared_database) _write_payload(reviewed) @@ -96,6 +107,13 @@ def plan( def apply( env_file: EnvFile = None, evidence_dir: EvidenceDir = None, + lock_coordination: LockCoordination = None, + acknowledge_previous_run: Annotated[ + str | None, + typer.Option( + help="OceanBase only: declare a DBA confirmed previous remote execution and background DDL ended; use its run token." + ), + ] = None, backup: Annotated[ BackupPolicy | None, typer.Option(help="PC native backup, a user-declared manual backup, or explicit no-backup risk."), @@ -122,14 +140,21 @@ def apply( ) -> None: """Review once, then stop writes, back up, migrate, verify, and optionally switch a local service.""" with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: - runner = deployment_runner(settings, evidence_dir=evidence_dir) + runner = deployment_runner( + settings, + evidence_dir=evidence_dir, + lock_coordination=lock_coordination, + acknowledge_previous_run=acknowledge_previous_run, + ) policy: BackupPolicy = backup or "auto" initial = runner.plan(backup_policy=policy, shared_database=shared_database) + _check_previous_run(initial, acknowledge_previous_run) if initial.state == "ready": _write_payload(runner.verify()) return service = _service_scope(_environment_file(env_file)) if manage_service else None reviewed = runner.plan(backup_policy=policy, service=service, shared_database=shared_database) + _check_previous_run(reviewed, acknowledge_previous_run) if policy == "auto" and not reviewed.backup_available and reviewed.state != "uninitialized": raise MigrationError( "backup_unsupported", @@ -168,9 +193,15 @@ def apply( _write_payload(result) -def _inspect(env_file: Path | None, *, evidence_dir: Path | None, verify: bool) -> None: +def _inspect( + env_file: Path | None, + *, + evidence_dir: Path | None, + lock_coordination: Literal["single-host"] | None, + verify: bool, +) -> None: with _operator_errors(), server_settings_context(env_file=_environment_file(env_file)) as settings: - runner = deployment_runner(settings, evidence_dir=evidence_dir) + runner = deployment_runner(settings, evidence_dir=evidence_dir, lock_coordination=lock_coordination) result = runner.verify() if verify else runner.plan() _write_payload(result) @@ -199,6 +230,14 @@ def _write_confirmation(reviewed: MigrationPlan, policy: BackupPolicy) -> None: ) typer.echo("Maintenance pauses writes. Stop all other Servers, Workers, SDK clients and automatic restarts first.") typer.echo("PC cannot automatically discover every node or client connected to this database.") + if reviewed.coordination and reviewed.coordination.get("mode") == "single-host": + typer.echo( + "All migration commands must use this fixed maintenance host and evidence directory. This is not a cross-host database lock." + ) + if reviewed.coordination.get("pending_run_id"): + typer.echo( + "The supplied previous-run token declares a DBA confirmed all previous remote execution and background DDL ended. PC does not verify that declaration." + ) if reviewed.shared_database: typer.echo("You are responsible for upgrading every affected shared-database node before restoring traffic.") if reviewed.service is not None: @@ -282,6 +321,17 @@ def _check_declarations( raise MigrationError("confirmation_required", "Skipping backup requires --accept-no-backup.") +def _check_previous_run(reviewed: MigrationPlan, acknowledged: str | None) -> None: + pending = (reviewed.coordination or {}).get("pending_run_id") + if pending is not None and acknowledged != pending: + raise MigrationError( + "recovery_required", + "A DBA must confirm previous remote execution and background DDL ended; provide its exact --acknowledge-previous-run token and review a new plan.", + ) + if pending is None and acknowledged is not None: + raise MigrationError("stale_ack", "There is no pending run matching --acknowledge-previous-run.") + + @contextmanager def _operator_errors() -> Iterator[None]: try: diff --git a/tests/builtin/persistence/test_database_migration_backends.py b/tests/builtin/persistence/test_database_migration_backends.py index cfc37465e8..a646f07564 100644 --- a/tests/builtin/persistence/test_database_migration_backends.py +++ b/tests/builtin/persistence/test_database_migration_backends.py @@ -32,12 +32,11 @@ from alembic import command from sqlalchemy import Connection, inspect, text from sqlalchemy.engine import make_url -from sqlalchemy.ext.asyncio import AsyncEngine from powercontext.builtin.persistence.migrations import MigrationBundle +from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections from powercontext.builtin.persistence.migrations.locking import local_migration_lock -from powercontext.builtin.persistence.migrations.oceanbase import oceanbase_migration_lock -from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile FIXTURE = Path(__file__).resolve().parents[2] / "fixtures/database_migrations" @@ -89,16 +88,6 @@ def exercise_revisions(connection: Connection, bundle: MigrationBundle) -> None: } -async def exercise_oceanbase(engine: AsyncEngine, database_name: str) -> None: - async with ( - engine.connect() as connection, - engine.connect() as contender, - oceanbase_migration_lock(connection, contender, database_id=database_name) as lock, - ): - await connection.run_sync(lambda sync: exercise_revisions(sync, MigrationBundle(FIXTURE))) - await lock.verify() - - @pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekDB probe not enabled") def test_real_seekdb_revision_operations(tmp_path: Path) -> None: async def scenario() -> None: @@ -116,7 +105,7 @@ async def scenario() -> None: not os.environ.get("POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL"), reason="dedicated real OceanBase target not configured", ) -def test_real_oceanbase_revision_operations_and_named_lock() -> None: +def test_real_oceanbase_revision_operations_and_single_host_coordination(tmp_path: Path) -> None: value = os.environ["POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL"] url = make_url(value) assert url.database and url.database.startswith("pc_migration_probe_"), ( @@ -124,8 +113,12 @@ def test_real_oceanbase_revision_operations_and_named_lock() -> None: ) config = OceanBaseConfig.model_validate({"url": value}) - async def scenario() -> None: - async with OceanBaseProfile.open(config, tables=()) as profile: - await exercise_oceanbase(profile.database.engine, str(url.database)) + adapter = MaintenanceConnections(config, evidence_directory=tmp_path / "evidence", lock_coordination="single-host") - asyncio.run(scenario()) + def exercise(connection, _identity, verify): + assert connection is not None + verify() + exercise_revisions(connection, MigrationBundle(FIXTURE)) + verify() + + adapter.run(exercise, writable=True) diff --git a/tests/builtin/persistence/test_database_migration_connections.py b/tests/builtin/persistence/test_database_migration_connections.py index bc5a806b4a..45dadfaf6e 100644 --- a/tests/builtin/persistence/test_database_migration_connections.py +++ b/tests/builtin/persistence/test_database_migration_connections.py @@ -17,7 +17,11 @@ from __future__ import annotations import os +import subprocess +import sys +import time from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime from pathlib import Path from types import SimpleNamespace @@ -25,6 +29,7 @@ import pytest from sqlalchemy import Connection, event +from sqlalchemy.engine import make_url from sqlalchemy.ext.asyncio import create_async_engine from powercontext.builtin.persistence.migrations import connections @@ -194,7 +199,18 @@ async def run_sync(self, operation): return operation(self.connection) -def test_oceanbase_routes_to_different_database_identity_before_write_are_rejected(monkeypatch) -> None: +def test_oceanbase_write_requires_an_explicit_single_host_coordinator() -> None: + config = OceanBaseConfig.model_validate({ + "url": "mysql+aoceanbase://probe:secret@example.invalid:2881/scratch?charset=utf8mb4" + }) + operation = Mock() + with pytest.raises(MigrationError) as error: + MaintenanceConnections(config).run(operation, writable=True) + assert error.value.code == "coordination_required" + operation.assert_not_called() + + +def test_oceanbase_routes_to_different_database_identity_before_write_are_rejected(tmp_path: Path, monkeypatch) -> None: sessions = iter((_RemoteSession(1), _RemoteSession(2))) engine = Mock() engine.connect.side_effect = lambda: next(sessions) @@ -209,7 +225,9 @@ async def dispose() -> None: "url": "mysql+aoceanbase://private-user:private-password@private-route:2881/scratch?charset=utf8mb4" }) with pytest.raises(MigrationError) as error: - MaintenanceConnections(config).run(operation, writable=True) + MaintenanceConnections(config, evidence_directory=tmp_path, lock_coordination="single-host").run( + operation, writable=True + ) assert error.value.code == "target_identity_unavailable" assert "private" not in str(error.value) operation.assert_not_called() @@ -253,3 +271,119 @@ def read(connection: Connection | None, identity: BackendIdentity, verify: Calla assert adapter.run(read) == written == missing[1] assert "preserved" not in caplog.text + + +@pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") +def test_real_seekdb_maintenance_excludes_another_process_and_reopens_after_release(tmp_path: Path) -> None: + pytest.importorskip("pylibseekdb") + path = tmp_path / "seekdb" + ready = tmp_path / "owner-ready" + code = """ +from pathlib import Path +import sys +from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections +from powercontext.builtin.persistence.seekdb import SeekDBConfig + +def own(connection, identity, verify): + verify() + connection.exec_driver_sql("CREATE TABLE pc_lock_probe (id INT PRIMARY KEY, value VARCHAR(32))") + connection.exec_driver_sql("INSERT INTO pc_lock_probe VALUES (1, 'preserved')") + connection.commit() + Path(sys.argv[2]).write_text("ready", encoding="utf-8") + sys.stdin.read() + verify() + +MaintenanceConnections(SeekDBConfig(path=Path(sys.argv[1]))).run(own, writable=True) +""" + process = subprocess.Popen( + [sys.executable, "-c", code, str(path), str(ready)], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + try: + deadline = time.monotonic() + 60 + while not ready.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(0.05) + assert ready.exists(), "The isolated seekdb owner did not become ready" + alias = tmp_path / "seekdb-alias" + alias.symlink_to(path, target_is_directory=True) + operation = Mock() + with pytest.raises(MigrationError) as locked: + MaintenanceConnections(SeekDBConfig(path=alias)).run(operation, writable=True) + assert locked.value.code == "migration_locked" + operation.assert_not_called() + finally: + try: + _, stderr = process.communicate(input="", timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=10) + pytest.fail("The isolated seekdb owner did not release its maintenance resources") + assert process.returncode == 0, stderr + + def read(connection: Connection | None, _identity: BackendIdentity, verify: Callable[[], None]) -> str: + assert connection is not None + verify() + return connection.exec_driver_sql("SELECT value FROM pc_lock_probe WHERE id = 1").scalar_one() + + assert MaintenanceConnections(SeekDBConfig(path=path)).run(read) == "preserved" + + +@pytest.mark.skipif( + not os.environ.get("POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL"), + reason="dedicated real OceanBase target not configured", +) +def test_real_oceanbase_single_host_coordinator_excludes_other_maintenance_and_survives_commit(tmp_path: Path) -> None: + """Run maintenance only on one host and evidence directory, without business DDL.""" + value = os.environ["POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL"] + url = make_url(value) + assert url.database and url.database.startswith("pc_migration_probe_"), ( + "Only a dedicated scratch database is allowed" + ) + config = OceanBaseConfig.model_validate({"url": value}) + adapter = MaintenanceConnections(config, evidence_directory=tmp_path, lock_coordination="single-host") + identity = adapter.run(lambda _connection, target, _verify: target) + assert identity.product == "oceanbase" + assert identity.database_name == url.database + + def diagnostic(error: MigrationError) -> str: + details = f"code={error.code}: {error}" + cause = error.__cause__ + if cause is not None: + details += f"; cause={type(cause).__name__}: {cause}" + for secret in (value, url.render_as_string(hide_password=False), url.password): + if secret: + details = details.replace(secret, "[redacted]") + return details + + def maintain(connection: Connection | None, target: BackendIdentity, verify: Callable[[], None]) -> BackendIdentity: + assert connection is not None + assert target == identity + verify() + contender = MaintenanceConnections(config, evidence_directory=tmp_path, lock_coordination="single-host") + operation = Mock() + with ThreadPoolExecutor(max_workers=1) as executor: + attempt = executor.submit(contender.run, operation, writable=True) + with pytest.raises(MigrationError) as locked: + attempt.result(timeout=30) + assert locked.value.code == "migration_locked", diagnostic(locked.value) + operation.assert_not_called() + connection.commit() + verify() + assert connection.exec_driver_sql("SELECT DATABASE()").scalar_one() == target.database_name + return target + + assert adapter.run(maintain, writable=True) == identity + + def reacquire( + connection: Connection | None, target: BackendIdentity, verify: Callable[[], None] + ) -> BackendIdentity: + assert connection is not None + verify() + connection.commit() + verify() + return target + + assert adapter.run(reacquire, writable=True) == identity diff --git a/tests/builtin/persistence/test_migration_locking.py b/tests/builtin/persistence/test_migration_locking.py new file mode 100644 index 0000000000..1da244e2d1 --- /dev/null +++ b/tests/builtin/persistence/test_migration_locking.py @@ -0,0 +1,110 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Migration callbacks require an OS-backed lock even after runtime fallback.""" + +from __future__ import annotations + +import errno +import sys +from pathlib import Path +from unittest.mock import Mock + +import pytest +from filelock import FileLock, SoftFileLock + +from powercontext.builtin.persistence.migrations.locking import local_migration_lock +from powercontext.builtin.persistence.migrations.models import MigrationError + + +@pytest.mark.parametrize("release_fails", [False, True]) +def test_runtime_soft_lock_fallback_blocks_the_operation_and_preserves_unsupported( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, release_fails: bool +) -> None: + target = tmp_path / "database" + operation = Mock() + original_release = SoftFileLock.release + + def fallback(lock, *args, **kwargs): + lock.__class__ = SoftFileLock + return SoftFileLock.acquire(lock, *args, **kwargs) + + def release(lock, force: bool = False) -> None: + was_locked = lock.is_locked + original_release(lock, force) + if was_locked and release_fails: + message = "Release failed after closing the lock." + raise OSError(message) + + with monkeypatch.context() as patch: + patch.setattr(FileLock, "acquire", fallback) + patch.setattr(SoftFileLock, "release", release) + with pytest.raises(MigrationError) as error, local_migration_lock(target): + operation() + + assert error.value.code == "migration_lock_unsupported" + operation.assert_not_called() + assert not target.exists() + assert not target.with_name(target.name + ".pc-migration.lock").exists() + with local_migration_lock(target): + pass + + +@pytest.mark.skipif(sys.platform == "win32", reason="flock capability detection is a Unix filesystem boundary") +def test_native_flock_unsupported_does_not_enter_a_migration(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import fcntl + + target = tmp_path / "database" + operation = Mock() + + def unsupported_flock(_descriptor: int, _operation: int) -> None: + raise OSError(errno.ENOSYS, "Native flock is unavailable on this filesystem.") + + with monkeypatch.context() as patch: + patch.setattr(fcntl, "flock", unsupported_flock) + with ( + pytest.warns(UserWarning, match="falling back to SoftFileLock"), + pytest.raises(MigrationError) as error, + local_migration_lock(target), + ): + operation() + + assert error.value.code == "migration_lock_unsupported" + operation.assert_not_called() + assert not target.exists() + assert not target.with_name(target.name + ".pc-migration.lock").exists() + with local_migration_lock(target): + pass + + +def test_cleanup_failure_does_not_replace_the_migration_error(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + target = tmp_path / "database" + original_release = FileLock.release + failure = MigrationError("verification_failed", "Migration verification failed.") + + def release(lock, force: bool = False) -> None: + was_locked = lock.is_locked + original_release(lock, force) + if was_locked: + message = "Release failed after closing the lock." + raise OSError(message) + + with monkeypatch.context() as patch: + patch.setattr(FileLock, "release", release) + with pytest.raises(MigrationError) as error, local_migration_lock(target): + raise failure + + assert error.value is failure + with local_migration_lock(target): + pass diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index bc61dc2b2d..f4b66f979c 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -12,45 +12,129 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Real seekdb maintenance acceptance on independent temporary instances.""" +"""Native MySQL-mode maintenance acceptance on isolated backend targets.""" from __future__ import annotations +import asyncio import json import os -from collections.abc import Callable +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass from pathlib import Path from typing import Literal from unittest.mock import Mock +from uuid import uuid4 import pytest +from pydantic import SecretStr, ValidationError from sqlalchemy import Connection, Engine, event, text +from sqlalchemy.engine import make_url +from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy.pool import NullPool from powercontext.builtin.persistence.migrations import mysql as mysql_migrations from powercontext.builtin.persistence.migrations.backup import BackupCapabilities, BackupProvider from powercontext.builtin.persistence.migrations.connections import BackendIdentity, MaintenanceConnections from powercontext.builtin.persistence.migrations.deployment import production_bundle -from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.models import MigrationError, MigrationPlan from powercontext.builtin.persistence.migrations.mysql import MySQLMigrationRunner +from powercontext.builtin.persistence.oceanbase import OceanBaseConfig +from powercontext.builtin.persistence.oceanbase import profile as oceanbase_profile from powercontext.builtin.persistence.seekdb import SeekDBConfig -pytestmark = pytest.mark.skipif( - os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled" -) +@dataclass(frozen=True) +class _BackendTarget: + config: SeekDBConfig | OceanBaseConfig + evidence_directory: Path -def _connections(tmp_path: Path) -> MaintenanceConnections: - pytest.importorskip("pylibseekdb") - return MaintenanceConnections(SeekDBConfig(path=tmp_path / "seekdb")) + def connections(self, *, acknowledge_previous_run: str | None = None) -> MaintenanceConnections: + return MaintenanceConnections( + self.config, + evidence_directory=self.evidence_directory, + lock_coordination="single-host" if isinstance(self.config, OceanBaseConfig) else None, + acknowledge_previous_run=acknowledge_previous_run, + ) + def runner(self, *, backup_factory=None, acknowledge_previous_run: str | None = None) -> MySQLMigrationRunner: + return MySQLMigrationRunner( + self.connections(acknowledge_previous_run=acknowledge_previous_run), + production_bundle(), + evidence_directory=self.evidence_directory, + backup_factory=backup_factory, + ) -def _runner(tmp_path: Path, *, backup_factory=None) -> MySQLMigrationRunner: - return MySQLMigrationRunner( - _connections(tmp_path), - production_bundle(), - evidence_directory=tmp_path / "evidence", - backup_factory=backup_factory, - ) + +@contextmanager +def _oceanbase_scratch_target(config: OceanBaseConfig, evidence_directory: Path) -> Iterator[_BackendTarget]: + # Never adopt or drop an existing database: only a successful CREATE grants ownership. + url = make_url(config.url.get_secret_value()) + database = f"pc_migration_probe_{uuid4().hex}" + owned = False + + async def manage(*, create: bool) -> None: + nonlocal owned + oceanbase_profile._register_official_dialect() + engine = create_async_engine( + config.url.get_secret_value(), + connect_args={"init_command": "SET autocommit = 0"}, + echo=False, + hide_parameters=True, + poolclass=NullPool, + ) + try: + async with engine.connect() as connection: + current = (await connection.exec_driver_sql("SELECT DATABASE()")).scalar_one() + if current != url.database: + pytest.fail("OceanBase scratch management database identity does not match its URL") + statement = f"CREATE DATABASE `{database}`" if create else f"DROP DATABASE `{database}`" + await connection.exec_driver_sql(statement) + owned = create + await connection.commit() + finally: + await engine.dispose() + + try: + try: + asyncio.run(manage(create=True)) + except Exception: + pytest.fail("OceanBase isolated test database creation failed", pytrace=False) + child_url = url.set(database=database) + child = config.model_copy(update={"url": SecretStr(child_url.render_as_string(hide_password=False))}) + yield _BackendTarget(child, evidence_directory) + finally: + if owned: + try: + asyncio.run(manage(create=False)) + except Exception: + pytest.fail("OceanBase owned test database cleanup failed", pytrace=False) + + +@pytest.fixture(params=("seekdb", "oceanbase"), ids=("seekdb", "oceanbase")) +def backend(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[_BackendTarget]: + if request.param == "seekdb": + if os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1": + pytest.skip("real seekdb probe not enabled") + pytest.importorskip("pylibseekdb") + yield _BackendTarget(SeekDBConfig(path=tmp_path / "seekdb"), tmp_path / "evidence") + return + value = os.environ.get("POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL") + if not value: + pytest.skip("real OceanBase probe not enabled") + try: + config = OceanBaseConfig(url=SecretStr(value)) + except ValidationError: + pytest.fail("OceanBase probe requires a valid official async-dialect URL", pytrace=False) + url = make_url(config.url.get_secret_value()) + if not url.database or not url.database.startswith("pc_migration_probe_"): + pytest.fail("OceanBase probe requires a dedicated pc_migration_probe_* management database", pytrace=False) + # Driver query options must not override the generated child database name. + if any(key.lower() in {"db", "database"} for key in url.query): + pytest.fail("OceanBase probe URL must not override the selected database", pytrace=False) + with _oceanbase_scratch_target(config, tmp_path / "evidence") as target: + yield target def _forbidden_automatic_backup(_connection, _identity, _directory): @@ -60,7 +144,7 @@ def _forbidden_automatic_backup(_connection, _identity, _directory): return provider -def _load_baseline(tmp_path: Path) -> None: +def _load_baseline(backend: _BackendTarget) -> None: bundle = production_bundle() schema = json.loads((bundle.snapshot_directory / "schemas/pre_dream.json").read_text()) @@ -84,7 +168,7 @@ def load(connection: Connection | None, _identity: BackendIdentity, _guard: Call ) connection.commit() - _connections(tmp_path).run(load, writable=True) + backend.connections().run(load, writable=True) def _apply(runner: MySQLMigrationRunner, *, policy: Literal["auto", "manual", "skip"] = "auto"): @@ -100,7 +184,20 @@ def _apply(runner: MySQLMigrationRunner, *, policy: Literal["auto", "manual", "s ) -def _read_state(tmp_path: Path): +def _acknowledge_known_completed_interruption( + backend: _BackendTarget, plan: MigrationPlan, *, backup_factory=None +) -> MySQLMigrationRunner: + # Only these tests' post-response/post-commit fault injections establish + # completion. An arbitrary interrupted remote operation cannot auto-resume. + pending = (plan.coordination or {}).get("pending_run_id") + if isinstance(backend.config, OceanBaseConfig): + assert pending + else: + assert pending is None + return backend.runner(backup_factory=backup_factory, acknowledge_previous_run=pending) + + +def _read_state(backend: _BackendTarget): def inspect(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]): assert connection is not None tables = {name for (name,) in connection.exec_driver_sql("SHOW TABLES")} @@ -122,70 +219,75 @@ def inspect(connection: Connection | None, _identity: BackendIdentity, _guard: C ) return {"tables": tables, "revision": revision, "columns": columns, "content": content, "tags": tags} - return _connections(tmp_path).run(inspect) + return backend.connections().run(inspect) -def test_seekdb_initialization_and_shared_evidence_noop(tmp_path: Path) -> None: - runner = _runner(tmp_path) +def test_mysql_initialization_and_shared_evidence_noop(backend: _BackendTarget) -> None: + runner = backend.runner() plan = runner.plan() assert plan.state == "uninitialized" - assert not (tmp_path / "seekdb").exists() + assert plan.coordination is not None + assert plan.coordination["mode"] == ( + "engine-directory" if isinstance(backend.config, SeekDBConfig) else "single-host" + ) + if isinstance(backend.config, SeekDBConfig): + assert not backend.config.path.exists() result = _apply(runner) assert result.state == "ready" assert result.changed assert result.backup_state == "not_required" assert result.revision == "p0003" - replacement = _runner(tmp_path) + replacement = backend.runner() assert replacement.plan().state == "ready" assert replacement.verify().revision == "p0003" assert not replacement.apply().changed @pytest.mark.parametrize("policy", ["manual", "skip"]) -def test_seekdb_operator_backup_choices_preserve_baseline_data_without_fork( - tmp_path: Path, policy: Literal["manual", "skip"] +def test_mysql_operator_backup_choices_preserve_baseline_data_without_fork( + backend: _BackendTarget, policy: Literal["manual", "skip"] ) -> None: - _load_baseline(tmp_path) - runner = _runner(tmp_path, backup_factory=_forbidden_automatic_backup) + _load_baseline(backend) + runner = backend.runner(backup_factory=_forbidden_automatic_backup) plan = runner.plan(backup_policy=policy) assert plan.source_revision == "p0001" assert plan.adopt_baseline with pytest.raises(MigrationError) as error: runner.apply(plan_id=plan.plan_id, accepted=True, maintenance_confirmed=True, backup_policy=policy) assert error.value.code == "backup_confirmation_required" - assert _read_state(tmp_path)["revision"] is None + assert _read_state(backend)["revision"] is None result = _apply(runner, policy=policy) assert result.backup_state == ("user_confirmed" if policy == "manual" else "skipped") - state = _read_state(tmp_path) + state = _read_state(backend) assert state["revision"] == "p0003" assert state["content"] == [(b'{"content":"preserved"}',)] assert state["tags"] == [("Keep",)] assert "memory_citations" in state["columns"]["pc_artifacts"] assert "memory_citations" in state["columns"]["pc_artifact_candidate_versions"] - assert _runner(tmp_path).verify().state == "ready" + assert backend.runner().verify().state == "ready" @pytest.mark.parametrize("table_fork", [False, True]) -def test_seekdb_unsupported_backup_does_not_write_source_schema(tmp_path: Path, table_fork: bool) -> None: - _load_baseline(tmp_path) - before = _read_state(tmp_path) +def test_mysql_unsupported_backup_does_not_write_source_schema(backend: _BackendTarget, table_fork: bool) -> None: + _load_baseline(backend) + before = _read_state(backend) def advertised_table_fork(connection, identity, directory): provider = _forbidden_automatic_backup(connection, identity, directory) provider.capabilities.side_effect = None provider.capabilities.return_value = BackupCapabilities( - available=True, method="fork_table", product="seekdb", recovery_workflow_accepted=True + available=True, method="fork_table", product=identity.product, recovery_workflow_accepted=True ) return provider - runner = _runner(tmp_path, backup_factory=advertised_table_fork if table_fork else None) + runner = backend.runner(backup_factory=advertised_table_fork if table_fork else None) plan = runner.plan() assert not plan.backup_available with pytest.raises(MigrationError) as error: _apply(runner) assert error.value.code == "backup_unsupported" - assert _read_state(tmp_path) == before - assert not (tmp_path / "evidence").exists() + assert _read_state(backend) == before + assert not list(backend.evidence_directory.rglob("maintenance.json")) class _InterruptedDDL(BaseException): @@ -207,86 +309,93 @@ def interrupt(_connection, _cursor, statement: str, _parameters, _context, _exec event.remove(Engine, "after_cursor_execute", interrupt) -def _interrupt_after_committed_citation_ddl(tmp_path: Path) -> None: - _load_baseline(tmp_path) - _interrupt_after_ddl(_runner(tmp_path), "ALTER TABLE pc_artifacts ADD COLUMN memory_citations") - state = _read_state(tmp_path) +def _interrupt_after_committed_citation_ddl(backend: _BackendTarget) -> None: + _load_baseline(backend) + _interrupt_after_ddl(backend.runner(), "ALTER TABLE pc_artifacts ADD COLUMN memory_citations") + state = _read_state(backend) assert state["revision"] == "p0001" assert "memory_citations" in state["columns"]["pc_artifacts"] assert "memory_citations" not in state["columns"]["pc_artifact_candidate_versions"] assert state["content"] == [(b'{"content":"preserved"}',)] -def test_seekdb_committed_partial_ddl_resumes_from_shared_original_evidence(tmp_path: Path) -> None: - _interrupt_after_committed_citation_ddl(tmp_path) - replacement = _runner(tmp_path) - assert replacement.plan(backup_policy="manual").state == "recovery_required" +def test_mysql_committed_partial_ddl_resumes_from_shared_original_evidence(backend: _BackendTarget) -> None: + _interrupt_after_committed_citation_ddl(backend) + replacement = backend.runner() + recovery = replacement.plan(backup_policy="manual") + assert recovery.state == "recovery_required" + replacement = _acknowledge_known_completed_interruption(backend, recovery) assert _apply(replacement, policy="manual").state == "ready" - state = _read_state(tmp_path) + state = _read_state(backend) assert state["revision"] == "p0003" assert state["content"] == [(b'{"content":"preserved"}',)] assert state["tags"] == [("Keep",)] assert replacement.verify().state == "ready" -def test_seekdb_missing_original_evidence_blocks_partial_ddl_recovery(tmp_path: Path) -> None: - _interrupt_after_committed_citation_ddl(tmp_path) - before = _read_state(tmp_path) - evidence_files = list((tmp_path / "evidence").rglob("maintenance.json")) +def test_mysql_missing_original_evidence_blocks_partial_ddl_recovery(backend: _BackendTarget) -> None: + _interrupt_after_committed_citation_ddl(backend) + before = _read_state(backend) + evidence_files = list(backend.evidence_directory.rglob("maintenance.json")) assert len(evidence_files) == 1 evidence_files[0].unlink() - replacement = _runner(tmp_path) + replacement = backend.runner() for operation in (lambda: replacement.plan(backup_policy="manual"), replacement.verify, replacement.apply): with pytest.raises(MigrationError) as error: operation() assert error.value.code == "recovery_required" - assert _read_state(tmp_path) == before + assert _read_state(backend) == before -def test_seekdb_empty_source_recovers_baseline_and_citation_ddl_without_probing_backup(tmp_path: Path) -> None: +def test_mysql_empty_source_recovers_baseline_and_citation_ddl_without_probing_backup(backend: _BackendTarget) -> None: _interrupt_after_ddl( - _runner(tmp_path, backup_factory=_forbidden_automatic_backup), "CREATE TABLE pc_artifacts", policy="auto" + backend.runner(backup_factory=_forbidden_automatic_backup), "CREATE TABLE pc_artifacts", policy="auto" ) - first = _read_state(tmp_path) + first = _read_state(backend) assert first["revision"] is None assert "pc_schema_revision" in first["tables"] assert "pc_artifacts" in first["tables"] assert "pc_artifact_heads" not in first["tables"] assert first["content"] == [] + recovery = backend.runner(backup_factory=_forbidden_automatic_backup).plan() _interrupt_after_ddl( - _runner(tmp_path, backup_factory=_forbidden_automatic_backup), + _acknowledge_known_completed_interruption(backend, recovery, backup_factory=_forbidden_automatic_backup), "ALTER TABLE pc_artifacts ADD COLUMN memory_citations", policy="auto", ) - second = _read_state(tmp_path) + second = _read_state(backend) assert second["revision"] == "p0001" assert "memory_citations" in second["columns"]["pc_artifacts"] assert "memory_citations" not in second["columns"]["pc_artifact_candidate_versions"] - replacement = _runner(tmp_path, backup_factory=_forbidden_automatic_backup) + recovery = backend.runner(backup_factory=_forbidden_automatic_backup).plan() + replacement = _acknowledge_known_completed_interruption( + backend, recovery, backup_factory=_forbidden_automatic_backup + ) result = _apply(replacement) assert result.backup_state == "not_required" assert replacement.verify().revision == "p0003" - assert _read_state(tmp_path)["content"] == [] + assert _read_state(backend)["content"] == [] -def test_seekdb_baseline_adoption_recovers_an_empty_committed_version_table(tmp_path: Path) -> None: - _load_baseline(tmp_path) - _interrupt_after_ddl(_runner(tmp_path), "CREATE TABLE pc_schema_revision") - state = _read_state(tmp_path) +def test_mysql_baseline_adoption_recovers_an_empty_committed_version_table(backend: _BackendTarget) -> None: + _load_baseline(backend) + _interrupt_after_ddl(backend.runner(), "CREATE TABLE pc_schema_revision") + state = _read_state(backend) assert "pc_schema_revision" in state["tables"] assert state["revision"] is None assert state["content"] == [(b'{"content":"preserved"}',)] - replacement = _runner(tmp_path) + replacement = backend.runner() recovered = replacement.plan(backup_policy="manual") assert recovered.state == "recovery_required" assert recovered.adopt_baseline + replacement = _acknowledge_known_completed_interruption(backend, recovered) assert _apply(replacement, policy="manual").state == "ready" assert replacement.verify().revision == "p0003" - assert _read_state(tmp_path)["tags"] == [("Keep",)] + assert _read_state(backend)["tags"] == [("Keep",)] -def test_seekdb_committed_revision_recovers_before_evidence_advance(tmp_path: Path, monkeypatch) -> None: - _load_baseline(tmp_path) +def test_mysql_committed_revision_recovers_before_evidence_advance(backend: _BackendTarget, monkeypatch) -> None: + _load_baseline(backend) write = mysql_migrations._Journal.write def interrupt(journal, evidence) -> None: @@ -297,13 +406,63 @@ def interrupt(journal, evidence) -> None: with monkeypatch.context() as patch: patch.setattr(mysql_migrations._Journal, "write", interrupt) with pytest.raises(_InterruptedDDL): - _apply(_runner(tmp_path), policy="manual") - state = _read_state(tmp_path) + _apply(backend.runner(), policy="manual") + state = _read_state(backend) assert state["revision"] == "p0002" assert "memory_citations" in state["columns"]["pc_artifacts"] assert "memory_citations" in state["columns"]["pc_artifact_candidate_versions"] - replacement = _runner(tmp_path) - assert replacement.plan(backup_policy="manual").source_revision == "p0002" + replacement = backend.runner() + recovery = replacement.plan(backup_policy="manual") + assert recovery.source_revision == "p0002" + replacement = _acknowledge_known_completed_interruption(backend, recovery) assert _apply(replacement, policy="manual").state == "ready" assert replacement.verify().revision == "p0003" - assert _read_state(tmp_path)["content"] == [(b'{"content":"preserved"}',)] + assert _read_state(backend)["content"] == [(b'{"content":"preserved"}',)] + + +def test_mysql_completed_head_with_an_interrupted_run_requires_exact_acknowledgement(backend: _BackendTarget) -> None: + if not isinstance(backend.config, OceanBaseConfig): + pytest.skip("embedded seekdb closes its engine and has no remote in-flight run receipt") + _load_baseline(backend) + assert _apply(backend.runner(), policy="manual").state == "ready" + assert backend.runner().verify().revision == "p0003" + + def committed_write_then_interrupt( + connection: Connection | None, _identity: BackendIdentity, verify: Callable[[], None] + ) -> None: + assert connection is not None + verify() + connection.exec_driver_sql("UPDATE pc_artifact_tags SET tag = 'Confirmed' WHERE scope_id = 's'") + connection.commit() + raise _InterruptedDDL + + with pytest.raises(_InterruptedDDL): + backend.connections().run(committed_write_then_interrupt, writable=True) + before = _read_state(backend) + assert before["revision"] == "p0003" + assert before["tags"] == [("Confirmed",)] + replacement = backend.runner(backup_factory=_forbidden_automatic_backup) + recovery = replacement.plan() + assert recovery.state == "recovery_required" + assert recovery.source_revision == "p0003" + assert recovery.revisions == () + pending = (recovery.coordination or {}).get("pending_run_id") + assert pending + for operation in (replacement.verify, replacement.apply): + with pytest.raises(MigrationError) as blocked: + operation() + assert blocked.value.code == "recovery_required" + wrong = backend.runner(backup_factory=_forbidden_automatic_backup, acknowledge_previous_run="wrong-token") + with pytest.raises(MigrationError) as blocked: + _apply(wrong) + assert blocked.value.code == "recovery_required" + assert _read_state(backend) == before + assert (backend.runner().plan().coordination or {})["pending_run_id"] == pending + acknowledged = backend.runner(backup_factory=_forbidden_automatic_backup, acknowledge_previous_run=pending) + result = _apply(acknowledged) + assert result.state == "ready" + assert not result.changed + assert _read_state(backend) == before + assert backend.runner().verify().revision == "p0003" + assert not backend.runner().apply().changed + assert "pending_run_id" not in (backend.runner().plan().coordination or {}) diff --git a/tests/builtin/persistence/test_oceanbase_migration_lock.py b/tests/builtin/persistence/test_oceanbase_migration_lock.py new file mode 100644 index 0000000000..5d8f7f6f08 --- /dev/null +++ b/tests/builtin/persistence/test_oceanbase_migration_lock.py @@ -0,0 +1,412 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Real coordinator OS locks and simulated migration connection shutdown. + +These tests verify local exclusion and explicit recovery acknowledgment, not +native OceanBase or ODP execution. No remote database or service is contacted. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import cast + +import pytest +from sqlalchemy import Connection +from sqlalchemy.ext.asyncio import AsyncConnection + +from powercontext.builtin.persistence.migrations import oceanbase +from powercontext.builtin.persistence.migrations.models import MigrationError +from powercontext.builtin.persistence.migrations.oceanbase import oceanbase_migration_lock, pending_run + +_DATABASE_ID = "scratch-database-identity" + + +class _Connection: + def __init__(self) -> None: + self.closed = False + self.invalidated = False + self.close_error = False + self.close_started = asyncio.Event() + self.close_allowed: asyncio.Event | None = None + self.close_receipt: Path | None = None + self.physical_session = 101 + + async def scalar(self, statement): + raise RuntimeError("Physical session and global PROCESS inspection are unavailable") # noqa: TRY003 + + async def commit(self) -> None: + # A proxy may route the next transaction to a different server session. + self.physical_session += 1 + + async def close(self) -> None: + self.close_started.set() + if self.close_allowed is not None: + await self.close_allowed.wait() + if self.close_error: + raise OSError("simulated close failure") # noqa: TRY003 + if self.close_receipt is not None: + assert self.close_receipt.is_file(), "The durable run receipt disappeared before connection close" + self.closed = True + + def connection(self) -> AsyncConnection: + return cast(AsyncConnection, self) + + def sync(self) -> Connection: + owner = self + + class Sync: + @property + def closed(self) -> bool: + return owner.closed + + @property + def invalidated(self) -> bool: + return owner.invalidated + + def scalar(self, statement): + raise RuntimeError("Physical session and global PROCESS inspection are unavailable") # noqa: TRY003 + + return cast(Connection, Sync()) + + +def _receipt(directory: Path) -> Path: + return directory / _DATABASE_ID / "active-run.json" + + +def test_commits_and_proxy_session_changes_keep_the_lock_until_successful_connection_close(tmp_path: Path) -> None: + owner = _Connection() + owner.close_receipt = _receipt(tmp_path) + + async def scenario() -> None: + async with oceanbase_migration_lock(owner.connection(), database_id=_DATABASE_ID, directory=tmp_path) as held: + await owner.commit() + await held.verify() + held.verify_sync(owner.sync()) + assert pending_run(tmp_path, _DATABASE_ID) is None + assert json.loads(_receipt(tmp_path).read_text())["database_id"] == _DATABASE_ID + competitor = _Connection() + with pytest.raises(MigrationError) as locked: + async with oceanbase_migration_lock( + competitor.connection(), database_id=_DATABASE_ID, directory=tmp_path + ): + pytest.fail("A second maintenance operation entered") + assert locked.value.code == "migration_locked" + assert owner.closed + assert not _receipt(tmp_path).exists() + assert pending_run(tmp_path, _DATABASE_ID) is None + assert (tmp_path / _DATABASE_ID / "migration.lock").is_file() + with pytest.raises(MigrationError) as released: + await held.verify() + assert released.value.code == "migration_lock_lost" + + asyncio.run(scenario()) + + +_OS_PROBE = """ +import os, sys +fd = os.open(sys.argv[1], os.O_RDWR) +try: + if os.name == 'posix': + import fcntl + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + elif os.name == 'nt': + import msvcrt + os.lseek(fd, 0, os.SEEK_SET) + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + else: + raise RuntimeError('unsupported test host') +except OSError: + print('locked') +else: + print('acquired') +finally: + os.close(fd) +""" + + +def test_the_os_lock_excludes_a_separate_process_and_reuses_its_inode(tmp_path: Path) -> None: + owner = _Connection() + path = tmp_path / _DATABASE_ID / "migration.lock" + + def probe() -> str: + result = subprocess.run( + [sys.executable, "-c", _OS_PROBE, str(path)], capture_output=True, text=True, timeout=10 + ) + assert result.returncode == 0, result.stderr + return result.stdout.strip() + + async def scenario() -> tuple[int, int]: + async with oceanbase_migration_lock(owner.connection(), database_id=_DATABASE_ID, directory=tmp_path): + assert probe() == "locked" + information = path.stat() + return information.st_dev, information.st_ino + + inode = asyncio.run(scenario()) + assert probe() == "acquired" + information = path.stat() + assert (information.st_dev, information.st_ino) == inode + + +@pytest.mark.parametrize("replacement", ["file", "parent"]) +@pytest.mark.skipif(os.name == "nt", reason="Windows prevents renaming an open locked file") +def test_replacing_the_lock_file_or_directory_invalidates_ownership(tmp_path: Path, replacement: str) -> None: + owner = _Connection() + + def replace(path: Path) -> None: + if replacement == "file": + path.rename(path.with_suffix(".original")) + path.touch() + else: + path.parent.rename(tmp_path / "old-database-directory") + path.parent.mkdir() + + async def scenario() -> None: + with pytest.raises(MigrationError) as changed: + async with oceanbase_migration_lock( + owner.connection(), database_id=_DATABASE_ID, directory=tmp_path + ) as held: + replace(Path(held.name)) + await held.verify() + assert changed.value.code == "migration_lock_lost" + assert owner.closed + retained = _receipt(tmp_path) if replacement == "file" else tmp_path / "old-database-directory/active-run.json" + assert retained.is_file() + + asyncio.run(scenario()) + + +def test_the_evidence_directory_is_bound_to_the_fixed_coordinator_host(tmp_path: Path, monkeypatch) -> None: + owner = _Connection() + monkeypatch.setattr(oceanbase.socket, "gethostname", lambda: "maintenance-host") + + async def bind() -> None: + async with oceanbase_migration_lock(owner.connection(), database_id=_DATABASE_ID, directory=tmp_path): + pass + + asyncio.run(bind()) + binding = (tmp_path / "coordinator.json").read_bytes() + monkeypatch.setattr(oceanbase.socket, "gethostname", lambda: "another-host") + other_owner = _Connection() + + async def reject() -> None: + with pytest.raises(MigrationError) as wrong_host: + async with oceanbase_migration_lock(other_owner.connection(), database_id=_DATABASE_ID, directory=tmp_path): + pytest.fail("A different coordinator host entered") + assert wrong_host.value.code == "recovery_required" + with pytest.raises(MigrationError) as wrong_host_plan: + pending_run(tmp_path, _DATABASE_ID) + assert wrong_host_plan.value.code == "recovery_required" + + asyncio.run(reject()) + assert (tmp_path / "coordinator.json").read_bytes() == binding + + +def test_interrupt_retains_the_run_and_only_exact_acknowledgment_permits_recovery(tmp_path: Path) -> None: + owner = _Connection() + interruption = KeyboardInterrupt("operator interrupted after committed DDL") + + async def interrupted() -> None: + async with oceanbase_migration_lock(owner.connection(), database_id=_DATABASE_ID, directory=tmp_path): + await owner.commit() + raise interruption + + with pytest.raises(KeyboardInterrupt) as preserved: + asyncio.run(interrupted()) + assert preserved.value is interruption + assert owner.closed + original_receipt = _receipt(tmp_path).read_bytes() + token = pending_run(tmp_path, _DATABASE_ID) + assert token is not None + + async def retry() -> None: + for acknowledgment in (None, "0" * 32): + with pytest.raises(MigrationError) as unresolved: + async with oceanbase_migration_lock( + _Connection().connection(), + database_id=_DATABASE_ID, + directory=tmp_path, + acknowledged_run=acknowledgment, + ): + pytest.fail("Recovery entered without acknowledgment of the exact interrupted run") + assert unresolved.value.code == "recovery_required" + assert _receipt(tmp_path).read_bytes() == original_receipt + recovered = _Connection() + async with oceanbase_migration_lock( + recovered.connection(), database_id=_DATABASE_ID, directory=tmp_path, acknowledged_run=token + ) as held: + await held.verify() + assert pending_run(tmp_path, _DATABASE_ID) == token + new_token = json.loads(_receipt(tmp_path).read_text())["token"] + assert new_token != token + assert recovered.closed + assert pending_run(tmp_path, _DATABASE_ID) is None + with pytest.raises(MigrationError) as stale: + async with oceanbase_migration_lock( + _Connection().connection(), database_id=_DATABASE_ID, directory=tmp_path, acknowledged_run=token + ): + pytest.fail("An acknowledgment was accepted after its run had been resolved") + assert stale.value.code == "stale_ack" + assert not _receipt(tmp_path).exists() + + asyncio.run(retry()) + + +@pytest.mark.parametrize("body_interrupt", [False, True]) +def test_failed_close_retains_the_receipt_and_preserves_an_original_interruption( + tmp_path: Path, body_interrupt: bool +) -> None: + owner = _Connection() + owner.close_error = True + interruption = KeyboardInterrupt("operator interrupted") + + async def maintain() -> None: + async with oceanbase_migration_lock(owner.connection(), database_id=_DATABASE_ID, directory=tmp_path): + if body_interrupt: + raise interruption + + if body_interrupt: + with pytest.raises(KeyboardInterrupt) as preserved: + asyncio.run(maintain()) + assert preserved.value is interruption + assert "OSError" in " ".join(getattr(interruption, "__notes__", [])) + else: + with pytest.raises(OSError, match="simulated close failure"): + asyncio.run(maintain()) + token = pending_run(tmp_path, _DATABASE_ID) + assert token is not None + + async def reject() -> None: + with pytest.raises(MigrationError) as blocked: + async with oceanbase_migration_lock( + _Connection().connection(), database_id=_DATABASE_ID, directory=tmp_path + ): + pytest.fail("Unclean shutdown was ignored") + assert blocked.value.code == "recovery_required" + assert pending_run(tmp_path, _DATABASE_ID) == token + + asyncio.run(reject()) + + +def test_guard_rejects_a_removed_run_receipt_before_further_work(tmp_path: Path) -> None: + owner = _Connection() + + async def scenario() -> None: + with pytest.raises(MigrationError) as missing: + async with oceanbase_migration_lock( + owner.connection(), database_id=_DATABASE_ID, directory=tmp_path + ) as held: + _receipt(tmp_path).unlink() + held.verify_sync(owner.sync()) + assert missing.value.code == "recovery_required" + assert owner.closed + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("during_close", [False, True]) +def test_cancellation_holds_the_lock_through_close_and_retains_the_run(tmp_path: Path, during_close: bool) -> None: + owner = _Connection() + + async def scenario() -> None: + entered = asyncio.Event() + owner.close_allowed = asyncio.Event() + + async def maintain() -> None: + async with oceanbase_migration_lock(owner.connection(), database_id=_DATABASE_ID, directory=tmp_path): + entered.set() + if not during_close: + await asyncio.Event().wait() + + task = asyncio.create_task(maintain()) + await entered.wait() + if not during_close: + task.cancel() + await owner.close_started.wait() + task.cancel() + await asyncio.sleep(0) + with pytest.raises(MigrationError) as locked: + async with oceanbase_migration_lock( + _Connection().connection(), database_id=_DATABASE_ID, directory=tmp_path + ): + pytest.fail("Cancellation released the lock before connection close") + assert locked.value.code == "migration_locked" + token = pending_run(tmp_path, _DATABASE_ID) + assert token is not None + owner.close_allowed.set() + with pytest.raises(asyncio.CancelledError): + await task + assert owner.closed + assert pending_run(tmp_path, _DATABASE_ID) == token + async with oceanbase_migration_lock( + _Connection().connection(), database_id=_DATABASE_ID, directory=tmp_path, acknowledged_run=token + ): + pass + assert pending_run(tmp_path, _DATABASE_ID) is None + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("previously_interrupted", [False, True]) +def test_locked_plan_recheck_sees_the_prior_run_while_independent_inspection_sees_the_current_run( + tmp_path: Path, previously_interrupted: bool +) -> None: + async def scenario() -> None: + if previously_interrupted: + with pytest.raises(RuntimeError, match="committed DDL interrupted"): + async with oceanbase_migration_lock( + _Connection().connection(), database_id=_DATABASE_ID, directory=tmp_path + ): + raise RuntimeError("committed DDL interrupted") # noqa: TRY003 + prior_token = pending_run(tmp_path, _DATABASE_ID) + # Create the inspection task before entering, matching an independent + # operator's context rather than inheriting the admitted run context. + inspect = asyncio.Event() + + async def independent_inspection() -> str | None: + await inspect.wait() + return pending_run(tmp_path, _DATABASE_ID) + + observer = asyncio.create_task(independent_inspection()) + async with oceanbase_migration_lock( + _Connection().connection(), database_id=_DATABASE_ID, directory=tmp_path, acknowledged_run=prior_token + ): + assert pending_run(tmp_path, _DATABASE_ID) == prior_token + inspect.set() + current_token = await observer + assert current_token is not None + assert current_token == json.loads(_receipt(tmp_path).read_text())["token"] + assert pending_run(tmp_path, _DATABASE_ID) is None + + asyncio.run(scenario()) + + +def test_readonly_inspection_is_side_effect_free_and_rejects_unknown_run_records(tmp_path: Path) -> None: + directory = tmp_path / "absent-coordinator" + assert pending_run(directory, _DATABASE_ID) is None + assert not directory.exists() + receipt = _receipt(tmp_path) + receipt.parent.mkdir() + receipt.write_text(json.dumps({"version": 1, "database_id": _DATABASE_ID, "token": "0" * 32})) + original = receipt.read_bytes() + with pytest.raises(MigrationError) as unknown: + pending_run(tmp_path, _DATABASE_ID) + assert unknown.value.code == "recovery_required" + assert receipt.read_bytes() == original diff --git a/tests/test_database_migration_cli.py b/tests/test_database_migration_cli.py index f2fb1c41d1..1d61ab0be9 100644 --- a/tests/test_database_migration_cli.py +++ b/tests/test_database_migration_cli.py @@ -17,6 +17,7 @@ import asyncio import json import os +import socket import sqlite3 from pathlib import Path from typing import Literal @@ -28,6 +29,8 @@ from powercontext.builtin.persistence.migrations.connections import BackendIdentity, MaintenanceConnections from powercontext.builtin.persistence.migrations.deployment import production_bundle +from powercontext.builtin.persistence.migrations.models import MigrationResult +from powercontext.builtin.persistence.migrations.mysql import MySQLMigrationRunner from powercontext.builtin.persistence.sqlite import SQLiteConfig, SQLiteProfile from powercontext.builtin.persistence.tables import BUILTIN_TABLES from powercontext.cli.app import create_cli @@ -72,6 +75,40 @@ def _invoke(command: list[str], action: str, tmp_path: Path, *options: str, prom ) +def _configure_oceanbase_inspection(tmp_path: Path, monkeypatch) -> tuple[list[str], list[str], Path]: + _database, command = _configure(tmp_path, monkeypatch) + (tmp_path / "deployment.env").write_text( + "POWERCONTEXT_SERVER_DATABASE_KIND=oceanbase\n" + "POWERCONTEXT_SERVER_DATABASE_URL=mysql+aoceanbase://operator:secret@database.invalid:2881/pc_probe" + "?charset=utf8mb4\n", + encoding="utf-8", + ) + identity = BackendIdentity("oceanbase", "simulated-oceanbase-database-identity", "pc_probe") + + def inspect_empty_database(_connections, operation, *, writable=False): + assert not writable, "This CLI fixture must never connect to or mutate a real database" + return operation(None, identity, lambda: None) + + monkeypatch.setattr(MaintenanceConnections, "run", inspect_empty_database) + evidence = tmp_path / "persistent-evidence" + options = ["--evidence-dir", str(evidence), "--lock-coordination", "single-host"] + return command, options, evidence / identity.database_id / "active-run.json" + + +def _record_pending_run(path: Path, token: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps({ + "version": 1, + "database_id": path.parent.name, + "hostname": socket.gethostname().casefold(), + "token": token, + "previous_run": None, + }), + encoding="utf-8", + ) + + def test_read_only_commands_never_create_missing_target(tmp_path, monkeypatch): database, command = _configure(tmp_path, monkeypatch) for action in ("status", "plan", "verify"): @@ -358,7 +395,10 @@ def test_missing_seekdb_is_inspected_without_initializing_the_engine(tmp_path, m @pytest.mark.parametrize("action", ["status", "plan", "apply", "verify"]) -def test_oceanbase_requires_persistent_evidence_before_connecting(tmp_path, monkeypatch, action): +@pytest.mark.parametrize("missing", ["evidence", "coordination"]) +def test_oceanbase_requires_evidence_and_explicit_coordination_before_connecting( + tmp_path, monkeypatch, action, missing +): _database, command = _configure(tmp_path, monkeypatch) private_value = "private-password-do-not-print" private_host = "private-database-host.invalid" @@ -370,20 +410,23 @@ def test_oceanbase_requires_persistent_evidence_before_connecting(tmp_path, monk ) def refuse_connection(*_args, **_kwargs): - pytest.fail("Missing persistent evidence must be rejected before contacting a database") + pytest.fail("Missing migration coordination must be rejected before contacting a database") monkeypatch.setattr(MaintenanceConnections, "run", refuse_connection) - result = _invoke(command, action, tmp_path) + evidence = tmp_path / "persistent-evidence" + options = ["--evidence-dir", str(evidence)] if missing == "coordination" else [] + result = _invoke(command, action, tmp_path, *options) assert result.exit_code == 1, result.output - assert json.loads(result.output)["error"] == "evidence_required" - assert "--evidence-dir" in result.output + assert json.loads(result.output)["error"] == f"{missing}_required" + assert ("--lock-coordination single-host" if missing == "coordination" else "--evidence-dir") in result.output assert private_value not in result.output assert private_host not in result.output assert "Traceback" not in result.output + assert not evidence.exists() @pytest.mark.parametrize("kind", ["seekdb", "oceanbase"]) -def test_remote_cli_uses_the_configured_maintenance_identity( +def test_remote_cli_uses_the_configured_maintenance_identity_and_coordination_scope( tmp_path, monkeypatch, kind: Literal["seekdb", "oceanbase"] ): _database, command = _configure(tmp_path, monkeypatch) @@ -407,6 +450,8 @@ def inspect_empty_database(_connections, operation, *, writable=False): evidence = tmp_path / "persistent-evidence" for action in ("status", "plan"): options = ["--evidence-dir", str(evidence)] + if kind == "oceanbase": + options.extend(["--lock-coordination", "single-host"]) if action == "plan": options.extend(["--backup", "manual"]) result = _invoke(command, action, tmp_path, *options) @@ -416,7 +461,141 @@ def inspect_empty_database(_connections, operation, *, writable=False): assert payload["state"] == "uninitialized" assert payload["readiness_scope"] == "registered_bundle" assert payload["server_ready"] is False + if kind == "oceanbase": + assert payload["coordination"] == { + "mode": "single-host", + "host": socket.gethostname(), + "directory": str(evidence.resolve() / identity.database_id), + } + else: + assert payload["coordination"] == { + "mode": "engine-directory", + "directory": str((tmp_path / "seekdb").resolve()), + } + if kind == "oceanbase": + original_plan = payload["plan_id"] + relocated = _invoke( + command, + "plan", + tmp_path, + "--evidence-dir", + str(tmp_path / "other-evidence"), + "--lock-coordination", + "single-host", + "--backup", + "manual", + ) + assert relocated.exit_code == 0, relocated.output + assert json.loads(relocated.output)["plan_id"] != original_plan + with monkeypatch.context() as patch: + patch.setattr(socket, "gethostname", lambda: "replacement-maintenance-host") + replanned = _invoke(command, "plan", tmp_path, *options) + assert replanned.exit_code == 0, replanned.output + assert json.loads(replanned.output)["plan_id"] != original_plan + refused = _invoke( + command, + "apply", + tmp_path, + *options, + "--yes", + "--plan-id", + original_plan, + "--backup-confirmed", + "--maintenance-confirmed", + ) + assert refused.exit_code == 1, refused.output + assert json.loads(refused.output)["error"] == "plan_changed" assert not evidence.exists() + assert not (tmp_path / "other-evidence").exists() + + +@pytest.mark.parametrize("kind", ["sqlite", "seekdb"]) +def test_previous_run_acknowledgement_is_rejected_for_other_backends_before_writing(tmp_path, monkeypatch, kind): + database, command = _configure(tmp_path, monkeypatch) + if kind == "seekdb": + (tmp_path / "deployment.env").write_text( + f"POWERCONTEXT_SERVER_DATABASE_KIND=seekdb\nPOWERCONTEXT_SERVER_DATABASE_PATH={tmp_path / 'seekdb'}\n", + encoding="utf-8", + ) + + def refuse_connection(*_args, **_kwargs): + pytest.fail("Unsupported acknowledgement must be rejected before opening a backend") + + monkeypatch.setattr(MaintenanceConnections, "run", refuse_connection) + result = _invoke(command, "apply", tmp_path, "--acknowledge-previous-run", uuid4().hex) + assert result.exit_code == 1, result.output + assert json.loads(result.output)["error"] == "unsupported_option" + assert not database.parent.exists() + assert not (tmp_path / "seekdb").exists() + + +@pytest.mark.parametrize("acknowledgement", [None, "incorrect-run-token"]) +def test_oceanbase_pending_run_requires_exact_acknowledgement_before_prompting(tmp_path, monkeypatch, acknowledgement): + command, options, receipt = _configure_oceanbase_inspection(tmp_path, monkeypatch) + _record_pending_run(receipt, uuid4().hex) + original = receipt.read_bytes() + monkeypatch.setattr(database_migration, "_interactive", lambda: True) + if acknowledgement is not None: + options.extend(["--acknowledge-previous-run", acknowledgement]) + result = _invoke(command, "apply", tmp_path, *options, "--backup", "manual", prompt_input="y\n") + assert result.exit_code == 1, result.output + assert json.loads(result.output)["error"] == "recovery_required" + assert "Accept this plan" not in result.output + assert receipt.read_bytes() == original + + +def test_oceanbase_rejects_a_stale_previous_run_acknowledgement(tmp_path, monkeypatch): + command, options, receipt = _configure_oceanbase_inspection(tmp_path, monkeypatch) + result = _invoke(command, "apply", tmp_path, *options, "--acknowledge-previous-run", uuid4().hex) + assert result.exit_code == 1, result.output + assert json.loads(result.output)["error"] == "stale_ack" + assert not receipt.parent.exists() + + +def test_oceanbase_plan_binds_pending_run_and_forwards_explicit_acknowledgement_in_one_confirmation( + tmp_path, monkeypatch +): + command, options, receipt = _configure_oceanbase_inspection(tmp_path, monkeypatch) + before = _invoke(command, "plan", tmp_path, *options, "--backup", "manual") + assert before.exit_code == 0, before.output + original_plan = json.loads(before.output) + token = uuid4().hex + _record_pending_run(receipt, token) + recorded = receipt.read_bytes() + planned = _invoke(command, "plan", tmp_path, *options, "--backup", "manual") + assert planned.exit_code == 0, planned.output + plan = json.loads(planned.output) + assert plan["state"] == "recovery_required" + assert plan["coordination"]["pending_run_id"] == token + assert plan["plan_id"] != original_plan["plan_id"] + verified = _invoke(command, "verify", tmp_path, *options) + assert verified.exit_code == 1, verified.output + assert json.loads(verified.output)["error"] == "recovery_required" + assert receipt.read_bytes() == recorded + + def accepted_migration(runner, **accepted): + assert runner.connections.acknowledge_previous_run == token + assert accepted["plan_id"] == plan["plan_id"] + return MigrationResult(revision=production_bundle().head, changed=True) + + monkeypatch.setattr(MySQLMigrationRunner, "apply", accepted_migration) + monkeypatch.setattr(database_migration, "_interactive", lambda: True) + result = _invoke( + command, + "apply", + tmp_path, + *options, + "--backup", + "manual", + "--acknowledge-previous-run", + token, + prompt_input="y\n", + ) + assert result.exit_code == 0, result.output + assert result.output.count("Accept this plan and the declarations above?") == 1 + assert "DBA confirmed all previous remote execution and background DDL ended" in result.output + assert "PC does not verify that declaration" in result.output + assert '"server_ready": false' in result.output def test_sqlite_commands_use_the_explicit_external_evidence_directory(tmp_path, monkeypatch): From f1a934e059218a1d812c2d7ad246e459e9a93b55 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 11:19:51 +0800 Subject: [PATCH 26/37] ci: capture stalled seekdb migration diagnostics --- .github/workflows/master.yml | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index cb27bdf829..15a1d1c958 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -157,13 +157,25 @@ jobs: - name: Test unified migrations with embedded seekdb env: POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" - run: >- - uv run --locked --extra seekdb pytest -q - tests/builtin/persistence/test_database_migration_backends.py - tests/builtin/persistence/test_database_backup_providers.py - tests/builtin/persistence/test_database_migration_connections.py - tests/builtin/persistence/test_mysql_migration_schema.py - tests/builtin/persistence/test_mysql_migration_runner.py -k seekdb + PYTHONUNBUFFERED: "1" + run: | + set -o pipefail + uv run --locked --extra seekdb pytest -vv \ + -o faulthandler_timeout=120 --durations=20 \ + tests/builtin/persistence/test_database_migration_backends.py \ + tests/builtin/persistence/test_database_backup_providers.py \ + tests/builtin/persistence/test_database_migration_connections.py \ + tests/builtin/persistence/test_mysql_migration_schema.py \ + tests/builtin/persistence/test_mysql_migration_runner.py -k seekdb \ + 2>&1 | tee seekdb-migrations.log + + - name: Upload seekdb migration diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: seekdb-migrations-${{ matrix.python-version }} + path: seekdb-migrations.log + if-no-files-found: ignore windows-unit-portability: timeout-minutes: 20 From 81d0a1115a86f97c28fc3abb41eb1bae1d2b3820 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 11:26:34 +0800 Subject: [PATCH 27/37] test: avoid blocking seekdb owner stderr --- .../test_database_migration_connections.py | 64 +++++++++++-------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/tests/builtin/persistence/test_database_migration_connections.py b/tests/builtin/persistence/test_database_migration_connections.py index 45dadfaf6e..8e3354ce48 100644 --- a/tests/builtin/persistence/test_database_migration_connections.py +++ b/tests/builtin/persistence/test_database_migration_connections.py @@ -278,6 +278,14 @@ def test_real_seekdb_maintenance_excludes_another_process_and_reopens_after_rele pytest.importorskip("pylibseekdb") path = tmp_path / "seekdb" ready = tmp_path / "owner-ready" + owner_log = tmp_path / "owner-stderr.log" + + def owner_diagnostic() -> str: + with owner_log.open("rb") as log: + log.seek(0, os.SEEK_END) + log.seek(max(0, log.tell() - 16_384)) + return log.read().decode("utf-8", errors="replace") + code = """ from pathlib import Path import sys @@ -295,33 +303,37 @@ def own(connection, identity, verify): MaintenanceConnections(SeekDBConfig(path=Path(sys.argv[1]))).run(own, writable=True) """ - process = subprocess.Popen( - [sys.executable, "-c", code, str(path), str(ready)], - stdin=subprocess.PIPE, - stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE, - text=True, - ) - try: - deadline = time.monotonic() + 60 - while not ready.exists() and process.poll() is None and time.monotonic() < deadline: - time.sleep(0.05) - assert ready.exists(), "The isolated seekdb owner did not become ready" - alias = tmp_path / "seekdb-alias" - alias.symlink_to(path, target_is_directory=True) - operation = Mock() - with pytest.raises(MigrationError) as locked: - MaintenanceConnections(SeekDBConfig(path=alias)).run(operation, writable=True) - assert locked.value.code == "migration_locked" - operation.assert_not_called() - finally: + with owner_log.open("w", encoding="utf-8") as log: + # Native startup can emit more stderr than an unread pipe can hold. + process = subprocess.Popen( + [sys.executable, "-c", code, str(path), str(ready)], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=log, + text=True, + ) try: - _, stderr = process.communicate(input="", timeout=30) - except subprocess.TimeoutExpired: - process.kill() - process.communicate(timeout=10) - pytest.fail("The isolated seekdb owner did not release its maintenance resources") - assert process.returncode == 0, stderr + deadline = time.monotonic() + 60 + while not ready.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(0.05) + assert ready.exists(), f"The isolated seekdb owner did not become ready\n{owner_diagnostic()}" + alias = tmp_path / "seekdb-alias" + alias.symlink_to(path, target_is_directory=True) + operation = Mock() + with pytest.raises(MigrationError) as locked: + MaintenanceConnections(SeekDBConfig(path=alias)).run(operation, writable=True) + assert locked.value.code == "migration_locked" + operation.assert_not_called() + finally: + try: + process.communicate(input="", timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.communicate(timeout=10) + pytest.fail( + f"The isolated seekdb owner did not release its maintenance resources\n{owner_diagnostic()}" + ) + assert process.returncode == 0, owner_diagnostic() def read(connection: Connection | None, _identity: BackendIdentity, verify: Callable[[], None]) -> str: assert connection is not None From 1f38aa545f10389a2f7c435dcf932e16ff31e762 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 11:28:47 +0800 Subject: [PATCH 28/37] test: cover noisy seekdb owner startup --- .../persistence/test_database_migration_connections.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/builtin/persistence/test_database_migration_connections.py b/tests/builtin/persistence/test_database_migration_connections.py index 8e3354ce48..7a56886e66 100644 --- a/tests/builtin/persistence/test_database_migration_connections.py +++ b/tests/builtin/persistence/test_database_migration_connections.py @@ -292,6 +292,10 @@ def owner_diagnostic() -> str: from powercontext.builtin.persistence.migrations.connections import MaintenanceConnections from powercontext.builtin.persistence.seekdb import SeekDBConfig +# Startup diagnostics must not block readiness even if no reader is attached. +sys.stderr.write("startup diagnostic\\n" * 65_536) +sys.stderr.flush() + def own(connection, identity, verify): verify() connection.exec_driver_sql("CREATE TABLE pc_lock_probe (id INT PRIMARY KEY, value VARCHAR(32))") From c7c8d9d5d3cfcd70f1b13a98e2dff249d3d9818e Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 11:48:39 +0800 Subject: [PATCH 29/37] test: reclaim native migration databases after each case --- .github/workflows/master.yml | 3 +++ .../persistence/test_database_backup_providers.py | 14 +++++++------- .../test_database_migration_backends.py | 4 ++-- .../test_database_migration_connections.py | 14 +++++++------- .../persistence/test_mysql_migration_runner.py | 6 +++--- .../persistence/test_mysql_migration_schema.py | 4 ++-- tests/conftest.py | 2 ++ 7 files changed, 26 insertions(+), 21 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 15a1d1c958..2d0b1c86c5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -155,11 +155,14 @@ jobs: run: make code-seekdb-test - name: Test unified migrations with embedded seekdb + timeout-minutes: 10 env: POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" PYTHONUNBUFFERED: "1" run: | set -o pipefail + trap 'df -h . /tmp' EXIT + df -h . /tmp uv run --locked --extra seekdb pytest -vv \ -o faulthandler_timeout=120 --durations=20 \ tests/builtin/persistence/test_database_migration_backends.py \ diff --git a/tests/builtin/persistence/test_database_backup_providers.py b/tests/builtin/persistence/test_database_backup_providers.py index b075d30095..4f56b239fe 100644 --- a/tests/builtin/persistence/test_database_backup_providers.py +++ b/tests/builtin/persistence/test_database_backup_providers.py @@ -233,12 +233,12 @@ def test_product_specific_fork_minimum_never_enables_unaccepted_recovery( @pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") -def test_real_seekdb_reports_engine_fork_version_without_claiming_recovery_acceptance(tmp_path: Path) -> None: +def test_real_seekdb_reports_engine_fork_version_without_claiming_recovery_acceptance(short_tmp_path: Path) -> None: from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile async def scenario() -> None: async with ( - SeekDBProfile.open(SeekDBConfig(path=tmp_path / "seekdb"), tables=()) as profile, + SeekDBProfile.open(SeekDBConfig(path=short_tmp_path / "seekdb"), tables=()) as profile, profile.database.engine.connect() as connection, ): await connection.exec_driver_sql("CREATE TABLE pc_artifacts (id INT PRIMARY KEY)") @@ -249,7 +249,7 @@ def inspect_provider(sync: Connection) -> None: sync, database_name="test", expected_product="seekdb", - directory=tmp_path / "manifests", + directory=short_tmp_path / "manifests", ) capability = provider.capabilities(context()) assert capability.product == "seekdb" @@ -266,12 +266,12 @@ def inspect_provider(sync: Connection) -> None: @pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") -def test_real_seekdb_fork_simple_table_survives_ddl_restart_and_original_name_restore(tmp_path: Path) -> None: +def test_real_seekdb_fork_simple_table_survives_ddl_restart_and_original_name_restore(short_tmp_path: Path) -> None: """Acceptance for one scratch-table shape, not the complete PC schema.""" from powercontext.builtin.persistence.seekdb import SeekDBConfig, SeekDBProfile async def scenario() -> None: - path = tmp_path / "seekdb" + path = short_tmp_path / "seekdb" config = SeekDBConfig(path=path) async with ( SeekDBProfile.open(config, tables=()) as profile, @@ -312,7 +312,7 @@ def create_provider_backup(sync: Connection) -> BackupRef: sync, database_name="test", expected_product="seekdb", - directory=tmp_path / "manifests", + directory=short_tmp_path / "manifests", acceptance=acceptance, ) assert provider.capabilities(context()).available @@ -333,7 +333,7 @@ def inspect_provider_backup(sync: Connection) -> None: sync, database_name="test", expected_product="seekdb", - directory=tmp_path / "manifests", + directory=short_tmp_path / "manifests", acceptance=acceptance, ) assert provider.inspect_backup(ref).state == "completed" diff --git a/tests/builtin/persistence/test_database_migration_backends.py b/tests/builtin/persistence/test_database_migration_backends.py index a646f07564..43a004d85d 100644 --- a/tests/builtin/persistence/test_database_migration_backends.py +++ b/tests/builtin/persistence/test_database_migration_backends.py @@ -89,9 +89,9 @@ def exercise_revisions(connection: Connection, bundle: MigrationBundle) -> None: @pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekDB probe not enabled") -def test_real_seekdb_revision_operations(tmp_path: Path) -> None: +def test_real_seekdb_revision_operations(short_tmp_path: Path) -> None: async def scenario() -> None: - path = tmp_path / "seekdb" + path = short_tmp_path / "seekdb" # The OS lock precedes engine startup and outlives engine shutdown. with local_migration_lock(path): async with SeekDBProfile.open(SeekDBConfig(path=path), tables=()) as profile: diff --git a/tests/builtin/persistence/test_database_migration_connections.py b/tests/builtin/persistence/test_database_migration_connections.py index 7a56886e66..e4c82676fa 100644 --- a/tests/builtin/persistence/test_database_migration_connections.py +++ b/tests/builtin/persistence/test_database_migration_connections.py @@ -235,10 +235,10 @@ async def dispose() -> None: @pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") def test_real_seekdb_maintenance_preserves_committed_data_without_initializing_business_tables( - tmp_path: Path, caplog + short_tmp_path: Path, caplog ) -> None: pytest.importorskip("pylibseekdb") - path = tmp_path / "seekdb" + path = short_tmp_path / "seekdb" adapter = MaintenanceConnections(SeekDBConfig(path=path, echo=True)) missing = adapter.run(lambda connection, identity, _verify: (connection, identity)) assert missing[0] is None @@ -274,11 +274,11 @@ def read(connection: Connection | None, identity: BackendIdentity, verify: Calla @pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") -def test_real_seekdb_maintenance_excludes_another_process_and_reopens_after_release(tmp_path: Path) -> None: +def test_real_seekdb_maintenance_excludes_another_process_and_reopens_after_release(short_tmp_path: Path) -> None: pytest.importorskip("pylibseekdb") - path = tmp_path / "seekdb" - ready = tmp_path / "owner-ready" - owner_log = tmp_path / "owner-stderr.log" + path = short_tmp_path / "seekdb" + ready = short_tmp_path / "owner-ready" + owner_log = short_tmp_path / "owner-stderr.log" def owner_diagnostic() -> str: with owner_log.open("rb") as log: @@ -321,7 +321,7 @@ def own(connection, identity, verify): while not ready.exists() and process.poll() is None and time.monotonic() < deadline: time.sleep(0.05) assert ready.exists(), f"The isolated seekdb owner did not become ready\n{owner_diagnostic()}" - alias = tmp_path / "seekdb-alias" + alias = short_tmp_path / "seekdb-alias" alias.symlink_to(path, target_is_directory=True) operation = Mock() with pytest.raises(MigrationError) as locked: diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index f4b66f979c..e8d1e13ee2 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -113,12 +113,12 @@ async def manage(*, create: bool) -> None: @pytest.fixture(params=("seekdb", "oceanbase"), ids=("seekdb", "oceanbase")) -def backend(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[_BackendTarget]: +def backend(request: pytest.FixtureRequest, short_tmp_path: Path) -> Iterator[_BackendTarget]: if request.param == "seekdb": if os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1": pytest.skip("real seekdb probe not enabled") pytest.importorskip("pylibseekdb") - yield _BackendTarget(SeekDBConfig(path=tmp_path / "seekdb"), tmp_path / "evidence") + yield _BackendTarget(SeekDBConfig(path=short_tmp_path / "seekdb"), short_tmp_path / "evidence") return value = os.environ.get("POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL") if not value: @@ -133,7 +133,7 @@ def backend(request: pytest.FixtureRequest, tmp_path: Path) -> Iterator[_Backend # Driver query options must not override the generated child database name. if any(key.lower() in {"db", "database"} for key in url.query): pytest.fail("OceanBase probe URL must not override the selected database", pytrace=False) - with _oceanbase_scratch_target(config, tmp_path / "evidence") as target: + with _oceanbase_scratch_target(config, short_tmp_path / "evidence") as target: yield target diff --git a/tests/builtin/persistence/test_mysql_migration_schema.py b/tests/builtin/persistence/test_mysql_migration_schema.py index f04773bd59..cc02c1d3f9 100644 --- a/tests/builtin/persistence/test_mysql_migration_schema.py +++ b/tests/builtin/persistence/test_mysql_migration_schema.py @@ -296,7 +296,7 @@ def test_portable_integrity_queries_reject_preexisting_invalid_rows(schema: MySQ @pytest.mark.skipif(os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1", reason="real seekdb probe not enabled") -def test_real_seekdb_frozen_schema_reflection_and_revision_recovery(tmp_path: Path) -> None: +def test_real_seekdb_frozen_schema_reflection_and_revision_recovery(short_tmp_path: Path) -> None: pytest.importorskip("pylibseekdb") schema = MySQLSchema(MigrationBundle(_RESOURCES)) @@ -344,4 +344,4 @@ def migrate(connection, _identity, guard) -> None: with pytest.raises(MigrationError, match="foreign key"): schema.validate_data(connection, "p0003") - MaintenanceConnections(SeekDBConfig(path=tmp_path / "seekdb")).run(migrate, writable=True) + MaintenanceConnections(SeekDBConfig(path=short_tmp_path / "seekdb")).run(migrate, writable=True) diff --git a/tests/conftest.py b/tests/conftest.py index 7feab57f05..f1c2b8b16d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,6 +30,8 @@ def short_tmp_path() -> Iterator[Path]: """Leave room for nested checkout caches and backup names on Windows.""" # pytest's user/session/test-name directories can exhaust MAX_PATH before # the fixture's cache, commit hash, and plugin files have been appended. + # Native database tests also need per-test cleanup to avoid accumulating + # preallocated engine logs across a complete CI session. with TemporaryDirectory(prefix="pc-") as directory: yield Path(directory) From 3570cc8e02ba0cf24fcf1b13075920374a08a964 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 12:36:49 +0800 Subject: [PATCH 30/37] ci: trace SQL in seekdb migration acceptance --- .github/workflows/master.yml | 8 ++++++-- tests/builtin/persistence/test_mysql_migration_runner.py | 7 ++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 2d0b1c86c5..2878b09a22 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -158,6 +158,7 @@ jobs: timeout-minutes: 10 env: POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" + POWERCONTEXT_TEST_MIGRATION_SQL_LOG: "1" PYTHONUNBUFFERED: "1" run: | set -o pipefail @@ -168,9 +169,12 @@ jobs: tests/builtin/persistence/test_database_migration_backends.py \ tests/builtin/persistence/test_database_backup_providers.py \ tests/builtin/persistence/test_database_migration_connections.py \ - tests/builtin/persistence/test_mysql_migration_schema.py \ - tests/builtin/persistence/test_mysql_migration_runner.py -k seekdb \ + tests/builtin/persistence/test_mysql_migration_schema.py -k seekdb \ 2>&1 | tee seekdb-migrations.log + uv run --locked --extra seekdb pytest -vv \ + -o faulthandler_timeout=120 --durations=20 --log-cli-level=INFO \ + tests/builtin/persistence/test_mysql_migration_runner.py -k seekdb \ + 2>&1 | tee -a seekdb-migrations.log - name: Upload seekdb migration diagnostics if: always() diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index e8d1e13ee2..257ce638bd 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -18,6 +18,7 @@ import asyncio import json +import logging import os from collections.abc import Callable, Iterator from contextlib import contextmanager @@ -113,7 +114,11 @@ async def manage(*, create: bool) -> None: @pytest.fixture(params=("seekdb", "oceanbase"), ids=("seekdb", "oceanbase")) -def backend(request: pytest.FixtureRequest, short_tmp_path: Path) -> Iterator[_BackendTarget]: +def backend( + request: pytest.FixtureRequest, short_tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> Iterator[_BackendTarget]: + if os.environ.get("POWERCONTEXT_TEST_MIGRATION_SQL_LOG") == "1": + caplog.set_level(logging.INFO, logger="sqlalchemy.engine") if request.param == "seekdb": if os.environ.get("POWERCONTEXT_TEST_MIGRATION_SEEKDB") != "1": pytest.skip("real seekdb probe not enabled") From 819329bc66dbb5ed15312562aa4b791e6fa85e59 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 14:07:42 +0800 Subject: [PATCH 31/37] fix(desktop): synchronize generated API contract --- .../src-tauri/src/transport/operations.json | 2 +- desktop/ui/src/generated/api.d.ts | 64 +++++++++++++++++++ desktop/ui/src/generated/operations.ts | 2 +- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/desktop/src-tauri/src/transport/operations.json b/desktop/src-tauri/src/transport/operations.json index 93eb65341a..dd777bd97d 100644 --- a/desktop/src-tauri/src/transport/operations.json +++ b/desktop/src-tauri/src/transport/operations.json @@ -1,5 +1,5 @@ { - "contractSha256": "9af88b2b779c372a21ae9f398d0ca75b333f2e5da2afa3626df879911a442edd", + "contractSha256": "790df0d51dd4ca600b34b77a4b6cce8b064305cbc65666e2813f96b23f3fd9fa", "operations": { "get_liveness": { "method": "GET", diff --git a/desktop/ui/src/generated/api.d.ts b/desktop/ui/src/generated/api.d.ts index d235ec9f2e..23524a45ab 100644 --- a/desktop/ui/src/generated/api.d.ts +++ b/desktop/ui/src/generated/api.d.ts @@ -596,6 +596,26 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/handoff/hint": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Prepare optional compact continuity hints from a selected Handoff + * @description Explicitly request untrusted historical orientation for a fresh session. Select an exact Revision, a complete transferred PreparedHandoff, or latest only after resolving the intended workstream Scope. Hints project existing Handoff fields and authorized evidence references without transcript summaries or model generation. They never replace full Handoff reads, evidence checks, current instructions, or live validation. The complete rendered UTF-8 hint is limited to max_bytes (at most 4000); omit it rather than truncate when budget or evidence is insufficient. Blocked hints include the full recorded state only when the complete hint fits. Authorization matches Continue: exact/latest require artifact.read and handoff.evidence.inspect on the selected Handoff, covering only its citation manifest without general Scope read; prepared requires scope.read. General evidence APIs retain their independent permission checks. Prepared selection provides no exact Revision reference; the receiver must retain the complete transferred PreparedHandoff. + */ + post: operations["prepare_handoff_hint"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/topic-memory/flush": { parameters: { query?: never; @@ -2753,6 +2773,19 @@ export interface components { prepared?: components["schemas"]["PreparedHandoff"]; revision?: components["schemas"]["ArtifactReference"]; }; + PrepareHandoffHintRequest: { + scope_id: string; + selection: components["schemas"]["HandoffSelection"]; + /** @description Required only for prepared selection; retain this complete value for continuation. */ + prepared?: components["schemas"]["PreparedHandoff"]; + /** @description Required only for exact selection. */ + revision?: components["schemas"]["ArtifactReference"]; + /** + * @description Complete rendered UTF-8 text budget including notice, boundaries, and exact references. + * @default 2000 + */ + max_bytes: number; + }; FinalizeHandoffRequest: { scope_id: string; draft: components["schemas"]["HandoffDraft"]; @@ -5522,6 +5555,37 @@ export interface operations { 503: components["responses"]["Unavailable"]; }; }; + prepare_handoff_hint: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["PrepareHandoffHintRequest"]; + }; + }; + responses: { + /** @description Bounded orientation text, or empty when no complete supported hint can be delivered. */ + 200: { + headers: { + "X-PowerContext-Request-ID": components["headers"]["RequestId"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["PreparedContext"]; + }; + }; + 401: components["responses"]["Unauthorized"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 422: components["responses"]["InvalidRequest"]; + 500: components["responses"]["InternalError"]; + 503: components["responses"]["Unavailable"]; + }; + }; flush_topic_memory: { parameters: { query?: never; diff --git a/desktop/ui/src/generated/operations.ts b/desktop/ui/src/generated/operations.ts index dedc8162bc..1aa51fee75 100644 --- a/desktop/ui/src/generated/operations.ts +++ b/desktop/ui/src/generated/operations.ts @@ -15,7 +15,7 @@ */ // Generated from openapi/powercontext.yaml. Do not edit. -export const contractSha256 = "9af88b2b779c372a21ae9f398d0ca75b333f2e5da2afa3626df879911a442edd"; +export const contractSha256 = "790df0d51dd4ca600b34b77a4b6cce8b064305cbc65666e2813f96b23f3fd9fa"; export const operations = { "get_liveness": { "method": "GET", From 3d418ef2f17bb0eeb27b5920940000a8a99b270c Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 14:27:16 +0800 Subject: [PATCH 32/37] fix(service): clear maintenance marker on uninstall retry --- src/powercontext/service/controller.py | 1 + tests/test_service_maintenance.py | 46 ++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/src/powercontext/service/controller.py b/src/powercontext/service/controller.py index c23fd3da1b..dabcce1280 100644 --- a/src/powercontext/service/controller.py +++ b/src/powercontext/service/controller.py @@ -661,6 +661,7 @@ def uninstall(self) -> ServiceStatus: registration.state is RegistrationState.NOT_INSTALLED and loaded.state is ManagerOwnershipState.NOT_LOADED ): + self.maintenance_path.unlink(missing_ok=True) return self.status() self._run_uninstall_stage("stop", self._adapter.stop) self._run_uninstall_stage("disable", self._adapter.disable) diff --git a/tests/test_service_maintenance.py b/tests/test_service_maintenance.py index d48070884f..5ebe4dcb85 100644 --- a/tests/test_service_maintenance.py +++ b/tests/test_service_maintenance.py @@ -37,6 +37,7 @@ ManagerState, ProbeResult, ProbeState, + RegistrationState, ServiceError, ) from powercontext.service.probe import probe_readiness @@ -163,6 +164,51 @@ def test_install_cannot_replace_a_service_with_an_unverified_migration(tmp_path: assert json.loads(controller.maintenance_path.read_text())["phase"] == "stopped" +@pytest.mark.parametrize("failed_stage", ["remove", "reload"]) +def test_uninstall_retry_clears_the_stop_marker_and_permits_reinstallation(tmp_path: Path, failed_stage: str) -> None: + controller, adapter = _installed(tmp_path) + controller.stop() + stopped_marker = controller.maintenance_path.read_bytes() + setattr(adapter, f"fail_{failed_stage}", True) + + with pytest.raises(ServiceError, match=f"uninstall failed during {failed_stage}"): + controller.uninstall() + + expected = RegistrationState.INSTALLED if failed_stage == "remove" else RegistrationState.NOT_INSTALLED + assert adapter.inspect().state is expected + assert controller.maintenance_path.read_bytes() == stopped_marker + setattr(adapter, f"fail_{failed_stage}", False) + + assert controller.uninstall().registration is RegistrationState.NOT_INSTALLED + assert not controller.maintenance_path.exists() + reinstalled = controller.install() + assert reinstalled.registration is RegistrationState.INSTALLED + assert reinstalled.manager is ManagerState.ACTIVE + assert reinstalled.server_liveness is LivenessState.LIVE + + +@pytest.mark.parametrize("ownership", [ManagerOwnershipState.FOREIGN, ManagerOwnershipState.UNKNOWN]) +def test_uninstall_retry_preserves_the_stop_marker_when_manager_ownership_cannot_be_verified( + tmp_path: Path, ownership: ManagerOwnershipState +) -> None: + controller, adapter = _installed(tmp_path) + controller.stop() + adapter.fail_reload = True + with pytest.raises(ServiceError, match="uninstall failed during reload"): + controller.uninstall() + marker = controller.maintenance_path.read_bytes() + adapter.fail_reload = False + manager_registration = ManagerRegistration(ownership, detail="manager ownership cannot be verified") + adapter.manager_registration_override = manager_registration + + with pytest.raises(ServiceError, match="manager ownership cannot be verified"): + controller.uninstall() + + assert adapter.inspect().state is RegistrationState.NOT_INSTALLED + assert adapter.loaded_registration() == manager_registration + assert controller.maintenance_path.read_bytes() == marker + + def test_successful_maintenance_switches_to_confirmed_executable_and_preserves_configuration(tmp_path: Path) -> None: controller, adapter = _installed(tmp_path) original = adapter.definition From 5d584bf83220740a90ba52ff268bb3564fe2812b Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 14:37:29 +0800 Subject: [PATCH 33/37] ci: isolate seekdb migration runner cases --- .github/workflows/master.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 2878b09a22..b1ae09517c 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -171,10 +171,24 @@ jobs: tests/builtin/persistence/test_database_migration_connections.py \ tests/builtin/persistence/test_mysql_migration_schema.py -k seekdb \ 2>&1 | tee seekdb-migrations.log - uv run --locked --extra seekdb pytest -vv \ - -o faulthandler_timeout=120 --durations=20 --log-cli-level=INFO \ + uv run --locked --extra seekdb pytest --collect-only -q --color=no \ tests/builtin/persistence/test_mysql_migration_runner.py -k seekdb \ - 2>&1 | tee -a seekdb-migrations.log + 2>&1 | tee seekdb-runner-collection.log | tee -a seekdb-migrations.log + mapfile -t seekdb_runner_nodes < <( + sed -n '/^tests\/builtin\/persistence\/test_mysql_migration_runner\.py::/p' \ + seekdb-runner-collection.log + ) + if [ "${#seekdb_runner_nodes[@]}" -eq 0 ]; then + echo "No seekdb migration runner tests were collected" >&2 + exit 1 + fi + for seekdb_runner_node in "${seekdb_runner_nodes[@]}"; do + timeout --signal=TERM --kill-after=10s 180s \ + uv run --locked --extra seekdb pytest -vv \ + -o faulthandler_timeout=120 --durations=20 --log-cli-level=INFO \ + "$seekdb_runner_node" -k seekdb \ + 2>&1 | tee -a seekdb-migrations.log + done - name: Upload seekdb migration diagnostics if: always() From 4ccaa6b00df4554a93e3fcb7bccf55a6416dfc29 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 14:54:16 +0800 Subject: [PATCH 34/37] test(desktop): wait for navigation focus before editing --- desktop/ui/tests/connections.test.tsx | 30 ++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/desktop/ui/tests/connections.test.tsx b/desktop/ui/tests/connections.test.tsx index 4de5e1f37f..f40ed9e27c 100644 --- a/desktop/ui/tests/connections.test.tsx +++ b/desktop/ui/tests/connections.test.tsx @@ -15,7 +15,7 @@ */ import { afterEach, beforeEach, expect, test, vi } from "vitest"; -import { act, cleanup, render, screen } from "@testing-library/react"; +import { act, cleanup, render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { App } from "../src/app/App"; import { Connections } from "../src/app/Connections"; @@ -148,23 +148,39 @@ test("a save from the previous connection editor cannot dismiss a new draft", as const user = userEvent.setup(); vi.mocked(desktopApi.state).mockResolvedValue(state()); let finish!: (next: DesktopState) => void; - vi.mocked(desktopApi.save).mockImplementationOnce( - () => - new Promise((resolve) => { - finish = resolve; - }), - ); + const pendingSave = new Promise((resolve) => { + finish = resolve; + }); + vi.mocked(desktopApi.save).mockReturnValueOnce(pendingSave); const confirm = vi.spyOn(window, "confirm").mockReturnValue(true); render(); await user.click(screen.getByRole("button", { name: "连接" })); + // Let navigation finish moving focus before keyboard input begins. + await waitFor(() => { + expect(document.activeElement).toBe( + screen.getByRole("heading", { level: 1, name: "连接" }), + ); + expect(screen.getByLabelText("连接名称").matches(":enabled")).toBe(true); + }); await user.type(screen.getByLabelText("连接名称"), "First"); await user.type( screen.getByLabelText("Server 地址"), "http://localhost:8000", ); await user.click(screen.getByRole("button", { name: "保存配置" })); + expect(desktopApi.save).toHaveBeenCalledWith( + expect.objectContaining({ + name: "First", + endpoint: "http://localhost:8000", + }), + ); await user.click(screen.getByRole("button", { name: "总览" })); await user.click(screen.getByRole("button", { name: "连接" })); + await waitFor(() => { + expect(document.activeElement).toBe( + screen.getByRole("heading", { level: 1, name: "连接" }), + ); + }); await user.type(screen.getByLabelText("连接名称"), "New draft"); await act(async () => { finish(state({ ...profile, name: "First" })); From a46ce3e830da38470dff03a258967f5f57a7dfb2 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 15:28:04 +0800 Subject: [PATCH 35/37] ci: preserve native seekdb logs on migration timeout --- .github/workflows/master.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index b1ae09517c..8be7d3101f 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -160,9 +160,19 @@ jobs: POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" POWERCONTEXT_TEST_MIGRATION_SQL_LOG: "1" PYTHONUNBUFFERED: "1" + TMPDIR: ${{ runner.temp }}/seekdb-migration run: | set -o pipefail - trap 'df -h . /tmp' EXIT + mkdir -p "$TMPDIR" + collect_diagnostics() { + df -h . /tmp + # A timed-out pytest process cannot run fixture cleanup. Retain + # bounded native logs from its isolated data directories as well + # as the Python stack and SQL trace. + find "$TMPDIR" -type f \( -name '*.log' -o -name '*.log.wf' \) \ + -print -exec tail -c 262144 {} \; > seekdb-native.log + } + trap collect_diagnostics EXIT df -h . /tmp uv run --locked --extra seekdb pytest -vv \ -o faulthandler_timeout=120 --durations=20 \ @@ -195,7 +205,9 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: seekdb-migrations-${{ matrix.python-version }} - path: seekdb-migrations.log + path: | + seekdb-migrations.log + seekdb-native.log if-no-files-found: ignore windows-unit-portability: From d76cdc0b7c5f4252c742018254c790fa1e93b09f Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 16:15:40 +0800 Subject: [PATCH 36/37] ci: retain seekdb transaction history in timeout diagnostics --- .github/workflows/master.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 8be7d3101f..bd3f0c4cc4 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -170,7 +170,7 @@ jobs: # bounded native logs from its isolated data directories as well # as the Python stack and SQL trace. find "$TMPDIR" -type f \( -name '*.log' -o -name '*.log.wf' \) \ - -print -exec tail -c 262144 {} \; > seekdb-native.log + -print -exec tail -c 8388608 {} \; > seekdb-native.log } trap collect_diagnostics EXIT df -h . /tmp From f92c0ee4a15ff1080add3585f1b1a955a1f81d0c Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 9 Oct 2026 16:31:09 +0800 Subject: [PATCH 37/37] fix(migrations): explicitly begin maintenance transactions --- .../persistence/migrations/connections.py | 3 ++ .../builtin/persistence/migrations/mysql.py | 2 ++ .../test_database_migration_connections.py | 5 ++++ .../test_mysql_migration_runner.py | 30 +++++++++++++++++++ 4 files changed, 40 insertions(+) diff --git a/src/powercontext/builtin/persistence/migrations/connections.py b/src/powercontext/builtin/persistence/migrations/connections.py index bc2ba8edfa..a66293b2da 100644 --- a/src/powercontext/builtin/persistence/migrations/connections.py +++ b/src/powercontext/builtin/persistence/migrations/connections.py @@ -149,6 +149,9 @@ def verify() -> None: verify() if sync.exec_driver_sql("SELECT DATABASE()").scalar_one() != identity.database_name: raise MigrationError("target_identity_unavailable", "The seekdb database identity changed.") + # MySQL's SQLAlchemy begin hook sends no SQL. Match the + # explicit transaction ownership used by AsyncDatabase. + sync.exec_driver_sql("START TRANSACTION") result = operation(sync, identity, verify) verify() return result diff --git a/src/powercontext/builtin/persistence/migrations/mysql.py b/src/powercontext/builtin/persistence/migrations/mysql.py index 9ef0708008..abfccff3a6 100644 --- a/src/powercontext/builtin/persistence/migrations/mysql.py +++ b/src/powercontext/builtin/persistence/migrations/mysql.py @@ -391,6 +391,7 @@ def _adopt( journal.write(evidence) connection.commit() guard() + connection.exec_driver_sql("START TRANSACTION") with ALEMBIC_CONTEXT_LOCK: command.stamp(config, revision) connection.commit() @@ -426,6 +427,7 @@ def upgrade(revision: str) -> None: config.attributes["execute_revision"] = upgrade connection.commit() guard() + connection.exec_driver_sql("START TRANSACTION") with ALEMBIC_CONTEXT_LOCK: command.upgrade(config, target) connection.commit() diff --git a/tests/builtin/persistence/test_database_migration_connections.py b/tests/builtin/persistence/test_database_migration_connections.py index e4c82676fa..05bc749720 100644 --- a/tests/builtin/persistence/test_database_migration_connections.py +++ b/tests/builtin/persistence/test_database_migration_connections.py @@ -111,6 +111,11 @@ def assert_locked() -> None: def configure_database(connection, _record) -> None: connection.create_function("DATABASE", 0, lambda: "test") + @event.listens_for(engine.sync_engine, "before_cursor_execute", retval=True) + def translate_transaction(_connection, _cursor, statement, parameters, _context, _many): + # This lifecycle double uses SQLite while the real adapter speaks MySQL. + return ("BEGIN" if statement == "START TRANSACTION" else statement), parameters + class Instance: def connection_options(self): return {} diff --git a/tests/builtin/persistence/test_mysql_migration_runner.py b/tests/builtin/persistence/test_mysql_migration_runner.py index 257ce638bd..45fc37ac30 100644 --- a/tests/builtin/persistence/test_mysql_migration_runner.py +++ b/tests/builtin/persistence/test_mysql_migration_runner.py @@ -227,6 +227,36 @@ def inspect(connection: Connection | None, _identity: BackendIdentity, _guard: C return backend.connections().run(inspect) +def test_mysql_maintenance_commits_rolls_back_and_reopens_for_ddl(backend: _BackendTarget) -> None: + def create(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + connection.exec_driver_sql("CREATE TABLE maintenance_probe (id INT PRIMARY KEY)") + connection.exec_driver_sql("INSERT INTO maintenance_probe VALUES (1)") + connection.commit() + connection.exec_driver_sql("INSERT INTO maintenance_probe VALUES (2)") + + backend.connections().run(create, writable=True) + + def change(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + assert connection.exec_driver_sql("SELECT id FROM maintenance_probe").all() == [(1,)] + connection.exec_driver_sql("ALTER TABLE maintenance_probe ADD COLUMN value INT DEFAULT 7") + connection.exec_driver_sql("INSERT INTO maintenance_probe (id) VALUES (3)") + connection.commit() + connection.exec_driver_sql("INSERT INTO maintenance_probe (id) VALUES (4)") + + backend.connections().run(change, writable=True) + + def inspect(connection: Connection | None, _identity: BackendIdentity, _guard: Callable[[], None]) -> None: + assert connection is not None + assert connection.exec_driver_sql("SELECT id, value FROM maintenance_probe ORDER BY id").all() == [ + (1, 7), + (3, 7), + ] + + backend.connections().run(inspect) + + def test_mysql_initialization_and_shared_evidence_noop(backend: _BackendTarget) -> None: runner = backend.runner() plan = runner.plan()