diff --git a/docs/en/rfcs/1560_recall_sufficiency_gate.md b/docs/en/rfcs/1560_recall_sufficiency_gate.md index d2873dc0fb..eccded208c 100644 --- a/docs/en/rfcs/1560_recall_sufficiency_gate.md +++ b/docs/en/rfcs/1560_recall_sufficiency_gate.md @@ -29,8 +29,8 @@ Three properties make the proposal safe to evaluate: - **Expansion cannot break the Builder invariant.** Merged candidates are re-selected down to each family's existing candidate ceiling using the allocators that already clamp round 0, so a cross-round union can never exceed the ceiling `build_scopes_result()` enforces (`prepared_context.py:164-169`). -- **A run that does not expand is byte-identical to today.** The no-expansion path is unchanged, which is the - regression guarantee. +- **A run that does not expand keeps its prepared content byte-identical.** The no-expansion path is unchanged, which + is the regression guarantee. - **Delivered size never exceeds `max_bytes`.** Expansion may use budget the round-0 candidate set left unused, so `content_bytes` can *increase* within the ceiling; it can never cross it. @@ -124,17 +124,35 @@ or comes from the counters this RFC adds; none requires a model call. | Signal | Computable from | What it detects | | --- | --- | --- | | Retrieved vs. admitted per family and channel | The new admission counters | A family's admission floor discarded nearly everything it retrieved. | -| Admitted candidates vs. that family's Builder ceiling | Round-zero hits and `prepared_context.py:103-105` | The family has room for a later round to contribute; a saturated family does not. | -| Top-1 score and the gap to the mean over admitted candidates | Score-bearing families only: Memory (`MemoryHit.score`) and Topic Memory (`TopicMemorySearchHit.score`) | One plausible hit surrounded by noise, or no clear winner. | +| Admitted candidates vs. that family's Builder ceiling | Round-zero hits and `prepared_context.py:103-105` | The family has room for a later round to contribute; a saturated family cannot grow, though a re-admitted hit may still change its membership. | +| Top vector cosine relevance and the family-local gap | Memory (`MemoryHit.relevance`) and Topic Memory (`TopicMemorySearchHit.relevance`), derived from unit-vector L2 distance; fused `score` remains a rank value | Whether each Scope's own first fused hit has usable semantic evidence. The gap uses only candidates above the round-zero semantic floor and is checked only when the top relevance is below the 0.70 strong-match threshold. | | Lexical coverage of the top candidates in the analyzer's token space | `analyze_text` / `fts_query_requirements` (`search.py:78`) | Hits matched on stopwords or on one shared token only. | | Number of participating families that returned at least one admitted candidate | Round-zero results | An assembly that selected three families and got results from one. | | Distinct evidence identities within a family | Family-specific identity (see below) | Many candidates that are really the same evidence. | -| Whether round zero's fit was budget-bound | The budget probe | Thin output caused by `max_bytes`, not by recall. | +| Whether the current round's fit was budget-bound | The budget probe over capped candidates | Thin output caused by `max_bytes`, not by recall. | -**Experience carries no score.** `ExperienceSearchHit` has exactly `artifact_ref` and `content` -(`artifacts/experience/search.py:26-30`), so the score-based signals apply only to the score-bearing families. +**Experience carries no vector relevance.** `ExperienceSearchHit` has exactly `artifact_ref` and `content` +(`artifacts/experience/search.py:26-30`), so relevance thresholds skip Experience and any Scope whose family's first +fused hit is FTS-only. Experience contributes its admission counters, its candidate count against its ceiling, and its family-coverage bit. +Cosine relevance is computed as `clamp(1 - d² / 2, -1, 1)` by the shared unit-vector L2 helper. A family contributes +scores only for Scopes whose own first fused hit has vector evidence; lower-ranked vector hits cannot turn an +FTS-only first hit into a weak score. Because Memory searches once per Scope, each Scope's fused ranking is judged +independently, so the verdict is invariant to the order Scopes are prepared in. `top_relevance` is the maximum known +cosine across those qualifying families. A gap requires at least two +scored candidates in the same family at or above the base semantic floor (`0.3`); fewer candidates leave that family's +gap unknown. Strong top relevance (`>= 0.70`) bypasses the gap check, while the configured minimum top score still +applies. Candidate-capture JSON retains the normalized fused rank at `assessments.candidates[].score` for external +tooling; that value does not enter the gate's relevance thresholds. + +The policy identifier is `powercontext.recall-gate.v2`. Version 1 used family-local normalized fused rank scores for +`top_score`, `mean_score` and `top_gap`; version 2 uses vector cosine relevance for those signals. Historical captures +with `policy_id: powercontext.recall-gate.v1` retain the rank-based interpretation and must not be relabelled as v2. +The inherited defaults `min_top_score=0.35` and `min_top_gap=0.02` were calibrated against the old rank scores. They are +**provisional values on the cosine scale and have not been recalibrated**; carrying the defaults forward does not +establish their suitability for cosine relevance. + **Evidence identity is family-specific.** A single Memory search returns many `MemoryHit` values that all share one `memory_ref` Artifact revision, because one Memory Revision holds many entries; the independent unit of evidence is the entry, identified by `entry_id` and `entry_version_id` (`memory/models.py:142-150`). Counting distinct Artifact @@ -161,6 +179,18 @@ section is `profile` is never expanded at all. | 1 | Lower the admission floor applied to what each participating searchable family's search already returned, within the configured expansion floor: the lexical-evidence requirement (`search.py:104`) and the cosine baseline (`memory/fusion.py:29`, `topic_memory/fusion.py:33`). | Round 0 assessed insufficient. | | 2 | Lower admission to the policy floor and accept the best available evidence. | Round 1 committed and round 1 assessed insufficient. | +`weak-top-1` records an insufficient score signal; it is not a terminal verdict. The signals are read off the fused +candidate window, and each family's search caps that window before the gate sees hits: a strong vector-only hit can +sit outside it until a lower lexical floor admits a second channel for the same identity and RRF moves it into +view. A weak top therefore still expands while a family reports recoverable candidates — the same rule that covers +candidate-count, family-coverage and lexical insufficiency — and expansion stops only on `sufficient`, an exhausted +recovery pool, or the spent round budget. When the last committed assessment is still `weak-top-1` once the rounds +are spent, the reason is kept instead of being rewritten to `at-max-rounds`. Within the window the baseline rules +are unchanged: scores below the round-zero floor cannot manufacture a gap, and a strong top (`>= 0.70`) still +exempts the gap check. A new budget probe assesses each expansion's capped candidate set before it is committed. +The committed set keeps the latest round's fused order inside each family's cap, so a hit that only entered the +window under a lowered floor is deliverable rather than trailing the round-zero prefix. + No further action is available in round 2. In particular, **raising `memory_rerank_candidate_limit` is not an expansion action**: `MemoryService` uses that bound to size the backend request, not only to rerank an existing pool (`coarse_limit` at `service.py:452`, then `candidate_limit=max(coarse_limit * 4, 32)` at `service.py:457`), so raising @@ -181,15 +211,19 @@ that a family the caller did not select is not searched and is not given output unselected family would violate that contract, so family membership is out of scope for expansion. If a caller omits `assembly` entirely, the Runtime's existing default family selection applies unchanged and is also not expanded. -**Expansion is a no-op for a family whose round 0 already saturated its ceiling.** Because a merged set keeps round 0 -as its prefix (see *Between rounds*), a family that already filled its candidate ceiling in round 0 cannot receive -later-round candidates. The gate's "admitted vs. ceiling" signal reports this, and the trace records the round as one -that changed nothing. The constraint in that case is downstream — budget, not recall — and the budget probe says so. +**A saturated family cannot grow, but its membership can still change.** The family's total stays at its ceiling: +the committed set keeps the latest round's fused order inside the cap (see *Between rounds*), so a hit that only +enters the window under a lowered floor displaces a lower-ranked same-scope candidate instead of trailing a full +round-zero prefix. The gate's "admitted vs. ceiling" signal still reports when no family has headroom, and the +budget probe still reports when the binding constraint is downstream — budget, not recall. ## What you can observe -Following the precedent of the RFC 0080 `rerank` trace, the gate result stays in the process and is **not** added to the -HTTP v1 response. +Following the precedent of the RFC 0080 `rerank` trace, the gate result stays in the process and is **not** added to +the HTTP v1 response. The `RecallEffort` trace carries `signals`: the aggregate gate signals of the last committed +assessment — candidate count, top vector relevance when a scored family exists, and lexical overlap — measured before +Builder ceilings and byte fitting. If the gate fails before its first assessment, `assessment` is `expansion-failed`, +`rounds` is 1, the count is from round zero, and `signals` is null. `RecallEffort` is produced by the expansion loop, which lives in `ScopedContextApplication._prepare` (`application.py:727`), not by the Builder. It is therefore **not** attached to `PreparedContextBuild`: `_prepare` @@ -219,22 +253,23 @@ class AdmissionCounts: @dataclass(frozen=True) class RecallEffort: - policy: str # versioned policy id, e.g. "powercontext.recall-gate.v1" + policy: str # versioned policy id, e.g. "powercontext.recall-gate.v2" assessment: str # final gate reason, e.g. "sufficient" | "thin-candidates" | "weak-top-1" - rounds: int # search passes actually executed: 1 + len(expansion_actions), so 1..3 + rounds: int # committed passes: round zero + len(expansion_actions), so 1..3 expansion_actions: tuple[str, ...] # committed rounds only; at most ("admission", "policy-floor") - candidates_by_round: tuple[int, ...] # candidates offered to the Builder per round; len == rounds - admission_by_family: tuple[AdmissionCounts, ...] # measured in the last executed round - added_embeddings: int # query embeddings paid by expansion rounds - added_generation_calls: int # extra RFC 0080 rerank calls paid by expansion rounds + candidates_by_round: tuple[int, ...] # accumulated pool before Builder ceilings; len == rounds + admission_by_family: tuple[AdmissionCounts, ...] # measured in the last committed round + added_embeddings: int # search-reported query embeddings in committed expansion rounds + added_generation_calls: int # search-reported extra rerank calls in committed expansion rounds truncated_items: int # delivered but cut; not counted today dropped_items: int # whole-item omissions across both fitting paths; not counted today dropped_below_min_bytes: int # subset of dropped_items: below _MIN_TRUNCATED_CONTENT_BYTES / _BODY_BYTES dropped_no_fitting_truncation: int # subset of dropped_items: no rendering fit at all ``` -`rounds` counts **search passes executed**, so a prepare that ran round 0 and two expansion rounds reports -`rounds: 3`, and one expansion round reports `rounds: 2`. `dropped_items` equals +`rounds` counts **committed search passes**: round zero and two committed expansions report `rounds: 3`, and one +committed expansion reports `rounds: 2`. A failed expansion does not add a round or change the committed signals; +work paid by that failed round is not included in the committed-only cost counters. `dropped_items` equals `dropped_below_min_bytes + dropped_no_fitting_truncation`, so a reader can tell "the budget could not fit this item" apart from "this item was too short to truncate into the remaining space". A single counter labelled "lost to the budget" would misreport the second cause, so both are reported and `_fit_entry`'s two `None` paths @@ -252,7 +287,8 @@ with no query text, no entry identity and no per-entry attribution. ## Example -A Codex hook asks for context with the default budget; the first round returns one weak Memory hit. +A caller asks for context with the default budget; the first round returns one Memory hit with cosine relevance 0.5 +and full lexical overlap. ```http POST /v1/context/prepare @@ -265,10 +301,10 @@ Content-Type: application/json } ``` -Round 0 returns three Memory candidates from a backend pool of 64, one with a usable score; admission admits three of -64, which is far below the Memory ceiling of 16, so the family has room to grow. The budget probe finds unused bytes -and no whole-item drops, so the output is recall-thin rather than budget-thin. The gate assesses `weak-top-1`, expands -once (admission floor lowered), and round 1 admits four more candidates that the round-0 floor had discarded from the +Round 0 admits one Memory candidate from a backend pool of 64, below the default minimum of two and the Memory ceiling +of 16, so the family has room to grow. The budget probe finds unused bytes +and no whole-item drops, so the output is recall-thin rather than budget-thin. The gate assesses `thin-candidates`, +expands once (admission floor lowered), and round 1 admits four more candidates that the round-0 floor had discarded from the pool the search had already retrieved. Selection and rendering then proceed exactly as today, inside the same 8000 bytes. @@ -349,28 +385,28 @@ those three entry points, described next. ## New components 1. **`RecallSufficiencyPolicy`** — a frozen, versioned value object holding the thresholds, the maximum round count, - the per-round admission floors, and the prompt-shape limits for the budget probe. Constructed from Runtime + the per-round admission floors. Constructed from Runtime configuration; defaults preserve today's behaviour when the feature is disabled. -2. **`RecallAdmissionPolicy`** — the value threaded into each searchable family's search to override its floor: - an optional `required_matches` (default: the value `fts_query_requirements` derives, `search.py:78`) and an - optional `min_semantic_similarity` (default: `0.3`, `memory/fusion.py:29`, `topic_memory/fusion.py:33`). Passing - `RecallAdmissionPolicy()` — both overrides `None` — reproduces today's behaviour exactly, which is what round 0 - does. +2. **`AdmissionFloor`** — the shared value threaded into each searchable family's search to override its lexical + coverage, minimum matched terms and minimum cosine similarity. Round zero passes `admission=None` to retain the + historical defaults: lexical coverage `0.25`, minimum matched terms `2` (bounded by query length), and cosine `0.3`. 3. **`AdmissionCounts`** — `(family, scope_id, retrieved, admitted)`, one per searchable family per scope per round. 4. **`RecallSufficiencyGate`** — a pure function. It takes the round's per-family views, the query, the policy, and a budget view: - `assess(*, query, families, budget, policy) -> GateAssessment`. No I/O, no model call, and no clock access beyond + `assess(candidates, query, policy, *, scope_has_content, budget, families_expected) -> GateAssessment`. + No I/O, no model call, and no clock access beyond what the candidates already carry. It returns `sufficient` plus a reason and the signal values that produced it. 5. **`RecallBudgetView`** — `max_bytes` together with the counters from a **budget probe**: one pass of the Builder's existing pure fitting code over the round's candidate set, discarding the rendered output and keeping `delivered_items`, `truncated_items`, `dropped_items` and `unused_bytes`. The gate needs this to distinguish budget-limited thinness from recall-limited thinness; without it the 512-byte edge case below is undecidable. -6. **`RecallExpander`** — a pure function `(round, policy) -> SearchPlan`, where `SearchPlan` carries only the two - admission overrides for the next round. It never names a family, so it cannot violate the assembly contract, and it +6. **`RecallExpander`** — a pure function `(round, policy) -> SearchPlan`, where `SearchPlan` carries only + `action` and `admission` for the next round. It never names a family, so it cannot violate the assembly contract, and it never sets `limit`, `mode`, or a rerank candidate bound. 7. **`RecallEffort`** — the trace value described above, delivered to the Runtime's optional sink. -All of these live under `src/powercontext/builtin/runtime/`. The gate, the expander and the budget probe reuse pure code +The policy, gate, expander, budget view and effort live under `src/powercontext/builtin/runtime/`; the shared floor and +admission counts live in `builtin/artifacts/search.py`. The gate, the expander and the budget probe reuse pure code and are tested directly without a database. ## Required internal plumbing @@ -378,11 +414,11 @@ and are tested directly without a database. The counts and the floor override have to reach the three admission sites. None of this is an HTTP change. **Memory.** `MemoryService.search` already materializes both sides of the boundary — `channels` from the backend, then -`admitted_fts` and `admitted_vector` (`service.py:463-465`). It gains an optional `admission: RecallAdmissionPolicy | +`admitted_fts` and `admitted_vector` (`service.py:463-465`). It gains an optional `admission: AdmissionFloor | None = None` keyword that replaces the derived required-match count and the `0.3` cosine baseline, and it reports `retrieved = len(channels.fts) + len(channels.vector)` together with `admitted = len(admitted_fts) + len(admitted_vector)`. The counts are exposed on an in-process-only field of `MemorySearchResult` -(`memory/models.py:165-169`). The HTTP contract is untouched because the response is built from `MemorySearchPage` +(`memory/models.py:165-169`). The Memory search response is untouched because it is built from `MemorySearchPage` (`runtime/models.py:183-189`, constructed at `application.py:1738` and `:1761`) by `search_response` (`server/mapping.py:788`), which enumerates its fields explicitly; the counters must stay off `MemorySearchPage` and out of `openapi/powercontext.yaml`, so `make api-generate` is not required. @@ -401,9 +437,8 @@ because `ContextAssemblySection.family` admits only `memory`, `experience`, `pro carrying the hits plus their `AdmissionCounts`, measured around `_admit_fts` and `_admit_vector` (`topic_memory/fusion.py:102-117`). -**Round 0 behaviour is unchanged.** Round 0 passes `RecallAdmissionPolicy()` and ignores the floor override; the only -difference is that it now observes and returns the counts. With the feature disabled, the counters are not even -collected. +**Round 0 behaviour is unchanged.** Round zero passes `admission=None` and uses the historical floors. Search results +carry admission counts regardless of whether the Runtime gate is enabled; the disabled gate does not assess or expand. ## Where the loop goes @@ -423,13 +458,16 @@ breaking the invariant the Builder enforces. later round contributes that are not already present, in round order, deduplicated by the family's evidence identity (*What the gate looks at*). On an identity collision the earlier round's occurrence is retained. Concretely, for `memory`: `merged = round0_hits + (new identities from round 1, in fusion order) + (new identities from round 2, in -fusion order)`. +fusion order)`. This accumulated order is what the gate scores — it keeps the per-Scope first-hit rule stable across +rounds — while the committed set the Builder receives is ordered by the latest round's fused ranking (*Reselection +rule* below). -**Reselection rule.** The merged set is then passed through the allocators that already clamp round 0 — -`_limit_memory_candidates` (`application.py:747`) and `_limit_experience_candidates` (`application.py:748-751`), which -round-robin the family's ceiling across scope groups — while Topic Memory keeps its single-scope ceiling check. Because -round 0 is a prefix of the merged sequence, a truncation can only ever remove candidates a later round added; a -candidate that round 0 produced is never displaced by expansion. +**Reselection rule.** The committed set is passed through the same kind of allocators that clamp round 0 — +`_limit_expanded_memory_candidates` and `_limit_expanded_experience_candidates`, which preserve each scope group's +round-zero quota and round-robin the family's remaining ceiling — while Topic Memory keeps its single-scope ceiling +check. Within a scope group the committed order is the latest round's fused order, followed by the identities that +round did not re-admit in their accumulated order: a candidate that only enters the fused window under a lowered +floor lands inside the cap rather than behind the round-zero prefix. Two consequences follow, and both are required properties rather than incidental effects: @@ -439,10 +477,10 @@ Two consequences follow, and both are required properties rather than incidental allocators instead of as an unbounded union: two individually valid rounds of 16 Memory hits can differ by one candidate, and their deduplicated union of 17 would otherwise raise `PreparedContextInvariantError("memory-candidate-limit")`. -- **A saturated family cannot grow.** If round 0 already filled a family's ceiling, the merged set truncates back to - round 0's candidates and that family's later rounds change nothing. Expansion can only add candidates to a family - whose round-0 admission count is below that family's ceiling — the signal the gate reads before deciding, and the - reason a no-op expansion is a normal, reported outcome. +- **A saturated family cannot grow.** The family total stays at its ceiling, so the checks at + `prepared_context.py:164-169` cannot be triggered by expansion. Its membership can still change, though: the + committed set keeps the latest round's fused order inside each scope's preserved quota, so a re-admitted candidate + displaces a lower-ranked same-scope candidate rather than being appended behind a full prefix. Consequently, a round returning *fewer* candidates than its predecessor is ordinary and expected rather than a degradation trigger. @@ -534,18 +572,19 @@ Expansion is committed **per round**, and a round is all-or-nothing: - Round 0 is committed unconditionally. If round 0 fails, prepare fails exactly as it does today. - A round `r >= 1` first collects every participating family's results for every scope into a **staged** set. The stage - is committed into the merged set only if every search in it succeeded. -- If any search in the round raises, the entire stage is discarded and the merged set reverts to the last committed - state. `RecallEffort.rounds` counts the passes actually executed, and `expansion_actions` records only committed + is committed only after every search, the budget probe and the gate assessment succeed. +- If any search, budget probe or assessment in the round raises, the entire stage is discarded and the returned + candidates and signals remain at the last committed state. `RecallEffort.rounds` and `expansion_actions` record only committed rounds. Without that boundary, "fail open" would not hold: expansion runs per family and per scope, so a later search can fail after earlier searches have already contributed candidates, and swallowing that error would return a partially expanded -result instead of today's round-0 result. With the boundary, a failed round yields exactly the previous committed +result. With the boundary, a failed round yields exactly the previous committed result. -The gate itself fails closed: if `assess` raises, or if configuration is absent, no expansion happens and prepare -proceeds with round 0. The gate never turns a successful prepare into a failure, and never changes `status`. Because +If the initial `assess` raises, prepare proceeds with round zero; a later assessment failure preserves earlier +committed expansions. Absent configuration skips the gate. The gate never turns a successful prepare into a failure, +and never changes `status`. Because the gate runs before selection, a failure costs at most an extra search, never a lost result. ## Edge cases @@ -563,8 +602,9 @@ the gate runs before selection, a failure costs at most an extra search, never a - **Profile-only request.** `profile` is a section family (`runtime/models.py:195`) but is read via `profiles.latest` (`application.py:752-759`) with no search and no admission floor, so it is never an expansion target and a profile-only request is never expanded. -- **A saturated family.** If a family's round-0 admission already equals its Builder ceiling, expansion cannot add - candidates to it; the round is recorded as one that changed nothing rather than being retried. +- **A saturated family.** If a family's round-0 admission already equals its Builder ceiling, expansion cannot grow + it; a later round can still change its membership when the lowered floor re-admits a higher-ranked hit, and is + otherwise recorded as a round that changed nothing. - **A caller-selected family set.** `assembly.sections` decides which families participate, so a family the caller did not select is never searched, is given no output budget, and is never a legal expansion target however thin round 0 was. If the caller selected Memory only, "add Experience" is not on the table. @@ -590,7 +630,7 @@ The feature is off by default and enabled by Runtime configuration. ## Testing Gate, expander and budget probe are tested as pure functions, including the signal computations for a family with no -score (Experience), a family whose admission admitted nothing, and a family already at its ceiling. +vector relevance (Experience), a family whose admission admitted nothing, and a family already at its ceiling. Runtime-level tests assert, for a fixed candidate set and a fixed budget: @@ -602,8 +642,10 @@ Runtime-level tests assert, for a fixed candidate set and a fixed budget: build successfully rather than raise `PreparedContextInvariantError("memory-candidate-limit")`, and no round can exceed a family's ceiling; - a round contributes no new identity → the earlier set stands and `candidates_by_round` is flat; -- a family saturated in round 0 receives no later-round candidate; -- no candidate produced by round 0 is ever displaced by expansion; +- a family saturated in round 0 cannot grow, though a re-admitted hit may still displace a lower-ranked same-scope + candidate inside the ceiling; +- the committed set keeps the latest round's fused order inside each scope's preserved quota, so a round-zero + candidate can be displaced only by a higher-ranked re-admitted hit; - a failure in any search of a round discards that round's entire stage and returns the previous committed result; - an empty Scope never expands; a profile-only request never expands; an explicitly selected family set is never widened; @@ -630,9 +672,10 @@ added latency. The feature is not enabled by default until that comparison shows calibrated, not merely switched on. - **Expanded output can be larger.** Within `max_bytes`, a successful expansion can deliver more bytes than round 0 would have, which is the intent but also means byte count is no longer a stable property of a query. -- **Expansion is inert for saturated families and for budget-bound queries.** The candidate ceiling caps what any - round can add, and the budget probe stops expansion when the budget is the constraint. The feature addresses exactly - one failure mode — thin candidates against a non-binding budget — and should not be sold as a general recall fix. +- **Expansion cannot grow saturated families and does not help budget-bound queries.** The candidate ceiling caps + what any round can deliver (membership may still change inside it), and the budget probe stops expansion when the + budget is the constraint. The feature addresses exactly one failure mode — thin candidates against a non-binding + budget — and should not be sold as a general recall fix. - **A new tunable surface.** Thresholds that decide "sufficient" are now joined by a second set that decides "relaxed enough". Both are easy to set wrong and hard to justify empirically. Shipping them as a versioned policy mitigates but does not remove this. diff --git a/docs/zh/rfcs/1560_recall_sufficiency_gate.md b/docs/zh/rfcs/1560_recall_sufficiency_gate.md index 92d7196525..1c3ace73cd 100644 --- a/docs/zh/rfcs/1560_recall_sufficiency_gate.md +++ b/docs/zh/rfcs/1560_recall_sufficiency_gate.md @@ -24,7 +24,7 @@ - **扩展不可能破坏 Builder 不变量。** 跨轮合并后的候选会沿用它已经用于截断第 0 轮的那两个分配器，重新选择到各类别 既有的候选上限之内，因此跨轮并集永远不会超过 `build_scopes_result()` 所强制的那道上限 （`prepared_context.py:164-169`）。 -- **不扩展的运行与今天逐字节相同。** 无扩展路径完全未变，这就是回归保证。 +- **不扩展的运行保持准备内容逐字节相同。** 无扩展路径完全未变，这就是回归保证。 - **交付体积永不越过 `max_bytes`。** 扩展可以使用第 0 轮候选集未用满的预算，因此 `content_bytes` 可以在这个上限 之内**变大**，但永远不能越过它。 @@ -108,17 +108,31 @@ Stage C 报告召回代价与省略情况 （进程内） | 信号 | 从哪来 | 检测什么 | | --- | --- | --- | | 各类别、各通道的取回数与准入数 | 新增的准入计数器 | 某个类别的准入下限把取回的内容几乎全部丢弃了。 | -| 准入候选数与该类别 Builder 上限之比 | 第 0 轮命中与 `prepared_context.py:103-105` | 该类别还有空间让后续轮次贡献；已饱和的类别没有。 | -| Top-1 分数及其与准入候选均分的差距 | 仅限带分数的类别：Memory（`MemoryHit.score`）与 Topic Memory（`TopicMemorySearchHit.score`） | 一条看似可用的命中被噪声包围，或根本没有明显胜出者。 | +| 准入候选数与该类别 Builder 上限之比 | 第 0 轮命中与 `prepared_context.py:103-105` | 该类别还有余量让后续轮次贡献；已饱和的类别不能增长，但重新准入的命中仍可能改变其成员构成。 | +| 顶部向量 cosine 相关度及类别内差距 | Memory（`MemoryHit.relevance`）与 Topic Memory（`TopicMemorySearchHit.relevance`），由单位向量 L2 距离换算；融合 `score` 仍是名次值 | 每个 Scope 各自的融合第一条是否有可用语义证据。差距只计算达到第 0 轮语义准入下限的候选，且只在顶部相关度低于 0.70 强匹配阈值时参与判断。 | | 头部候选在 analyzer 词元空间中的词项覆盖度 | `analyze_text` / `fts_query_requirements`（`search.py:78`） | 命中只是靠停用词或某一个共现 token 匹配上的。 | | 至少返回一条准入候选的参与类别数量 | 第 0 轮结果 | 选了三个类别，只有一个有结果。 | | 类别内的不同证据身份数 | 类别专属身份（见下表） | 多条候选其实是同一份证据。 | -| 第 0 轮的拟合是否被预算卡住 | 预算探测结果 | 偏薄输出是由 `max_bytes` 造成的，而不是召回。 | +| 当前轮次的拟合是否被预算卡住 | 对按类别上限截断的候选进行预算探测 | 偏薄输出是由 `max_bytes` 造成的，而不是召回。 | -**Experience 没有分数。** `ExperienceSearchHit` 只有 `artifact_ref` 与 `content` -（`artifacts/experience/search.py:26-30`），因此基于分数的信号只对带分数的类别适用。Experience 只贡献它的准入计数、 +**Experience 没有向量相关度。** `ExperienceSearchHit` 只有 `artifact_ref` 与 `content` +（`artifacts/experience/search.py:26-30`），因此相关度阈值跳过 Experience，以及某类别下融合第一条只有 FTS 证据的 Scope。Experience 只贡献它的准入计数、 它相对自身上限的候选数，以及它的"有结果"位。 +cosine 相关度由共享的单位向量 L2 helper 按 `clamp(1 - d² / 2, -1, 1)` 换算。只有当某 Scope 自己的融合第一条命中 +具有向量证据时，该类别的该 Scope 才贡献分数；下位向量命中不能把 FTS-only 的第一条变成弱分数。Memory 按 Scope +分别搜索，每个 Scope 的融合排序独立判定，因此判定结论与 Scope 的 prepare 顺序无关。`top_relevance` 是这些合格 +类别中已知 cosine 的最大值。差距要求同类别至少有两个 scored 候选达到基础语义准入下限（`0.3`）；不足两个时， +该类别的差距未知。 +强匹配（顶部相关度 `>= 0.70`）跳过差距检查，但仍应用配置的最低顶部相关度。候选捕获 JSON 保留 +`assessments.candidates[].score` 的归一化融合名次值供外部工具链使用；该值不参与闸门的相关度阈值判断。 + +当前 policy 标识为 `powercontext.recall-gate.v2`。v1 的 `top_score`、`mean_score` 与 `top_gap` 基于类别内归一化 +融合名次分；v2 的这些信号改为基于向量 cosine 相关度。历史捕获中 +`policy_id: powercontext.recall-gate.v1` 仍按名次分口径解释，不得重新标为 v2。 +继承的默认值 `min_top_score=0.35` 与 `min_top_gap=0.02` 针对旧名次分标定，在 cosine 尺度上是 +**暂定值，尚未重新标定**；沿用默认值不代表已证明它们适合 cosine 相关度。 + **证据身份是类别专属的。** 一次 Memory 搜索返回的多个 `MemoryHit` 共享同一个 `memory_ref` Artifact revision，因为一个 Memory Revision 装有多个条目；真正独立的证据单位是条目，由 `entry_id` 与 `entry_version_id` 标识 （`memory/models.py:142-150`）。因此统计"不同 Artifact revision 数"会把任意 Memory-only 结果都压成一个来源。闸门 @@ -143,6 +157,15 @@ Memory Revision 装有多个条目；真正独立的证据单位是条目，由 | 1 | 在配置的扩展下限范围内，降低施加于各可搜索类别搜索**已经取回**的候选上的准入下限：词项证据要求（`search.py:104`）与余弦基线（`memory/fusion.py:29`、`topic_memory/fusion.py:33`）。 | 第 0 轮判定不充分。 | | 2 | 把准入降到 policy 下限，接受当前最好的证据。 | 第 1 轮已提交且第 1 轮判定不充分。 | +`weak-top-1` 记录的是分数信号不足的判定，不是终止结论。分数信号读取的是融合后的候选窗口，而各类别的检索在闸门 +看到命中之前就按上限截断了窗口：一条仅有向量证据的强命中可能位于窗口之外，直到更低的词项下限为同一身份补进 +第二个通道、RRF 把它带进窗口。因此弱顶部在类别报告有可恢复候选时仍会扩展——与候选数、类别覆盖、词项不足适用 +同一规则——扩展只在判定为 `sufficient`、可恢复池耗尽或轮次用尽时停止。轮次用尽后，若最后一次已提交评估仍是 +`weak-top-1`，该原因保留，不改写为 `at-max-rounds`。窗口内的基线规则不变：低于第 0 轮下限的分数无法伪造 gap， +强顶部（`>= 0.70`）仍豁免 gap 检查。每轮扩展提交前，都会对按类别上限截断后的候选集重新做预算探测。 +已提交的候选集在各类别上限内按最新一轮的融合序排列，因此在降低下限后才进入窗口的命中会被交付， +而不是排在第 0 轮前缀之后被截掉。 + 第 2 轮没有更多可选动作。特别地，**提高 `memory_rerank_candidate_limit` 不是扩展动作**：`MemoryService` 用这个界 来决定后端请求的规模，而不只是对已有候选池做重排（`coarse_limit` 见 `service.py:452`，随后 `candidate_limit=max(coarse_limit * 4, 32)` 见 `service.py:457`）。把它从 30 提到 100 会把后端候选池从 120 扩到 400， @@ -159,13 +182,17 @@ Memory Revision 装有多个条目；真正独立的证据单位是条目，由 也不分配输出预算。静默搜索一个未选择的类别会违反该契约，因此类别成员不属于扩展范围。调用方完全省略 `assembly` 时，Runtime 沿用现有的默认类别选择，同样不扩展。 -**对于第 0 轮已经填满自身上限的类别，扩展是空操作。** 因为合并后的集合以第 0 轮为前缀（见**轮次之间**），一个在 -第 0 轮就把候选上限填满的类别无法再接纳后续轮次的候选。闸门的"准入数 vs 上限"信号会报出这一点，trace 会把该轮记成 -"什么都没改变的一轮"。这种情形下的约束在下游——是预算，而不是召回——预算探测会给出结论。 +**已饱和的类别无法增长，但其成员仍可能变化。** 类别总量保持在上限：已提交集合在上限内按最新一轮的融合序排列 +（见**轮次之间**），所以在降低下限后才进入窗口的命中会顶掉排名更低的同 Scope 候选，而不是排在已满的第 0 轮前缀 +之后。闸门的"准入数 vs 上限"信号仍会报出各类别是否还有余量，当约束在下游——是预算而不是召回——时，预算探测仍会给出 +结论。 ## 可以观测到什么 沿用 RFC 0080 `rerank` trace 的先例，闸门结果留在进程内，**不**进入 HTTP v1 响应。 +`RecallEffort` trace 携带 `signals`：最后一次已提交评估的聚合闸门信号——候选数、有打分类别时的顶部向量相关度、 +词项覆盖度——计算时尚未应用 Builder 类别上限和字节拟合。若闸门在首次评估完成之前失败，`assessment` 为 +`expansion-failed`，`rounds` 为 1，候选数来自第 0 轮，`signals` 为 null。 `RecallEffort` 由扩展循环产出，而循环位于 `ScopedContextApplication._prepare`（`application.py:727`），不在 Builder 内。 因此它**不**挂在 `PreparedContextBuild` 上：`_prepare` 返回的是 `build.context`（`application.py:812`），构建结果的其余 @@ -194,21 +221,22 @@ class AdmissionCounts: @dataclass(frozen=True) class RecallEffort: - policy: str # 带版本的 policy id，如 "powercontext.recall-gate.v1" + policy: str # 带版本的 policy id，如 "powercontext.recall-gate.v2" assessment: str # 最终闸门原因，如 "sufficient" | "thin-candidates" | "weak-top-1" - rounds: int # 实际执行的搜索轮次：1 + len(expansion_actions)，即 1..3 + rounds: int # 已提交轮次：第 0 轮 + len(expansion_actions)，即 1..3 expansion_actions: tuple[str, ...] # 仅已提交的轮次；最多 ("admission", "policy-floor") - candidates_by_round: tuple[int, ...] # 每一轮交给 Builder 的候选数；len == rounds - admission_by_family: tuple[AdmissionCounts, ...] # 在最后执行的轮次中测量 - added_embeddings: int # 扩展轮次额外付出的 query embedding 次数 - added_generation_calls: int # 扩展轮次额外付出的 RFC 0080 rerank 调用次数 + candidates_by_round: tuple[int, ...] # 应用 Builder 上限前的累计候选数；len == rounds + admission_by_family: tuple[AdmissionCounts, ...] # 在最后已提交的轮次中测量 + added_embeddings: int # 已提交扩展轮次由搜索报告的 query embedding 次数 + added_generation_calls: int # 已提交扩展轮次由搜索报告的额外 rerank 调用次数 truncated_items: int # 已交付但被截断；当前未计数 dropped_items: int # 两条拟合路径上整条省略的总数；当前未计数 dropped_below_min_bytes: int # dropped_items 的子集：低于 _MIN_TRUNCATED_CONTENT_BYTES / _BODY_BYTES dropped_no_fitting_truncation: int # dropped_items 的子集：没有任何渲染能放下 ``` -`rounds` 统计的是**实际执行的搜索轮次**：跑了第 0 轮加两轮扩展的 prepare 报 `rounds: 3`，只扩展一次报 `rounds: 2`。 +`rounds` 统计的是**已提交的搜索轮次**：第 0 轮加两轮已提交扩展报 `rounds: 3`，只提交一次扩展报 `rounds: 2`。 +失败的扩展不增加轮次，也不改变已提交信号；失败轮次实际付出的代价不计入仅统计已提交轮次的代价计数器。 `dropped_items` 等于 `dropped_below_min_bytes + dropped_no_fitting_truncation`，因此读者可以区分"预算放不下这条" 与"这条太短，无法截断进剩余空间"。若把两者合成一个标注为"整条输给预算"的计数器，就会误报第二种原因，所以两种原因 分别报告，`_fit_entry` 的两条 `None` 路径（`prepared_context.py:462-463`、`:465-485`）与 assembly 路径 @@ -223,7 +251,7 @@ class RecallEffort: ## 示例 -Codex Hook 用默认预算请求上下文，第一轮只返回一条很弱的 Memory 命中。 +调用方用默认预算请求上下文；第一轮只返回一条 cosine 相关度 0.5、词项完全覆盖的 Memory 命中。 ```http POST /v1/context/prepare @@ -236,9 +264,9 @@ Content-Type: application/json } ``` -第 0 轮从 64 条后端候选池中返回三条 Memory 候选，其中一条分数可用；准入从 64 条里只放进三条，远低于 Memory 的 -上限 16，因此该类别还有增长空间。预算探测发现仍有未用字节、且没有整条丢弃，所以这是**召回**偏薄而不是**预算**偏薄。 -闸门判定为 `weak-top-1`，扩展一次（降低准入下限），第 1 轮又放进四条候选——它们本就在第 0 轮搜索已取回的候选池里， +第 0 轮从 64 条后端候选池中准入一条 Memory 候选，低于默认最低候选数 2 和 Memory 上限 16，因此该类别还有增长空间。 +预算探测发现仍有未用字节、且没有整条丢弃，所以这是**召回**偏薄而不是**预算**偏薄。 +闸门判定为 `thin-candidates`，扩展一次（降低准入下限），第 1 轮又放进四条候选——它们本就在第 0 轮搜索已取回的候选池里， 只是被当时的下限丢弃了。随后选择与渲染完全按现有逻辑进行，仍在同样的 8000 字节内。 如果第 1 轮没有贡献任何新候选，prepare 会交付第 0 轮的结果——也就是今天的行为——`RecallEffort` 会报 @@ -310,24 +338,25 @@ Builder 的上限本身（`application.py:742-743`、`:761`），与上文一致 ## 新增组件 -1. **`RecallSufficiencyPolicy`** —— 冻结的、带版本的值对象，持有阈值、最大轮数、每轮扩展的准入下限，以及预算探测的 - 提示规模上限。由 Runtime 配置构造；功能关闭时默认值保持当前行为。 -2. **`RecallAdmissionPolicy`** —— 传给每个可搜索类别搜索、用于覆盖其下限的值：可选的 `required_matches` - （默认取 `fts_query_requirements` 推导值，`search.py:78`）与可选的 `min_semantic_similarity` - （默认 `0.3`，`memory/fusion.py:29`、`topic_memory/fusion.py:33`）。传 `RecallAdmissionPolicy()`——两个覆盖都为 - `None`——精确复现今天的行为，第 0 轮就是这么做的。 +1. **`RecallSufficiencyPolicy`** —— 冻结的、带版本的值对象，持有阈值、最大轮数和每轮扩展的准入下限。 + 由 Runtime 配置构造；功能关闭时默认值保持当前行为。 +2. **`AdmissionFloor`** —— 传给每个可搜索类别搜索的共享值，用于覆盖词项覆盖率、最低匹配词项数与最低 cosine。 + 第 0 轮传 `admission=None`，保留历史默认值：词项覆盖率 `0.25`、最低匹配词项数 `2`（受 query 长度约束）、 + cosine `0.3`。 3. **`AdmissionCounts`** —— `(family, scope_id, retrieved, admitted)`，每个可搜索类别、每个 Scope、每一轮一个。 4. **`RecallSufficiencyGate`** —— 纯函数。输入是该轮各类别的视图、query、policy 与一个预算视图： - `assess(*, query, families, budget, policy) -> GateAssessment`。无 I/O、无模型调用、除候选自身已携带的信息外不访问 + `assess(candidates, query, policy, *, scope_has_content, budget, families_expected) -> GateAssessment`。 + 无 I/O、无模型调用、除候选自身已携带的信息外不访问 时钟。返回 `sufficient`，以及原因与产生该判断的信号取值。 5. **`RecallBudgetView`** —— `max_bytes` 连同一次**预算探测**得到的计数：用 Builder 已有的纯拟合代码在该轮候选集上 跑一遍，丢弃渲染结果，只保留 `delivered_items`、`truncated_items`、`dropped_items` 与 `unused_bytes`。闸门需要它 来区分"预算造成的偏薄"与"召回造成的偏薄"；没有它，下面 512 字节的边界条件无法判定。 -6. **`RecallExpander`** —— 纯函数 `(round, policy) -> SearchPlan`，`SearchPlan` 只携带下一轮的两项准入覆盖。它不涉及 +6. **`RecallExpander`** —— 纯函数 `(round, policy) -> SearchPlan`，`SearchPlan` 只携带下一轮的 `action` 与 `admission`。它不涉及 类别，因此不可能违反 assembly 契约，也绝不设置 `limit`、`mode` 或 rerank 候选界。 7. **`RecallEffort`** —— 上文描述的 trace 值，交付给 Runtime 的可选 sink。 -以上都位于 `src/powercontext/builtin/runtime/` 下。闸门、扩展器与预算探测复用纯逻辑，可以直接测试，不需要数据库。 +policy、闸门、扩展器、预算视图与 effort 位于 `src/powercontext/builtin/runtime/` 下；共享的下限与准入计数位于 +`builtin/artifacts/search.py`。闸门、扩展器与预算探测复用纯逻辑，可以直接测试，不需要数据库。 ## 所需内部传递机制 @@ -335,10 +364,10 @@ Builder 的上限本身（`application.py:742-743`、`:761`），与上文一致 **Memory。** `MemoryService.search` 本来就已经把边界两侧的东西都物化了——后端的 `channels`，以及随后的 `admitted_fts` 与 `admitted_vector`（`service.py:463-465`）。它新增一个可选关键字参数 -`admission: RecallAdmissionPolicy | None = None`，用于替换推导出的词项匹配数与 `0.3` 余弦基线，并报出 +`admission: AdmissionFloor | None = None`，用于替换推导出的词项匹配数与 `0.3` 余弦基线，并报出 `retrieved = len(channels.fts) + len(channels.vector)` 与 `admitted = len(admitted_fts) + len(admitted_vector)`。计数通过 `MemorySearchResult`（`memory/models.py:165-169`）上的 -仅进程内字段暴露。HTTP 契约不受影响，因为响应是由 `search_response`（`server/mapping.py:788`）从 +仅进程内字段暴露。Memory 搜索响应不受影响，因为它由 `search_response`（`server/mapping.py:788`）从 `MemorySearchPage`（`runtime/models.py:183-189`，构造于 `application.py:1738` 与 `:1761`）构建的，而它显式枚举自己的 字段；这些计数器必须留在 `MemorySearchPage` 之外、也不进入 `openapi/powercontext.yaml`，因此不需要执行 `make api-generate`。 @@ -354,8 +383,8 @@ Builder 的上限本身（`application.py:742-743`、`:761`），与上文一致 **Topic Memory。** `_topic_memory_hits`（`application.py:890`）同样返回裸元组；它需要改为返回一个同时携带命中与其 `AdmissionCounts` 的结果值，计数在 `_admit_fts` 与 `_admit_vector`（`topic_memory/fusion.py:102-117`）两侧测量。 -**第 0 轮行为不变。** 第 0 轮传 `RecallAdmissionPolicy()`，忽略下限覆盖；唯一区别是它现在会观测并返回计数。功能 -关闭时，计数器根本不会被采集。 +**第 0 轮行为不变。** 第 0 轮传 `admission=None`，使用历史默认下限。搜索结果始终携带准入计数，无论 Runtime +闸门是否启用；关闭的闸门不进行评估或扩展。 ## 循环放在哪里 @@ -371,12 +400,14 @@ Builder 只接收一份候选集，所以必须说清它来自哪些轮次，而 **合并规则。** 在每个类别、每个 Scope 分组内，合并后的集合是：第 0 轮的候选，之后按轮次顺序追加后续各轮贡献的、 此前不存在的身份，并按该类别的证据身份去重（见**闸门看什么**）。身份冲突时保留较早轮次的那一条。以 `memory` 为例： -`merged = round0_hits + (第 1 轮的新身份，按融合顺序) + (第 2 轮的新身份，按融合顺序)`。 +`merged = round0_hits + (第 1 轮的新身份，按融合顺序) + (第 2 轮的新身份，按融合顺序)`。这个累积序就是闸门打分的 +顺序——它让按 Scope 的首条规则在各轮之间保持稳定——而交给 Builder 的已提交集合按**最新一轮**的融合序排列（见下面的 +**重新选择规则**）。 -**重新选择规则。** 合并后的集合随后通过**已经用于截断第 0 轮**的那两个分配器——`_limit_memory_candidates` -（`application.py:747`）与 `_limit_experience_candidates`（`application.py:748-751`），它们把类别上限按 scope 分组做 -round-robin 分配——Topic Memory 则保留它自己的单 Scope 上限检查。因为第 0 轮是合并序列的前缀，截断只可能移除后续轮次 -新增的候选；第 0 轮产生的候选永远不会被扩展挤掉。 +**重新选择规则。** 已提交的集合随后通过与截断第 0 轮同类的分配器——`_limit_expanded_memory_candidates` 与 +`_limit_expanded_experience_candidates`，它们保留每个 Scope 分组的第 0 轮配额、并把类别剩余上限按分组做 round-robin +分配——Topic Memory 则保留它自己的单 Scope 上限检查。在每个 Scope 分组内，已提交顺序是该轮次的融合序，其后是该轮 +未重新准入的身份按累积序排列：在降低下限后才进入融合窗口的候选因此落在上限之内，而不是排在第 0 轮前缀之后被截掉。 由此得到两个必须成立的性质，而不是偶然的副作用： @@ -385,9 +416,9 @@ round-robin 分配——Topic Memory 则保留它自己的单 Scope 上限检查 扩展触发。这正是把合并定义在同一批分配器上、而不是定义成无界并集的原因：两轮各自合法、各含 16 条 Memory 命中的 结果可能只差一条候选，而它们去重后的 17 条并集会让 `build_scopes_result()` 抛出 `PreparedContextInvariantError("memory-candidate-limit")`。 -- **已饱和的类别不可能增长。** 如果第 0 轮已经填满某类别的上限，合并集合会被截断回第 0 轮的候选，该类别的后续轮次不会 - 改变任何东西。扩展只能给那些第 0 轮准入数低于自身上限的类别增加候选——这正是闸门在做决定之前读取的信号，也是"空 - 操作扩展"属于正常且会被报告的结果的原因。 +- **已饱和的类别不可能增长。** 类别总量保持在上限之内，因此 `prepared_context.py:164-169` 的检查不可能被扩展触发。 + 但它的成员仍可能变化：已提交集合在每个 Scope 保留的配额内按最新一轮融合序排列，所以重新准入的候选会顶掉排名更低 + 的同 Scope 候选，而不是被追加到已满的前缀之后。 因此，某一轮返回的候选**少于**上一轮是正常且预期的，不构成降级触发条件。 @@ -465,16 +496,16 @@ Experience 与 Topic Memory 的搜索不做 rerank。因此上表的生成调用 扩展是**按轮次**提交的，且一轮是全有或全无： - 第 0 轮无条件提交。如果第 0 轮失败，prepare 与今天一样失败。 -- 轮次 `r >= 1` 先把该轮所有参与类别、所有 Scope 的结果收集到一个**暂存**集合。只有当其中每一次搜索都成功时，暂存 - 集合才会合并进已提交集合。 -- 如果该轮中任何一次搜索抛错，整个暂存集合被丢弃，已提交集合回退到上一次提交的状态。`RecallEffort.rounds` 统计实际 - 执行的轮次，`expansion_actions` 只记录已提交的轮次。 +- 轮次 `r >= 1` 先把该轮所有参与类别、所有 Scope 的结果收集到一个**暂存**集合。只有当其中每一次搜索、预算探测和 + 闸门评估都成功后，才提交该轮。 +- 如果该轮中任何一次搜索、预算探测或评估抛错，整个暂存集合被丢弃，返回的候选与信号保持上一次已提交的状态。 + `RecallEffort.rounds` 与 `expansion_actions` 都只记录已提交的轮次。 没有这道边界，"fail-open"就不成立：扩展按类别和 Scope 执行，后面的搜索可能在前面几次搜索已经把候选贡献进来之后才 -失败，而把这个错误吞掉就会返回一个**部分扩展**的结果，而不是今天的第 0 轮结果。有了这道边界，一个失败的轮次得到的 +失败，而把这个错误吞掉就会返回一个**部分扩展**的结果。有了这道边界，一个失败的轮次得到的 正是上一次已提交的结果。 -闸门本身是失败即不扩展的：如果 `assess` 抛错，或配置缺失，则不执行扩展，prepare 带着第 0 轮继续。闸门永远不会把一次 +首次 `assess` 抛错时，prepare 带着第 0 轮继续；后续评估失败时保留此前已提交的扩展。配置缺失时跳过闸门。闸门永远不会把一次 成功的 prepare 变成失败，也不会改变 `status`。因为闸门在选择之前运行，一次失败最多多花一次搜索，永远不会丢掉已有 结果。 @@ -490,8 +521,8 @@ Experience 与 Topic Memory 的搜索不做 rerank。因此上表的生成调用 revision。 - **只选 Profile 的请求。** `profile` 是 section 类别（`runtime/models.py:195`），但通过 `profiles.latest` 读取 （`application.py:752-759`），既不搜索也不经过准入下限，因此它不是扩展目标，只选 profile 的请求永不扩展。 -- **已饱和的类别。** 如果某类别第 0 轮的准入数已经等于其 Builder 上限，扩展无法再给它增加候选；该轮被记录为 - "什么都没改变"，而不是被重试。 +- **已饱和的类别。** 如果某类别第 0 轮的准入数已经等于其 Builder 上限，扩展无法再让它增长；但当降低后的下限重新 + 准入一条排名更高的命中时，后续轮次仍可能改变它的成员构成，否则该轮被记录为"什么都没改变"。 - **指定了 `assembly` 的请求。** 扩展不得引入未选择的类别。若调用方只选择了 Memory，则"补 Experience"不是一个 合法动作，无论第 0 轮多薄。 - **没有向量部署的 Scope。** hybrid 不可用，`mode` 本来就是 `fts`，余弦基线也不适用；此时只有词项证据要求可以放宽， @@ -512,7 +543,7 @@ Experience 与 Topic Memory 的搜索不做 rerank。因此上表的生成调用 ## 测试 -闸门、扩展器与预算探测作为纯函数测试，覆盖：无分数类别（Experience）的信号计算、准入数为零的类别，以及已经处于上限的 +闸门、扩展器与预算探测作为纯函数测试，覆盖：无向量相关度类别（Experience）的信号计算、准入数为零的类别，以及已经处于上限的 类别。 Runtime 层测试在固定候选集与固定预算下断言： @@ -523,8 +554,8 @@ Runtime 层测试在固定候选集与固定预算下断言： - **合并集合永不破坏 Builder 不变量**：两轮各含 16 条 Memory 命中、只差一条候选时，必须成功构建，而不是抛出 `PreparedContextInvariantError("memory-candidate-limit")`；任何轮次都不得超出类别上限； - 一轮没有贡献新身份 → 沿用先前的候选集，且 `candidates_by_round` 持平； -- 第 0 轮已饱和的类别不再接收后续轮次的候选； -- 第 0 轮产生的候选永远不会被扩展挤掉； +- 第 0 轮已饱和的类别不能增长，但重新准入的命中仍可能在上限内顶掉排名更低的同 Scope 候选； +- 已提交集合在每个 Scope 保留的配额内按最新一轮融合序排列，第 0 轮的候选只会被排名更高的重新准入命中顶掉； - 一轮中任何一次搜索失败都会丢弃该轮整个暂存集合，并返回上一次已提交的结果； - 空 Scope 永不扩展；只选 profile 的请求永不扩展；显式选定的类别集合永不被扩大； - 准入计数被报出，且未配置 sink 时不产生任何副作用。 @@ -546,8 +577,9 @@ OceanBase 上各跑一遍，报告任务成功率、注入字节数、`truncated 它们可能挤掉默认下限本可交付的证据。这是该特性默认关闭、且必须先标定而不能直接打开的主要原因。 - **扩展后的输出可能更大。** 在 `max_bytes` 之内，一次成功的扩展可以比第 0 轮交付更多字节，这正是它的意图，但也 意味着字节数不再是某个 query 的稳定属性。 -- **对已饱和类别与被预算卡住的查询，扩展是无效的。** 候选上限封顶了任何轮次能增加的量，而预算探测会在预算是约束时 - 停止扩展。该特性只针对一种失败模式——候选偏薄而预算未满——不应被当成通用的召回修复方案来宣传。 +- **扩展无法增大已饱和类别，也帮不了被预算卡住的查询。** 候选上限封顶了任何轮次能交付的量（上限内的成员仍可能 + 变化），而预算探测会在预算是约束时停止扩展。该特性只针对一种失败模式——候选偏薄而预算未满——不应被当成通用的 + 召回修复方案来宣传。 - **新增可调参数面。** 决定"是否充分"的阈值，现在又加上了决定"放宽到哪"的一组阈值。两者都容易设错，也很难用经验 证据证明。以带版本 policy 的形式发布可以缓解但不能消除这一点。 - **可能掩盖检索缺陷。** 如果第一轮偏薄的原因是索引或 embedding 有问题，用同样机制再跑一轮往往同样偏薄， diff --git a/src/powercontext/builtin/artifacts/memory/fusion.py b/src/powercontext/builtin/artifacts/memory/fusion.py index 3182f6325a..bebe70d585 100644 --- a/src/powercontext/builtin/artifacts/memory/fusion.py +++ b/src/powercontext/builtin/artifacts/memory/fusion.py @@ -23,7 +23,7 @@ MemoryHit, MemoryMatchedBy, ) -from powercontext.builtin.artifacts.search import AdmissionFloor, admits_fts_text +from powercontext.builtin.artifacts.search import AdmissionFloor, admits_fts_text, unit_l2_cosine_similarity _RRF_CONSTANT = 60 _MIN_SEMANTIC_SIMILARITY = 0.3 @@ -56,14 +56,10 @@ def admit_vector_candidates( return tuple( candidate for candidate in candidates - if candidate.distance is not None and _unit_l2_cosine_similarity(candidate.distance) >= baseline + if candidate.distance is not None and unit_l2_cosine_similarity(candidate.distance) >= baseline ) -def _unit_l2_cosine_similarity(distance: float) -> float: - return max(-1.0, min(1.0, 1.0 - distance**2 / 2.0)) - - def fuse_rankings( *, fts: Sequence[MemoryChannelHit], @@ -78,6 +74,7 @@ def fuse_rankings( candidates: dict[_HitIdentity, MemoryChannelHit] = {} scores: dict[_HitIdentity, float] = {} channels: dict[_HitIdentity, set[MemoryMatchedBy]] = {} + relevance: dict[_HitIdentity, float] = {} for channel, ranking in (("fts", fts), ("vector", vector)): seen: set[_HitIdentity] = set() @@ -89,6 +86,9 @@ def fuse_rankings( candidates.setdefault(identity, candidate) scores[identity] = scores.get(identity, 0.0) + 1.0 / (_RRF_CONSTANT + rank) channels.setdefault(identity, set()).add(channel) + if channel == "vector" and candidate.distance is not None: + similarity = unit_l2_cosine_similarity(candidate.distance) + relevance[identity] = max(relevance.get(identity, -1.0), similarity) ordered = sorted( candidates, @@ -107,6 +107,7 @@ def fuse_rankings( text=candidates[identity].text, score=scores[identity], matched_by=tuple(channel for channel in ("fts", "vector") if channel in channels[identity]), + relevance=relevance.get(identity), ) for identity in ordered ) diff --git a/src/powercontext/builtin/artifacts/memory/models.py b/src/powercontext/builtin/artifacts/memory/models.py index e7b5c553a6..c8b13cedaa 100644 --- a/src/powercontext/builtin/artifacts/memory/models.py +++ b/src/powercontext/builtin/artifacts/memory/models.py @@ -213,6 +213,7 @@ class MemoryHit(BaseModel): text: str score: float matched_by: tuple[MemoryMatchedBy, ...] + relevance: float | None = Field(default=None, ge=-1.0, le=1.0, allow_inf_nan=False, exclude=True) class MemoryRerankTrace(BaseModel): diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index fb158749c2..6d80bc2524 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Shared deterministic lexical analysis for built-in Artifact projections.""" +"""Shared deterministic lexical and vector admission for built-in Artifact projections.""" from __future__ import annotations @@ -107,10 +107,8 @@ class AdmissionFloor: module and ``0.3`` mirrored by ``memory/fusion.py`` and ``topic_memory/fusion.py`` — so that a ``floor=None`` / ``admission=None`` call reproduces today's behaviour bit for bit. - This type plays the ``RecallAdmissionPolicy`` role described by RFC 1560: it is the value - threaded into each searchable family's search to override its floor. Passing ``None`` - (the historical default) is therefore equivalent to the RFC's ``RecallAdmissionPolicy()`` - with both overrides unset, which is exactly what round 0 does. + RFC 1560 threads this value into each searchable family's search to override its floor. + Round zero passes ``None`` to keep the historical defaults. """ lexical_coverage: float = _FTS_MIN_QUERY_COVERAGE @@ -182,6 +180,12 @@ def lexical_search_score(raw: object, metric: object, /) -> tuple[float, Channel ) +def unit_l2_cosine_similarity(distance: float) -> float: + """Convert L2 distance between unit vectors to cosine similarity.""" + + return max(-1.0, min(1.0, 1.0 - distance**2 / 2.0)) + + @dataclass(frozen=True) class AdmissionCounts: """Per-family, per-scope admission accounting for one search. @@ -370,4 +374,5 @@ def _is_cjk(character: str) -> bool: "fts_match_query", "fts_query_requirements", "lexical_search_score", + "unit_l2_cosine_similarity", ] diff --git a/src/powercontext/builtin/artifacts/topic_memory/fusion.py b/src/powercontext/builtin/artifacts/topic_memory/fusion.py index dd19e58be7..c1507f9958 100644 --- a/src/powercontext/builtin/artifacts/topic_memory/fusion.py +++ b/src/powercontext/builtin/artifacts/topic_memory/fusion.py @@ -29,6 +29,7 @@ analyze_text, analyze_text_with_spans, lexical_search_score, + unit_l2_cosine_similarity, ) from powercontext.builtin.artifacts.topic_memory.models import ( TopicMemoryChannelHit, @@ -134,6 +135,7 @@ def _fuse_topic_memory_rankings( matched_by=tuple(channel for channel in _CHANNEL_ORDER if channel in evidence[hit.key]), retrieval_score=hit.score, channel_scores=_channel_scores(evidence[hit.key]) if include_scores else None, + relevance=_relevance(evidence[hit.key]), ) for hit in selected ), @@ -155,6 +157,17 @@ def _channel_scores(evidence: dict[str, TopicMemoryChannelHit]) -> dict[str, Cha return result +def _relevance(evidence: dict[str, TopicMemoryChannelHit]) -> float | None: + """Return the best cosine similarity among the admitted vector channels, if any.""" + + similarities = [ + unit_l2_cosine_similarity(hit.distance) + for name, hit in evidence.items() + if name.endswith("_vector") and hit.distance is not None + ] + return max(similarities, default=None) + + def _retrieved_count( channels: TopicMemorySearchChannels, channel: TopicMemoryMatchedBy, @@ -216,7 +229,7 @@ def admits_topic_memory_vector_distance( """ baseline = _MIN_SEMANTIC_SIMILARITY if floor is None else floor.min_semantic_similarity - return max(-1.0, min(1.0, 1.0 - distance**2 / 2.0)) >= baseline + return unit_l2_cosine_similarity(distance) >= baseline def _snippet(query: str, value: str, *, lexical: bool) -> str: diff --git a/src/powercontext/builtin/artifacts/topic_memory/models.py b/src/powercontext/builtin/artifacts/topic_memory/models.py index 49496525aa..879ef13013 100644 --- a/src/powercontext/builtin/artifacts/topic_memory/models.py +++ b/src/powercontext/builtin/artifacts/topic_memory/models.py @@ -215,6 +215,7 @@ class TopicMemorySearchHit(BaseModel): matched_by: tuple[TopicMemoryMatchedBy, ...] retrieval_score: float | None = Field(default=None, ge=0, le=1, allow_inf_nan=False, exclude=True) channel_scores: dict[str, ChannelScore] | None = Field(default=None, exclude=True) + relevance: float | None = Field(default=None, ge=-1.0, le=1.0, allow_inf_nan=False, exclude=True) @dataclass(frozen=True) diff --git a/src/powercontext/builtin/runtime/application.py b/src/powercontext/builtin/runtime/application.py index cb301f545d..91c732d6c2 100644 --- a/src/powercontext/builtin/runtime/application.py +++ b/src/powercontext/builtin/runtime/application.py @@ -24,7 +24,7 @@ from datetime import UTC, datetime from pathlib import Path from time import perf_counter -from typing import TYPE_CHECKING, Any, Protocol +from typing import TYPE_CHECKING, Any, Protocol, TypeVar from pydantic import BaseModel, JsonValue, ValidationError @@ -245,6 +245,7 @@ MEMORY_FAMILY, REASON_AT_MAX_ROUNDS, REASON_EXPANSION_FAILED, + REASON_WEAK_TOP_ONE, TOPIC_MEMORY_FAMILY, RecallEffort, RecallExpander, @@ -911,7 +912,7 @@ async def _prepare_build( The controlled expansion loop lives here. It runs at most ``policy.max_rounds`` extra searches, each of which only lowers the admission floor for the families the caller already selected — never a new family, a larger ``limit``, or a different ``mode``. - Every gate or expansion error degrades to the round-zero candidate set. + Every gate or expansion error preserves the last committed candidate set. Returns ``(build, effort)``. The RFC 1560 trace is **not** a field of the build: ``_prepare`` returns ``build.context`` and discards the rest, so a field there would @@ -1086,11 +1087,13 @@ async def _gated_recall_effort( # noqa: C901 - the bounded expansion loop is in """Run the bounded expansion loop and return the winning candidates plus the trace. Candidates accumulate across rounds; only new identities are ever added, so an earlier - round's candidate is never removed. Any error returns the round-zero candidates unchanged + round's candidate is never removed. Any error returns the last committed candidates unchanged and records ``expansion-failed`` — the gate can never turn a successful prepare into a failure. For a consistent signal basis, ``candidates_by_round`` counts the *un-truncated* accumulated pool (no family is clamped while the gate is consulting it); the Builder - ceilings are applied once, on the returned candidates only. + ceilings are applied once, on the returned candidates only. Committed candidates keep the + latest round's fused order inside those ceilings, so evidence surfaced by a later round is + deliverable instead of trailing the round-zero prefix. """ gate = RecallSufficiencyGate() @@ -1110,6 +1113,7 @@ async def _gated_recall_effort( # noqa: C901 - the bounded expansion loop is in memory_hits=_flatten_memory_hits(memory_candidates), topic_memory_hits=topic_memory_hits, experience_hits=_flatten_experience_hits(experience_candidates), + memory_scope_ids=tuple(group.scope_id for group in memory_candidates for _hit in group.hits), ) round_zero_count = len(candidates) candidates_by_round = [round_zero_count] @@ -1117,6 +1121,10 @@ async def _gated_recall_effort( # noqa: C901 - the bounded expansion loop is in added_embeddings = 0 added_generation_calls = 0 admission_by_family = list(round_zero.admissions) + last_signals = None + committed_memory = memory_candidates + committed_experience = experience_candidates + committed_topic = topic_memory_hits try: budget = builder.probe_budget( request=request, @@ -1134,6 +1142,7 @@ async def _gated_recall_effort( # noqa: C901 - the bounded expansion loop is in budget=budget, families_expected=families_expected, ) + last_signals = assessment.signals while not assessment.sufficient and families_recoverable > 0 and len(expansions) < policy.max_rounds: plan = expander.plan(len(expansions) + 1, policy) issued = await self._recall_round( @@ -1145,6 +1154,7 @@ async def _gated_recall_effort( # noqa: C901 - the bounded expansion loop is in reuse=reuse, topic_reuse=topic_reuse, ) + memory_ranks: dict[Any, int] = {} for group in issued.memory: _ensure_memory_head_stable( group.scope_id, group.memory_ref, memory_ref_by_scope.get(group.scope_id) @@ -1152,62 +1162,76 @@ async def _gated_recall_effort( # noqa: C901 - the bounded expansion loop is in bucket = memory_hits_by_scope.setdefault(group.scope_id, []) for hit in group.hits: identity = _memory_identity(group.scope_id, hit) + memory_ranks[identity] = len(memory_ranks) if identity in seen_memory: continue seen_memory.add(identity) bucket.append(hit) + experience_ranks: dict[Any, int] = {} for group in issued.experience: bucket = experience_hits_by_scope.setdefault(group.scope_id, []) for hit in group.hits: identity = _experience_identity(group.scope_id, hit) + experience_ranks[identity] = len(experience_ranks) if identity in seen_experience: continue seen_experience.add(identity) bucket.append(hit) + topic_ranks = {_topic_identity(hit): index for index, hit in enumerate(issued.topic_memory.hits)} for hit in issued.topic_memory.hits: identity = _topic_identity(hit) if identity in seen_topic: continue seen_topic.add(identity) accumulated_topic.append(hit) - expansions.append(plan.action) - added_embeddings += issued.embedding_calls - added_generation_calls += issued.generation_calls - admission_by_family = list(issued.admissions) - families_recoverable = _families_with_recoverable_candidates(families, issued.admissions) candidates = build_recall_candidates( memory_hits=_flatten_scope_memory(memory_hits_by_scope, scope_ids), topic_memory_hits=tuple(accumulated_topic), experience_hits=_flatten_scope_experience(experience_hits_by_scope, scope_ids), + memory_scope_ids=tuple( + scope_id for scope_id in scope_ids for _hit in memory_hits_by_scope.get(scope_id, ()) + ), + ) + staged_memory = _limit_expanded_memory_candidates( + [ + PreparedMemoryCandidates( + scope_id=scope_id, + memory_ref=memory_ref_by_scope.get(scope_id), + hits=_latest_round_order( + memory_hits_by_scope.get(scope_id, ()), + memory_ranks, + lambda hit, s=scope_id: _memory_identity(s, hit), + ), + ) + for scope_id in scope_ids + ], + memory_candidates, + builder.memory_candidate_limit, + ) + staged_topic = _latest_round_order(accumulated_topic, topic_ranks, _topic_identity)[ + : builder.topic_memory_candidate_limit + ] + staged_experience = _limit_expanded_experience_candidates( + [ + PreparedExperienceCandidates( + scope_id=scope_id, + hits=_latest_round_order( + experience_hits_by_scope.get(scope_id, ()), + experience_ranks, + lambda hit, s=scope_id: _experience_identity(s, hit), + ), + ) + for scope_id in scope_ids + ], + experience_candidates, + builder.experience_candidate_limit, ) - candidates_by_round.append(len(candidates)) budget = builder.probe_budget( request=request, current_scope_id=self.scope_id, - memory_candidates=_limit_expanded_memory_candidates( - [ - PreparedMemoryCandidates( - scope_id=scope_id, - memory_ref=memory_ref_by_scope.get(scope_id), - hits=tuple(memory_hits_by_scope.get(scope_id, ())), - ) - for scope_id in scope_ids - ], - memory_candidates, - builder.memory_candidate_limit, - ), - topic_memory_hits=tuple(accumulated_topic[: builder.topic_memory_candidate_limit]), - experience_candidates=_limit_expanded_experience_candidates( - [ - PreparedExperienceCandidates( - scope_id=scope_id, - hits=tuple(experience_hits_by_scope.get(scope_id, ())), - ) - for scope_id in scope_ids - ], - experience_candidates, - builder.experience_candidate_limit, - ), + memory_candidates=staged_memory, + topic_memory_hits=staged_topic, + experience_candidates=staged_experience, profile_candidates=profile_candidates, ) assessment = gate.assess( @@ -1218,13 +1242,29 @@ async def _gated_recall_effort( # noqa: C901 - the bounded expansion loop is in budget=budget, families_expected=families_expected, ) - if not assessment.sufficient and families_recoverable > 0 and len(expansions) >= policy.max_rounds: + committed_memory = staged_memory + committed_experience = staged_experience + committed_topic = staged_topic + expansions.append(plan.action) + added_embeddings += issued.embedding_calls + added_generation_calls += issued.generation_calls + admission_by_family = list(issued.admissions) + families_recoverable = _families_with_recoverable_candidates(families, issued.admissions) + candidates_by_round.append(len(candidates)) + last_signals = assessment.signals + if ( + not assessment.sufficient + and assessment.reason != REASON_WEAK_TOP_ONE + and families_recoverable > 0 + and len(expansions) >= policy.max_rounds + ): assessment = replace(assessment, reason=REASON_AT_MAX_ROUNDS) + reason = assessment.reason except Exception as error: log_safely( logger, logging.ERROR, - "Recall sufficiency expansion failed; keeping the round-zero candidates", + "Recall sufficiency expansion failed; keeping the last committed candidates", exc_info=error, extra={ "event": "context.recall_gate.expansion_failed", @@ -1232,54 +1272,20 @@ async def _gated_recall_effort( # noqa: C901 - the bounded expansion loop is in "unit": "context", }, ) - return ( - memory_candidates, - experience_candidates, - topic_memory_hits, - recall_effort( - policy=policy, - assessment=REASON_EXPANSION_FAILED, - expansion_actions=expansions, - candidates_by_round=candidates_by_round, - admission_by_family=admission_by_family, - added_embeddings=added_embeddings, - added_generation_calls=added_generation_calls, - ), - ) - capped_topic = tuple(accumulated_topic[: builder.topic_memory_candidate_limit]) + reason = REASON_EXPANSION_FAILED return ( - _limit_expanded_memory_candidates( - [ - PreparedMemoryCandidates( - scope_id=scope_id, - memory_ref=memory_ref_by_scope.get(scope_id), - hits=tuple(memory_hits_by_scope.get(scope_id, ())), - ) - for scope_id in scope_ids - ], - memory_candidates, - builder.memory_candidate_limit, - ), - _limit_expanded_experience_candidates( - [ - PreparedExperienceCandidates( - scope_id=scope_id, - hits=tuple(experience_hits_by_scope.get(scope_id, ())), - ) - for scope_id in scope_ids - ], - experience_candidates, - builder.experience_candidate_limit, - ), - capped_topic, + committed_memory, + committed_experience, + committed_topic, recall_effort( policy=policy, - assessment=assessment.reason, + assessment=reason, expansion_actions=expansions, candidates_by_round=candidates_by_round, admission_by_family=admission_by_family, added_embeddings=added_embeddings, added_generation_calls=added_generation_calls, + signals=last_signals, ), ) @@ -1658,6 +1664,22 @@ def _topic_identity(hit: TopicMemorySearchHit) -> tuple[str, int]: return (hit.artifact_ref.artifact_id, hit.artifact_ref.revision) +_HitT = TypeVar("_HitT") + + +def _latest_round_order( + hits: Sequence[_HitT], + ranks: Mapping[Any, int], + identity: Callable[[_HitT], Any], +) -> tuple[_HitT, ...]: + """Order hits by the latest round's fused ranking, keeping the accumulated order for + identities the latest round did not re-admit. A fused hit that only entered the window + under a lowered floor must rank inside the delivery cap, not behind the round-zero prefix.""" + if not ranks: + return tuple(hits) + return tuple(sorted(hits, key=lambda hit: ranks.get(identity(hit), len(ranks)))) + + def _admission_keyword(admission: AdmissionFloor | None) -> dict[str, Any]: """Forward ``admission`` only when set, keeping today's exact downstream calls otherwise.""" diff --git a/src/powercontext/builtin/runtime/recall_sufficiency.py b/src/powercontext/builtin/runtime/recall_sufficiency.py index 13b0cb06ab..534c5d2963 100644 --- a/src/powercontext/builtin/runtime/recall_sufficiency.py +++ b/src/powercontext/builtin/runtime/recall_sufficiency.py @@ -29,6 +29,7 @@ from powercontext.builtin.artifacts.experience import ExperienceSearchHit, experience_search_text from powercontext.builtin.artifacts.memory import MemoryHit from powercontext.builtin.artifacts.search import ( + DEFAULT_ADMISSION_FLOOR, AdmissionCounts, AdmissionFloor, analyze_text, @@ -42,7 +43,7 @@ from powercontext.builtin.runtime.prepared_context import PreparedContextOmissions # ── Policy identifier and reason vocabulary ──────────────────────────────────────────────── -POLICY_ID = "powercontext.recall-gate.v1" +POLICY_ID = "powercontext.recall-gate.v2" REASON_SUFFICIENT = "sufficient" REASON_NO_CONTENT = "no-content" @@ -72,16 +73,20 @@ _RRF_CONSTANT = 60 # Topic Memory relevance is already normalized against its reachable upper bound. _TOPIC_SCORE_SCALE = 100.0 +_STRONG_TOP_RELEVANCE = 0.7 @dataclass(frozen=True) class RecallCandidate: """One family-local retrieval result, projected for model-free assessment. - ``score`` is a family-local relevance normalized into ``[0.0, 1.0]`` for the families that - expose a real score. Families that expose only presence/counts (Experience) carry ``0.0`` - and never take part in the score signals — inventing a confidence for them would make the - gate less honest, not more. + ``score`` retains the family-local normalized RRF rank value for external gate-capture JSON + consumers of ``assessments.candidates[].score``. ``relevance`` is the cosine similarity of the best admitted vector + channel, when one exists. Only relevance enters the gate's score thresholds. + + ``scope_id`` identifies the Scope an independent fusion run belongs to. Memory searches + once per participating Scope, so each Scope's hits form their own fused ranking; the + gate's "first fused hit" rule is applied per Scope, never across Scope boundaries. """ family: str @@ -91,19 +96,19 @@ class RecallCandidate: entry_version_id: str | None score: float text: str + relevance: float | None = None + scope_id: str | None = None @dataclass(frozen=True) class RecallSignals: """Cheap, model-free signals derived from the accumulated candidate set. - Score caveat (a known limitation, not a polished story): Memory's fused score is a - reciprocal-rank score, so its discrimination is compressed by rank. The normalized Memory - score saturates near ``1.0`` for a single strong channel hit, which is why the real - load-bearing signals in v1 are ``candidate_count``, family coverage and ``lexical_overlap``; - the score signal is driven mainly by Topic Memory. ``lexical_overlap`` is the query-term - recall of the single best candidate (its overlap is the maximum over candidates), not the - share of candidates sharing at least one term. + ``top_score``, ``mean_score`` and ``top_gap`` measure vector cosine relevance, regardless + of the families' different fused-score scales. ``lexical_overlap`` is the query-term + recall of the single best candidate (its overlap is the maximum over candidates). + A family contributes to these score signals only for Scopes whose own first fused hit + has vector evidence. The gap uses candidates above the round-zero semantic floor. ``distinct_source_count`` counts family-specific evidence identities via :func:`candidate_identity` — a Memory entry (``memory_ref`` + ``entry_id`` + @@ -124,6 +129,7 @@ class RecallSignals: lexical_overlap: float families_expected: int = 0 scored_families: int = 0 + gap_families: int = 0 @dataclass(frozen=True) @@ -272,8 +278,9 @@ class RecallEffort: admission totals. There is no query text, no entry id and no per-entry attribution, so the value cannot leak evidence through a trace. - ``rounds`` counts the search passes actually executed, so it is ``1 + len(expansion_actions)`` - (1..3): round 0 always runs and each *committed* expansion adds one. ``candidates_by_round`` + ``rounds`` counts assessed, committed search passes, so it is ``1 + len(expansion_actions)`` + (1..3): round 0 always runs and each committed expansion adds one. A failed pass that did + not finish assessment does not add a round. ``candidates_by_round`` holds the accumulated candidate-pool size the gate saw after each committed round, so ``len(candidates_by_round) == rounds``; it measures the **un-truncated** accumulated pool the gate assessed, not the subset the Builder finally selected (the Builder is called once, @@ -306,6 +313,7 @@ class RecallEffort: dropped_items: int = 0 dropped_below_min_bytes: int = 0 dropped_no_fitting_truncation: int = 0 + signals: RecallSignals | None = None def recall_effort( @@ -318,6 +326,7 @@ def recall_effort( added_embeddings: int = 0, added_generation_calls: int = 0, omissions: PreparedContextOmissions | None = None, + signals: RecallSignals | None = None, ) -> RecallEffort: """Assemble one :class:`RecallEffort`, folding a build's omission counts into it. @@ -341,6 +350,7 @@ def recall_effort( dropped_items=0 if omissions is None else omissions.dropped_items, dropped_below_min_bytes=0 if omissions is None else omissions.dropped_below_min_bytes, dropped_no_fitting_truncation=0 if omissions is None else omissions.dropped_no_fitting_truncation, + signals=signals, ) @@ -392,11 +402,17 @@ def build_recall_candidates( memory_hits: Sequence[MemoryHit], topic_memory_hits: Sequence[TopicMemorySearchHit], experience_hits: Sequence[ExperienceSearchHit], + memory_scope_ids: Sequence[str | None] | None = None, ) -> tuple[RecallCandidate, ...]: - """Project the participating families' hits into a flat, ordered candidate tuple.""" + """Project the participating families' hits into a flat, ordered candidate tuple. + + ``memory_scope_ids``, when given, must parallel ``memory_hits`` and records which Scope's + independent fusion run produced each hit; the gate scores each Scope's fused ranking + separately. Omitting it treats all Memory hits as one fusion run. + """ candidates: list[RecallCandidate] = [] - for hit in memory_hits: + for index, hit in enumerate(memory_hits): candidates.append( RecallCandidate( family=MEMORY_FAMILY, @@ -406,6 +422,8 @@ def build_recall_candidates( entry_version_id=hit.entry_version_id, score=_normalize_memory_score(hit), text=hit.text, + relevance=hit.relevance, + scope_id=None if memory_scope_ids is None else memory_scope_ids[index], ) ) for topic_hit in topic_memory_hits: @@ -418,6 +436,7 @@ def build_recall_candidates( entry_version_id=None, score=_normalize_topic_score(topic_hit), text="\n".join(part for part in (topic_hit.title, topic_hit.summary, topic_hit.snippet) if part), + relevance=topic_hit.relevance, ) ) for experience_hit in experience_hits: @@ -446,7 +465,7 @@ def _normalize_memory_score(hit: MemoryHit) -> float: def _normalize_topic_score(hit: TopicMemorySearchHit) -> float: - """Normalize Topic Memory relevance into ``[0.0, 1.0]``.""" + """Normalize a Topic Memory fused rank score into ``[0.0, 1.0]`` for captures.""" return max(0.0, min(1.0, hit.score / _TOPIC_SCORE_SCALE)) @@ -468,8 +487,8 @@ def assess( """Assess one candidate set. No I/O, no clock, no model; deterministic in its inputs. ``budget`` carries the result of the round's budget probe, or ``None`` when no probe - was run. The gate does not perform the probe itself; the Runtime does it once, on - round 0, and reuses the view for every round. + was run. The gate does not perform the probe itself; the Runtime probes each round's + capped candidate set before committing its assessment. """ signals = _build_signals(candidates, query, families_expected) @@ -483,8 +502,10 @@ def assess( return GateAssessment(sufficient=False, reason=REASON_THIN_CANDIDATES, signals=signals) if families_expected > 1 and signals.family_count < families_expected: return GateAssessment(sufficient=False, reason=REASON_THIN_FAMILIES, signals=signals) - if signals.scored_families > 0 and ( - signals.top_score < policy.min_top_score or signals.top_gap < policy.min_top_gap + if (signals.scored_families > 0 and signals.top_score < policy.min_top_score) or ( + signals.gap_families > 0 + and signals.top_score < _STRONG_TOP_RELEVANCE + and signals.top_gap < policy.min_top_gap ): return GateAssessment(sufficient=False, reason=REASON_WEAK_TOP_ONE, signals=signals) if signals.lexical_overlap < policy.min_lexical_overlap: @@ -514,7 +535,7 @@ def _build_signals( families_with_candidates = len({candidate.family for candidate in candidates}) distinct_source_count = len({candidate_identity(candidate) for candidate in candidates}) lexical_overlap = _lexical_overlap(candidates, query) - top_score, mean_score, top_gap, scored_families = _score_signals(candidates) + top_score, mean_score, top_gap, scored_families, gap_families = _score_signals(candidates) return RecallSignals( candidate_count=candidate_count, family_count=families_with_candidates, @@ -525,6 +546,7 @@ def _build_signals( lexical_overlap=lexical_overlap, families_expected=families_expected, scored_families=scored_families, + gap_families=gap_families, ) @@ -547,33 +569,47 @@ def _lexical_overlap(candidates: Sequence[RecallCandidate], query: str) -> float return best -def _score_signals(candidates: Sequence[RecallCandidate]) -> tuple[float, float, float, int]: - """Aggregate score signals over the families that expose a real relevance score. +def _score_signals(candidates: Sequence[RecallCandidate]) -> tuple[float, float, float, int, int]: + """Aggregate cosine relevance over families with admitted vector hits. - Returns ``(top_score, mean_score, top_gap, scored_families)``. ``top_score`` is the maximum - family top and ``top_gap`` the maximum family-local ``top - mean``. ``mean_score`` is the mean - of all scored candidates pooled across scored families — a different basis from the - family-local ``top_gap`` — and is reported **for observation only; it never takes part in the - verdict**. When no family exposes a score the caller must skip the score signal entirely - (``scored_families == 0``), so the aggregate values here stay ``0.0`` rather than fabricated. + Each Scope runs an independent fusion, so the "first fused hit" rule is applied per + Scope: a Scope's hits contribute score signals only when that Scope's first family hit + has vector evidence. This keeps the verdict invariant to the order Scopes are prepared + in. The gap uses only candidates above the round-zero semantic admission floor, so + lowering that floor cannot manufacture a larger gap. ``mean_score`` pools all known + vector scores for observation only. A family with fewer than two baseline scores cannot + produce a gap. """ - top_score = 0.0 + top_score = -1.0 top_gap = 0.0 scored_families = 0 + gap_families = 0 all_scores: list[float] = [] for family in _SCORING_FAMILIES: - family_scores = [candidate.score for candidate in candidates if candidate.family == family] + scope_groups: dict[str | None, list[RecallCandidate]] = {} + for candidate in candidates: + if candidate.family == family: + scope_groups.setdefault(candidate.scope_id, []).append(candidate) + family_scores = [ + candidate.relevance + for group in scope_groups.values() + if group[0].relevance is not None + for candidate in group + if candidate.relevance is not None + ] if not family_scores: continue scored_families += 1 family_top = max(family_scores) - family_mean = sum(family_scores) / len(family_scores) top_score = max(top_score, family_top) - top_gap = max(top_gap, family_top - family_mean) + baseline_scores = [score for score in family_scores if score >= DEFAULT_ADMISSION_FLOOR.min_semantic_similarity] + if len(baseline_scores) > 1: + gap_families += 1 + top_gap = max(top_gap, max(baseline_scores) - sum(baseline_scores) / len(baseline_scores)) all_scores.extend(family_scores) mean_score = sum(all_scores) / len(all_scores) if all_scores else 0.0 - return top_score, mean_score, top_gap, scored_families + return (top_score if all_scores else 0.0), mean_score, top_gap, scored_families, gap_families __all__ = [ diff --git a/tests/builtin/artifacts/memory/test_fusion.py b/tests/builtin/artifacts/memory/test_fusion.py index 168b5d5c25..27803e7351 100644 --- a/tests/builtin/artifacts/memory/test_fusion.py +++ b/tests/builtin/artifacts/memory/test_fusion.py @@ -62,6 +62,15 @@ def test_single_channel_uses_public_rrf_score_and_identity_order() -> None: assert tuple(hit.entry_id for hit in hits) == ("zulu", "alpha") assert tuple(hit.score for hit in hits) == pytest.approx((1 / 61, 1 / 62)) assert all(hit.matched_by == ("fts",) for hit in hits) + assert all(hit.relevance is None for hit in hits) + + +def test_vector_relevance_survives_fusion_with_fts_first() -> None: + fts = channel_hit("alpha", text="alpha beta") + vector = channel_hit("alpha", text="alpha beta", distance=1.0) + hit = fuse_rankings(fts=(fts,), vector=(vector,), limit=1)[0] + assert hit.relevance == pytest.approx(0.5) + assert hit.score == pytest.approx(2 / 61) def test_duplicate_channel_rows_contribute_only_the_first_rank() -> None: diff --git a/tests/builtin/artifacts/topic_memory/test_fusion.py b/tests/builtin/artifacts/topic_memory/test_fusion.py index 98a0775e5f..de556943cb 100644 --- a/tests/builtin/artifacts/topic_memory/test_fusion.py +++ b/tests/builtin/artifacts/topic_memory/test_fusion.py @@ -50,6 +50,20 @@ def test_fts_snippet_centers_its_window_on_analyzer_match(match_offset: int) -> assert len(result[0].snippet) <= 480 +def test_topic_vector_relevance_survives_fusion_with_fts() -> None: + fts = TopicMemoryChannelHit( + artifact_ref=ArtifactRef(family="topic-memory", artifact_id="topic-1", revision=1), + title="alpha beta", + summary="alpha beta", + channel="topic_fts", + ) + vector = fts.model_copy(update={"channel": "topic_vector", "distance": 1.0}) + hit = fuse_topic_memory_rankings( + "alpha beta", TopicMemorySearchChannels(topic_fts=(fts,), topic_vector=(vector,)), 1 + )[0] + assert hit.relevance == pytest.approx(0.5) + + def test_vector_snippet_uses_a_stable_chunk_local_window() -> None: detail = f"{'prefix ' * 150}stable-center{' suffix' * 150}" hit = TopicMemoryChannelHit( diff --git a/tests/builtin/runtime/test_recall_sufficiency.py b/tests/builtin/runtime/test_recall_sufficiency.py index be2b65aed1..2e88ea71a4 100644 --- a/tests/builtin/runtime/test_recall_sufficiency.py +++ b/tests/builtin/runtime/test_recall_sufficiency.py @@ -100,6 +100,7 @@ def _topic_candidate( artifact_id: str = "topic", revision: int = 1, text: str = "zzz", + relevance: float | None = None, ) -> RecallCandidate: return RecallCandidate( family="topic-memory", @@ -109,6 +110,7 @@ def _topic_candidate( entry_version_id=None, score=score, text=text, + relevance=relevance, ) @@ -143,6 +145,7 @@ def _topic_hit( title: str = "Title", summary: str = "Summary", snippet: str | None = None, + relevance: float | None = None, ) -> TopicMemorySearchHit: return TopicMemorySearchHit( artifact_ref=ArtifactRef(family="topic-memory", artifact_id=artifact_id, revision=revision), @@ -151,6 +154,7 @@ def _topic_hit( snippet=snippet, score=score, matched_by=("topic_fts",), + relevance=relevance, ) @@ -287,7 +291,10 @@ def test_gate_reports_thin_families_when_a_selected_family_returned_nothing() -> def test_gate_reports_weak_top_one_for_a_weak_scoring_family() -> None: policy = RecallSufficiencyPolicy(min_top_score=0.35) - candidates = (_topic_candidate(0.2), _topic_candidate(0.1, artifact_id="topic-2")) + candidates = ( + _topic_candidate(1.0, relevance=0.2), + _topic_candidate(0.9, artifact_id="topic-2", relevance=0.1), + ) assessment = RecallSufficiencyGate().assess( candidates, "alpha beta gamma delta", @@ -300,9 +307,143 @@ def test_gate_reports_weak_top_one_for_a_weak_scoring_family() -> None: assert assessment.signals.scored_families == 1 +def test_gate_reports_thin_candidates_even_when_the_top_is_weak() -> None: + """A thin and weak set still expands: lowering admission can recover + round-zero-rejected evidence, so ``weak-top-1`` must not stop expansion + before the thin check runs.""" + policy = RecallSufficiencyPolicy(min_candidates=3) + candidates = ( + _topic_candidate(1.0, relevance=0.2), + _topic_candidate(0.9, artifact_id="topic-2", relevance=0.1), + ) + assessment = RecallSufficiencyGate().assess( + candidates, + "alpha beta gamma delta", + policy, + scope_has_content=True, + families_expected=1, + ) + assert assessment.sufficient is False + assert assessment.reason == REASON_THIN_CANDIDATES + + +def _scoped_memory_candidate( + text: str, + *, + scope_id: str, + relevance: float | None, +) -> RecallCandidate: + return RecallCandidate( + family="memory", + artifact_id=f"memory-{scope_id}", + revision=1, + entry_id="entry", + entry_version_id="entry-v1", + score=1.0, + text=text, + relevance=relevance, + scope_id=scope_id, + ) + + +def test_gate_scores_each_scope_by_its_own_first_fused_hit() -> None: + """Each Scope's fusion run applies the first-hit rule independently of Scope order.""" + policy = RecallSufficiencyPolicy() + scope_a = ( + _scoped_memory_candidate("alpha beta", scope_id="scope-a", relevance=None), + _scoped_memory_candidate("alpha beta", scope_id="scope-a", relevance=0.9), + ) + scope_b = (_scoped_memory_candidate("alpha beta", scope_id="scope-b", relevance=0.31),) + + gate = RecallSufficiencyGate() + a_first = gate.assess(scope_a + scope_b, "alpha beta", policy, scope_has_content=True, families_expected=1) + b_first = gate.assess(scope_b + scope_a, "alpha beta", policy, scope_has_content=True, families_expected=1) + + assert a_first.reason == b_first.reason == REASON_WEAK_TOP_ONE + assert a_first.signals.top_score == b_first.signals.top_score == pytest.approx(0.31) + assert a_first.signals.scored_families == b_first.signals.scored_families == 1 + + +def test_build_recall_candidates_wires_scope_ids_into_the_per_scope_rule() -> None: + """Deleting ``memory_scope_ids`` must change observable behavior: without it every + Memory hit shares one group and the flat first hit decides scoring for all Scopes.""" + fts_first = MemoryHit( + memory_ref=MEMORY_REF, + entry_id="a-fts", + entry_version_id="a-fts-v1", + text="alpha beta", + score=1 / 61, + matched_by=("fts",), + ) + deep_vector = MemoryHit( + memory_ref=MEMORY_REF, + entry_id="a-vector", + entry_version_id="a-vector-v1", + text="alpha beta", + score=1 / 62, + matched_by=("vector",), + relevance=0.9, + ) + ref_vector = MemoryHit( + memory_ref=ArtifactRef(family="memory", artifact_id="memory-b", revision=1), + entry_id="b-vector", + entry_version_id="b-vector-v1", + text="alpha beta", + score=1 / 61, + matched_by=("vector",), + relevance=0.31, + ) + gate = RecallSufficiencyGate() + policy = RecallSufficiencyPolicy() + + scoped = build_recall_candidates( + memory_hits=(fts_first, deep_vector, ref_vector), + memory_scope_ids=("scope-a", "scope-a", "scope-b"), + topic_memory_hits=(), + experience_hits=(), + ) + unscoped = build_recall_candidates( + memory_hits=(fts_first, deep_vector, ref_vector), + topic_memory_hits=(), + experience_hits=(), + ) + + scoped_assessment = gate.assess(scoped, "alpha beta", policy, scope_has_content=True, families_expected=1) + unscoped_assessment = gate.assess(unscoped, "alpha beta", policy, scope_has_content=True, families_expected=1) + + assert scoped_assessment.reason == REASON_WEAK_TOP_ONE + assert scoped_assessment.signals.top_score == pytest.approx(0.31) + assert unscoped_assessment.reason != REASON_WEAK_TOP_ONE + assert unscoped_assessment.signals.scored_families == 0 + + +def test_gate_lexical_and_family_skip_for_a_scope_without_vector_evidence() -> None: + """A Scope whose fused top is FTS-only contributes no scores; the verdict still uses the + candidates it admitted for counts and lexical overlap.""" + policy = RecallSufficiencyPolicy(min_top_score=0.35, min_top_gap=0.02, min_lexical_overlap=0.5) + candidates = ( + _scoped_memory_candidate("alpha beta", scope_id="scope-a", relevance=None), + _scoped_memory_candidate("alpha beta", scope_id="scope-b", relevance=0.8), + _scoped_memory_candidate("alpha beta", scope_id="scope-b", relevance=0.5), + ) + assessment = RecallSufficiencyGate().assess( + candidates, + "alpha beta", + policy, + scope_has_content=True, + families_expected=1, + ) + assert assessment.sufficient is True + assert assessment.signals.candidate_count == 3 + assert assessment.signals.top_score == pytest.approx(0.8) + + def test_gate_reports_weak_lexical_when_the_best_candidate_misses_query_terms() -> None: policy = RecallSufficiencyPolicy(min_top_score=0.35, min_top_gap=0.02, min_lexical_overlap=0.5) - candidates = (_topic_candidate(1.0), _topic_candidate(0.5, artifact_id="topic-2")) + candidates = ( + _topic_candidate(1.0, relevance=0.8), + _topic_candidate(0.5, artifact_id="topic-2", relevance=0.5), + ) assessment = RecallSufficiencyGate().assess( candidates, "alpha beta gamma delta", @@ -318,7 +459,10 @@ def test_gate_reports_weak_lexical_when_the_best_candidate_misses_query_terms() def test_gate_reports_sufficient_when_every_signal_passes() -> None: policy = RecallSufficiencyPolicy() text = "alpha beta gamma delta" - candidates = (_topic_candidate(1.0, text=text), _topic_candidate(0.5, artifact_id="topic-2", text=text)) + candidates = ( + _topic_candidate(1.0, text=text, relevance=0.8), + _topic_candidate(0.5, artifact_id="topic-2", text=text, relevance=0.5), + ) assessment = RecallSufficiencyGate().assess( candidates, text, @@ -440,10 +584,10 @@ def test_experience_only_candidates_never_expose_a_fabricated_score() -> None: assert assessment.reason != REASON_WEAK_TOP_ONE -def test_topic_candidates_use_their_normalized_relevance_as_top_score() -> None: +def test_topic_candidates_use_vector_relevance_as_top_score() -> None: candidates = build_recall_candidates( memory_hits=(), - topic_memory_hits=(_topic_hit(score=42.0),), + topic_memory_hits=(_topic_hit(score=42.0, relevance=0.78),), experience_hits=(), ) signals = ( @@ -458,7 +602,77 @@ def test_topic_candidates_use_their_normalized_relevance_as_top_score() -> None: .signals ) assert signals.scored_families == 1 - assert signals.top_score == pytest.approx(0.42) + assert candidates[0].score == pytest.approx(0.42) + assert signals.top_score == pytest.approx(0.78) + + +def test_gate_thresholds_use_cosine_relevance_independently_of_fused_rank() -> None: + policy = RecallSufficiencyPolicy(min_candidates=2, min_top_score=0.35, min_top_gap=0.02) + strong_rank_weak_cosine = ( + _topic_candidate(1.0, text="alpha beta", relevance=0.3), + _topic_candidate(0.5, artifact_id="topic-2", text="alpha beta", relevance=0.1), + ) + weak_rank_strong_cosine = ( + _topic_candidate(0.01, text="alpha beta", relevance=0.8), + _topic_candidate(0.005, artifact_id="topic-2", text="alpha beta", relevance=0.5), + ) + flat_cosine = ( + _topic_candidate(1.0, text="alpha beta", relevance=0.8), + _topic_candidate(0.5, artifact_id="topic-2", text="alpha beta", relevance=0.8), + ) + gate = RecallSufficiencyGate() + assert gate.assess(strong_rank_weak_cosine, "alpha beta", policy).reason == REASON_WEAK_TOP_ONE + assert gate.assess(weak_rank_strong_cosine, "alpha beta", policy).reason == REASON_SUFFICIENT + assert gate.assess(flat_cosine, "alpha beta", policy).reason == REASON_SUFFICIENT + + +def test_lower_floor_candidates_cannot_manufacture_a_top_gap() -> None: + policy = RecallSufficiencyPolicy(min_top_score=0.35, min_top_gap=0.02) + baseline = ( + _topic_candidate(1.0, text="alpha beta", relevance=0.4), + _topic_candidate(0.9, artifact_id="topic-2", text="alpha beta", relevance=0.39), + ) + expanded = (*baseline, _topic_candidate(0.5, artifact_id="topic-3", text="alpha beta", relevance=0.15)) + gate = RecallSufficiencyGate() + before = gate.assess(baseline, "alpha beta", policy) + after = gate.assess(expanded, "alpha beta", policy) + assert before.reason == after.reason == REASON_WEAK_TOP_ONE + assert after.signals.top_gap == pytest.approx(before.signals.top_gap) + + +@pytest.mark.parametrize("fts_competitor", [False, True], ids=["single-candidate", "fts-competitor"]) +def test_one_scored_candidate_skips_gap_even_with_fts_competitors(fts_competitor: bool) -> None: + candidates = [_topic_candidate(1.0, text="alpha beta", relevance=0.5)] + if fts_competitor: + candidates.append(_topic_candidate(0.9, artifact_id="topic-2", text="alpha beta")) + assessment = RecallSufficiencyGate().assess( + candidates, "alpha beta", RecallSufficiencyPolicy(min_candidates=len(candidates)) + ) + assert assessment.reason == REASON_SUFFICIENT + assert assessment.signals.scored_families == 1 + assert assessment.signals.gap_families == 0 + + +def test_fts_first_family_skips_score_threshold_despite_lower_vector_hit() -> None: + candidates = ( + _topic_candidate(1.0, text="alpha beta"), + _topic_candidate(0.9, artifact_id="topic-2", text="alpha beta", relevance=0.31), + ) + assessment = RecallSufficiencyGate().assess(candidates, "alpha beta", RecallSufficiencyPolicy()) + assert assessment.reason == REASON_SUFFICIENT + assert assessment.signals.scored_families == 0 + + +def test_fts_only_candidates_skip_relevance_thresholds() -> None: + candidates = build_recall_candidates( + memory_hits=(_memory_hit(score=1 / 61, matched_by=("fts",)),), + topic_memory_hits=(_topic_hit(score=100.0, artifact_id="topic-2", title="alpha", summary="beta"),), + experience_hits=(), + ) + assessment = RecallSufficiencyGate().assess(candidates, "alpha beta", RecallSufficiencyPolicy()) + assert assessment.reason == REASON_SUFFICIENT + assert assessment.signals.scored_families == 0 + assert assessment.signals.gap_families == 0 def test_memory_single_channel_top_rank_normalizes_to_one() -> None: @@ -479,6 +693,22 @@ def test_memory_two_channel_hit_normalizes_against_its_wider_upper_bound() -> No assert candidates[0].score == pytest.approx(0.5) +def test_memory_vector_relevance_reaches_gate_without_changing_rank_score() -> None: + hits = ( + _memory_hit(score=2 / 61, matched_by=("fts", "vector"), text="alpha beta").model_copy( + update={"relevance": 0.3} + ), + _memory_hit(score=1 / 62, matched_by=("vector",), text="alpha beta").model_copy( + update={"entry_id": "other", "relevance": 0.1} + ), + ) + candidates = build_recall_candidates(memory_hits=hits, topic_memory_hits=(), experience_hits=()) + assessment = RecallSufficiencyGate().assess(candidates, "alpha beta", RecallSufficiencyPolicy()) + assert candidates[0].score == pytest.approx(1.0) + assert assessment.signals.top_score == pytest.approx(0.3) + assert assessment.reason == REASON_WEAK_TOP_ONE + + # ── lexical_overlap semantics ─────────────────────────────────────────────────────────────── @@ -616,7 +846,7 @@ def test_effort_rounds_is_one_plus_the_committed_expansions() -> None: for effort in (none, one, two): assert effort.rounds == 1 + len(effort.expansion_actions) assert len(effort.candidates_by_round) == effort.rounds - assert effort.policy == POLICY_ID + assert effort.policy == "powercontext.recall-gate.v2" def test_effort_folds_the_builder_omission_counts_and_keeps_their_sum() -> None: diff --git a/tests/e2e/test_recall_sufficiency_gate.py b/tests/e2e/test_recall_sufficiency_gate.py index 426275c201..b790d8c5ad 100644 --- a/tests/e2e/test_recall_sufficiency_gate.py +++ b/tests/e2e/test_recall_sufficiency_gate.py @@ -23,10 +23,12 @@ from __future__ import annotations import asyncio +import math import sqlite3 import time from collections.abc import AsyncIterator from contextlib import asynccontextmanager +from dataclasses import replace from datetime import date, datetime, timedelta, timezone from pathlib import Path from typing import Any, cast @@ -59,7 +61,11 @@ ScopedContextApplication, _RecallRoundOutcome, ) -from powercontext.builtin.runtime.prepared_context import PreparedContextBuild, PreparedMemoryCandidates +from powercontext.builtin.runtime.prepared_context import ( + PreparedContextBuild, + PreparedContextBuilder, + PreparedMemoryCandidates, +) from powercontext.builtin.runtime.recall_sufficiency import ( MEMORY_FAMILY, REASON_AT_MAX_ROUNDS, @@ -140,13 +146,19 @@ async def counted( @asynccontextmanager async def _runtime( - database: Path, runtime: RuntimeConfig | None = None, *, in_memory: bool = False + database: Path, + runtime: RuntimeConfig | None = None, + *, + in_memory: bool = False, + embedding_model: Any = None, ) -> AsyncIterator[BuiltinRuntime]: config = BuiltinConfig( database=SQLiteConfig() if in_memory else SQLiteConfig(url=f"sqlite+aiosqlite:///{database}"), runtime=runtime if runtime is not None else RuntimeConfig(), ) - async with open_builtin_runtime(config, scheduler_path=database.with_suffix(".scheduler.db")) as opened: + async with open_builtin_runtime( + config, scheduler_path=database.with_suffix(".scheduler.db"), embedding_model=embedding_model + ) as opened: yield opened @@ -178,10 +190,15 @@ async def _seed_topic_memories(runtime: BuiltinRuntime, scope_id: str, count: in ) -async def _create_scope(runtime: BuiltinRuntime, key: str) -> str: +async def _create_scope(runtime: BuiltinRuntime, key: str, *, references: tuple[str, ...] = ()) -> str: assert runtime.scopes is not None created = await runtime.scopes.create( - ScopeDraft(title="Gate", summary="Recall gate acceptance", idempotency_key=key) + ScopeDraft( + title="Gate", + summary="Recall gate acceptance", + idempotency_key=key, + context_references=references, + ) ) return created.scope_id @@ -254,6 +271,220 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_sqlite_vector_prepare_reports_cosine_and_expands_past_a_weak_top(tmp_path, monkeypatch) -> None: + embedding_profile = EmbeddingProfile( + profile_id="recall-gate-vector", model="deterministic", dimension=2, distance="l2", normalization="unit" + ) + + class DeterministicEmbedding: + profile = embedding_profile + + async def embed(self, texts: tuple[str, ...], /) -> EmbeddingResult: + similarities = tuple( + 1.0 if text == _QUERY else 0.2 if text == "unrelated vector evidence" else 0.31 for text in texts + ) + return EmbeddingResult( + vectors=tuple((similarity, math.sqrt(1.0 - similarity**2)) for similarity in similarities) + ) + + log = _RecallRoundLog() + log.install(monkeypatch) + + async def scenario() -> None: + database = tmp_path / "vector-gate.db" + async with _runtime( + database, + RuntimeConfig(recall_gate_enabled=True, recall_gate_min_candidates=1), + embedding_model=DeterministicEmbedding(), + ) as runtime: + scope_id = await _create_scope(runtime, "vector-gate") + await _seed(runtime, scope_id, ["alpha beta gamma vector evidence", "unrelated vector evidence"]) + build, effort = await _prepare_build(runtime, scope_id, _memory_request()) + + # weak-top-1 is not terminal: the recoverable below-floor hit is re-admitted by one + # expansion round, and the reason stays weak-top-1 once nothing remains recoverable. + assert effort is not None + assert effort.assessment == "weak-top-1" + assert effort.rounds == 2 + assert effort.signals is not None + assert effort.signals.top_score == pytest.approx(0.31, abs=0.005) + assert len(log.calls) == 2 + assert "unrelated vector evidence" in (build.context.content or "") + memory_admission = next(count for count in effort.admission_by_family if count.family == MEMORY_FAMILY) + assert memory_admission.retrieved == memory_admission.admitted + + asyncio.run(scenario()) + + +def test_bounded_expansion_surfaces_a_strong_hit_hidden_by_the_fusion_limit(tmp_path, monkeypatch) -> None: + """The fused window is capped before the gate sees hits, so a weak top cannot be + declared unimprovable: a strong vector-only hit outside the window gains an FTS + channel under the lowered floor and RRF moves it into view.""" + embedding_profile = EmbeddingProfile( + profile_id="recall-gate-hidden", model="deterministic", dimension=2, distance="l2", normalization="unit" + ) + weak_texts = tuple(f"alpha beta gamma weak {index:02d}" for index in range(16)) + strong_text = "alpha window hidden" + cosines = {_QUERY: 1.0, strong_text: 0.90} + cosines.update({text: 0.31 + index * 0.001 for index, text in enumerate(weak_texts)}) + + class DeterministicEmbedding: + profile = embedding_profile + + async def embed(self, texts: tuple[str, ...], /) -> EmbeddingResult: + return EmbeddingResult( + vectors=tuple((cosine := cosines.get(text, 0.0), math.sqrt(1.0 - cosine * cosine)) for text in texts) + ) + + log = _RecallRoundLog() + log.install(monkeypatch) + + # Capture the committed candidates handed to the Builder: the verdict's evidence must + # be inside the delivery cap, not trailing the round-zero prefix. The section render + # limit (<= 8 items) is a separate request-level display cap and may still clip it. + delivered: list[PreparedMemoryCandidates] = [] + original_build = PreparedContextBuilder.build_scopes_result + + def spying_build(self: PreparedContextBuilder, **kwargs: Any) -> PreparedContextBuild: + delivered.extend(kwargs["memory_candidates"]) + return original_build(self, **kwargs) + + monkeypatch.setattr(PreparedContextBuilder, "build_scopes_result", spying_build) + + async def scenario() -> None: + database = tmp_path / "hidden-hit.db" + async with _runtime( + database, + RuntimeConfig(recall_gate_enabled=True), + embedding_model=DeterministicEmbedding(), + ) as runtime: + scope_id = await _create_scope(runtime, "hidden-hit") + await _seed(runtime, scope_id, [*weak_texts, strong_text]) + _, effort = await _prepare_build(runtime, scope_id, _memory_request()) + + # Round 0: the sixteen dual-channel weak hits fill the fusion window and truncate + # the vector-only 0.90 hit, so the gate sees a weak top. Round 1's lower lexical + # floor admits the strong hit's FTS row; the dual-channel hit then enters the + # window, the top score reflects it, and the committed candidates keep the latest + # round's fused order so the surfaced hit is actually delivered to the Builder. + assert effort is not None + assert effort.assessment == REASON_SUFFICIENT + assert effort.rounds == 2 + assert effort.candidates_by_round == (16, 17) + assert effort.signals is not None + assert effort.signals.top_score == pytest.approx(0.90, abs=0.005) + assert len(log.calls) == 2 + assert any(strong_text in hit.text for group in delivered for hit in group.hits) + + asyncio.run(scenario()) + + +def test_expansion_fts_only_hit_preserves_round_zero_scored_family(tmp_path, monkeypatch) -> None: + embedding_profile = EmbeddingProfile( + profile_id="recall-gate-first-hit", model="deterministic", dimension=2, distance="l2", normalization="unit" + ) + + class DeterministicEmbedding: + profile = embedding_profile + + async def embed(self, texts: tuple[str, ...], /) -> EmbeddingResult: + vectors = {_QUERY: (1.0, 0.0), "alpha beta gamma evidence": (0.5, math.sqrt(0.75))} + return EmbeddingResult(vectors=tuple(vectors.get(text, (0.0, 1.0)) for text in texts)) + + original = ScopedContextApplication._recall_round + expansion_first_relevance = [] + + async def fts_first_expansion(*args: Any, **kwargs: Any) -> _RecallRoundOutcome: + result = await original(*args, **kwargs) + if kwargs["admission"] is None: + return result + # Put the new FTS-only hit first in the returned round. Accumulation must still append it + # after round zero's vector-backed first hit rather than replace the family's first hit. + memory = tuple(replace(group, hits=tuple(reversed(group.hits))) for group in result.memory) + expansion_first_relevance.extend(group.hits[0].relevance for group in memory if group.hits) + return replace(result, memory=memory) + + monkeypatch.setattr(ScopedContextApplication, "_recall_round", fts_first_expansion) + + async def scenario() -> None: + database = tmp_path / "first-hit.db" + request = _memory_request() + async with _runtime( + database, + RuntimeConfig(recall_gate_enabled=True, recall_gate_max_rounds=0), + embedding_model=DeterministicEmbedding(), + ) as runtime: + scope_id = await _create_scope(runtime, "first-hit") + await _seed(runtime, scope_id, ["alpha beta gamma evidence", "alpha solo evidence"]) + _before_build, before_effort = await _prepare_build(runtime, scope_id, request) + + async with _runtime( + database, + RuntimeConfig(recall_gate_enabled=True), + embedding_model=DeterministicEmbedding(), + ) as runtime: + after_build, after_effort = await _prepare_build(runtime, scope_id, request) + + assert before_effort is not None + assert before_effort.candidates_by_round == (1,) + assert before_effort.signals is not None + assert before_effort.signals.top_score == pytest.approx(0.5, abs=0.005) + assert expansion_first_relevance == [None] + assert after_effort is not None + assert after_effort.rounds == 2 + assert after_effort.candidates_by_round == (1, 2) + assert after_effort.signals is not None + assert after_effort.signals.top_score == before_effort.signals.top_score + assert after_effort.assessment == REASON_SUFFICIENT + assert "alpha solo evidence" in (after_build.context.content or "") + + asyncio.run(scenario()) + + +def test_scoped_first_hit_rule_applies_per_scope_through_the_runtime(tmp_path) -> None: + embedding_profile = EmbeddingProfile( + profile_id="recall-gate-scopes", model="deterministic", dimension=2, distance="l2", normalization="unit" + ) + + class DeterministicEmbedding: + profile = embedding_profile + + async def embed(self, texts: tuple[str, ...], /) -> EmbeddingResult: + vectors = {_QUERY: (1.0, 0.0), "alpha beta gamma vector evidence": (0.31, math.sqrt(1 - 0.31**2))} + return EmbeddingResult(vectors=tuple(vectors.get(text, (0.0, 1.0)) for text in texts)) + + async def scenario() -> None: + database = tmp_path / "scoped-first-hit.db" + async with _runtime( + database, + RuntimeConfig(recall_gate_enabled=True, recall_gate_min_candidates=1), + embedding_model=DeterministicEmbedding(), + ) as runtime: + referenced = await _create_scope(runtime, "referenced") + await _seed(runtime, referenced, ["alpha beta gamma vector evidence"]) + # The primary Scope's fused top is FTS-only; the referenced Scope's fused top + # carries cosine 0.31. Each Scope's first hit must be judged on its own fusion + # run, so the verdict is weak-top-1 on the referenced evidence rather than the + # whole family going unscored. weak-top-1 is not terminal: the lowered floor + # still re-admits the primary Scope's floor-dropped "alpha solo" entry. + primary = await _create_scope(runtime, "primary", references=(referenced,)) + await _seed(runtime, primary, ["alpha beta gamma fts evidence", "alpha solo fts evidence"]) + build, effort = await _prepare_build(runtime, primary, _memory_request()) + + assert effort is not None + assert effort.assessment == "weak-top-1" + # The primary Scope's zero-cosine rows keep the vector channel recoverable, so the + # bounded expansion spends both rounds; the last committed weak-top-1 reason is kept. + assert effort.rounds == 3 + assert effort.candidates_by_round == (2, 3, 3) + assert effort.signals is not None + assert effort.signals.scored_families == 1 + assert effort.signals.top_score == pytest.approx(0.31, abs=0.005) + assert "alpha solo fts evidence" in (build.context.content or "") + + asyncio.run(scenario()) + + def test_one_expansion_round_re_admits_a_candidate_blocked_at_round_zero(tmp_path, monkeypatch) -> None: log = _RecallRoundLog() log.install(monkeypatch) @@ -594,7 +825,8 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_later_round_failure_preserves_committed_expansion_trace(tmp_path, monkeypatch) -> None: +@pytest.mark.parametrize("failure_phase", ["search", "assessment"]) +def test_later_round_failure_preserves_committed_expansion(tmp_path, monkeypatch, failure_phase) -> None: async def scenario() -> None: database = tmp_path / "later-failure.db" request = _memory_request() @@ -604,6 +836,15 @@ async def scenario() -> None: await _seed(disabled, scope_id, ["alpha beta gamma evidence", "alpha solo marker", "beta gamma marker"]) baseline, _baseline_effort = await _prepare_build(disabled, scope_id, request) + async with _runtime( + database, + RuntimeConfig(recall_gate_enabled=True, recall_gate_min_candidates=100, recall_gate_max_rounds=1), + ) as runtime: + committed, committed_effort = await _prepare_build(runtime, scope_id, request) + assert committed_effort is not None + assert committed_effort.candidates_by_round == (2, 3) + assert committed.context.content != baseline.context.content + original = ScopedContextApplication._recall_round calls = 0 @@ -620,7 +861,7 @@ async def fails_on_second_expansion( ) -> Any: nonlocal calls calls += 1 - if calls == 3: + if calls == 3 and failure_phase == "search": raise RuntimeError("later expansion failed") # noqa: TRY003 result = await original( application, @@ -651,6 +892,14 @@ async def fails_on_second_expansion( return result monkeypatch.setattr(ScopedContextApplication, "_recall_round", fails_on_second_expansion) + original_assess = RecallSufficiencyGate.assess + + def fails_on_second_assessment(*args: Any, **kwargs: Any) -> Any: + if calls == 3 and failure_phase == "assessment": + raise RuntimeError("later assessment failed") # noqa: TRY003 + return original_assess(*args, **kwargs) + + monkeypatch.setattr(RecallSufficiencyGate, "assess", fails_on_second_assessment) async with _runtime( database, RuntimeConfig(recall_gate_enabled=True, recall_gate_min_candidates=100), @@ -662,9 +911,12 @@ async def fails_on_second_expansion( assert effort.rounds == 2 assert effort.expansion_actions == ("admission",) assert len(effort.candidates_by_round) == 2 + assert effort.signals is not None + assert effort.signals.candidate_count == effort.candidates_by_round[-1] + assert effort.signals == committed_effort.signals assert calls == 3 - assert build.context.content == baseline.context.content - assert build.origins == baseline.origins + assert build.context.content == committed.context.content + assert build.origins == committed.origins asyncio.run(scenario())