Continue patching UOM APIs and Candidate APIs.

Author: ShikiSuen

Package.swift

@@ -24,7 +24,9 @@ let package = Package(
     ),
     .target(
       name: "MegrezTestComponents",
-      dependencies: []
+      dependencies: [
+        "Megrez",
+      ]
     ),
     .testTarget(
       name: "MegrezTests",

Sources/Megrez/3_KeyValuePaired.swift

@@ -169,12 +169,14 @@ extension Megrez.Compositor {
   ///   - candidate: 指定用來覆寫為的候選字（詞音鍵值配對）。
   ///   - location: 游標位置。
   ///   - overrideType: 指定覆寫行為。
+  ///   - enforceRetokenization: 是否強制重新分詞，對所有重疊節點施作重置與降權，以避免殘留舊節點狀態。
   ///   - perceptionHandler: 覆寫成功後用於回傳觀測智慧的回呼。
   /// - Returns: 該操作是否成功執行。
   @discardableResult
   public func overrideCandidate(
     _ candidate: Megrez.KeyValuePaired, at location: Int,
     overrideType: Megrez.Node.OverrideType = .withHighScore,
+    enforceRetokenization: Bool = false,
     perceptionHandler: ((Megrez.PerceptionIntel) -> ())? = nil
   )
     -> Bool {
@@ -183,6 +185,7 @@ extension Megrez.Compositor {
       at: location,
       value: candidate.value,
       type: overrideType,
+      enforceRetokenization: enforceRetokenization,
       perceptionHandler: perceptionHandler
     )
   }
@@ -194,12 +197,14 @@ extension Megrez.Compositor {
   ///   - candidate: 指定用來覆寫為的候選字（字串）。
   ///   - location: 游標位置。
   ///   - overrideType: 指定覆寫行為。
+  ///   - enforceRetokenization: 是否強制重新分詞，對所有重疊節點施作重置與降權，以避免殘留舊節點狀態。
   ///   - perceptionHandler: 覆寫成功後用於回傳觀測智慧的回呼。
   /// - Returns: 該操作是否成功執行。
   @discardableResult
   public func overrideCandidateLiteral(
     _ candidate: String,
     at location: Int, overrideType: Megrez.Node.OverrideType = .withHighScore,
+    enforceRetokenization: Bool = false,
     perceptionHandler: ((Megrez.PerceptionIntel) -> ())? = nil
   )
     -> Bool {
@@ -208,6 +213,7 @@ extension Megrez.Compositor {
       at: location,
       value: candidate,
       type: overrideType,
+      enforceRetokenization: enforceRetokenization,
       perceptionHandler: perceptionHandler
     )
   }
@@ -220,13 +226,15 @@ extension Megrez.Compositor {
   ///   - location: 游標位置。
   ///   - value: 資料值。
   ///   - type: 指定覆寫行為。
+  ///   - enforceRetokenization: 是否強制重新分詞，對所有重疊節點施作重置與降權，以避免殘留舊節點狀態。
   ///   - perceptionHandler: 覆寫成功後用於回傳觀測智慧的回呼。
   /// - Returns: 該操作是否成功執行。
   internal func overrideCandidateAgainst(
     keyArray: [String]?,
     at location: Int,
     value: String,
     type: Megrez.Node.OverrideType,
+    enforceRetokenization: Bool,
     perceptionHandler: ((Megrez.PerceptionIntel) -> ())? = nil
   )
     -> Bool {
@@ -296,20 +304,39 @@ extension Megrez.Compositor {
       overridden.location + overridden.node.segLength
     )
 
-    for i in overriddenRange {
-      let overlappingNodes = fetchOverlappingNodes(at: i)
-
-      for anchor in overlappingNodes where anchor.node != overridden.node {
-        // 檢查是否需要重設節點
-        let shouldReset = shouldResetNode(
-          anchor: anchor.node,
-          overriddenNode: overridden.node
-        )
-
-        if shouldReset {
-          anchor.node.reset()
-        } else {
-          anchor.node.overridingScore /= 4
+    if enforceRetokenization {
+      let overriddenNodeRef = overridden.node
+      let demotionScore = -Swift.max(1.0, Swift.abs(overriddenNodeRef.overridingScore))
+      for i in overriddenRange {
+        let overlappingNodes = fetchOverlappingNodes(at: i)
+        for anchor in overlappingNodes where anchor.node !== overriddenNodeRef
+          && anchor.location <= overridden.location {
+          if shouldResetNode(anchor: anchor.node, overriddenNode: overriddenNodeRef) {
+            anchor.node.reset()
+          }
+          anchor.node.overrideStatus = .init(
+            overridingScore: demotionScore,
+            currentOverrideType: .withHighScore,
+            currentUnigramIndex: anchor.node.currentUnigramIndex
+          )
+        }
+      }
+    } else {
+      for i in overriddenRange {
+        let overlappingNodes = fetchOverlappingNodes(at: i)
+
+        for anchor in overlappingNodes where anchor.node != overridden.node {
+          // 檢查是否需要重設節點
+          let shouldReset = shouldResetNode(
+            anchor: anchor.node,
+            overriddenNode: overridden.node
+          )
+
+          if shouldReset {
+            anchor.node.reset()
+          } else {
+            anchor.node.overridingScore /= 4
+          }
         }
       }
     }
@@ -403,7 +430,12 @@ extension Megrez {
 
     // 選擇用來形成觀測 key 的資料源：長→短使用 after，其他使用 before
     let keySource = isBreakingUp ? currentAssembled : previouslyAssembled
-    let keyCursor = afterHit.range.upperBound // 與現有 key 生成方法對齊
+    let keyCursorRaw = Swift.max(
+      afterHit.range.lowerBound,
+      Swift.min(cursor, afterHit.range.upperBound - 1)
+    )
+    guard keySource.totalKeyCount > 0 else { return nil }
+    let keyCursor = Swift.max(0, Swift.min(keyCursorRaw, keySource.totalKeyCount - 1))
 
     guard let keyGen = keySource.generateKeyForPerception(cursor: keyCursor) else { return nil }
 

Sources/Megrez/7_Unigram.swift

@@ -44,7 +44,7 @@ extension Megrez {
 
     /// 將當前單元圖列印成一個字串。
     public var description: String {
-      "(" + value.description + "," + String(score) + ")"
+      "(\(keyArray.joined(separator: "-")),\(value),\(score))"
     }
 
     /// 單元圖的淺層複製品（保持相同的索引鍵陣列）。

Sources/Megrez/8_GramInPath.swift

@@ -151,6 +151,7 @@ extension Array where Element == Megrez.GramInPath {
   ///
   /// - Remark: 除非有專門指定游標，否則身為 `[GramInPath]` 自身的
   /// 「陣列最尾端」（也就是打字方向上最前方）的那個 Gram 會被當成 Head。
+  /// 若游標落在多字詞節點內，Head 會選擇游標右方（文字輸入方向上的下一個）讀音。
   /// - Parameters:
   ///   - cursor: 指定用於當作 head 的游標位置；若為 nil 則取陣列尾端。
   ///   - maxContext: 最多向前取用的上下文節點數（含 head），預設 3。
@@ -159,41 +160,83 @@ extension Array where Element == Megrez.GramInPath {
     maxContext: Int = 3
   )
     -> (ngramKey: String, candidate: String, headReading: String)? {
-    let perceptedGIP: Megrez.GramInPath?
-    if let cursor, (0 ..< self.totalKeyCount).contains(cursor) {
-      perceptedGIP = findGram(at: cursor)?.gram
+    guard maxContext > 0 else { return nil }
+
+    let headInfo: (pair: Megrez.GramInPath, range: Range<Int>)?
+    let resolvedCursor: Int
+    if let cursor, (0 ..< totalKeyCount).contains(cursor), let hit = findGram(at: cursor) {
+      headInfo = (pair: hit.gram, range: hit.range)
+      resolvedCursor = Swift.max(
+        hit.range.lowerBound,
+        Swift.min(cursor, hit.range.upperBound - 1)
+      )
+    } else if let tail = last {
+      let lowerBound = totalKeyCount - tail.keyArray.count
+      headInfo = (pair: tail, range: lowerBound ..< totalKeyCount)
+      resolvedCursor = Swift.max(lowerBound, totalKeyCount - 1)
     } else {
-      perceptedGIP = last
+      headInfo = nil
+      resolvedCursor = 0
     }
-    guard let perceptedGIP else { return nil }
-    var arrGIPs = self
-    while arrGIPs.last?.gram !== perceptedGIP.gram { arrGIPs.removeLast() }
-    var isHead = true
-    var outputCells = [String]()
-    loopProc: while !arrGIPs.isEmpty, let frontendPair = arrGIPs.last {
-      defer { arrGIPs = arrGIPs.dropLast() }
-
-      func makeNGramKeyCell(isHead: Bool) -> String? {
-        // 字音數與字數不一致的內容會被拋棄。
-        guard !frontendPair.isReadingMismatched else { return nil }
-        guard !frontendPair.value.isEmpty else { return nil }
-        guard !frontendPair.keyArray.joined().isEmpty else { return nil }
-        let keyChain = frontendPair.keyArray.joined(separator: "-")
-        guard !keyChain.contains("_") else { return nil }
-        // 前置單元只記錄讀音，在其後的單元則同時記錄讀音與字詞
-        return isHead ? keyChain : "(\(keyChain):\(frontendPair.value))"
+
+    guard let headInfo else { return nil }
+    let headPair = headInfo.pair
+    let headRange = headInfo.range
+    let headKeyOffset = Swift.max(0, resolvedCursor - headRange.lowerBound)
+    guard headPair.keyArray.indices.contains(headKeyOffset) else { return nil }
+
+    let placeholder = "()"
+    let readingSeparator = Megrez.Compositor.theSeparator
+
+    func isPunctuation(_ pair: Megrez.GramInPath) -> Bool {
+      guard let firstReading = pair.keyArray.first else { return false }
+      return firstReading.first == "_"
+    }
+
+    func combinedString(for pair: Megrez.GramInPath) -> String? {
+      guard !pair.value.isEmpty else { return nil }
+      guard !pair.keyArray.isEmpty else { return nil }
+      let reading = pair.joinedCurrentKey(by: readingSeparator)
+      guard !reading.isEmpty else { return nil }
+      return "(\(reading),\(pair.value))"
+    }
+
+    guard let headIndex = lastIndex(where: { $0.gram === headPair.gram })
+      ?? firstIndex(of: headPair)
+    else { return nil }
+
+    var resultCells = [String](repeating: placeholder, count: maxContext)
+    resultCells[maxContext - 1] = combinedString(for: headPair) ?? placeholder
+
+    var contextSlot = maxContext - 2
+    var currentIndex = headIndex - 1
+    var encounteredPunctuation = false
+
+    while contextSlot >= 0 {
+      guard currentIndex >= 0 else {
+        resultCells[contextSlot] = placeholder
+        contextSlot -= 1
+        continue
+      }
+      let currentPair = self[currentIndex]
+      currentIndex -= 1
+
+      if encounteredPunctuation || isPunctuation(currentPair) {
+        encounteredPunctuation = true
+        resultCells[contextSlot] = placeholder
+        contextSlot -= 1
+        continue
       }
 
-      guard let keyCellStr = makeNGramKeyCell(isHead: isHead) else { break loopProc }
-      outputCells.insert(keyCellStr, at: 0)
-      if outputCells.count >= maxContext { break loopProc }
-      if isHead { isHead = false }
+      resultCells[contextSlot] = combinedString(for: currentPair) ?? placeholder
+      contextSlot -= 1
     }
-    guard !outputCells.isEmpty else { return nil }
+
+    let headReading = headPair.keyArray[headKeyOffset]
     return (
-      "(\(outputCells.joined(separator: ",")))",
-      perceptedGIP.gram.value,
-      perceptedGIP.joinedCurrentKey(by: "-")
+      resultCells.joined(separator: "&"),
+      headPair.value,
+      headReading
     )
   }
 }

Tests/MegrezTests/MegrezTests.swift

@@ -652,39 +652,54 @@ final class MegrezTestsAdvanced: XCTestCase {
   func test18_Composer_UOMMarginalCaseTest() throws {
     let lm = SimpleLM(input: MegrezTestComponents.strLMSampleData_SaisoukiNoGaika)
     let compositor = Megrez.Compositor(with: lm)
-    ["zai4", "chuang4", "shi4", "de5", "kai3", "ge1"].forEach {
+    let readingKeys = ["zai4", "chuang4", "shi4", "de5", "kai3", "ge1"]
+    readingKeys.forEach {
       _ = compositor.insertKey($0)
     }
     compositor.assemble()
     let assembledBefore = compositor.assembledSentence.map(\.value).joined(separator: " ")
     XCTAssertTrue("再 創 是的 凱歌" == assembledBefore)
+    // 測試此時生成的 keyForQueryingData 是否正確
+    let cursorShi = 2
+    let cursorShiDe = 3
+    let keyForQueryingDataAt2 = compositor.assembledSentence
+      .generateKeyForPerception(cursor: cursorShi)
+    XCTAssertEqual(keyForQueryingDataAt2?.ngramKey, "(zai4,再)&(chuang4,創)&(shi4-de5,是的)")
+    XCTAssertEqual(keyForQueryingDataAt2?.headReading, "shi4")
+    let keyForQueryingDataAt3 = compositor.assembledSentence
+      .generateKeyForPerception(cursor: cursorShiDe)
+    XCTAssertEqual(keyForQueryingDataAt3?.ngramKey, "(zai4,再)&(chuang4,創)&(shi4-de5,是的)")
+    XCTAssertEqual(keyForQueryingDataAt3?.headReading, "de5")
     // 應能提供『是的』『似的』『凱歌』等候選
     let pairsAtShiDeEnd = compositor.fetchCandidates(at: 4, filter: .endAt)
     XCTAssertTrue(pairsAtShiDeEnd.map(\.value).contains("是的"))
     XCTAssertTrue(pairsAtShiDeEnd.map(\.value).contains("似的"))
     // 模擬使用者把『是』改為『世』，再合成：觀測應為 shortToLong
     var obsCaptured: Megrez.PerceptionIntel?
-    let cursorShi = 2
     _ = compositor.overrideCandidate(
       .init(keyArray: ["shi4"], value: "世"),
-      at: cursorShi
-    ) {
-      obsCaptured = $0
-    }
+      at: cursorShi,
+      overrideType: .withHighScore,
+      enforceRetokenization: true,
+      perceptionHandler: { obsCaptured = $0 }
+    )
+    XCTAssertEqual(obsCaptured?.ngramKey, "(zai4,再)&(chuang4,創)&(shi4,世)")
     // compositor.assemble() <- 已經組句了。
     let assembledAfter = compositor.assembledSentence.map(\.value).joined(separator: " ")
     XCTAssertTrue("再 創 世 的 凱歌" == assembledAfter)
-    let cursorShiDe = 3
+    // ====
     let prevAssembly = compositor.assembledSentence
     obsCaptured = nil
     let overrideSucceeded = compositor.overrideCandidate(
       .init(keyArray: ["shi4", "de5"], value: "是的"),
-      at: cursorShiDe
-    ) {
-      obsCaptured = $0
-    }
+      at: cursorShiDe,
+      overrideType: .withHighScore,
+      enforceRetokenization: true,
+      perceptionHandler: { obsCaptured = $0 }
+    )
+    XCTAssertEqual(obsCaptured?.ngramKey, "(chuang4,創)&(shi4,世)&(de5,的)")
     XCTAssertTrue(overrideSucceeded)
-    let currentAssembly = compositor.assemble()
+    let currentAssembly = compositor.assembledSentence
     guard let afterHit = currentAssembly.findGram(at: cursorShiDe) else {
       XCTFail("Expected current gram at cursor \(cursorShiDe)")
       return
@@ -701,5 +716,27 @@ final class MegrezTestsAdvanced: XCTestCase {
     XCTAssertNotNil(obsCaptured)
     XCTAssertEqual(obsCaptured?.scenario, .shortToLong)
     XCTAssertEqual(obsCaptured?.candidate, "是的")
+
+    // 測試 POM 建議的候選覆寫
+    compositor.clear()
+    readingKeys.prefix(4).forEach {
+      _ = compositor.insertKey($0)
+    }
+
+    let pomSuggestedCandidate = Megrez.KeyValuePaired((["shi4"], "世", -0.07449307430679043))
+    let pomSuggestedCandidateOverrideCursor = 2
+    // let forceHighScoreOverride = false
+    // overrideType: forceHighScoreOverride ? .withHighScore : .withTopUnigramScore
+    compositor.overrideCandidate(
+      pomSuggestedCandidate,
+      at: pomSuggestedCandidateOverrideCursor,
+      overrideType: .withTopUnigramScore,
+      enforceRetokenization: true
+    )
+    compositor.assemble()
+    let assembledByPOM = compositor.assembledSentence
+      .map(\.value)
+      .joined(separator: " ")
+    XCTAssertEqual("再 創 世 的", assembledByPOM)
   }
 }