Meeting speaker diarization — feature spec & exploration

.gitignore

@@ -15,4 +15,8 @@ DerivedData/
 
 # Secrets
 secrets
-private_keys
+private_keys
+
+# Editor/agent local config
+.rules
+.agents/

Sources/WisprApp/Models/MeetingTranscript.swift

@@ -8,9 +8,30 @@
 import Foundation
 
 /// Identifies the audio source / speaker in a meeting transcript.
-enum MeetingSpeaker: String, Sendable, Equatable, Hashable {
-    case you = "You"
-    case others = "Others"
+///
+/// Speaker indices are stored **0-based** internally. The +1 transform to
+/// "Speaker 1", "Speaker 2", etc. happens only inside `displayName`.
+enum MeetingSpeaker: Sendable, Equatable, Hashable {
+    case you
+    /// Remote participant. `speakerIndex` is nil during Sortformer's cold-start
+    /// window or when diarization is disabled — renders as plain "Others".
+    case others(speakerIndex: Int?)
+
+    /// User-visible label for transcript display and plain-text export.
+    var displayName: String {
+        switch self {
+        case .you: return "You"
+        case .others(.none): return "Others"
+        case .others(.some(let index)): return "Speaker \(index + 1)"
+        }
+    }
+
+    /// Whether this speaker is a remote participant (system-audio track),
+    /// regardless of the resolved diarization index.
+    var isRemote: Bool {
+        if case .others = self { return true }
+        return false
+    }
 }
 
 /// A single timestamped entry in a meeting transcript.
@@ -37,6 +58,141 @@ struct MeetingTranscript: Sendable, Equatable {
         self.startTime = startTime
     }
 
+    // MARK: - Echo Suppression
+
+    /// How far back (in seconds) to look for a matching entry on the other
+    /// audio track when detecting microphone echo. Mic and system chunks are
+    /// transcribed independently and on different chunk boundaries, so the two
+    /// copies of the same utterance can land a few seconds apart.
+    static let echoSuppressionWindow: TimeInterval = 8
+
+    /// Minimum normalized similarity (0...1) between two entries' text for them
+    /// to be considered the same utterance. ASR of speaker-echo is usually a
+    /// near-perfect copy, so a high threshold keeps false positives rare.
+    static let echoSimilarityThreshold = 0.8
+
+    /// Utterances shorter than this (in words) are not echo-suppressed. Short
+    /// backchannels ("yes", "right", "okay") are commonly said by both sides,
+    /// so suppressing them would risk dropping genuine speech.
+    static let minimumEchoWordCount = 3
+
+    /// Appends `entry`, suppressing microphone echo of remote (system) audio.
+    ///
+    /// In meeting mode without headphones, remote participants' speech plays out
+    /// of the speakers and leaks back into the microphone, producing a duplicate
+    /// "You" entry that mirrors an "Others" entry (see issue #65). When a "You"
+    /// entry closely matches a recent "Others" entry — or a newly arriving
+    /// "Others" entry matches a recent "You" entry — the microphone copy is
+    /// treated as echo: the utterance is attributed to the remote participant
+    /// and the "You" copy is dropped.
+    ///
+    /// Handles both arrival orders (mic-first or system-first) since the two
+    /// transcription paths run concurrently.
+    ///
+    /// - Returns: `true` if `entry` was added to the transcript, `false` if it
+    ///   was suppressed as microphone echo.
+    @discardableResult
+    mutating func appendSuppressingEcho(_ entry: MeetingTranscriptEntry) -> Bool {
+        let normalized = Self.normalizedForComparison(entry.text)
+        guard normalized.split(separator: " ").count >= Self.minimumEchoWordCount else {
+            entries.append(entry)
+            return true
+        }
+
+        switch entry.speaker {
+        case .you:
+            // A mic entry echoing a recent remote utterance is dropped.
+            if recentEchoMatchIndex(of: normalized, isRemote: true, before: entry.timestamp) != nil
+            {
+                return false
+            }
+            entries.append(entry)
+            return true
+
+        case .others:
+            // A remote utterance that a recent mic entry already echoed: drop the
+            // earlier mic echo and keep this (authoritative) remote entry.
+            if let echoIndex = recentEchoMatchIndex(
+                of: normalized, isRemote: false, before: entry.timestamp)
+            {
+                entries.remove(at: echoIndex)
+            }
+            entries.append(entry)
+            return true
+        }
+    }
+
+    /// Finds the most recent entry within `echoSuppressionWindow` of `timestamp`
+    /// that originates from the requested track (`isRemote`) and whose text is
+    /// similar enough to `normalizedText` to be the same utterance.
+    private func recentEchoMatchIndex(
+        of normalizedText: String,
+        isRemote: Bool,
+        before timestamp: Date
+    ) -> Int? {
+        for index in entries.indices.reversed() {
+            let candidate = entries[index]
+            // Entries are appended in chronological order, so once we pass the
+            // window boundary every earlier entry is also out of range.
+            if timestamp.timeIntervalSince(candidate.timestamp) > Self.echoSuppressionWindow {
+                break
+            }
+            guard candidate.speaker.isRemote == isRemote else { continue }
+            let candidateNormalized = Self.normalizedForComparison(candidate.text)
+            if Self.similarity(normalizedText, candidateNormalized) >= Self.echoSimilarityThreshold
+            {
+                return index
+            }
+        }
+        return nil
+    }
+
+    /// Lowercases, strips punctuation, and collapses whitespace so that two
+    /// transcriptions of the same speech compare equal despite cosmetic
+    /// differences (capitalization, trailing punctuation, extra spaces).
+    static func normalizedForComparison(_ text: String) -> String {
+        let scalars = text.lowercased().unicodeScalars.map { scalar -> Character in
+            CharacterSet.alphanumerics.contains(scalar) ? Character(scalar) : " "
+        }
+        return String(scalars)
+            .split(separator: " ", omittingEmptySubsequences: true)
+            .joined(separator: " ")
+    }
+
+    /// Similarity of two strings in 0...1, derived from Levenshtein edit distance
+    /// normalized by the longer string's length (1 = identical).
+    static func similarity(_ a: String, _ b: String) -> Double {
+        if a.isEmpty && b.isEmpty { return 1 }
+        let lhs = Array(a)
+        let rhs = Array(b)
+        let maxLength = max(lhs.count, rhs.count)
+        guard maxLength > 0 else { return 1 }
+        return 1 - Double(levenshtein(lhs, rhs)) / Double(maxLength)
+    }
+
+    /// Standard two-row Levenshtein edit distance.
+    private static func levenshtein(_ a: [Character], _ b: [Character]) -> Int {
+        if a.isEmpty { return b.count }
+        if b.isEmpty { return a.count }
+
+        var previous = Array(0...b.count)
+        var current = [Int](repeating: 0, count: b.count + 1)
+
+        for i in 1...a.count {
+            current[0] = i
+            for j in 1...b.count {
+                let cost = a[i - 1] == b[j - 1] ? 0 : 1
+                current[j] = min(
+                    previous[j] + 1,  // deletion
+                    current[j - 1] + 1,  // insertion
+                    previous[j - 1] + cost  // substitution
+                )
+            }
+            swap(&previous, &current)
+        }
+        return previous[b.count]
+    }
+
     /// Shared time formatter for transcript display (HH:mm:ss).
     private static let timeFormatter: DateFormatter = {
         let formatter = DateFormatter()
@@ -53,7 +209,7 @@ struct MeetingTranscript: Sendable, Equatable {
     func asPlainText() -> String {
         entries.map { entry in
             let time = Self.formatTime(entry.timestamp)
-            return "[\(time)] \(entry.speaker.rawValue): \(entry.text)"
+            return "[\(time)] \(entry.speaker.displayName): \(entry.text)"
         }.joined(separator: "\n")
     }
 

Sources/WisprApp/Services/MeetingAudioEngine.swift

@@ -19,6 +19,13 @@ import ScreenCaptureKit
 import WisprCore
 import os
 
+/// A timestamped audio chunk from the system-audio capture path.
+/// `startTime` is seconds elapsed since capture began (cumulative sample count / sampleRate).
+struct SystemAudioChunk: Sendable {
+    let samples: [Float]
+    let startTime: TimeInterval
+}
+
 /// Actor responsible for capturing both microphone and system audio simultaneously.
 ///
 /// Uses `AVAudioEngine` for microphone input and `SCStream` (ScreenCaptureKit)
@@ -36,9 +43,11 @@ actor MeetingAudioEngine {
 
     private var micBuffer: [Float] = []
     private var systemBuffer: [Float] = []
+    private var systemSamplesTotal: Int = 0
+    private var systemChunkStartSamples: Int = 0
 
     private var micContinuation: AsyncStream<[Float]>.Continuation?
-    private var systemContinuation: AsyncStream<[Float]>.Continuation?
+    private var systemContinuation: AsyncStream<SystemAudioChunk>.Continuation?
     private var micLevelContinuation: AsyncStream<Float>.Continuation?
     private var systemLevelContinuation: AsyncStream<Float>.Continuation?
 
@@ -54,12 +63,20 @@ actor MeetingAudioEngine {
 
     /// The audio chunk streams created at capture start.
     private var _micAudioStream: AsyncStream<[Float]>?
-    private var _systemAudioStream: AsyncStream<[Float]>?
+    private var _systemAudioStream: AsyncStream<SystemAudioChunk>?
 
     /// The chunk size in samples before yielding to the transcription stream.
     /// ~5 seconds of audio at 16kHz = 80,000 samples.
     private let chunkSize = 80_000
 
+    /// System-audio chunk size. Shorter than the mic chunk (~3s) so each
+    /// transcript entry spans less time, reducing how often a single entry
+    /// flattens a speaker change into one diarization label.
+    private let systemChunkSize = 48_000
+
+    /// Audio sample rate used for both capture and diarization (Hz).
+    static let sampleRate: Int = 16_000
+
     // MARK: - Public Interface
 
     /// Starts dual audio capture (microphone + system audio).
@@ -86,7 +103,7 @@ actor MeetingAudioEngine {
         _micAudioStream = micStream
         micContinuation = micCont
 
-        let (sysStream, sysCont) = AsyncStream.makeStream(of: [Float].self)
+        let (sysStream, sysCont) = AsyncStream.makeStream(of: SystemAudioChunk.self)
         _systemAudioStream = sysStream
         systemContinuation = sysCont
 
@@ -131,9 +148,9 @@ actor MeetingAudioEngine {
     }
 
     /// Returns the system audio chunk stream created during `startCapture()`.
-    var systemAudioStream: AsyncStream<[Float]> {
+    var systemAudioStream: AsyncStream<SystemAudioChunk> {
         if let stream = _systemAudioStream { return stream }
-        let (stream, cont) = AsyncStream.makeStream(of: [Float].self)
+        let (stream, cont) = AsyncStream.makeStream(of: SystemAudioChunk.self)
         cont.finish()
         return stream
     }
@@ -145,7 +162,8 @@ actor MeetingAudioEngine {
             micBuffer.removeAll()
         }
         if !systemBuffer.isEmpty {
-            systemContinuation?.yield(systemBuffer)
+            let startTime = Double(systemChunkStartSamples) / Double(Self.sampleRate)
+            systemContinuation?.yield(SystemAudioChunk(samples: systemBuffer, startTime: startTime))
             systemBuffer.removeAll()
         }
         micContinuation?.finish()
@@ -299,7 +317,7 @@ actor MeetingAudioEngine {
         config.minimumFrameInterval = CMTime(value: 1, timescale: 1)
         config.capturesAudio = true
         config.excludesCurrentProcessAudio = true
-        config.sampleRate = 16000
+        config.sampleRate = Self.sampleRate
         config.channelCount = 1
 
         let (systemBridgeStream, systemBridgeCont) = AsyncStream.makeStream(of: [Float].self)
@@ -335,13 +353,17 @@ actor MeetingAudioEngine {
         let normalizedLevel = min(max(rms * 5.0, 0.0), 1.0)
         systemLevelContinuation?.yield(normalizedLevel)
 
+        systemSamplesTotal += samples.count
         systemBuffer.append(contentsOf: samples)
-        if systemBuffer.count >= chunkSize {
-            let chunk = Array(systemBuffer.prefix(chunkSize))
-            systemBuffer.removeFirst(min(chunkSize, systemBuffer.count))
+        if systemBuffer.count >= systemChunkSize {
+            let chunk = Array(systemBuffer.prefix(systemChunkSize))
+            systemBuffer.removeFirst(min(systemChunkSize, systemBuffer.count))
+            let startTime = Double(systemChunkStartSamples) / Double(Self.sampleRate)
+            systemChunkStartSamples = systemSamplesTotal - systemBuffer.count
             Log.audioEngine.debug(
-                "MeetingAudioEngine — yielding system chunk of \(chunk.count) samples")
-            systemContinuation?.yield(chunk)
+                "MeetingAudioEngine — yielding system chunk of \(chunk.count) samples at t=\(String(format: "%.2f", startTime))s"
+            )
+            systemContinuation?.yield(SystemAudioChunk(samples: chunk, startTime: startTime))
         }
     }
 
@@ -353,6 +375,8 @@ actor MeetingAudioEngine {
         systemStream = nil
         systemStreamOutput = nil
         systemBuffer.removeAll()
+        systemSamplesTotal = 0
+        systemChunkStartSamples = 0
         systemContinuation?.finish()
         systemContinuation = nil
         systemLevelContinuation?.finish()