From e930543b7b4ba27ecd2f4d4d679c02cc42fa24b6 Mon Sep 17 00:00:00 2001 From: Gabriel Bruno <242597616+gbrunoo@users.noreply.github.com> Date: Fri, 19 Jun 2026 23:05:48 +0200 Subject: [PATCH 1/5] Add speaker diarization on meeting Others track MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the spec from PR #64: splits the system-audio ("Others") track into per-speaker labels (Speaker 1, Speaker 2, …) using FluidAudio's SortformerDiarizer. The mic track stays "You". - MeetingSpeaker becomes .you / .others(speakerIndex: Int?) with a displayName computed property (0-based storage, +1 only at display). - New MeetingDiarizer actor wraps SortformerDiarizer: warmUp() loads the CoreML model, ingest() feeds chunks, dominantSpeaker(in:) resolves the speaker over a time window. Sortformer slots map to monotonic per-meeting indices. Failures degrade gracefully to plain "Others". - MeetingAudioEngine exposes a single sampleRate constant and emits the system-audio stream as timestamped SystemAudioChunk values. - MeetingStateManager warms up the diarizer on start (when enabled), routes system chunks through ingest + dominantSpeaker, resets on stop. - MeetingTranscriptView uses displayName with a per-speaker color palette. - SettingsStore gains meetingDiarizationEnabled (default false, opt-in); Settings adds a Meeting section toggle. - Cold-start nil speaker renders as "Others" until the diarizer warms up. Tests: MeetingDiarizerTests for the pre-init no-op contract, updated MeetingTranscript/AudioEngine tests for the new APIs. Diarization quality is verified via the manual E2E plan in the spec. --- .../WisprApp/Models/MeetingTranscript.swift | 22 +- .../Services/MeetingAudioEngine.swift | 37 +++- .../WisprApp/Services/MeetingDiarizer.swift | 152 ++++++++++++++ .../Services/MeetingStateManager.swift | 56 ++++- Sources/WisprApp/Services/SettingsStore.swift | 135 ++++++++---- .../UI/Meeting/MeetingTranscriptView.swift | 21 +- .../WisprApp/UI/Settings/SettingsView.swift | 192 +++++++++++------- Sources/WisprApp/Utilities/SFSymbols.swift | 3 + Sources/WisprApp/wisprApp.swift | 8 +- Sources/WisprCore/Utilities/Logger.swift | 1 + Sources/WisprCore/Utilities/ModelPaths.swift | 35 ++-- wispr.xcodeproj/project.pbxproj | 4 + wisprTests/MeetingAudioEngineTests.swift | 2 +- wisprTests/MeetingDiarizerTests.swift | 59 ++++++ wisprTests/MeetingStateManagerTests.swift | 14 +- 15 files changed, 589 insertions(+), 152 deletions(-) create mode 100644 Sources/WisprApp/Services/MeetingDiarizer.swift create mode 100644 wisprTests/MeetingDiarizerTests.swift diff --git a/Sources/WisprApp/Models/MeetingTranscript.swift b/Sources/WisprApp/Models/MeetingTranscript.swift index fbe9cc2..823a6f9 100644 --- a/Sources/WisprApp/Models/MeetingTranscript.swift +++ b/Sources/WisprApp/Models/MeetingTranscript.swift @@ -8,9 +8,23 @@ import Foundation /// Identifies the audio source / speaker in a meeting transcript. -enum MeetingSpeaker: String, Sendable, Equatable, Hashable { - case you = "You" - case others = "Others" +/// +/// Speaker indices are stored **0-based** internally. The +1 transform to +/// "Speaker 1", "Speaker 2", etc. happens only inside `displayName`. +enum MeetingSpeaker: Sendable, Equatable, Hashable { + case you + /// Remote participant. `speakerIndex` is nil during Sortformer's cold-start + /// window or when diarization is disabled — renders as plain "Others". + case others(speakerIndex: Int?) + + /// User-visible label for transcript display and plain-text export. + var displayName: String { + switch self { + case .you: return "You" + case .others(.none): return "Others" + case .others(.some(let index)): return "Speaker \(index + 1)" + } + } } /// A single timestamped entry in a meeting transcript. @@ -53,7 +67,7 @@ struct MeetingTranscript: Sendable, Equatable { func asPlainText() -> String { entries.map { entry in let time = Self.formatTime(entry.timestamp) - return "[\(time)] \(entry.speaker.rawValue): \(entry.text)" + return "[\(time)] \(entry.speaker.displayName): \(entry.text)" }.joined(separator: "\n") } diff --git a/Sources/WisprApp/Services/MeetingAudioEngine.swift b/Sources/WisprApp/Services/MeetingAudioEngine.swift index 345d444..beaf314 100644 --- a/Sources/WisprApp/Services/MeetingAudioEngine.swift +++ b/Sources/WisprApp/Services/MeetingAudioEngine.swift @@ -19,6 +19,13 @@ import ScreenCaptureKit import WisprCore import os +/// A timestamped audio chunk from the system-audio capture path. +/// `startTime` is seconds elapsed since capture began (cumulative sample count / sampleRate). +struct SystemAudioChunk: Sendable { + let samples: [Float] + let startTime: TimeInterval +} + /// Actor responsible for capturing both microphone and system audio simultaneously. /// /// Uses `AVAudioEngine` for microphone input and `SCStream` (ScreenCaptureKit) @@ -36,9 +43,11 @@ actor MeetingAudioEngine { private var micBuffer: [Float] = [] private var systemBuffer: [Float] = [] + private var systemSamplesTotal: Int = 0 + private var systemChunkStartSamples: Int = 0 private var micContinuation: AsyncStream<[Float]>.Continuation? - private var systemContinuation: AsyncStream<[Float]>.Continuation? + private var systemContinuation: AsyncStream.Continuation? private var micLevelContinuation: AsyncStream.Continuation? private var systemLevelContinuation: AsyncStream.Continuation? @@ -54,12 +63,15 @@ actor MeetingAudioEngine { /// The audio chunk streams created at capture start. private var _micAudioStream: AsyncStream<[Float]>? - private var _systemAudioStream: AsyncStream<[Float]>? + private var _systemAudioStream: AsyncStream? /// The chunk size in samples before yielding to the transcription stream. /// ~5 seconds of audio at 16kHz = 80,000 samples. private let chunkSize = 80_000 + /// Audio sample rate used for both capture and diarization (Hz). + static let sampleRate: Int = 16_000 + // MARK: - Public Interface /// Starts dual audio capture (microphone + system audio). @@ -86,7 +98,7 @@ actor MeetingAudioEngine { _micAudioStream = micStream micContinuation = micCont - let (sysStream, sysCont) = AsyncStream.makeStream(of: [Float].self) + let (sysStream, sysCont) = AsyncStream.makeStream(of: SystemAudioChunk.self) _systemAudioStream = sysStream systemContinuation = sysCont @@ -131,9 +143,9 @@ actor MeetingAudioEngine { } /// Returns the system audio chunk stream created during `startCapture()`. - var systemAudioStream: AsyncStream<[Float]> { + var systemAudioStream: AsyncStream { if let stream = _systemAudioStream { return stream } - let (stream, cont) = AsyncStream.makeStream(of: [Float].self) + let (stream, cont) = AsyncStream.makeStream(of: SystemAudioChunk.self) cont.finish() return stream } @@ -145,7 +157,8 @@ actor MeetingAudioEngine { micBuffer.removeAll() } if !systemBuffer.isEmpty { - systemContinuation?.yield(systemBuffer) + let startTime = Double(systemChunkStartSamples) / Double(Self.sampleRate) + systemContinuation?.yield(SystemAudioChunk(samples: systemBuffer, startTime: startTime)) systemBuffer.removeAll() } micContinuation?.finish() @@ -299,7 +312,7 @@ actor MeetingAudioEngine { config.minimumFrameInterval = CMTime(value: 1, timescale: 1) config.capturesAudio = true config.excludesCurrentProcessAudio = true - config.sampleRate = 16000 + config.sampleRate = Self.sampleRate config.channelCount = 1 let (systemBridgeStream, systemBridgeCont) = AsyncStream.makeStream(of: [Float].self) @@ -335,13 +348,17 @@ actor MeetingAudioEngine { let normalizedLevel = min(max(rms * 5.0, 0.0), 1.0) systemLevelContinuation?.yield(normalizedLevel) + systemSamplesTotal += samples.count systemBuffer.append(contentsOf: samples) if systemBuffer.count >= chunkSize { let chunk = Array(systemBuffer.prefix(chunkSize)) systemBuffer.removeFirst(min(chunkSize, systemBuffer.count)) + let startTime = Double(systemChunkStartSamples) / Double(Self.sampleRate) + systemChunkStartSamples = systemSamplesTotal - systemBuffer.count Log.audioEngine.debug( - "MeetingAudioEngine — yielding system chunk of \(chunk.count) samples") - systemContinuation?.yield(chunk) + "MeetingAudioEngine — yielding system chunk of \(chunk.count) samples at t=\(String(format: "%.2f", startTime))s" + ) + systemContinuation?.yield(SystemAudioChunk(samples: chunk, startTime: startTime)) } } @@ -353,6 +370,8 @@ actor MeetingAudioEngine { systemStream = nil systemStreamOutput = nil systemBuffer.removeAll() + systemSamplesTotal = 0 + systemChunkStartSamples = 0 systemContinuation?.finish() systemContinuation = nil systemLevelContinuation?.finish() diff --git a/Sources/WisprApp/Services/MeetingDiarizer.swift b/Sources/WisprApp/Services/MeetingDiarizer.swift new file mode 100644 index 0000000..07a9452 --- /dev/null +++ b/Sources/WisprApp/Services/MeetingDiarizer.swift @@ -0,0 +1,152 @@ +// +// MeetingDiarizer.swift +// wispr +// +// Actor wrapping FluidAudio's SortformerDiarizer for real-time speaker +// diarization on the system-audio ("Others") track. +// +// Only the system-audio track is diarized. The mic track keeps the "You" +// label. Diarization is gated by SettingsStore.meetingDiarizationEnabled +// and requires a one-time Sortformer model download (~30 MB). +// + +import FluidAudio +import Foundation +import WisprCore +import os + +/// Actor that wraps `SortformerDiarizer` and exposes a simple ingest/query API +/// for attributing transcription chunks to individual remote speakers. +/// +/// Speaker slot indices from Sortformer (0-based, up to 4) are mapped to +/// **monotonically assigned** per-meeting indices so that the first speaker +/// heard always becomes index 0 ("Speaker 1"), the second becomes index 1 +/// ("Speaker 2"), and so on regardless of which Sortformer slot they occupy. +/// +/// Cold-start: Sortformer needs a few hundred ms of audio before its timeline +/// is populated. `dominantSpeaker(in:)` returns `nil` for the first chunk or +/// two; callers should treat `nil` as "unknown" and render as plain "Others". +actor MeetingDiarizer { + + // MARK: - Internal segment cache + + /// A resolved speaker segment (fully internal, Sendable value type). + private struct Segment { + let assignedIndex: Int + let startTime: Float + let endTime: Float + } + + // MARK: - State + + private let sortformer = SortformerDiarizer() + private var isInitialized = false + + /// Accumulated finalized speaker segments from all processed chunks. + private var finalizedSegments: [Segment] = [] + /// Latest tentative segments (replaced on each process() call). + private var tentativeSegments: [Segment] = [] + + /// Sortformer slot → per-meeting monotonic index (0-based). + private var slotToAssigned: [Int: Int] = [:] + private var nextAssigned: Int = 0 + + // MARK: - Public Interface + + /// Downloads/loads the Sortformer CoreML model and initializes the diarizer. + /// + /// Safe to call multiple times — returns immediately if already initialized. + /// Throws if the model download or compilation fails. + func warmUp() async throws { + guard !isInitialized else { return } + + Log.diarizer.info("MeetingDiarizer — loading Sortformer model") + let models = try await SortformerModels.loadFromHuggingFace( + config: .default, + cacheDirectory: ModelPaths.sortformer + ) + sortformer.initialize(models: models) + isInitialized = true + Log.diarizer.info("MeetingDiarizer — Sortformer initialized") + } + + /// Feeds a system-audio chunk into the diarizer. + /// + /// The `startTime` is used for logging only; the diarizer computes its own + /// timeline from the sample count. Errors are logged and swallowed so that + /// a diarizer failure never interrupts transcription. + func ingest(_ samples: [Float], at startTime: TimeInterval) { + guard isInitialized else { return } + do { + sortformer.addAudio(samples) + if let update = try sortformer.process() { + // Accumulate newly finalized segments + for seg in update.finalizedSegments { + let idx = assignedIndex(for: seg.speakerIndex) + finalizedSegments.append( + Segment(assignedIndex: idx, startTime: seg.startTime, endTime: seg.endTime) + ) + } + // Replace tentative segments with the latest snapshot + tentativeSegments = update.tentativeSegments.map { seg in + Segment( + assignedIndex: assignedIndex(for: seg.speakerIndex), + startTime: seg.startTime, + endTime: seg.endTime + ) + } + } + } catch { + Log.diarizer.warning( + "MeetingDiarizer — ingest error at t=\(String(format: "%.2f", startTime))s: \(error.localizedDescription)" + ) + } + } + + /// Returns the 0-based assigned index of the speaker who dominated the + /// given time window, or `nil` if no speaker data is available yet. + /// + /// Finalized segments are queried first; tentative segments serve as + /// a best-effort fallback for the most recent window. + func dominantSpeaker(in window: ClosedRange) -> Int? { + guard isInitialized else { return nil } + let wStart = Float(window.lowerBound) + let wEnd = Float(window.upperBound) + + // Search finalized first, then tentative + for source in [finalizedSegments, tentativeSegments] { + var totals: [Int: Float] = [:] + for seg in source { + let s = max(seg.startTime, wStart) + let e = min(seg.endTime, wEnd) + if e > s { totals[seg.assignedIndex, default: 0] += (e - s) } + } + if let best = totals.max(by: { $0.value < $1.value }) { + return best.key + } + } + return nil + } + + /// Resets all internal diarization state for a new meeting session. + func reset() { + sortformer.reset() + finalizedSegments.removeAll() + tentativeSegments.removeAll() + slotToAssigned.removeAll() + nextAssigned = 0 + Log.diarizer.debug("MeetingDiarizer — reset") + } + + // MARK: - Private + + /// Returns the per-meeting monotonic index for a Sortformer slot, creating + /// a new assignment if the slot hasn't been seen before this session. + private func assignedIndex(for slot: Int) -> Int { + if let existing = slotToAssigned[slot] { return existing } + let idx = nextAssigned + slotToAssigned[slot] = idx + nextAssigned += 1 + return idx + } +} diff --git a/Sources/WisprApp/Services/MeetingStateManager.swift b/Sources/WisprApp/Services/MeetingStateManager.swift index 62d746b..e9ab715 100644 --- a/Sources/WisprApp/Services/MeetingStateManager.swift +++ b/Sources/WisprApp/Services/MeetingStateManager.swift @@ -63,6 +63,14 @@ final class MeetingStateManager { private let transcriptionEngine: any TranscriptionEngine private let settingsStore: SettingsStore + /// Optional speaker-diarization engine for the system-audio ("Others") track. + /// `nil` disables diarization entirely (e.g. in tests). When present, it is + /// only warmed up if `settingsStore.meetingDiarizationEnabled` is true. + private let meetingDiarizer: MeetingDiarizer? + + /// Whether diarization is active for the current session (set at startMeeting). + private var diarizationActive = false + // MARK: - Tasks private var recordingTask: Task? @@ -72,11 +80,13 @@ final class MeetingStateManager { init( meetingAudioEngine: MeetingAudioEngine, transcriptionEngine: any TranscriptionEngine, - settingsStore: SettingsStore + settingsStore: SettingsStore, + meetingDiarizer: MeetingDiarizer? = nil ) { self.meetingAudioEngine = meetingAudioEngine self.transcriptionEngine = transcriptionEngine self.settingsStore = settingsStore + self.meetingDiarizer = meetingDiarizer } // MARK: - Meeting Lifecycle @@ -90,6 +100,21 @@ final class MeetingStateManager { transcript = MeetingTranscript() errorMessage = nil + // Warm up the diarizer when enabled. On failure, degrade gracefully to + // source-based labels (the same fallback used when system audio is + // unavailable) rather than blocking the meeting. + diarizationActive = false + if settingsStore.meetingDiarizationEnabled, let diarizer = meetingDiarizer { + do { + try await diarizer.warmUp() + diarizationActive = true + } catch { + Log.stateManager.warning( + "MeetingStateManager — diarizer warmup failed: \(error.localizedDescription). Continuing without diarization." + ) + } + } + do { let (micLevels, systemLevels) = try await meetingAudioEngine.startCapture() @@ -126,6 +151,10 @@ final class MeetingStateManager { recordingTask = nil await meetingAudioEngine.stopCapture() + if diarizationActive { + await meetingDiarizer?.reset() + diarizationActive = false + } meetingState = .idle micLevel = 0 @@ -183,18 +212,35 @@ final class MeetingStateManager { private func transcribeSystemAudio() async { let audioStream = await meetingAudioEngine.systemAudioStream let language = settingsStore.languageMode + let diarizer = diarizationActive ? meetingDiarizer : nil - for await chunk in audioStream { + for await audioChunk in audioStream { guard !Task.isCancelled else { break } - guard chunk.count >= 8000 else { continue } + guard audioChunk.samples.count >= 8000 else { continue } + + // Feed the chunk to the diarizer before transcribing so its timeline + // covers this window by the time we query the dominant speaker. + await diarizer?.ingest(audioChunk.samples, at: audioChunk.startTime) do { - let result = try await transcriptionEngine.transcribe(chunk, language: language) + let result = try await transcriptionEngine.transcribe( + audioChunk.samples, language: language) let text = result.text.trimmingCharacters(in: .whitespacesAndNewlines) guard !text.isEmpty else { continue } + // Resolve the dominant speaker over the chunk's time window. + // nil (cold-start or diarization disabled) renders as "Others". + var speakerIndex: Int? + if let diarizer { + let chunkEnd = + audioChunk.startTime + + Double(audioChunk.samples.count) / Double(MeetingAudioEngine.sampleRate) + speakerIndex = await diarizer.dominantSpeaker( + in: audioChunk.startTime...chunkEnd) + } + transcript.entries.append( - MeetingTranscriptEntry(speaker: .others, text: text) + MeetingTranscriptEntry(speaker: .others(speakerIndex: speakerIndex), text: text) ) } catch { if case WisprError.emptyTranscription = error { continue } diff --git a/Sources/WisprApp/Services/SettingsStore.swift b/Sources/WisprApp/Services/SettingsStore.swift index 086d234..2218457 100644 --- a/Sources/WisprApp/Services/SettingsStore.swift +++ b/Sources/WisprApp/Services/SettingsStore.swift @@ -5,10 +5,10 @@ // Settings persistence using UserDefaults // -import WisprCore import Foundation import Observation import ServiceManagement +import WisprCore import os @MainActor @@ -16,21 +16,33 @@ import os final class SettingsStore { // MARK: - Hotkey Settings var hotkeyKeyCode: UInt32 { - didSet { guard !isLoading else { return }; defaults.set(Int(hotkeyKeyCode), forKey: Keys.hotkeyKeyCode) } + didSet { + guard !isLoading else { return } + defaults.set(Int(hotkeyKeyCode), forKey: Keys.hotkeyKeyCode) + } } var hotkeyModifiers: UInt32 { - didSet { guard !isLoading else { return }; defaults.set(Int(hotkeyModifiers), forKey: Keys.hotkeyModifiers) } + didSet { + guard !isLoading else { return } + defaults.set(Int(hotkeyModifiers), forKey: Keys.hotkeyModifiers) + } } // MARK: - Audio Settings var selectedAudioDeviceUID: String? { - didSet { guard !isLoading else { return }; defaults.set(selectedAudioDeviceUID, forKey: Keys.selectedAudioDeviceUID) } + didSet { + guard !isLoading else { return } + defaults.set(selectedAudioDeviceUID, forKey: Keys.selectedAudioDeviceUID) + } } // MARK: - Model Settings var activeModelName: String { - didSet { guard !isLoading else { return }; defaults.set(activeModelName, forKey: Keys.activeModelName) } + didSet { + guard !isLoading else { return } + defaults.set(activeModelName, forKey: Keys.activeModelName) + } } // MARK: - Language Settings @@ -45,7 +57,10 @@ final class SettingsStore { // MARK: - General Settings var showRecordingOverlay: Bool { - didSet { guard !isLoading else { return }; defaults.set(showRecordingOverlay, forKey: Keys.showRecordingOverlay) } + didSet { + guard !isLoading else { return } + defaults.set(showRecordingOverlay, forKey: Keys.showRecordingOverlay) + } } var launchAtLogin: Bool { @@ -56,57 +71,94 @@ final class SettingsStore { } var onboardingCompleted: Bool { - didSet { guard !isLoading else { return }; defaults.set(onboardingCompleted, forKey: Keys.onboardingCompleted) } + didSet { + guard !isLoading else { return } + defaults.set(onboardingCompleted, forKey: Keys.onboardingCompleted) + } } var onboardingLastStep: Int { - didSet { guard !isLoading else { return }; defaults.set(onboardingLastStep, forKey: Keys.onboardingLastStep) } + didSet { + guard !isLoading else { return } + defaults.set(onboardingLastStep, forKey: Keys.onboardingLastStep) + } } - + // MARK: - Dictation Mode /// When true, hotkey toggles recording on/off (press once to start, press again to stop). /// When false, uses push-to-talk (hold to record, release to stop). var handsFreeMode: Bool { - didSet { guard !isLoading else { return }; defaults.set(handsFreeMode, forKey: Keys.handsFreeMode) } + didSet { + guard !isLoading else { return } + defaults.set(handsFreeMode, forKey: Keys.handsFreeMode) + } + } + + /// When true, the "Others" track is split into per-speaker labels + /// (Speaker 1, Speaker 2, …) using on-device Sortformer diarization. + /// Requires a one-time model download (~30 MB). Defaults to false. + var meetingDiarizationEnabled: Bool { + didSet { + guard !isLoading else { return } + defaults.set(meetingDiarizationEnabled, forKey: Keys.meetingDiarizationEnabled) + } } /// When true, plays short audio cues on recording start/stop. var soundFeedbackEnabled: Bool { - didSet { guard !isLoading else { return }; defaults.set(soundFeedbackEnabled, forKey: Keys.soundFeedbackEnabled) } + didSet { + guard !isLoading else { return } + defaults.set(soundFeedbackEnabled, forKey: Keys.soundFeedbackEnabled) + } } // MARK: - Auto-Suffix Settings /// When true, appends `autoSuffixText` to transcribed text before insertion. var autoSuffixEnabled: Bool { - didSet { guard !isLoading else { return }; defaults.set(autoSuffixEnabled, forKey: Keys.autoSuffixEnabled) } + didSet { + guard !isLoading else { return } + defaults.set(autoSuffixEnabled, forKey: Keys.autoSuffixEnabled) + } } /// The suffix string appended to transcribed text when `autoSuffixEnabled` is true. var autoSuffixText: String { - didSet { guard !isLoading else { return }; defaults.set(autoSuffixText, forKey: Keys.autoSuffixText) } + didSet { + guard !isLoading else { return } + defaults.set(autoSuffixText, forKey: Keys.autoSuffixText) + } } // MARK: - Filler Word Removal Settings /// When true, removes common filler words (um, uh, ah, etc.) from transcriptions before insertion. var removeFillerWords: Bool { - didSet { guard !isLoading else { return }; defaults.set(removeFillerWords, forKey: Keys.removeFillerWords) } + didSet { + guard !isLoading else { return } + defaults.set(removeFillerWords, forKey: Keys.removeFillerWords) + } } // MARK: - Auto-Send Enter Settings /// When true, simulates an Enter/Return keystroke after text insertion. var autoSendEnterEnabled: Bool { - didSet { guard !isLoading else { return }; defaults.set(autoSendEnterEnabled, forKey: Keys.autoSendEnterEnabled) } + didSet { + guard !isLoading else { return } + defaults.set(autoSendEnterEnabled, forKey: Keys.autoSendEnterEnabled) + } } // MARK: - AI Text Correction Settings /// When true, applies on-device AI text correction after filler word removal. var aiTextCorrectionEnabled: Bool { - didSet { guard !isLoading else { return }; defaults.set(aiTextCorrectionEnabled, forKey: Keys.aiTextCorrectionEnabled) } + didSet { + guard !isLoading else { return } + defaults.set(aiTextCorrectionEnabled, forKey: Keys.aiTextCorrectionEnabled) + } } /// The correction style used by AI text correction. @@ -131,6 +183,7 @@ final class SettingsStore { static let onboardingCompleted = "onboardingCompleted" static let onboardingLastStep = "onboardingLastStep" static let handsFreeMode = "handsFreeMode" + static let meetingDiarizationEnabled = "meetingDiarizationEnabled" static let soundFeedbackEnabled = "soundFeedbackEnabled" static let autoSuffixEnabled = "autoSuffixEnabled" static let autoSuffixText = "autoSuffixText" @@ -139,14 +192,14 @@ final class SettingsStore { static let aiTextCorrectionEnabled = "aiTextCorrectionEnabled" static let aiTextCorrectionStyle = "aiTextCorrectionStyle" } - + // MARK: - Default Values /// Single source of truth for all setting defaults. /// Referenced by `init`, `restoreDefaults()`, and tests. enum Defaults { - static let hotkeyKeyCode: UInt32 = 49 // Space - static let hotkeyModifiers: UInt32 = 2048 // Option + static let hotkeyKeyCode: UInt32 = 49 // Space + static let hotkeyModifiers: UInt32 = 2048 // Option static let selectedAudioDeviceUID: String? = nil static let activeModelName: String = ModelInfo.KnownID.tiny static let languageMode: TranscriptionLanguage = .autoDetect @@ -155,6 +208,7 @@ final class SettingsStore { static let onboardingCompleted: Bool = false static let onboardingLastStep: Int = 0 static let handsFreeMode: Bool = false + static let meetingDiarizationEnabled: Bool = false static let soundFeedbackEnabled: Bool = false static let autoSuffixEnabled: Bool = false static let autoSuffixText: String = " " @@ -167,11 +221,11 @@ final class SettingsStore { // MARK: - Dependencies private let defaults: UserDefaults private var isLoading = false - + // MARK: - Initialization init(defaults: UserDefaults = .standard) { self.defaults = defaults - + // Initialize with defaults self.hotkeyKeyCode = Defaults.hotkeyKeyCode self.hotkeyModifiers = Defaults.hotkeyModifiers @@ -183,6 +237,7 @@ final class SettingsStore { self.onboardingCompleted = Defaults.onboardingCompleted self.onboardingLastStep = Defaults.onboardingLastStep self.handsFreeMode = Defaults.handsFreeMode + self.meetingDiarizationEnabled = Defaults.meetingDiarizationEnabled self.soundFeedbackEnabled = Defaults.soundFeedbackEnabled self.autoSuffixEnabled = Defaults.autoSuffixEnabled self.autoSuffixText = Defaults.autoSuffixText @@ -209,6 +264,7 @@ final class SettingsStore { showRecordingOverlay = Defaults.showRecordingOverlay launchAtLogin = Defaults.launchAtLogin handsFreeMode = Defaults.handsFreeMode + meetingDiarizationEnabled = Defaults.meetingDiarizationEnabled soundFeedbackEnabled = Defaults.soundFeedbackEnabled autoSuffixEnabled = Defaults.autoSuffixEnabled autoSuffixText = Defaults.autoSuffixText @@ -217,7 +273,7 @@ final class SettingsStore { aiTextCorrectionEnabled = Defaults.aiTextCorrectionEnabled aiTextCorrectionStyle = Defaults.aiTextCorrectionStyle } - + // MARK: - Persistence /// Persists all current values to UserDefaults without forcing a disk flush. @@ -234,6 +290,7 @@ final class SettingsStore { defaults.set(onboardingCompleted, forKey: Keys.onboardingCompleted) defaults.set(onboardingLastStep, forKey: Keys.onboardingLastStep) defaults.set(handsFreeMode, forKey: Keys.handsFreeMode) + defaults.set(meetingDiarizationEnabled, forKey: Keys.meetingDiarizationEnabled) defaults.set(soundFeedbackEnabled, forKey: Keys.soundFeedbackEnabled) defaults.set(autoSuffixEnabled, forKey: Keys.autoSuffixEnabled) defaults.set(autoSuffixText, forKey: Keys.autoSuffixText) @@ -257,49 +314,54 @@ final class SettingsStore { save() defaults.synchronize() } - + func load() { isLoading = true defer { isLoading = false } - + // Load hotkey settings let storedKeyCode = defaults.integer(forKey: Keys.hotkeyKeyCode) if storedKeyCode != 0 || defaults.object(forKey: Keys.hotkeyKeyCode) != nil { self.hotkeyKeyCode = UInt32(storedKeyCode) } - + let storedModifiers = defaults.integer(forKey: Keys.hotkeyModifiers) if storedModifiers != 0 || defaults.object(forKey: Keys.hotkeyModifiers) != nil { self.hotkeyModifiers = UInt32(storedModifiers) } - + // Load audio settings self.selectedAudioDeviceUID = defaults.string(forKey: Keys.selectedAudioDeviceUID) - + // Load model settings if let modelName = defaults.string(forKey: Keys.activeModelName) { self.activeModelName = modelName } - + // Load language mode if let data = defaults.data(forKey: Keys.languageMode), - let decoded = try? JSONDecoder().decode(TranscriptionLanguage.self, from: data) { + let decoded = try? JSONDecoder().decode(TranscriptionLanguage.self, from: data) + { self.languageMode = decoded } - + // Load general settings if defaults.object(forKey: Keys.showRecordingOverlay) != nil { self.showRecordingOverlay = defaults.bool(forKey: Keys.showRecordingOverlay) } self.launchAtLogin = SMAppService.mainApp.status == .enabled self.onboardingCompleted = defaults.bool(forKey: Keys.onboardingCompleted) - + self.onboardingLastStep = defaults.integer(forKey: Keys.onboardingLastStep) - + if defaults.object(forKey: Keys.handsFreeMode) != nil { self.handsFreeMode = defaults.bool(forKey: Keys.handsFreeMode) } + if defaults.object(forKey: Keys.meetingDiarizationEnabled) != nil { + self.meetingDiarizationEnabled = defaults.bool(forKey: Keys.meetingDiarizationEnabled) + } + if defaults.object(forKey: Keys.soundFeedbackEnabled) != nil { self.soundFeedbackEnabled = defaults.bool(forKey: Keys.soundFeedbackEnabled) } @@ -329,13 +391,14 @@ final class SettingsStore { } if let data = defaults.data(forKey: Keys.aiTextCorrectionStyle), - let decoded = try? JSONDecoder().decode(CorrectionStyle.self, from: data) { + let decoded = try? JSONDecoder().decode(CorrectionStyle.self, from: data) + { self.aiTextCorrectionStyle = decoded } } - + // MARK: - Launch at Login - + /// Registers or unregisters the app as a login item using ServiceManagement. /// After the operation, reads back the actual system state so the toggle /// always reflects reality. @@ -355,7 +418,7 @@ final class SettingsStore { } catch { Log.app.error("Failed to \(enabled ? "register" : "unregister") login item: \(error)") } - + // Always sync back to the actual system state. // The source of truth for launch-at-login is ServiceManagement, not UserDefaults, // so no explicit defaults.set is needed — load() reads from SMAppService.mainApp.status. diff --git a/Sources/WisprApp/UI/Meeting/MeetingTranscriptView.swift b/Sources/WisprApp/UI/Meeting/MeetingTranscriptView.swift index 6a2b8f2..a9bf5ab 100644 --- a/Sources/WisprApp/UI/Meeting/MeetingTranscriptView.swift +++ b/Sources/WisprApp/UI/Meeting/MeetingTranscriptView.swift @@ -187,10 +187,10 @@ struct MeetingTranscriptView: View { .frame(width: 50, alignment: .trailing) // Speaker badge - Text(entry.speaker.rawValue) + Text(entry.speaker.displayName) .font(.caption.weight(.semibold)) - .foregroundStyle(entry.speaker == .you ? .blue : .green) - .frame(width: 48, alignment: .leading) + .foregroundStyle(speakerColor(entry.speaker)) + .frame(width: 56, alignment: .leading) // Text Text(entry.text) @@ -206,6 +206,21 @@ struct MeetingTranscriptView: View { MeetingTranscript.formatTime(date) } + /// Color for a speaker badge. "You" is blue; each diarized remote speaker + /// gets a distinct color, cycling for indices beyond the palette. Unknown + /// (cold-start / diarization off) remote speech is gray. + private func speakerColor(_ speaker: MeetingSpeaker) -> Color { + switch speaker { + case .you: + return .blue + case .others(.none): + return .gray + case .others(.some(let index)): + let palette: [Color] = [.green, .orange, .purple, .pink] + return palette[index % palette.count] + } + } + // MARK: - Footer private var footerBar: some View { diff --git a/Sources/WisprApp/UI/Settings/SettingsView.swift b/Sources/WisprApp/UI/Settings/SettingsView.swift index c2a9338..1d58183 100644 --- a/Sources/WisprApp/UI/Settings/SettingsView.swift +++ b/Sources/WisprApp/UI/Settings/SettingsView.swift @@ -6,8 +6,8 @@ // Recognition, After Transcription, Feedback, and General. // -import WisprCore import SwiftUI +import WisprCore import os // MARK: - Reusable Components @@ -42,20 +42,25 @@ struct SettingsView: View { enum AccessibilityHints { // Shortcut section static let hotkeyShortcut = "Activate to record a new hotkey combination" - static let handsFreeMode = "When enabled, press the hotkey once to start recording and again to stop. When disabled, hold the hotkey to record." + static let handsFreeMode = + "When enabled, press the hotkey once to start recording and again to stop. When disabled, hold the hotkey to record." // Audio Device section static let inputDevice = "Select the microphone to use for recording" // Recognition section static let activeModel = "Select the speech recognition model to use" - static let autoDetectLanguage = "When enabled, Wispr automatically detects the spoken language" + static let autoDetectLanguage = + "When enabled, Wispr automatically detects the spoken language" static let languagePicker = "Select the language for speech transcription" - static let alwaysUseLanguage = "When enabled, always transcribes in the selected language instead of detecting per-recording" + static let alwaysUseLanguage = + "When enabled, always transcribes in the selected language instead of detecting per-recording" // After Transcription section - static let removeFillerWords = "When enabled, removes filler words like um, uh, and ah from transcriptions" - static let aiTextCorrection = "When enabled, uses on-device AI to correct grammar and improve transcription fluency. All processing stays on your Mac." + static let removeFillerWords = + "When enabled, removes filler words like um, uh, and ah from transcriptions" + static let aiTextCorrection = + "When enabled, uses on-device AI to correct grammar and improve transcription fluency. All processing stays on your Mac." static let autoInsertSuffix = "When enabled, appends a suffix to transcribed text" static let autoSendEnter = "When enabled, simulates pressing Enter after text insertion" @@ -63,6 +68,10 @@ struct SettingsView: View { static let soundFeedback = "When enabled, plays audio cues when recording starts and stops" static let showRecordingOverlay = "When enabled, a floating overlay appears while recording" + // Meeting section + static let meetingDiarization = + "When enabled, the meeting transcript labels remote participants as Speaker 1, Speaker 2, and so on using on-device diarization" + // General section static let launchAtLogin = "When enabled, Wispr starts automatically when you log in" static let restoreDefaults = "Resets all settings to their original values" @@ -72,7 +81,8 @@ struct SettingsView: View { @Environment(UpdateChecker.self) private var updateChecker: UpdateChecker @Environment(StateManager.self) private var stateManager: StateManager @Environment(HotkeyMonitor.self) private var hotkeyMonitor: HotkeyMonitor - @Environment(TextCorrectionService.self) private var textCorrectionService: TextCorrectionService + @Environment(TextCorrectionService.self) private var textCorrectionService: + TextCorrectionService @State private var audioDevices: [AudioInputDevice] = [] @State private var whisperModels: [ModelInfo] = [] @@ -101,6 +111,7 @@ struct SettingsView: View { recognitionSection afterTranscriptionSection feedbackSection + meetingSection generalSection } .formStyle(.grouped) @@ -130,7 +141,8 @@ struct SettingsView: View { hotkeyError = nil } catch { hotkeyError = error.localizedDescription - Log.hotkey.error("Settings — failed to re-register hotkey: \(error.localizedDescription)") + Log.hotkey.error( + "Settings — failed to re-register hotkey: \(error.localizedDescription)") } } } @@ -160,9 +172,12 @@ struct SettingsView: View { } if settingsStore.hotkeyKeyCode == HotkeyMonitor.fnKeyCode - && settingsStore.hotkeyModifiers == 0 { + && settingsStore.hotkeyModifiers == 0 + { Label { - Text("The Globe key may conflict with macOS features like the emoji picker or input source switching. If dictation doesn't start, go to System Settings → Keyboard → \"Press 🌐 key to\" and select \"Do Nothing\".") + Text( + "The Globe key may conflict with macOS features like the emoji picker or input source switching. If dictation doesn't start, go to System Settings → Keyboard → \"Press 🌐 key to\" and select \"Do Nothing\"." + ) } icon: { Image(systemName: SFSymbols.info) .foregroundStyle(.blue) @@ -245,7 +260,8 @@ struct SettingsView: View { .accessibilityHint(AccessibilityHints.activeModel) .onChange(of: selectedModelId) { _, newModelId in guard newModelId != settingsStore.activeModelName, - !newModelId.isEmpty else { return } + !newModelId.isEmpty + else { return } activatingModelId = newModelId } .task(id: activatingModelId) { @@ -359,6 +375,28 @@ struct SettingsView: View { } } + // MARK: - Meeting Section + + private var meetingSection: some View { + Section { + @Bindable var store = settingsStore + Toggle("Identify Individual Speakers", isOn: $store.meetingDiarizationEnabled) + .accessibilityHint(AccessibilityHints.meetingDiarization) + + Text( + "Splits the \"Others\" track into Speaker 1, Speaker 2, … using on-device diarization. Downloads a small model on first use. Experimental." + ) + .font(.caption) + .foregroundStyle(theme.secondaryTextColor) + } header: { + SectionHeader( + title: "Meeting", + systemImage: SFSymbols.meeting, + tint: .indigo + ) + } + } + // MARK: - General Section private var generalSection: some View { @@ -403,13 +441,16 @@ struct SettingsView: View { ) } } - + // MARK: - Version Info - + /// Returns the app version string in the format "1.0.0 (123)" where 123 is the build number. private var appVersion: String { - let version = Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "Unknown" - let build = Bundle.main.object(forInfoDictionaryKey: "CFBundleVersion") as? String ?? "Unknown" + let version = + Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String + ?? "Unknown" + let build = + Bundle.main.object(forInfoDictionaryKey: "CFBundleVersion") as? String ?? "Unknown" return "\(version) (\(build))" } @@ -491,72 +532,73 @@ struct SettingsView: View { // MARK: - Preview #if DEBUG -private struct SettingsPreview: View { - @State private var settingsStore: SettingsStore - @State private var theme = PreviewMocks.makeTheme() - @State private var updateChecker = PreviewMocks.makeUpdateChecker() - @State private var stateManager: StateManager - @State private var textCorrectionService = TextCorrectionService() - - private let whisperService: any TranscriptionEngine + private struct SettingsPreview: View { + @State private var settingsStore: SettingsStore + @State private var theme = PreviewMocks.makeTheme() + @State private var updateChecker = PreviewMocks.makeUpdateChecker() + @State private var stateManager: StateManager + @State private var textCorrectionService = TextCorrectionService() + + private let whisperService: any TranscriptionEngine + + init( + autoSuffixEnabled: Bool = false, + autoSendEnterEnabled: Bool = false, + languageSpecific: Bool = false, + languagePinned: Bool = false + ) { + let store = PreviewMocks.makeSettingsStore() + store.autoSuffixEnabled = autoSuffixEnabled + store.autoSendEnterEnabled = autoSendEnterEnabled + if languagePinned { + store.languageMode = .pinned(code: "en") + } else if languageSpecific { + store.languageMode = .specific(code: "en") + } + self._settingsStore = State(initialValue: store) + + let service = PreviewMocks.makeWhisperService() + self.whisperService = service + self._stateManager = State( + initialValue: PreviewMocks.makeStateManager( + settingsStore: store, + whisperService: service + )) + } - init( - autoSuffixEnabled: Bool = false, - autoSendEnterEnabled: Bool = false, - languageSpecific: Bool = false, - languagePinned: Bool = false - ) { - let store = PreviewMocks.makeSettingsStore() - store.autoSuffixEnabled = autoSuffixEnabled - store.autoSendEnterEnabled = autoSendEnterEnabled - if languagePinned { - store.languageMode = .pinned(code: "en") - } else if languageSpecific { - store.languageMode = .specific(code: "en") + var body: some View { + SettingsView( + audioEngine: PreviewMocks.makeAudioEngine(), + whisperService: whisperService + ) + .environment(settingsStore) + .environment(theme) + .environment(updateChecker) + .environment(stateManager) + .environment(HotkeyMonitor()) + .environment(textCorrectionService) } - self._settingsStore = State(initialValue: store) - - let service = PreviewMocks.makeWhisperService() - self.whisperService = service - self._stateManager = State(initialValue: PreviewMocks.makeStateManager( - settingsStore: store, - whisperService: service - )) } - var body: some View { - SettingsView( - audioEngine: PreviewMocks.makeAudioEngine(), - whisperService: whisperService - ) - .environment(settingsStore) - .environment(theme) - .environment(updateChecker) - .environment(stateManager) - .environment(HotkeyMonitor()) - .environment(textCorrectionService) + #Preview("Settings") { + SettingsPreview() } -} -#Preview("Settings") { - SettingsPreview() -} - -#Preview("Settings - Dark") { - SettingsPreview() - .preferredColorScheme(.dark) -} + #Preview("Settings - Dark") { + SettingsPreview() + .preferredColorScheme(.dark) + } -#Preview("Settings - Suffix & Language Expanded") { - SettingsPreview(autoSuffixEnabled: true, languageSpecific: true) -} + #Preview("Settings - Suffix & Language Expanded") { + SettingsPreview(autoSuffixEnabled: true, languageSpecific: true) + } -#Preview("Settings - All Toggles On") { - SettingsPreview( - autoSuffixEnabled: true, - autoSendEnterEnabled: true, - languageSpecific: true, - languagePinned: true - ) -} + #Preview("Settings - All Toggles On") { + SettingsPreview( + autoSuffixEnabled: true, + autoSendEnterEnabled: true, + languageSpecific: true, + languagePinned: true + ) + } #endif diff --git a/Sources/WisprApp/Utilities/SFSymbols.swift b/Sources/WisprApp/Utilities/SFSymbols.swift index 8812a37..09f65bf 100644 --- a/Sources/WisprApp/Utilities/SFSymbols.swift +++ b/Sources/WisprApp/Utilities/SFSymbols.swift @@ -58,6 +58,9 @@ enum SFSymbols { /// Stop button icon for meeting transcription. static let stopFill = "stop.fill" + /// Icon for the meeting / speaker-diarization settings section. + static let meeting = "person.2.wave.2" + /// Checkmark (filled circle) for success / active states. static let checkmark = "checkmark.circle.fill" diff --git a/Sources/WisprApp/wisprApp.swift b/Sources/WisprApp/wisprApp.swift index cbaf1f2..43f01c7 100644 --- a/Sources/WisprApp/wisprApp.swift +++ b/Sources/WisprApp/wisprApp.swift @@ -8,8 +8,8 @@ // Requirements: 5.6, 13.1, 13.12, 13.16 // -import WisprCore import SwiftUI +import WisprCore import os /// Main application entry point for Wispr. @@ -95,6 +95,9 @@ final class WisprAppDelegate: NSObject, NSApplicationDelegate, NSWindowDelegate /// Meeting audio engine for dual capture (mic + system audio). let meetingAudioEngine = MeetingAudioEngine() + /// Speaker diarization engine for the meeting "Others" track. + let meetingDiarizer = MeetingDiarizer() + /// Central state coordinator — depends on all services above. private(set) var stateManager: StateManager? @@ -171,7 +174,8 @@ final class WisprAppDelegate: NSObject, NSApplicationDelegate, NSWindowDelegate let msm = MeetingStateManager( meetingAudioEngine: meetingAudioEngine, transcriptionEngine: whisperService, - settingsStore: settingsStore + settingsStore: settingsStore, + meetingDiarizer: meetingDiarizer ) meetingStateManager = msm diff --git a/Sources/WisprCore/Utilities/Logger.swift b/Sources/WisprCore/Utilities/Logger.swift index a9b4694..d70baa3 100644 --- a/Sources/WisprCore/Utilities/Logger.swift +++ b/Sources/WisprCore/Utilities/Logger.swift @@ -24,4 +24,5 @@ public nonisolated enum Log { public static let updateChecker = Logger(subsystem: subsystem, category: "UpdateChecker") public static let hotkey = Logger(subsystem: subsystem, category: "HotkeyMonitor") public static let textCorrection = Logger(subsystem: subsystem, category: "TextCorrection") + public static let diarizer = Logger(subsystem: subsystem, category: "Diarizer") } diff --git a/Sources/WisprCore/Utilities/ModelPaths.swift b/Sources/WisprCore/Utilities/ModelPaths.swift index 5a8a936..440daa4 100644 --- a/Sources/WisprCore/Utilities/ModelPaths.swift +++ b/Sources/WisprCore/Utilities/ModelPaths.swift @@ -37,10 +37,12 @@ public nonisolated enum ModelPaths { if isSandboxed { // Inside the sandbox (GUI app): FileManager automatically redirects // to ~/Library/Containers//Data/Library/Application Support/ - guard let appSupport = FileManager.default.urls( - for: .applicationSupportDirectory, - in: .userDomainMask - ).first else { + guard + let appSupport = FileManager.default.urls( + for: .applicationSupportDirectory, + in: .userDomainMask + ).first + else { fatalError("Application Support directory unavailable — cannot store models") } return appSupport.appendingPathComponent("wispr", isDirectory: true) @@ -59,10 +61,12 @@ public nonisolated enum ModelPaths { // Container doesn't exist yet (GUI never launched). Fall back to the // standard Application Support path so the CLI can still start up, // though no models will be found until the GUI downloads them. - guard let appSupport = FileManager.default.urls( - for: .applicationSupportDirectory, - in: .userDomainMask - ).first else { + guard + let appSupport = FileManager.default.urls( + for: .applicationSupportDirectory, + in: .userDomainMask + ).first + else { fatalError("Application Support directory unavailable — cannot store models") } return appSupport.appendingPathComponent("wispr", isDirectory: true) @@ -97,6 +101,11 @@ public nonisolated enum ModelPaths { models } + /// Sortformer streaming diarizer model cache: `/models/sortformer/` + public static var sortformer: URL { + models.appendingPathComponent("sortformer", isDirectory: true) + } + /// URL of the GUI app's UserDefaults plist. /// Inside the sandbox this is the standard location; outside (CLI) it /// points into the GUI's container so the CLI reads current values @@ -104,10 +113,12 @@ public nonisolated enum ModelPaths { public static var guiDefaultsPlist: URL { let isSandboxed = ProcessInfo.processInfo.environment["APP_SANDBOX_CONTAINER_ID"] != nil if isSandboxed { - guard let prefs = FileManager.default.urls( - for: .libraryDirectory, - in: .userDomainMask - ).first?.appendingPathComponent("Preferences/com.stormacq.mac.wispr.plist") else { + guard + let prefs = FileManager.default.urls( + for: .libraryDirectory, + in: .userDomainMask + ).first?.appendingPathComponent("Preferences/com.stormacq.mac.wispr.plist") + else { fatalError("Library directory unavailable") } return prefs diff --git a/wispr.xcodeproj/project.pbxproj b/wispr.xcodeproj/project.pbxproj index 4e41caa..94e0bd8 100644 --- a/wispr.xcodeproj/project.pbxproj +++ b/wispr.xcodeproj/project.pbxproj @@ -81,6 +81,7 @@ 1C74D4222FA6394300208826 /* WhisperKit in Frameworks */ = {isa = PBXBuildFile; productRef = 00C110102F60A00100000001 /* WhisperKit */; }; 1C74D4232FA6394300208826 /* FluidAudio in Frameworks */ = {isa = PBXBuildFile; productRef = 00C110112F60A00100000002 /* FluidAudio */; }; E4782D9F9369B23177982DC0 /* MeetingAudioEngine.swift in Sources */ = {isa = PBXBuildFile; fileRef = 85566A381EF49B2D8D58E9FE /* MeetingAudioEngine.swift */; }; + D1AD12340000000000000001 /* MeetingDiarizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = D1AD12340000000000000002 /* MeetingDiarizer.swift */; }; 0AF8E6B5B2AC76E3983230BF /* MeetingStateManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = DAA7524D483751DC8BD496D3 /* MeetingStateManager.swift */; }; EAABFC5DC0A565D10326809B /* MeetingTranscript.swift in Sources */ = {isa = PBXBuildFile; fileRef = F6CE95739BEEBDC90BA22323 /* MeetingTranscript.swift */; }; 5871C9411CAF50B12419F9DA /* MeetingTranscriptView.swift in Sources */ = {isa = PBXBuildFile; fileRef = F66665AA7D70423B4954925F /* MeetingTranscriptView.swift */; }; @@ -206,6 +207,7 @@ 1C74D2FC2FA632B700208826 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = ""; }; 1C74D2FD2FA632B700208826 /* wisprApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = wisprApp.swift; sourceTree = ""; }; 85566A381EF49B2D8D58E9FE /* MeetingAudioEngine.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingAudioEngine.swift; sourceTree = ""; }; + D1AD12340000000000000002 /* MeetingDiarizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingDiarizer.swift; sourceTree = ""; }; DAA7524D483751DC8BD496D3 /* MeetingStateManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingStateManager.swift; sourceTree = ""; }; F6CE95739BEEBDC90BA22323 /* MeetingTranscript.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingTranscript.swift; sourceTree = ""; }; F66665AA7D70423B4954925F /* MeetingTranscriptView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MeetingTranscriptView.swift; sourceTree = ""; }; @@ -398,6 +400,7 @@ children = ( 1C74D2D32FA632B700208826 /* AudioEngine.swift */, 85566A381EF49B2D8D58E9FE /* MeetingAudioEngine.swift */, + D1AD12340000000000000002 /* MeetingDiarizer.swift */, DAA7524D483751DC8BD496D3 /* MeetingStateManager.swift */, 1C74D2D42FA632B700208826 /* HotkeyMonitor.swift */, 1C74D2D52FA632B700208826 /* PermissionManager.swift */, @@ -745,6 +748,7 @@ 1C74D3452FA632B700208826 /* OnboardingAccessibilityStep.swift in Sources */, 1C74D3462FA632B700208826 /* AudioEngine.swift in Sources */, E4782D9F9369B23177982DC0 /* MeetingAudioEngine.swift in Sources */, + D1AD12340000000000000001 /* MeetingDiarizer.swift in Sources */, 0AF8E6B5B2AC76E3983230BF /* MeetingStateManager.swift in Sources */, EAABFC5DC0A565D10326809B /* MeetingTranscript.swift in Sources */, AEAC6284FE7B470A7CB12A52 /* TranscriptDocument.swift in Sources */, diff --git a/wisprTests/MeetingAudioEngineTests.swift b/wisprTests/MeetingAudioEngineTests.swift index 6eaa3c2..4ad2da3 100644 --- a/wisprTests/MeetingAudioEngineTests.swift +++ b/wisprTests/MeetingAudioEngineTests.swift @@ -42,7 +42,7 @@ struct MeetingAudioEngineTests { let engine = MeetingAudioEngine() let stream = await engine.systemAudioStream - var chunks: [[Float]] = [] + var chunks: [SystemAudioChunk] = [] for await chunk in stream { chunks.append(chunk) } diff --git a/wisprTests/MeetingDiarizerTests.swift b/wisprTests/MeetingDiarizerTests.swift new file mode 100644 index 0000000..ab8e2b6 --- /dev/null +++ b/wisprTests/MeetingDiarizerTests.swift @@ -0,0 +1,59 @@ +// +// MeetingDiarizerTests.swift +// wisprTests +// +// Unit tests for MeetingDiarizer's safe pre-initialization contract. +// +// Note: Exercising real diarization quality requires loading the ~30 MB +// Sortformer CoreML model and feeding multi-speaker audio, which is covered +// by the manual end-to-end verification plan in +// `.kiro/specs/meeting-speaker-diarization/design.md` rather than unit tests. +// These tests verify the actor is safe to call before (or without) warmUp(). +// + +import Foundation +import Testing + +@testable import WisprApp + +@Suite("MeetingDiarizer Tests") +struct MeetingDiarizerTests { + + @Test("dominantSpeaker returns nil before warmUp") + func testDominantSpeakerNilBeforeWarmUp() async { + let diarizer = MeetingDiarizer() + let result = await diarizer.dominantSpeaker(in: 0.0...5.0) + #expect(result == nil) + } + + @Test("ingest before warmUp is a safe no-op") + func testIngestBeforeWarmUpIsNoOp() async { + let diarizer = MeetingDiarizer() + let samples = [Float](repeating: 0.1, count: 16_000) + + // Should not crash or throw — ingest guards on isInitialized. + await diarizer.ingest(samples, at: 0.0) + + // Still no speaker data available. + let result = await diarizer.dominantSpeaker(in: 0.0...1.0) + #expect(result == nil) + } + + @Test("reset before warmUp is a safe no-op") + func testResetBeforeWarmUpIsNoOp() async { + let diarizer = MeetingDiarizer() + + // Should not crash on a fresh, uninitialized diarizer. + await diarizer.reset() + + let result = await diarizer.dominantSpeaker(in: 0.0...5.0) + #expect(result == nil) + } + + @Test("dominantSpeaker handles zero-width window before warmUp") + func testDominantSpeakerZeroWidthWindow() async { + let diarizer = MeetingDiarizer() + let result = await diarizer.dominantSpeaker(in: 2.5...2.5) + #expect(result == nil) + } +} diff --git a/wisprTests/MeetingStateManagerTests.swift b/wisprTests/MeetingStateManagerTests.swift index 5114cb6..9f6653f 100644 --- a/wisprTests/MeetingStateManagerTests.swift +++ b/wisprTests/MeetingStateManagerTests.swift @@ -66,7 +66,8 @@ struct MeetingTranscriptTests { MeetingTranscriptEntry(speaker: .you, text: "Hello world", timestamp: timestamp1) ) transcript.entries.append( - MeetingTranscriptEntry(speaker: .others, text: "Hi there", timestamp: timestamp2) + MeetingTranscriptEntry( + speaker: .others(speakerIndex: nil), text: "Hi there", timestamp: timestamp2) ) let plainText = transcript.asPlainText() @@ -99,10 +100,13 @@ struct MeetingTranscriptTests { #expect(entry1.id != entry2.id) } - @Test("MeetingSpeaker raw values are correct") - func testMeetingSpeakerRawValues() { - #expect(MeetingSpeaker.you.rawValue == "You") - #expect(MeetingSpeaker.others.rawValue == "Others") + @Test("MeetingSpeaker display names are correct") + func testMeetingSpeakerDisplayNames() { + #expect(MeetingSpeaker.you.displayName == "You") + #expect(MeetingSpeaker.others(speakerIndex: nil).displayName == "Others") + #expect(MeetingSpeaker.others(speakerIndex: 0).displayName == "Speaker 1") + #expect(MeetingSpeaker.others(speakerIndex: 1).displayName == "Speaker 2") + #expect(MeetingSpeaker.others(speakerIndex: 3).displayName == "Speaker 4") } } From 5047cf74d16fc8ffea700b2ef0245e7981038e46 Mon Sep 17 00:00:00 2001 From: Gabriel Bruno <242597616+gbrunoo@users.noreply.github.com> Date: Sat, 20 Jun 2026 00:06:55 +0200 Subject: [PATCH 2/5] Fix unresponsive Start Meeting button during diarizer warmup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The diarizer warmUp() (model load/download, several seconds) ran before meetingState flipped to .recording, so the button stayed on 'Start Meeting' with no feedback, and the meetingState==.idle guard was still true during the gap — a second click double-triggered startCapture(). - Add an isStarting re-entrancy guard. - Start capture and flip to .recording immediately for instant feedback; warm up the diarizer concurrently inside the recording task group. - Check diarizationActive per-chunk so diarization engages once warmup completes mid-meeting. - MeetingDiarizer now tracks the meeting-relative time of its first ingested chunk and offsets segment times, keeping speaker windows aligned when the diarizer starts after the meeting has begun. - stopMeeting resets the diarizer unconditionally (safe no-op) to avoid a race with the concurrent warmup task. --- .../WisprApp/Services/MeetingDiarizer.swift | 20 +++++-- .../Services/MeetingStateManager.swift | 56 ++++++++++++------- 2 files changed, 52 insertions(+), 24 deletions(-) diff --git a/Sources/WisprApp/Services/MeetingDiarizer.swift b/Sources/WisprApp/Services/MeetingDiarizer.swift index 07a9452..988dc60 100644 --- a/Sources/WisprApp/Services/MeetingDiarizer.swift +++ b/Sources/WisprApp/Services/MeetingDiarizer.swift @@ -47,6 +47,12 @@ actor MeetingDiarizer { /// Latest tentative segments (replaced on each process() call). private var tentativeSegments: [Segment] = [] + /// Meeting-relative start time of the first ingested chunk. Sortformer's + /// internal timeline begins at 0 when it receives its first audio, so we + /// add this offset to align segment times with the engine's meeting clock + /// (the diarizer may start mid-meeting once its model finishes warming up). + private var baseTime: TimeInterval? + /// Sortformer slot → per-meeting monotonic index (0-based). private var slotToAssigned: [Int: Int] = [:] private var nextAssigned: Int = 0 @@ -77,22 +83,27 @@ actor MeetingDiarizer { /// a diarizer failure never interrupts transcription. func ingest(_ samples: [Float], at startTime: TimeInterval) { guard isInitialized else { return } + if baseTime == nil { baseTime = startTime } + let offset = Float(baseTime ?? 0) do { sortformer.addAudio(samples) if let update = try sortformer.process() { - // Accumulate newly finalized segments + // Accumulate newly finalized segments (shifted to meeting time) for seg in update.finalizedSegments { let idx = assignedIndex(for: seg.speakerIndex) finalizedSegments.append( - Segment(assignedIndex: idx, startTime: seg.startTime, endTime: seg.endTime) + Segment( + assignedIndex: idx, + startTime: seg.startTime + offset, + endTime: seg.endTime + offset) ) } // Replace tentative segments with the latest snapshot tentativeSegments = update.tentativeSegments.map { seg in Segment( assignedIndex: assignedIndex(for: seg.speakerIndex), - startTime: seg.startTime, - endTime: seg.endTime + startTime: seg.startTime + offset, + endTime: seg.endTime + offset ) } } @@ -135,6 +146,7 @@ actor MeetingDiarizer { tentativeSegments.removeAll() slotToAssigned.removeAll() nextAssigned = 0 + baseTime = nil Log.diarizer.debug("MeetingDiarizer — reset") } diff --git a/Sources/WisprApp/Services/MeetingStateManager.swift b/Sources/WisprApp/Services/MeetingStateManager.swift index e9ab715..13f53bb 100644 --- a/Sources/WisprApp/Services/MeetingStateManager.swift +++ b/Sources/WisprApp/Services/MeetingStateManager.swift @@ -71,6 +71,10 @@ final class MeetingStateManager { /// Whether diarization is active for the current session (set at startMeeting). private var diarizationActive = false + /// Guards against re-entrant `startMeeting()` while capture is being set up + /// (the window between the first `await` and `meetingState = .recording`). + private var isStarting = false + // MARK: - Tasks private var recordingTask: Task? @@ -93,31 +97,22 @@ final class MeetingStateManager { /// Starts a new meeting transcription session. func startMeeting() async { - guard meetingState == .idle else { return } + guard meetingState == .idle, !isStarting else { return } + isStarting = true + defer { isStarting = false } Log.stateManager.debug("MeetingStateManager — starting meeting") transcript = MeetingTranscript() errorMessage = nil - - // Warm up the diarizer when enabled. On failure, degrade gracefully to - // source-based labels (the same fallback used when system audio is - // unavailable) rather than blocking the meeting. diarizationActive = false - if settingsStore.meetingDiarizationEnabled, let diarizer = meetingDiarizer { - do { - try await diarizer.warmUp() - diarizationActive = true - } catch { - Log.stateManager.warning( - "MeetingStateManager — diarizer warmup failed: \(error.localizedDescription). Continuing without diarization." - ) - } - } do { let (micLevels, systemLevels) = try await meetingAudioEngine.startCapture() + // Flip to recording immediately so the UI is responsive. The diarizer + // (if enabled) warms up concurrently below — chunks that arrive before + // it's ready simply render as "Others". meetingState = .recording isWindowVisible = true @@ -125,6 +120,7 @@ final class MeetingStateManager { await withTaskGroup(of: Void.self) { group in group.addTask { await self.consumeMicLevels(micLevels) } group.addTask { await self.consumeSystemLevels(systemLevels) } + group.addTask { await self.warmUpDiarizerIfEnabled() } group.addTask { await self.transcribeMicAudio() } group.addTask { await self.transcribeSystemAudio() } group.addTask { await self.runTimer() } @@ -138,6 +134,24 @@ final class MeetingStateManager { } } + /// Warms up the diarizer in the background when enabled, so it's ready for + /// upcoming system-audio chunks without blocking the recording start. + /// On failure, degrades gracefully to source-based "Others" labels. + private func warmUpDiarizerIfEnabled() async { + guard settingsStore.meetingDiarizationEnabled, let diarizer = meetingDiarizer else { + return + } + do { + try await diarizer.warmUp() + guard !Task.isCancelled else { return } + diarizationActive = true + } catch { + Log.stateManager.warning( + "MeetingStateManager — diarizer warmup failed: \(error.localizedDescription). Continuing without diarization." + ) + } + } + /// Stops the meeting and finalizes the transcript. func stopMeeting() async { guard meetingState == .recording else { return } @@ -151,10 +165,10 @@ final class MeetingStateManager { recordingTask = nil await meetingAudioEngine.stopCapture() - if diarizationActive { - await meetingDiarizer?.reset() - diarizationActive = false - } + // reset() is a safe no-op if the diarizer was never warmed up, so call it + // unconditionally to avoid a race with the concurrent warmup task. + await meetingDiarizer?.reset() + diarizationActive = false meetingState = .idle micLevel = 0 @@ -212,12 +226,14 @@ final class MeetingStateManager { private func transcribeSystemAudio() async { let audioStream = await meetingAudioEngine.systemAudioStream let language = settingsStore.languageMode - let diarizer = diarizationActive ? meetingDiarizer : nil for await audioChunk in audioStream { guard !Task.isCancelled else { break } guard audioChunk.samples.count >= 8000 else { continue } + // Re-check per chunk: the diarizer may finish warming up mid-meeting. + let diarizer = diarizationActive ? meetingDiarizer : nil + // Feed the chunk to the diarizer before transcribing so its timeline // covers this window by the time we query the dominant speaker. await diarizer?.ingest(audioChunk.samples, at: audioChunk.startTime) From a7c53f21e4874a245d481e885a011a351d4a70ba Mon Sep 17 00:00:00 2001 From: Gabriel Bruno <242597616+gbrunoo@users.noreply.github.com> Date: Sat, 20 Jun 2026 00:16:50 +0200 Subject: [PATCH 3/5] Improve diarization precision: balancedV2 config + finer system chunks - Switch Sortformer from the default fastV2_1 config to balancedV2: same ~1s latency but a much larger FIFO (188 vs 40) for better quality, and v2 weights handle overlapping/high-speaker-count audio better than v2.1 (which is documented to degrade and likely caused phantom speakers). - Give the system-audio path its own shorter chunk size (~3s vs the mic's ~5s) so each transcript entry spans less time, reducing how often a speaker change inside one entry gets flattened to a single label. --- Sources/WisprApp/Services/MeetingAudioEngine.swift | 11 ++++++++--- Sources/WisprApp/Services/MeetingDiarizer.swift | 9 +++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Sources/WisprApp/Services/MeetingAudioEngine.swift b/Sources/WisprApp/Services/MeetingAudioEngine.swift index beaf314..7b63d0e 100644 --- a/Sources/WisprApp/Services/MeetingAudioEngine.swift +++ b/Sources/WisprApp/Services/MeetingAudioEngine.swift @@ -69,6 +69,11 @@ actor MeetingAudioEngine { /// ~5 seconds of audio at 16kHz = 80,000 samples. private let chunkSize = 80_000 + /// System-audio chunk size. Shorter than the mic chunk (~3s) so each + /// transcript entry spans less time, reducing how often a single entry + /// flattens a speaker change into one diarization label. + private let systemChunkSize = 48_000 + /// Audio sample rate used for both capture and diarization (Hz). static let sampleRate: Int = 16_000 @@ -350,9 +355,9 @@ actor MeetingAudioEngine { systemSamplesTotal += samples.count systemBuffer.append(contentsOf: samples) - if systemBuffer.count >= chunkSize { - let chunk = Array(systemBuffer.prefix(chunkSize)) - systemBuffer.removeFirst(min(chunkSize, systemBuffer.count)) + if systemBuffer.count >= systemChunkSize { + let chunk = Array(systemBuffer.prefix(systemChunkSize)) + systemBuffer.removeFirst(min(systemChunkSize, systemBuffer.count)) let startTime = Double(systemChunkStartSamples) / Double(Self.sampleRate) systemChunkStartSamples = systemSamplesTotal - systemBuffer.count Log.audioEngine.debug( diff --git a/Sources/WisprApp/Services/MeetingDiarizer.swift b/Sources/WisprApp/Services/MeetingDiarizer.swift index 988dc60..9046395 100644 --- a/Sources/WisprApp/Services/MeetingDiarizer.swift +++ b/Sources/WisprApp/Services/MeetingDiarizer.swift @@ -39,7 +39,12 @@ actor MeetingDiarizer { // MARK: - State - private let sortformer = SortformerDiarizer() + /// Sortformer configuration. `.balancedV2` uses the v2 weights with a large + /// FIFO (188) for better quality at the same ~1s latency as the fast config, + /// and handles overlapping/high-speaker-count audio better than v2.1. + private static let config: SortformerConfig = .balancedV2 + + private let sortformer = SortformerDiarizer(config: MeetingDiarizer.config) private var isInitialized = false /// Accumulated finalized speaker segments from all processed chunks. @@ -68,7 +73,7 @@ actor MeetingDiarizer { Log.diarizer.info("MeetingDiarizer — loading Sortformer model") let models = try await SortformerModels.loadFromHuggingFace( - config: .default, + config: Self.config, cacheDirectory: ModelPaths.sortformer ) sortformer.initialize(models: models) From 8bfbf52c81dcb1bb1f1c60c39482ac91745b7ff8 Mon Sep 17 00:00:00 2001 From: Gabriel Bruno <242597616+gbrunoo@users.noreply.github.com> Date: Tue, 23 Jun 2026 13:43:13 +0200 Subject: [PATCH 4/5] Add .rules and .agents/ to gitignore Local editor config and agent skills should not be committed to the repo. --- .gitignore | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 233ecdb..51f0f01 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,8 @@ DerivedData/ # Secrets secrets -private_keys \ No newline at end of file +private_keys + +# Editor/agent local config +.rules +.agents/ From bbeb10e048af197545cabb9b8b91e8f86341dab8 Mon Sep 17 00:00:00 2001 From: Gabriel Bruno <242597616+gbrunoo@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:57:57 +0200 Subject: [PATCH 5/5] Suppress microphone echo of remote audio in meetings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes issue #65: without headphones, remote participants' speech plays out of the speakers and leaks back into the microphone, so the same utterance was transcribed twice — once as 'Others' (system audio) and once as 'You' (mic echo). - MeetingTranscript.appendSuppressingEcho() de-duplicates across the two tracks: a 'You' entry that closely matches a recent 'Others' entry is dropped, and an arriving 'Others' entry removes a prior 'You' echo (handles either arrival order). Matching uses normalized text + Levenshtein similarity within an 8s window, skipping short (<3 word) backchannels to avoid dropping genuine speech. - MeetingStateManager routes both transcription paths through record(), which applies suppression when enabled (text is never logged). - SettingsStore gains meetingEchoSuppressionEnabled (default true, opt-out); Settings adds a 'Suppress Microphone Echo' toggle. - Tests: MeetingEchoSuppressionTests for the de-dup contract and SettingsStore default/persistence/restore coverage. --- .../WisprApp/Models/MeetingTranscript.swift | 142 ++++++++++++++++++ .../Services/MeetingStateManager.swift | 21 ++- Sources/WisprApp/Services/SettingsStore.swift | 21 +++ .../WisprApp/UI/Settings/SettingsView.swift | 11 ++ wisprTests/MeetingStateManagerTests.swift | 118 +++++++++++++++ wisprTests/SettingsStoreTests.swift | 28 ++++ 6 files changed, 337 insertions(+), 4 deletions(-) diff --git a/Sources/WisprApp/Models/MeetingTranscript.swift b/Sources/WisprApp/Models/MeetingTranscript.swift index 823a6f9..54aafa9 100644 --- a/Sources/WisprApp/Models/MeetingTranscript.swift +++ b/Sources/WisprApp/Models/MeetingTranscript.swift @@ -25,6 +25,13 @@ enum MeetingSpeaker: Sendable, Equatable, Hashable { case .others(.some(let index)): return "Speaker \(index + 1)" } } + + /// Whether this speaker is a remote participant (system-audio track), + /// regardless of the resolved diarization index. + var isRemote: Bool { + if case .others = self { return true } + return false + } } /// A single timestamped entry in a meeting transcript. @@ -51,6 +58,141 @@ struct MeetingTranscript: Sendable, Equatable { self.startTime = startTime } + // MARK: - Echo Suppression + + /// How far back (in seconds) to look for a matching entry on the other + /// audio track when detecting microphone echo. Mic and system chunks are + /// transcribed independently and on different chunk boundaries, so the two + /// copies of the same utterance can land a few seconds apart. + static let echoSuppressionWindow: TimeInterval = 8 + + /// Minimum normalized similarity (0...1) between two entries' text for them + /// to be considered the same utterance. ASR of speaker-echo is usually a + /// near-perfect copy, so a high threshold keeps false positives rare. + static let echoSimilarityThreshold = 0.8 + + /// Utterances shorter than this (in words) are not echo-suppressed. Short + /// backchannels ("yes", "right", "okay") are commonly said by both sides, + /// so suppressing them would risk dropping genuine speech. + static let minimumEchoWordCount = 3 + + /// Appends `entry`, suppressing microphone echo of remote (system) audio. + /// + /// In meeting mode without headphones, remote participants' speech plays out + /// of the speakers and leaks back into the microphone, producing a duplicate + /// "You" entry that mirrors an "Others" entry (see issue #65). When a "You" + /// entry closely matches a recent "Others" entry — or a newly arriving + /// "Others" entry matches a recent "You" entry — the microphone copy is + /// treated as echo: the utterance is attributed to the remote participant + /// and the "You" copy is dropped. + /// + /// Handles both arrival orders (mic-first or system-first) since the two + /// transcription paths run concurrently. + /// + /// - Returns: `true` if `entry` was added to the transcript, `false` if it + /// was suppressed as microphone echo. + @discardableResult + mutating func appendSuppressingEcho(_ entry: MeetingTranscriptEntry) -> Bool { + let normalized = Self.normalizedForComparison(entry.text) + guard normalized.split(separator: " ").count >= Self.minimumEchoWordCount else { + entries.append(entry) + return true + } + + switch entry.speaker { + case .you: + // A mic entry echoing a recent remote utterance is dropped. + if recentEchoMatchIndex(of: normalized, isRemote: true, before: entry.timestamp) != nil + { + return false + } + entries.append(entry) + return true + + case .others: + // A remote utterance that a recent mic entry already echoed: drop the + // earlier mic echo and keep this (authoritative) remote entry. + if let echoIndex = recentEchoMatchIndex( + of: normalized, isRemote: false, before: entry.timestamp) + { + entries.remove(at: echoIndex) + } + entries.append(entry) + return true + } + } + + /// Finds the most recent entry within `echoSuppressionWindow` of `timestamp` + /// that originates from the requested track (`isRemote`) and whose text is + /// similar enough to `normalizedText` to be the same utterance. + private func recentEchoMatchIndex( + of normalizedText: String, + isRemote: Bool, + before timestamp: Date + ) -> Int? { + for index in entries.indices.reversed() { + let candidate = entries[index] + // Entries are appended in chronological order, so once we pass the + // window boundary every earlier entry is also out of range. + if timestamp.timeIntervalSince(candidate.timestamp) > Self.echoSuppressionWindow { + break + } + guard candidate.speaker.isRemote == isRemote else { continue } + let candidateNormalized = Self.normalizedForComparison(candidate.text) + if Self.similarity(normalizedText, candidateNormalized) >= Self.echoSimilarityThreshold + { + return index + } + } + return nil + } + + /// Lowercases, strips punctuation, and collapses whitespace so that two + /// transcriptions of the same speech compare equal despite cosmetic + /// differences (capitalization, trailing punctuation, extra spaces). + static func normalizedForComparison(_ text: String) -> String { + let scalars = text.lowercased().unicodeScalars.map { scalar -> Character in + CharacterSet.alphanumerics.contains(scalar) ? Character(scalar) : " " + } + return String(scalars) + .split(separator: " ", omittingEmptySubsequences: true) + .joined(separator: " ") + } + + /// Similarity of two strings in 0...1, derived from Levenshtein edit distance + /// normalized by the longer string's length (1 = identical). + static func similarity(_ a: String, _ b: String) -> Double { + if a.isEmpty && b.isEmpty { return 1 } + let lhs = Array(a) + let rhs = Array(b) + let maxLength = max(lhs.count, rhs.count) + guard maxLength > 0 else { return 1 } + return 1 - Double(levenshtein(lhs, rhs)) / Double(maxLength) + } + + /// Standard two-row Levenshtein edit distance. + private static func levenshtein(_ a: [Character], _ b: [Character]) -> Int { + if a.isEmpty { return b.count } + if b.isEmpty { return a.count } + + var previous = Array(0...b.count) + var current = [Int](repeating: 0, count: b.count + 1) + + for i in 1...a.count { + current[0] = i + for j in 1...b.count { + let cost = a[i - 1] == b[j - 1] ? 0 : 1 + current[j] = min( + previous[j] + 1, // deletion + current[j - 1] + 1, // insertion + previous[j - 1] + cost // substitution + ) + } + swap(&previous, ¤t) + } + return previous[b.count] + } + /// Shared time formatter for transcript display (HH:mm:ss). private static let timeFormatter: DateFormatter = { let formatter = DateFormatter() diff --git a/Sources/WisprApp/Services/MeetingStateManager.swift b/Sources/WisprApp/Services/MeetingStateManager.swift index 13f53bb..ef6b231 100644 --- a/Sources/WisprApp/Services/MeetingStateManager.swift +++ b/Sources/WisprApp/Services/MeetingStateManager.swift @@ -212,9 +212,7 @@ final class MeetingStateManager { let text = result.text.trimmingCharacters(in: .whitespacesAndNewlines) guard !text.isEmpty else { continue } - transcript.entries.append( - MeetingTranscriptEntry(speaker: .you, text: text) - ) + record(MeetingTranscriptEntry(speaker: .you, text: text)) } catch { if case WisprError.emptyTranscription = error { continue } Log.stateManager.warning( @@ -255,7 +253,7 @@ final class MeetingStateManager { in: audioChunk.startTime...chunkEnd) } - transcript.entries.append( + record( MeetingTranscriptEntry(speaker: .others(speakerIndex: speakerIndex), text: text) ) } catch { @@ -267,6 +265,21 @@ final class MeetingStateManager { } } + /// Adds a transcript entry, applying microphone echo suppression when the + /// setting is enabled. Echo suppression drops mic ("You") transcriptions that + /// duplicate a recent remote ("Others") transcription — the result of remote + /// speech leaking from the speakers into the mic (issue #65). + private func record(_ entry: MeetingTranscriptEntry) { + guard settingsStore.meetingEchoSuppressionEnabled else { + transcript.entries.append(entry) + return + } + if !transcript.appendSuppressingEcho(entry) { + Log.stateManager.debug( + "MeetingStateManager — suppressed microphone echo of remote audio") + } + } + // MARK: - Audio Level Consumption private func consumeMicLevels(_ stream: AsyncStream) async { diff --git a/Sources/WisprApp/Services/SettingsStore.swift b/Sources/WisprApp/Services/SettingsStore.swift index 2218457..81c9e00 100644 --- a/Sources/WisprApp/Services/SettingsStore.swift +++ b/Sources/WisprApp/Services/SettingsStore.swift @@ -105,6 +105,17 @@ final class SettingsStore { } } + /// When true, microphone transcriptions that duplicate a recent system-audio + /// ("Others") transcription are suppressed. Without headphones, remote + /// participants' speech leaks from the speakers into the mic and would + /// otherwise be transcribed twice (see issue #65). Defaults to true. + var meetingEchoSuppressionEnabled: Bool { + didSet { + guard !isLoading else { return } + defaults.set(meetingEchoSuppressionEnabled, forKey: Keys.meetingEchoSuppressionEnabled) + } + } + /// When true, plays short audio cues on recording start/stop. var soundFeedbackEnabled: Bool { didSet { @@ -184,6 +195,7 @@ final class SettingsStore { static let onboardingLastStep = "onboardingLastStep" static let handsFreeMode = "handsFreeMode" static let meetingDiarizationEnabled = "meetingDiarizationEnabled" + static let meetingEchoSuppressionEnabled = "meetingEchoSuppressionEnabled" static let soundFeedbackEnabled = "soundFeedbackEnabled" static let autoSuffixEnabled = "autoSuffixEnabled" static let autoSuffixText = "autoSuffixText" @@ -209,6 +221,7 @@ final class SettingsStore { static let onboardingLastStep: Int = 0 static let handsFreeMode: Bool = false static let meetingDiarizationEnabled: Bool = false + static let meetingEchoSuppressionEnabled: Bool = true static let soundFeedbackEnabled: Bool = false static let autoSuffixEnabled: Bool = false static let autoSuffixText: String = " " @@ -238,6 +251,7 @@ final class SettingsStore { self.onboardingLastStep = Defaults.onboardingLastStep self.handsFreeMode = Defaults.handsFreeMode self.meetingDiarizationEnabled = Defaults.meetingDiarizationEnabled + self.meetingEchoSuppressionEnabled = Defaults.meetingEchoSuppressionEnabled self.soundFeedbackEnabled = Defaults.soundFeedbackEnabled self.autoSuffixEnabled = Defaults.autoSuffixEnabled self.autoSuffixText = Defaults.autoSuffixText @@ -265,6 +279,7 @@ final class SettingsStore { launchAtLogin = Defaults.launchAtLogin handsFreeMode = Defaults.handsFreeMode meetingDiarizationEnabled = Defaults.meetingDiarizationEnabled + meetingEchoSuppressionEnabled = Defaults.meetingEchoSuppressionEnabled soundFeedbackEnabled = Defaults.soundFeedbackEnabled autoSuffixEnabled = Defaults.autoSuffixEnabled autoSuffixText = Defaults.autoSuffixText @@ -291,6 +306,7 @@ final class SettingsStore { defaults.set(onboardingLastStep, forKey: Keys.onboardingLastStep) defaults.set(handsFreeMode, forKey: Keys.handsFreeMode) defaults.set(meetingDiarizationEnabled, forKey: Keys.meetingDiarizationEnabled) + defaults.set(meetingEchoSuppressionEnabled, forKey: Keys.meetingEchoSuppressionEnabled) defaults.set(soundFeedbackEnabled, forKey: Keys.soundFeedbackEnabled) defaults.set(autoSuffixEnabled, forKey: Keys.autoSuffixEnabled) defaults.set(autoSuffixText, forKey: Keys.autoSuffixText) @@ -362,6 +378,11 @@ final class SettingsStore { self.meetingDiarizationEnabled = defaults.bool(forKey: Keys.meetingDiarizationEnabled) } + if defaults.object(forKey: Keys.meetingEchoSuppressionEnabled) != nil { + self.meetingEchoSuppressionEnabled = defaults.bool( + forKey: Keys.meetingEchoSuppressionEnabled) + } + if defaults.object(forKey: Keys.soundFeedbackEnabled) != nil { self.soundFeedbackEnabled = defaults.bool(forKey: Keys.soundFeedbackEnabled) } diff --git a/Sources/WisprApp/UI/Settings/SettingsView.swift b/Sources/WisprApp/UI/Settings/SettingsView.swift index 1d58183..abd08cd 100644 --- a/Sources/WisprApp/UI/Settings/SettingsView.swift +++ b/Sources/WisprApp/UI/Settings/SettingsView.swift @@ -71,6 +71,8 @@ struct SettingsView: View { // Meeting section static let meetingDiarization = "When enabled, the meeting transcript labels remote participants as Speaker 1, Speaker 2, and so on using on-device diarization" + static let meetingEchoSuppression = + "When enabled, speech from remote participants that leaks into your microphone is not transcribed a second time as your own speech" // General section static let launchAtLogin = "When enabled, Wispr starts automatically when you log in" @@ -388,6 +390,15 @@ struct SettingsView: View { ) .font(.caption) .foregroundStyle(theme.secondaryTextColor) + + Toggle("Suppress Microphone Echo", isOn: $store.meetingEchoSuppressionEnabled) + .accessibilityHint(AccessibilityHints.meetingEchoSuppression) + + Text( + "Avoids transcribing remote participants twice when their voice leaks from your speakers into the microphone. Turn off if you use headphones and want every microphone word kept." + ) + .font(.caption) + .foregroundStyle(theme.secondaryTextColor) } header: { SectionHeader( title: "Meeting", diff --git a/wisprTests/MeetingStateManagerTests.swift b/wisprTests/MeetingStateManagerTests.swift index 9f6653f..08cfc02 100644 --- a/wisprTests/MeetingStateManagerTests.swift +++ b/wisprTests/MeetingStateManagerTests.swift @@ -110,6 +110,124 @@ struct MeetingTranscriptTests { } } +// MARK: - Echo Suppression Tests + +/// Tests for the microphone-echo de-duplication that fixes issue #65: remote +/// participants' speech leaking from the speakers into the mic and being +/// transcribed twice (once as "Others", once as "You"). +@Suite("Meeting Echo Suppression Tests") +struct MeetingEchoSuppressionTests { + + /// Builds an entry at a fixed offset (seconds) from a shared base time so + /// tests can control the echo time window precisely. + private func entry( + _ speaker: MeetingSpeaker, _ text: String, at offset: TimeInterval, base: Date + ) -> MeetingTranscriptEntry { + MeetingTranscriptEntry( + speaker: speaker, text: text, timestamp: base.addingTimeInterval(offset)) + } + + @Test("Mic entry echoing a recent remote entry is suppressed") + func testMicEchoOfRemoteIsSuppressed() { + let base = Date() + var transcript = MeetingTranscript() + + let added1 = transcript.appendSuppressingEcho( + entry(.others(speakerIndex: 0), "Let's review the quarterly numbers", at: 0, base: base) + ) + let added2 = transcript.appendSuppressingEcho( + entry(.you, "Let's review the quarterly numbers.", at: 1, base: base)) + + #expect(added1 == true) + #expect(added2 == false) + #expect(transcript.entries.count == 1) + #expect(transcript.entries[0].speaker.isRemote) + } + + @Test("Remote entry removes a prior mic echo (system arrives second)") + func testRemoteRemovesPriorMicEcho() { + let base = Date() + var transcript = MeetingTranscript() + + transcript.appendSuppressingEcho( + entry(.you, "Can everyone hear me clearly", at: 0, base: base)) + transcript.appendSuppressingEcho( + entry(.others(speakerIndex: 1), "Can everyone hear me clearly?", at: 1, base: base)) + + #expect(transcript.entries.count == 1) + #expect(transcript.entries[0].speaker == .others(speakerIndex: 1)) + } + + @Test("Distinct utterances on both tracks are both kept") + func testDistinctUtterancesKept() { + let base = Date() + var transcript = MeetingTranscript() + + transcript.appendSuppressingEcho( + entry(.others(speakerIndex: 0), "What did you think of the proposal", at: 0, base: base) + ) + transcript.appendSuppressingEcho( + entry(.you, "I thought it was a strong start overall", at: 1, base: base)) + + #expect(transcript.entries.count == 2) + } + + @Test("Matches outside the time window are not suppressed") + func testOutsideWindowNotSuppressed() { + let base = Date() + var transcript = MeetingTranscript() + let beyond = MeetingTranscript.echoSuppressionWindow + 2 + + transcript.appendSuppressingEcho( + entry(.others(speakerIndex: 0), "Please send the report by Friday", at: 0, base: base)) + let added = transcript.appendSuppressingEcho( + entry(.you, "Please send the report by Friday", at: beyond, base: base)) + + #expect(added == true) + #expect(transcript.entries.count == 2) + } + + @Test("Short utterances are not echo-suppressed") + func testShortUtterancesNotSuppressed() { + let base = Date() + var transcript = MeetingTranscript() + + transcript.appendSuppressingEcho( + entry(.others(speakerIndex: 0), "Sounds good", at: 0, base: base)) + let added = transcript.appendSuppressingEcho(entry(.you, "Sounds good", at: 1, base: base)) + + #expect(added == true) + #expect(transcript.entries.count == 2) + } + + @Test("Cosmetic differences still count as the same utterance") + func testCosmeticDifferencesMatch() { + let base = Date() + var transcript = MeetingTranscript() + + transcript.appendSuppressingEcho( + entry(.others(speakerIndex: 0), "So, what's our next step here?", at: 0, base: base)) + let added = transcript.appendSuppressingEcho( + entry(.you, "so what is our next step here", at: 1, base: base)) + + // Minor wording ("what's" vs "what is") keeps similarity high enough. + #expect(added == false) + #expect(transcript.entries.count == 1) + } + + @Test("normalizedForComparison strips case, punctuation, and extra whitespace") + func testNormalization() { + #expect( + MeetingTranscript.normalizedForComparison(" Hello, WORLD!! ") == "hello world") + } + + @Test("similarity is 1 for identical strings and lower for different ones") + func testSimilarity() { + #expect(MeetingTranscript.similarity("hello world", "hello world") == 1.0) + #expect(MeetingTranscript.similarity("hello world", "goodbye planet") < 0.5) + } +} + // MARK: - MeetingStateManager Tests @MainActor diff --git a/wisprTests/SettingsStoreTests.swift b/wisprTests/SettingsStoreTests.swift index 52cecd4..c069a4a 100644 --- a/wisprTests/SettingsStoreTests.swift +++ b/wisprTests/SettingsStoreTests.swift @@ -306,6 +306,34 @@ struct SettingsStoreTests { #expect(store.removeFillerWords == false) } + // MARK: - Meeting Echo Suppression Tests + + @Test("SettingsStore meetingEchoSuppressionEnabled defaults to true") + func testMeetingEchoSuppressionDefault() { + let defaults = createTestDefaults() + let store = SettingsStore(defaults: defaults) + #expect(store.meetingEchoSuppressionEnabled == true) + } + + @Test("SettingsStore persists meetingEchoSuppressionEnabled") + func testMeetingEchoSuppressionPersistence() { + let defaults = createTestDefaults() + let store = SettingsStore(defaults: defaults) + store.meetingEchoSuppressionEnabled = false + + let newStore = SettingsStore(defaults: defaults) + #expect(newStore.meetingEchoSuppressionEnabled == false, "meetingEchoSuppressionEnabled should persist") + } + + @Test("SettingsStore restoreDefaults resets meetingEchoSuppressionEnabled to true") + func testRestoreDefaultsResetsMeetingEchoSuppression() { + let defaults = createTestDefaults() + let store = SettingsStore(defaults: defaults) + store.meetingEchoSuppressionEnabled = false + store.restoreDefaults() + #expect(store.meetingEchoSuppressionEnabled == true) + } + // MARK: - Property-Based Tests // Feature: auto-suffix-insertion, Property 1: Settings persistence round-trip