From 26a64b8f3057c01ac5fefbb2a4375eab8fe12770 Mon Sep 17 00:00:00 2001 From: ComputelessComputer <63365510+ComputelessComputer@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:24:20 +0900 Subject: [PATCH 1/2] Show live transcript in standalone note windows Expose active capture snapshots and attach standalone note routes to listener events. --- .../src/routes/app/-note.$sessionId.test.tsx | 17 ++++ .../src/routes/app/note.$sessionId.tsx | 10 ++ .../store/zustand/listener/general-live.ts | 94 +++++++++++++++++++ .../store/zustand/listener/general.test.ts | 68 +++++++++++++- .../src/store/zustand/listener/general.ts | 9 ++ crates/listener-core/src/actors/root.rs | 40 ++++++-- crates/listener-core/src/lib.rs | 9 ++ plugins/transcription/build.rs | 1 + plugins/transcription/js/bindings.gen.ts | 9 ++ .../commands/get_capture_snapshot.toml | 13 +++ .../permissions/autogenerated/reference.md | 27 ++++++ .../transcription/permissions/default.toml | 1 + .../permissions/schemas/schema.json | 16 +++- plugins/transcription/src/api.rs | 22 +++++ plugins/transcription/src/lib.rs | 2 + .../transcription/src/listener/commands.rs | 10 +- plugins/transcription/src/listener/ext.rs | 43 ++++++++- 17 files changed, 375 insertions(+), 16 deletions(-) create mode 100644 plugins/transcription/permissions/autogenerated/commands/get_capture_snapshot.toml diff --git a/apps/desktop/src/routes/app/-note.$sessionId.test.tsx b/apps/desktop/src/routes/app/-note.$sessionId.test.tsx index af33a063f1..d99f873a45 100644 --- a/apps/desktop/src/routes/app/-note.$sessionId.test.tsx +++ b/apps/desktop/src/routes/app/-note.$sessionId.test.tsx @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { resetTabsStore } from "~/store/zustand/tabs/test-utils"; const mocks = vi.hoisted(() => ({ + attachLiveSession: vi.fn(), close: vi.fn(), })); @@ -13,7 +14,16 @@ vi.mock("@tauri-apps/api/window", () => ({ }), })); +vi.mock("~/stt/contexts", () => ({ + useListener: ( + selector: (state: { + attachLiveSession: typeof mocks.attachLiveSession; + }) => unknown, + ) => selector({ attachLiveSession: mocks.attachLiveSession }), +})); + import { + useAttachStandaloneNoteToLiveSession, useCloseStandaloneNoteWindowOnEscape, useStandaloneNoteTab, } from "./note.$sessionId"; @@ -22,6 +32,7 @@ import { useTabs } from "~/store/zustand/tabs"; describe("standalone note window route", () => { beforeEach(() => { + mocks.attachLiveSession.mockClear(); mocks.close.mockClear(); resetTabsStore(); }); @@ -71,6 +82,12 @@ describe("standalone note window route", () => { expect(result.current.state.view).toEqual({ type: "raw" }); }); + + it("attaches the standalone note to live session events", () => { + renderHook(() => useAttachStandaloneNoteToLiveSession("session-1")); + + expect(mocks.attachLiveSession).toHaveBeenCalledWith("session-1"); + }); }); function dispatchKeyDown(key: string) { diff --git a/apps/desktop/src/routes/app/note.$sessionId.tsx b/apps/desktop/src/routes/app/note.$sessionId.tsx index 7bcc9c93f8..a6b24d52dd 100644 --- a/apps/desktop/src/routes/app/note.$sessionId.tsx +++ b/apps/desktop/src/routes/app/note.$sessionId.tsx @@ -6,6 +6,7 @@ import { ClassicMainLayout } from "~/main/layout"; import { TabContentNote } from "~/session"; import { StandaloneWindowShell } from "~/shared/window-shell"; import { type Tab, useTabs } from "~/store/zustand/tabs"; +import { useListener } from "~/stt/contexts"; export const Route = createFileRoute("/app/note/$sessionId")({ component: Component, @@ -14,6 +15,7 @@ export const Route = createFileRoute("/app/note/$sessionId")({ function Component() { const { sessionId } = Route.useParams(); useCloseStandaloneNoteWindowOnEscape(); + useAttachStandaloneNoteToLiveSession(sessionId); const tab = useStandaloneNoteTab(sessionId); return ( @@ -27,6 +29,14 @@ function Component() { ); } +export function useAttachStandaloneNoteToLiveSession(sessionId: string) { + const attachLiveSession = useListener((state) => state.attachLiveSession); + + useEffect(() => { + void attachLiveSession(sessionId); + }, [attachLiveSession, sessionId]); +} + export function useStandaloneNoteTab(sessionId: string) { const tab = useMemo( () => diff --git a/apps/desktop/src/store/zustand/listener/general-live.ts b/apps/desktop/src/store/zustand/listener/general-live.ts index 54b2ff63c5..2eac73b960 100644 --- a/apps/desktop/src/store/zustand/listener/general-live.ts +++ b/apps/desktop/src/store/zustand/listener/general-live.ts @@ -12,6 +12,7 @@ import { type CaptureDataEvent, type CaptureConfigUpdate, type CaptureLifecycleEvent, + type CaptureSnapshot, type CaptureParams, type CaptureStatusEvent, type LiveTranscriptDelta, @@ -255,6 +256,13 @@ export const startLiveSession = ( targetSessionId: string, params: CaptureParams, ): Promise => { + clearLiveEventUnlisteners( + get().live.eventUnlistenersBySession[targetSessionId], + ); + setLiveState(set, (live) => { + delete live.eventUnlistenersBySession[targetSessionId]; + }); + const handlers = createSessionEventHandlers(set, get, targetSessionId); const program = Effect.gen(function* () { @@ -339,6 +347,92 @@ export const startLiveSession = ( ); }; +export const attachLiveSession = ( + set: StoreApi["setState"], + get: StoreApi["getState"], + targetSessionId: string, +): Promise => { + const currentLive = get().live; + if (currentLive.eventUnlistenersBySession[targetSessionId]) { + return Promise.resolve(); + } + + const handlers = createSessionEventHandlers(set, get, targetSessionId); + + const program = Effect.gen(function* () { + const unlisteners = yield* listenToAllSessionEvents(handlers); + setLiveState(set, (live) => { + live.eventUnlistenersBySession[targetSessionId] = unlisteners; + }); + + const snapshot = yield* fromResult(listenerCommands.getCaptureSnapshot()); + applyCaptureSnapshot(set, get, targetSessionId, snapshot); + }); + + return Effect.runPromiseExit(program).then((exit) => + Exit.match(exit, { + onFailure: (cause) => { + console.error("[listener] failed to attach live session:", cause); + clearLiveEventUnlisteners( + get().live.eventUnlistenersBySession[targetSessionId], + ); + setLiveState(set, (live) => { + delete live.eventUnlistenersBySession[targetSessionId]; + }); + }, + onSuccess: () => undefined, + }), + ); +}; + +function applyCaptureSnapshot( + set: StoreApi["setState"], + get: StoreApi["getState"], + targetSessionId: string, + snapshot: CaptureSnapshot, +) { + if ( + snapshot.state === "active" && + snapshot.activeSessionId === targetSessionId + ) { + const currentLive = get().live; + const intervalId = + currentLive.sessionId === targetSessionId && currentLive.intervalId + ? currentLive.intervalId + : setInterval(() => { + setLiveState(set, (live) => { + if ( + live.sessionId === targetSessionId && + live.status === "active" + ) { + live.seconds += 1; + } + }); + }, 1000); + + setLiveState(set, (live) => { + markLiveActive( + live, + targetSessionId, + intervalId, + snapshot.requestedLiveTranscription ?? true, + snapshot.liveTranscriptionActive ?? true, + null, + ); + }); + return; + } + + if ( + snapshot.state === "finalizing" && + snapshot.finalizingSessionIds.includes(targetSessionId) + ) { + setLiveState(set, (live) => { + markLiveFinalizing(live, targetSessionId); + }); + } +} + export const stopLiveSession = ( set: StoreApi["setState"], get: StoreApi["getState"], diff --git a/apps/desktop/src/store/zustand/listener/general.test.ts b/apps/desktop/src/store/zustand/listener/general.test.ts index dd894df851..e1e7ee5a44 100644 --- a/apps/desktop/src/store/zustand/listener/general.test.ts +++ b/apps/desktop/src/store/zustand/listener/general.test.ts @@ -3,12 +3,20 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; const { getIdentifierMock, + getCaptureSnapshotMock, + listenCaptureDataMock, + listenCaptureLifecycleMock, + listenCaptureStatusMock, runEventHooksMock, setRecordingIndicatorMock, stopCaptureMock, vaultBaseMock, } = vi.hoisted(() => ({ getIdentifierMock: vi.fn(), + getCaptureSnapshotMock: vi.fn(), + listenCaptureDataMock: vi.fn(), + listenCaptureLifecycleMock: vi.fn(), + listenCaptureStatusMock: vi.fn(), runEventHooksMock: vi.fn(), setRecordingIndicatorMock: vi.fn(), stopCaptureMock: vi.fn(), @@ -45,6 +53,7 @@ vi.mock("@hypr/plugin-settings", () => ({ vi.mock("@hypr/plugin-transcription", () => ({ commands: { + getCaptureSnapshot: getCaptureSnapshotMock, setMicMuted: vi.fn(), startCapture: vi.fn(), startTranscription: vi.fn(), @@ -54,13 +63,13 @@ vi.mock("@hypr/plugin-transcription", () => ({ }, events: { captureDataEvent: { - listen: vi.fn(), + listen: listenCaptureDataMock, }, captureLifecycleEvent: { - listen: vi.fn(), + listen: listenCaptureLifecycleMock, }, captureStatusEvent: { - listen: vi.fn(), + listen: listenCaptureStatusMock, }, }, })); @@ -79,6 +88,19 @@ describe("General Listener Slice", () => { store = createListenerStore(); vi.clearAllMocks(); getIdentifierMock.mockResolvedValue("com.hyprnote.stable"); + getCaptureSnapshotMock.mockResolvedValue({ + status: "ok", + data: { + activeSessionId: null, + finalizingSessionIds: [], + liveTranscriptionActive: null, + requestedLiveTranscription: null, + state: "inactive", + }, + }); + listenCaptureDataMock.mockResolvedValue(() => {}); + listenCaptureLifecycleMock.mockResolvedValue(() => {}); + listenCaptureStatusMock.mockResolvedValue(() => {}); runEventHooksMock.mockResolvedValue({ status: "ok", data: null }); setRecordingIndicatorMock.mockResolvedValue({ status: "ok", data: null }); stopCaptureMock.mockResolvedValue({ status: "ok", data: null }); @@ -632,6 +654,46 @@ describe("General Listener Slice", () => { expect(typeof start).toBe("function"); }); + test("attachLiveSession hydrates the active native capture for the same session", async () => { + getCaptureSnapshotMock.mockResolvedValueOnce({ + status: "ok", + data: { + activeSessionId: "session-a", + finalizingSessionIds: [], + liveTranscriptionActive: true, + requestedLiveTranscription: true, + state: "active", + }, + }); + + await store.getState().attachLiveSession("session-a"); + + expect(store.getState().getSessionMode("session-a")).toBe("active"); + expect(store.getState().live.sessionId).toBe("session-a"); + expect(store.getState().live.liveTranscriptionActive).toBe(true); + expect(listenCaptureLifecycleMock).toHaveBeenCalledTimes(1); + expect(listenCaptureStatusMock).toHaveBeenCalledTimes(1); + expect(listenCaptureDataMock).toHaveBeenCalledTimes(1); + }); + + test("attachLiveSession does not mark a different active native capture as current", async () => { + getCaptureSnapshotMock.mockResolvedValueOnce({ + status: "ok", + data: { + activeSessionId: "session-b", + finalizingSessionIds: [], + liveTranscriptionActive: true, + requestedLiveTranscription: true, + state: "active", + }, + }); + + await store.getState().attachLiveSession("session-a"); + + expect(store.getState().getSessionMode("session-a")).toBe("inactive"); + expect(store.getState().live.sessionId).toBeNull(); + }); + test("start returns false while another session is active", async () => { store.setState((state) => mutate(state, (draft) => { diff --git a/apps/desktop/src/store/zustand/listener/general.ts b/apps/desktop/src/store/zustand/listener/general.ts index fb5f612487..839af348c7 100644 --- a/apps/desktop/src/store/zustand/listener/general.ts +++ b/apps/desktop/src/store/zustand/listener/general.ts @@ -10,6 +10,7 @@ import type { TranscriptionParams } from "@hypr/plugin-transcription"; import type { BatchActions, BatchState } from "./batch"; import { runBatchSession } from "./general-batch"; import { + attachLiveSession, startLiveSession, stopLiveSession, updateLiveSessionConfig, @@ -41,6 +42,7 @@ export type GeneralActions = { }, ) => Promise; stop: () => void; + attachLiveSession: (sessionId: string) => Promise; setMuted: (value: boolean) => void; setTriggerAppIds: (appIds: string[] | null) => void; updateCaptureConfig: ( @@ -118,6 +120,13 @@ export const createGeneralSlice = < stop: () => { stopLiveSession(set, get); }, + attachLiveSession: async (sessionId) => { + if (!sessionId) { + return; + } + + await attachLiveSession(set, get, sessionId); + }, setMuted: (value) => { set((state) => mutate(state, (draft) => { diff --git a/crates/listener-core/src/actors/root.rs b/crates/listener-core/src/actors/root.rs index dd8f2bf86d..07745be8f7 100644 --- a/crates/listener-core/src/actors/root.rs +++ b/crates/listener-core/src/actors/root.rs @@ -14,7 +14,7 @@ use crate::actors::{ SessionConfigUpdate, SessionContext, SessionMsg, SessionParams, session_span, spawn_session_supervisor, }; -use crate::{ListenerRuntime, SessionLifecycleEvent, StartSessionError, State}; +use crate::{ListenerRuntime, SessionLifecycleEvent, Snapshot, StartSessionError, State}; use hypr_audio::AudioProvider; pub enum RootMsg { @@ -22,6 +22,7 @@ pub enum RootMsg { UpdateSessionConfig(SessionConfigUpdate, RpcReplyPort<()>), StopSession(RpcReplyPort<()>), GetState(RpcReplyPort), + GetSnapshot(RpcReplyPort), } pub struct RootArgs { @@ -85,14 +86,10 @@ impl Actor for RootActor { let _ = reply.send(()); } RootMsg::GetState(reply) => { - let fsm_state = if state.active_supervisor.is_some() { - State::Active - } else if !state.finalizing_sessions.is_empty() { - State::Finalizing - } else { - State::Inactive - }; - let _ = reply.send(fsm_state); + let _ = reply.send(root_snapshot(state).state); + } + RootMsg::GetSnapshot(reply) => { + let _ = reply.send(root_snapshot(state)); } } Ok(()) @@ -117,6 +114,31 @@ impl Actor for RootActor { } } +fn root_snapshot(state: &RootState) -> Snapshot { + let active_session_id = state + .active_supervisor + .as_ref() + .and(state.active_session_id.clone()); + let finalizing_session_ids = state + .finalizing_sessions + .keys() + .cloned() + .collect::>(); + let state = if active_session_id.is_some() { + State::Active + } else if !finalizing_session_ids.is_empty() { + State::Finalizing + } else { + State::Inactive + }; + + Snapshot { + state, + active_session_id, + finalizing_session_ids, + } +} + async fn start_session_impl( root_cell: ActorCell, mut params: SessionParams, diff --git a/crates/listener-core/src/lib.rs b/crates/listener-core/src/lib.rs index 3d7bae1986..3b87ca8ae2 100644 --- a/crates/listener-core/src/lib.rs +++ b/crates/listener-core/src/lib.rs @@ -16,6 +16,15 @@ pub enum State { Finalizing, } +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[cfg_attr(feature = "specta", derive(specta::Type))] +#[serde(rename_all = "camelCase")] +pub struct Snapshot { + pub state: State, + pub active_session_id: Option, + pub finalizing_session_ids: Vec, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] #[cfg_attr(feature = "specta", derive(specta::Type))] #[serde(rename_all = "camelCase")] diff --git a/plugins/transcription/build.rs b/plugins/transcription/build.rs index 8a536e8556..46c1603b17 100644 --- a/plugins/transcription/build.rs +++ b/plugins/transcription/build.rs @@ -6,6 +6,7 @@ const COMMANDS: &[&str] = &[ "start_capture", "stop_capture", "get_capture_state", + "get_capture_snapshot", "is_supported_languages_live", "suggest_providers_for_languages_live", "list_documented_language_codes_live", diff --git a/plugins/transcription/js/bindings.gen.ts b/plugins/transcription/js/bindings.gen.ts index b462581ceb..30c714759d 100644 --- a/plugins/transcription/js/bindings.gen.ts +++ b/plugins/transcription/js/bindings.gen.ts @@ -70,6 +70,14 @@ async getCaptureState() : Promise> { else return { status: "error", error: e as any }; } }, +async getCaptureSnapshot() : Promise> { + try { + return { status: "ok", data: await TAURI_INVOKE("plugin:transcription|get_capture_snapshot") }; +} catch (e) { + if(e instanceof Error) throw e; + else return { status: "error", error: e as any }; +} +}, async isSupportedLanguagesLive(provider: string, model: string | null, languages: string[]) : Promise> { try { return { status: "ok", data: await TAURI_INVOKE("plugin:transcription|is_supported_languages_live", { provider, model, languages }) }; @@ -204,6 +212,7 @@ export type CaptureConfigUpdate = { session_id: string; languages: string[]; par export type CaptureDataEvent = { type: "audio_amplitude"; session_id: string; mic: number; speaker: number } | { type: "mic_muted"; session_id: string; value: boolean } | { type: "transcript_delta"; session_id: string; delta: LiveTranscriptDelta } | { type: "transcript_segment_delta"; session_id: string; delta: LiveTranscriptSegmentDelta } export type CaptureLifecycleEvent = { type: "started"; session_id: string; requested_live_transcription: boolean; live_transcription_active: boolean; degraded: DegradedError | null } | { type: "finalizing"; session_id: string } | { type: "stopped"; session_id: string; audio_path: string | null; requested_live_transcription: boolean; live_transcription_active: boolean; error: string | null } export type CaptureParams = { session_id: string; languages: string[]; onboarding: boolean; model: string; base_url: string; api_key: string; keywords: string[]; transcription_mode?: TranscriptionMode | null; participant_human_ids?: string[]; self_human_id?: string | null } +export type CaptureSnapshot = { state: CaptureState; activeSessionId: string | null; finalizingSessionIds: string[]; requestedLiveTranscription: boolean | null; liveTranscriptionActive: boolean | null } export type CaptureState = "active" | "finalizing" | "inactive" export type CaptureStatusEvent = { type: "audio_initializing"; session_id: string } | { type: "audio_ready"; session_id: string; device: string | null } | { type: "connecting"; session_id: string } | { type: "connected"; session_id: string; adapter: string } | { type: "audio_error"; session_id: string; error: string; device: string | null; is_fatal: boolean } | { type: "connection_error"; session_id: string; error: string } export type ChannelProfile = "DirectMic" | "RemoteParty" | "MixedCapture" diff --git a/plugins/transcription/permissions/autogenerated/commands/get_capture_snapshot.toml b/plugins/transcription/permissions/autogenerated/commands/get_capture_snapshot.toml new file mode 100644 index 0000000000..07910396a0 --- /dev/null +++ b/plugins/transcription/permissions/autogenerated/commands/get_capture_snapshot.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-get-capture-snapshot" +description = "Enables the get_capture_snapshot command without any pre-configured scope." +commands.allow = ["get_capture_snapshot"] + +[[permission]] +identifier = "deny-get-capture-snapshot" +description = "Denies the get_capture_snapshot command without any pre-configured scope." +commands.deny = ["get_capture_snapshot"] diff --git a/plugins/transcription/permissions/autogenerated/reference.md b/plugins/transcription/permissions/autogenerated/reference.md index 23447a89e7..dda66cb95f 100644 --- a/plugins/transcription/permissions/autogenerated/reference.md +++ b/plugins/transcription/permissions/autogenerated/reference.md @@ -11,6 +11,7 @@ Default permissions for the plugin - `allow-get-mic-muted` - `allow-set-mic-muted` - `allow-get-capture-state` +- `allow-get-capture-snapshot` - `allow-is-supported-languages-live` - `allow-suggest-providers-for-languages-live` - `allow-list-documented-language-codes-live` @@ -62,6 +63,32 @@ Denies the export_to_vtt command without any pre-configured scope. +`transcription:allow-get-capture-snapshot` + + + + +Enables the get_capture_snapshot command without any pre-configured scope. + + + + + + + +`transcription:deny-get-capture-snapshot` + + + + +Denies the get_capture_snapshot command without any pre-configured scope. + + + + + + + `transcription:allow-get-capture-state` diff --git a/plugins/transcription/permissions/default.toml b/plugins/transcription/permissions/default.toml index aeebef74e7..fb03a81321 100644 --- a/plugins/transcription/permissions/default.toml +++ b/plugins/transcription/permissions/default.toml @@ -8,6 +8,7 @@ permissions = [ "allow-get-mic-muted", "allow-set-mic-muted", "allow-get-capture-state", + "allow-get-capture-snapshot", "allow-is-supported-languages-live", "allow-suggest-providers-for-languages-live", "allow-list-documented-language-codes-live", diff --git a/plugins/transcription/permissions/schemas/schema.json b/plugins/transcription/permissions/schemas/schema.json index 028a99deab..92d48c11fa 100644 --- a/plugins/transcription/permissions/schemas/schema.json +++ b/plugins/transcription/permissions/schemas/schema.json @@ -306,6 +306,18 @@ "const": "deny-export-to-vtt", "markdownDescription": "Denies the export_to_vtt command without any pre-configured scope." }, + { + "description": "Enables the get_capture_snapshot command without any pre-configured scope.", + "type": "string", + "const": "allow-get-capture-snapshot", + "markdownDescription": "Enables the get_capture_snapshot command without any pre-configured scope." + }, + { + "description": "Denies the get_capture_snapshot command without any pre-configured scope.", + "type": "string", + "const": "deny-get-capture-snapshot", + "markdownDescription": "Denies the get_capture_snapshot command without any pre-configured scope." + }, { "description": "Enables the get_capture_state command without any pre-configured scope.", "type": "string", @@ -523,10 +535,10 @@ "markdownDescription": "Denies the suggest_providers_for_languages_live command without any pre-configured scope." }, { - "description": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-list-microphone-devices`\n- `allow-get-current-microphone-device`\n- `allow-start-capture`\n- `allow-stop-capture`\n- `allow-get-mic-muted`\n- `allow-set-mic-muted`\n- `allow-get-capture-state`\n- `allow-is-supported-languages-live`\n- `allow-suggest-providers-for-languages-live`\n- `allow-list-documented-language-codes-live`\n- `allow-render-transcript-segments`\n- `allow-start-transcription`\n- `allow-stop-transcription`\n- `allow-run-denoise`\n- `allow-parse-subtitle`\n- `allow-export-to-vtt`\n- `allow-is-supported-languages-batch`\n- `allow-suggest-providers-for-languages-batch`\n- `allow-list-documented-language-codes-batch`", + "description": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-list-microphone-devices`\n- `allow-get-current-microphone-device`\n- `allow-start-capture`\n- `allow-stop-capture`\n- `allow-get-mic-muted`\n- `allow-set-mic-muted`\n- `allow-get-capture-state`\n- `allow-get-capture-snapshot`\n- `allow-is-supported-languages-live`\n- `allow-suggest-providers-for-languages-live`\n- `allow-list-documented-language-codes-live`\n- `allow-render-transcript-segments`\n- `allow-start-transcription`\n- `allow-stop-transcription`\n- `allow-run-denoise`\n- `allow-parse-subtitle`\n- `allow-export-to-vtt`\n- `allow-is-supported-languages-batch`\n- `allow-suggest-providers-for-languages-batch`\n- `allow-list-documented-language-codes-batch`", "type": "string", "const": "default", - "markdownDescription": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-list-microphone-devices`\n- `allow-get-current-microphone-device`\n- `allow-start-capture`\n- `allow-stop-capture`\n- `allow-get-mic-muted`\n- `allow-set-mic-muted`\n- `allow-get-capture-state`\n- `allow-is-supported-languages-live`\n- `allow-suggest-providers-for-languages-live`\n- `allow-list-documented-language-codes-live`\n- `allow-render-transcript-segments`\n- `allow-start-transcription`\n- `allow-stop-transcription`\n- `allow-run-denoise`\n- `allow-parse-subtitle`\n- `allow-export-to-vtt`\n- `allow-is-supported-languages-batch`\n- `allow-suggest-providers-for-languages-batch`\n- `allow-list-documented-language-codes-batch`" + "markdownDescription": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-list-microphone-devices`\n- `allow-get-current-microphone-device`\n- `allow-start-capture`\n- `allow-stop-capture`\n- `allow-get-mic-muted`\n- `allow-set-mic-muted`\n- `allow-get-capture-state`\n- `allow-get-capture-snapshot`\n- `allow-is-supported-languages-live`\n- `allow-suggest-providers-for-languages-live`\n- `allow-list-documented-language-codes-live`\n- `allow-render-transcript-segments`\n- `allow-start-transcription`\n- `allow-stop-transcription`\n- `allow-run-denoise`\n- `allow-parse-subtitle`\n- `allow-export-to-vtt`\n- `allow-is-supported-languages-batch`\n- `allow-suggest-providers-for-languages-batch`\n- `allow-list-documented-language-codes-batch`" } ] } diff --git a/plugins/transcription/src/api.rs b/plugins/transcription/src/api.rs index 0ff6f2d32d..9b6beeac38 100644 --- a/plugins/transcription/src/api.rs +++ b/plugins/transcription/src/api.rs @@ -9,6 +9,16 @@ pub enum CaptureState { Inactive, } +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, specta::Type)] +#[serde(rename_all = "camelCase")] +pub struct CaptureSnapshot { + pub state: CaptureState, + pub active_session_id: Option, + pub finalizing_session_ids: Vec, + pub requested_live_transcription: Option, + pub live_transcription_active: Option, +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, specta::Type)] pub struct CaptureParams { pub session_id: String, @@ -242,6 +252,18 @@ impl From for CaptureState { } } +impl From for CaptureSnapshot { + fn from(value: listener::Snapshot) -> Self { + Self { + state: CaptureState::from(value.state), + active_session_id: value.active_session_id, + finalizing_session_ids: value.finalizing_session_ids, + requested_live_transcription: None, + live_transcription_active: None, + } + } +} + impl From for CaptureStatusEvent { fn from(value: listener::SessionProgressEvent) -> Self { match value { diff --git a/plugins/transcription/src/lib.rs b/plugins/transcription/src/lib.rs index c71fe4d38f..0a6019bc62 100644 --- a/plugins/transcription/src/lib.rs +++ b/plugins/transcription/src/lib.rs @@ -75,6 +75,7 @@ fn make_specta_builder() -> tauri_specta::Builder { listener::commands::stop_capture::, listener::commands::update_capture_config::, listener::commands::get_capture_state::, + listener::commands::get_capture_snapshot::, listener::commands::is_supported_languages_live::, listener::commands::suggest_providers_for_languages_live::, listener::commands::list_documented_language_codes_live::, @@ -117,6 +118,7 @@ pub fn init() -> tauri::plugin::TauriPlugin { let audio = app.state::>().inner().clone(); let session_state_cache: SessionStateCache = Arc::new(StdMutex::new(HashMap::new())); + app.manage(session_state_cache.clone()); let runtime = Arc::new(listener::TauriRuntime { app: app_handle.clone(), session_state_cache, diff --git a/plugins/transcription/src/listener/commands.rs b/plugins/transcription/src/listener/commands.rs index a2e2ffe055..1c8e6ed9b9 100644 --- a/plugins/transcription/src/listener/commands.rs +++ b/plugins/transcription/src/listener/commands.rs @@ -2,7 +2,7 @@ use owhisper_client::AdapterKind; use std::str::FromStr; use crate::listener::ListenerPluginExt; -use crate::{CaptureConfigUpdate, CaptureParams, CaptureState}; +use crate::{CaptureConfigUpdate, CaptureParams, CaptureSnapshot, CaptureState}; use hypr_transcript::{RenderTranscriptRequest, RenderedTranscriptSegment}; use hypr_transcription_core::listener2 as listener2_core; @@ -81,6 +81,14 @@ pub async fn get_capture_state( Ok(app.listener().get_capture_state().await) } +#[tauri::command] +#[specta::specta] +pub async fn get_capture_snapshot( + app: tauri::AppHandle, +) -> Result { + Ok(app.listener().get_capture_snapshot().await) +} + #[tauri::command] #[specta::specta] pub async fn is_supported_languages_live( diff --git a/plugins/transcription/src/listener/ext.rs b/plugins/transcription/src/listener/ext.rs index 6da3a6c778..e68d3e4c48 100644 --- a/plugins/transcription/src/listener/ext.rs +++ b/plugins/transcription/src/listener/ext.rs @@ -1,6 +1,6 @@ use ractor::{ActorRef, call_t, registry}; -use crate::{CaptureConfigUpdate, CaptureParams, CaptureState}; +use crate::{CaptureConfigUpdate, CaptureParams, CaptureSnapshot, CaptureState, SessionStateCache}; use hypr_transcription_core::listener::{ StartSessionError, actors::{RootActor, RootMsg, SessionParams, SourceActor, SourceMsg}, @@ -47,6 +47,47 @@ impl<'a, R: tauri::Runtime, M: tauri::Manager> Listener<'a, R, M> { } } + #[tracing::instrument(skip_all)] + pub async fn get_capture_snapshot(&self) -> CaptureSnapshot { + let mut snapshot = if let Some(cell) = registry::where_is(RootActor::name()) { + let actor: ActorRef = cell.into(); + match call_t!(actor, RootMsg::GetSnapshot, 100) { + Ok(snapshot) => CaptureSnapshot::from(snapshot), + Err(_) => CaptureSnapshot { + state: CaptureState::Inactive, + active_session_id: None, + finalizing_session_ids: vec![], + requested_live_transcription: None, + live_transcription_active: None, + }, + } + } else { + CaptureSnapshot { + state: CaptureState::Inactive, + active_session_id: None, + finalizing_session_ids: vec![], + requested_live_transcription: None, + live_transcription_active: None, + } + }; + + let session_id = snapshot + .active_session_id + .as_ref() + .or_else(|| snapshot.finalizing_session_ids.first()); + if let Some(session_id) = session_id + && let Some((requested, active)) = self + .manager + .try_state::() + .and_then(|cache| cache.lock().ok()?.get(session_id).copied()) + { + snapshot.requested_live_transcription = Some(requested); + snapshot.live_transcription_active = Some(active); + } + + snapshot + } + #[tracing::instrument(skip_all)] pub async fn get_mic_muted(&self) -> bool { if let Some(cell) = registry::where_is(SourceActor::name()) { From 9fcb56b320d004bc0f35aa53bdc6fa59294b2f44 Mon Sep 17 00:00:00 2001 From: ComputelessComputer <63365510+ComputelessComputer@users.noreply.github.com> Date: Tue, 30 Jun 2026 16:38:14 +0900 Subject: [PATCH 2/2] Fix live listener attach lifecycle Prevent overlapping attach calls from leaking listeners and clear stale timers when snapshots switch active sessions. --- .../src/routes/app/-note.$sessionId.test.tsx | 32 +++- .../src/routes/app/note.$sessionId.tsx | 9 +- .../store/zustand/listener/general-live.ts | 49 +++++- .../store/zustand/listener/general.test.ts | 165 ++++++++++++++++++ 4 files changed, 245 insertions(+), 10 deletions(-) diff --git a/apps/desktop/src/routes/app/-note.$sessionId.test.tsx b/apps/desktop/src/routes/app/-note.$sessionId.test.tsx index d99f873a45..4a13a1198c 100644 --- a/apps/desktop/src/routes/app/-note.$sessionId.test.tsx +++ b/apps/desktop/src/routes/app/-note.$sessionId.test.tsx @@ -6,6 +6,12 @@ import { resetTabsStore } from "~/store/zustand/tabs/test-utils"; const mocks = vi.hoisted(() => ({ attachLiveSession: vi.fn(), close: vi.fn(), + listenerState: { + attachLiveSession: vi.fn(), + live: { + eventUnlistenersBySession: {} as Record void)[]>, + }, + }, })); vi.mock("@tauri-apps/api/window", () => ({ @@ -15,11 +21,8 @@ vi.mock("@tauri-apps/api/window", () => ({ })); vi.mock("~/stt/contexts", () => ({ - useListener: ( - selector: (state: { - attachLiveSession: typeof mocks.attachLiveSession; - }) => unknown, - ) => selector({ attachLiveSession: mocks.attachLiveSession }), + useListener: (selector: (state: typeof mocks.listenerState) => unknown) => + selector(mocks.listenerState), })); import { @@ -32,6 +35,8 @@ import { useTabs } from "~/store/zustand/tabs"; describe("standalone note window route", () => { beforeEach(() => { + mocks.listenerState.attachLiveSession = mocks.attachLiveSession; + mocks.listenerState.live.eventUnlistenersBySession = {}; mocks.attachLiveSession.mockClear(); mocks.close.mockClear(); resetTabsStore(); @@ -88,6 +93,23 @@ describe("standalone note window route", () => { expect(mocks.attachLiveSession).toHaveBeenCalledWith("session-1"); }); + + it("reattaches after standalone live session events are removed", () => { + const { rerender } = renderHook(() => + useAttachStandaloneNoteToLiveSession("session-1"), + ); + expect(mocks.attachLiveSession).toHaveBeenCalledTimes(1); + + mocks.listenerState.live.eventUnlistenersBySession = { + "session-1": [vi.fn()], + }; + rerender(); + expect(mocks.attachLiveSession).toHaveBeenCalledTimes(1); + + mocks.listenerState.live.eventUnlistenersBySession = {}; + rerender(); + expect(mocks.attachLiveSession).toHaveBeenCalledTimes(2); + }); }); function dispatchKeyDown(key: string) { diff --git a/apps/desktop/src/routes/app/note.$sessionId.tsx b/apps/desktop/src/routes/app/note.$sessionId.tsx index a6b24d52dd..9509b8aa6b 100644 --- a/apps/desktop/src/routes/app/note.$sessionId.tsx +++ b/apps/desktop/src/routes/app/note.$sessionId.tsx @@ -31,10 +31,17 @@ function Component() { export function useAttachStandaloneNoteToLiveSession(sessionId: string) { const attachLiveSession = useListener((state) => state.attachLiveSession); + const hasLiveSessionEvents = useListener((state) => + Boolean(state.live.eventUnlistenersBySession[sessionId]), + ); useEffect(() => { + if (hasLiveSessionEvents) { + return; + } + void attachLiveSession(sessionId); - }, [attachLiveSession, sessionId]); + }, [attachLiveSession, hasLiveSessionEvents, sessionId]); } export function useStandaloneNoteTab(sessionId: string) { diff --git a/apps/desktop/src/store/zustand/listener/general-live.ts b/apps/desktop/src/store/zustand/listener/general-live.ts index 2eac73b960..eb48eea7c4 100644 --- a/apps/desktop/src/store/zustand/listener/general-live.ts +++ b/apps/desktop/src/store/zustand/listener/general-live.ts @@ -357,10 +357,28 @@ export const attachLiveSession = ( return Promise.resolve(); } + const pendingUnlisteners: (() => void)[] = []; + let registeredUnlisteners = pendingUnlisteners; + setLiveState(set, (live) => { + live.eventUnlistenersBySession[targetSessionId] = pendingUnlisteners; + if (!live.sessionId) { + live.sessionId = targetSessionId; + } + }); + const handlers = createSessionEventHandlers(set, get, targetSessionId); const program = Effect.gen(function* () { const unlisteners = yield* listenToAllSessionEvents(handlers); + if ( + get().live.eventUnlistenersBySession[targetSessionId] !== + pendingUnlisteners + ) { + clearLiveEventUnlisteners(unlisteners); + return; + } + + registeredUnlisteners = unlisteners; setLiveState(set, (live) => { live.eventUnlistenersBySession[targetSessionId] = unlisteners; }); @@ -373,11 +391,20 @@ export const attachLiveSession = ( Exit.match(exit, { onFailure: (cause) => { console.error("[listener] failed to attach live session:", cause); - clearLiveEventUnlisteners( - get().live.eventUnlistenersBySession[targetSessionId], - ); + clearLiveEventUnlisteners(registeredUnlisteners); setLiveState(set, (live) => { - delete live.eventUnlistenersBySession[targetSessionId]; + if ( + live.eventUnlistenersBySession[targetSessionId] === + registeredUnlisteners + ) { + delete live.eventUnlistenersBySession[targetSessionId]; + } + if ( + live.sessionId === targetSessionId && + live.status === "inactive" + ) { + live.sessionId = null; + } }); }, onSuccess: () => undefined, @@ -396,6 +423,10 @@ function applyCaptureSnapshot( snapshot.activeSessionId === targetSessionId ) { const currentLive = get().live; + if (currentLive.sessionId !== targetSessionId) { + clearLiveInterval(currentLive.intervalId); + } + const intervalId = currentLive.sessionId === targetSessionId && currentLive.intervalId ? currentLive.intervalId @@ -428,9 +459,19 @@ function applyCaptureSnapshot( snapshot.finalizingSessionIds.includes(targetSessionId) ) { setLiveState(set, (live) => { + if (!live.sessionId) { + live.sessionId = targetSessionId; + } markLiveFinalizing(live, targetSessionId); }); + return; } + + setLiveState(set, (live) => { + if (live.sessionId === targetSessionId && live.status === "inactive") { + live.sessionId = null; + } + }); } export const stopLiveSession = ( diff --git a/apps/desktop/src/store/zustand/listener/general.test.ts b/apps/desktop/src/store/zustand/listener/general.test.ts index e1e7ee5a44..3d76acd30e 100644 --- a/apps/desktop/src/store/zustand/listener/general.test.ts +++ b/apps/desktop/src/store/zustand/listener/general.test.ts @@ -676,6 +676,46 @@ describe("General Listener Slice", () => { expect(listenCaptureDataMock).toHaveBeenCalledTimes(1); }); + test("attachLiveSession ignores overlapping attaches for the same session", async () => { + await Promise.all([ + store.getState().attachLiveSession("session-a"), + store.getState().attachLiveSession("session-a"), + ]); + + expect(listenCaptureLifecycleMock).toHaveBeenCalledTimes(1); + expect(listenCaptureStatusMock).toHaveBeenCalledTimes(1); + expect(listenCaptureDataMock).toHaveBeenCalledTimes(1); + expect(getCaptureSnapshotMock).toHaveBeenCalledTimes(1); + }); + + test("attachLiveSession clears the previous session interval before applying an active snapshot", async () => { + const intervalId = setInterval(() => {}, 1000); + const clearIntervalSpy = vi.spyOn(globalThis, "clearInterval"); + getCaptureSnapshotMock.mockResolvedValueOnce({ + status: "ok", + data: { + activeSessionId: "session-b", + finalizingSessionIds: [], + liveTranscriptionActive: true, + requestedLiveTranscription: true, + state: "active", + }, + }); + store.setState((state) => + mutate(state, (draft) => { + markLiveActive(draft.live, "session-a", intervalId, true, true, null); + }), + ); + + await store.getState().attachLiveSession("session-b"); + + expect(clearIntervalSpy).toHaveBeenCalledWith(intervalId); + expect(store.getState().live.sessionId).toBe("session-b"); + + clearInterval(store.getState().live.intervalId); + clearIntervalSpy.mockRestore(); + }); + test("attachLiveSession does not mark a different active native capture as current", async () => { getCaptureSnapshotMock.mockResolvedValueOnce({ status: "ok", @@ -694,6 +734,131 @@ describe("General Listener Slice", () => { expect(store.getState().live.sessionId).toBeNull(); }); + test("attachLiveSession hydrates finalizing native capture for the same session", async () => { + let dataHandler: + | ((event: { + payload: { + session_id: string; + type: "transcript_segment_delta"; + delta: unknown; + }; + }) => void) + | undefined; + listenCaptureDataMock.mockImplementationOnce((handler) => { + dataHandler = handler; + return Promise.resolve(() => {}); + }); + getCaptureSnapshotMock.mockResolvedValueOnce({ + status: "ok", + data: { + activeSessionId: null, + finalizingSessionIds: ["session-a"], + liveTranscriptionActive: null, + requestedLiveTranscription: null, + state: "finalizing", + }, + }); + + await store.getState().attachLiveSession("session-a"); + + expect(store.getState().live.sessionId).toBe("session-a"); + expect(store.getState().getSessionMode("session-a")).toBe("finalizing"); + + dataHandler?.({ + payload: { + delta: { + removed_ids: [], + upserts: [ + { + end_ms: 1000, + id: "segment-1", + key: { + channel: "DirectMic", + }, + start_ms: 0, + text: "hello", + words: [], + }, + ], + }, + session_id: "session-a", + type: "transcript_segment_delta", + }, + }); + + expect(store.getState().liveSegments).toMatchObject([ + { id: "segment-1", text: "hello" }, + ]); + }); + + test("attachLiveSession accepts segment events before snapshot hydration", async () => { + let dataHandler: + | ((event: { + payload: { + session_id: string; + type: "transcript_segment_delta"; + delta: unknown; + }; + }) => void) + | undefined; + let resolveSnapshot: + | ((value: Awaited>) => void) + | undefined; + listenCaptureDataMock.mockImplementationOnce((handler) => { + dataHandler = handler; + return Promise.resolve(() => {}); + }); + getCaptureSnapshotMock.mockReturnValueOnce( + new Promise((resolve) => { + resolveSnapshot = resolve; + }), + ); + + const attachPromise = store.getState().attachLiveSession("session-a"); + await vi.waitFor(() => { + expect(dataHandler).toBeDefined(); + }); + + dataHandler?.({ + payload: { + delta: { + removed_ids: [], + upserts: [ + { + end_ms: 1000, + id: "segment-before-snapshot", + key: { + channel: "DirectMic", + }, + start_ms: 0, + text: "early", + words: [], + }, + ], + }, + session_id: "session-a", + type: "transcript_segment_delta", + }, + }); + + expect(store.getState().liveSegments).toMatchObject([ + { id: "segment-before-snapshot", text: "early" }, + ]); + + resolveSnapshot?.({ + status: "ok", + data: { + activeSessionId: null, + finalizingSessionIds: [], + liveTranscriptionActive: null, + requestedLiveTranscription: null, + state: "inactive", + }, + }); + await attachPromise; + expect(store.getState().live.sessionId).toBeNull(); + }); + test("start returns false while another session is active", async () => { store.setState((state) => mutate(state, (draft) => {