From 25445a0bfb0699a425c859dfa55d8f040d6205c2 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 5 Mar 2025 16:11:32 +0000 Subject: [PATCH 1/5] quick transcription prototype --- packages/core/src/components/chat.ts | 13 +++++--- packages/core/src/components/textStream.ts | 2 ++ packages/core/src/observables/dataChannel.ts | 2 +- .../participant/ParticipantTile.tsx | 31 +++++++++++++++++++ packages/react/src/hooks/useTranscription.ts | 17 ++++++++++ 5 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 packages/react/src/hooks/useTranscription.ts diff --git a/packages/core/src/components/chat.ts b/packages/core/src/components/chat.ts index 6689f0476..6214b8a6d 100644 --- a/packages/core/src/components/chat.ts +++ b/packages/core/src/components/chat.ts @@ -8,6 +8,7 @@ import { sendMessage, setupDataMessageHandler, } from '../observables/dataChannel'; +import { log } from '../logger'; /** @public */ export type { ChatMessage }; @@ -165,10 +166,14 @@ export function setupChat(room: Room, options?: ChatOptions) { ...chatMsg, ignoreLegacy: serverSupportsDataStreams(), }); - await sendMessage(room.localParticipant, encodedLegacyMsg, { - reliable: true, - topic: legacyTopic, - }); + try { + await sendMessage(room.localParticipant, encodedLegacyMsg, { + reliable: true, + topic: legacyTopic, + }); + } catch (error) { + log.info('could not send message in legacy chat format', error); + } return chatMsg; } finally { isSending$.next(false); diff --git a/packages/core/src/components/textStream.ts b/packages/core/src/components/textStream.ts index 83841ef28..1ea99ccd3 100644 --- a/packages/core/src/components/textStream.ts +++ b/packages/core/src/components/textStream.ts @@ -67,6 +67,7 @@ export function setupTextStream(room: Room, topic: string): Observable { + console.log('accumulatedText', accumulatedText); // Find and update the stream in our array const index = textStreams.findIndex((stream) => stream.streamInfo.id === reader.info.id); if (index !== -1) { @@ -97,6 +98,7 @@ export function setupTextStream(room: Room, topic: string): Observable { + room.unregisterTextStreamHandler(topic); textStreamsSubject.complete(); getObservableCache().delete(cacheKey); }); diff --git a/packages/core/src/observables/dataChannel.ts b/packages/core/src/observables/dataChannel.ts index e86d5ff25..cff44e91e 100644 --- a/packages/core/src/observables/dataChannel.ts +++ b/packages/core/src/observables/dataChannel.ts @@ -12,7 +12,7 @@ import { createChatObserver, createDataObserver } from './room'; import { ReceivedChatMessage } from '../components/chat'; export const DataTopic = { - CHAT: 'lk.chat', + CHAT: 'lk.chat-temp', } as const; /** @deprecated */ diff --git a/packages/react/src/components/participant/ParticipantTile.tsx b/packages/react/src/components/participant/ParticipantTile.tsx index 5d0706e0b..c49eeae3f 100644 --- a/packages/react/src/components/participant/ParticipantTile.tsx +++ b/packages/react/src/components/participant/ParticipantTile.tsx @@ -22,6 +22,7 @@ import { VideoTrack } from './VideoTrack'; import { AudioTrack } from './AudioTrack'; import { useParticipantTile } from '../../hooks'; import { useIsEncrypted } from '../../hooks/useIsEncrypted'; +import { useTranscription } from '../../hooks/useTranscription'; /** * The `ParticipantContextIfNeeded` component only creates a `ParticipantContext` @@ -71,6 +72,7 @@ export interface ParticipantTileProps extends React.HTMLAttributes void; } @@ -100,12 +102,15 @@ export const ParticipantTile: ( children, onParticipantClick, disableSpeakingIndicator, + showTranscription = true, ...htmlProps }: ParticipantTileProps, ref, ) { const trackReference = useEnsureTrackRef(trackRef); + const { activeTranscription } = useTranscription(trackReference.participant.identity); + const { elementProps } = useParticipantTile({ htmlProps, disableSpeakingIndicator, @@ -184,6 +189,32 @@ export const ParticipantTile: ( )} + {showTranscription && activeTranscription && ( +
+
+ {activeTranscription.text} +
+
+ )} diff --git a/packages/react/src/hooks/useTranscription.ts b/packages/react/src/hooks/useTranscription.ts new file mode 100644 index 000000000..9f053596f --- /dev/null +++ b/packages/react/src/hooks/useTranscription.ts @@ -0,0 +1,17 @@ +import * as React from 'react'; +import { useTextStream } from './useTextStream'; + +export function useTranscription(participantIdentity: string) { + const { textStreams } = useTextStream('lk.chat'); + + const filteredMessages = React.useMemo( + () => textStreams.filter((stream) => stream.participantInfo.identity === participantIdentity), + [textStreams, participantIdentity], + ); + + const activeTranscription = React.useMemo(() => { + return filteredMessages.at(-1); + }, [filteredMessages]); + + return { activeTranscription: activeTranscription, transcriptions: filteredMessages }; +} From 1c72ae2ada6f4f980d7cfd0c86d09d1db85ea3e5 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 27 Mar 2025 17:31:16 +0100 Subject: [PATCH 2/5] Add useTranscription hook with filtering based on identities and track ids --- .../react/src/hooks/useTrackTranscription.ts | 22 ++----------- packages/react/src/hooks/useTranscription.ts | 31 +++++++++++++------ 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/packages/react/src/hooks/useTrackTranscription.ts b/packages/react/src/hooks/useTrackTranscription.ts index 809dd37b4..8c917330b 100644 --- a/packages/react/src/hooks/useTrackTranscription.ts +++ b/packages/react/src/hooks/useTrackTranscription.ts @@ -35,7 +35,7 @@ const TRACK_TRANSCRIPTION_DEFAULTS = { } as const satisfies TrackTranscriptionOptions; /** - * @returns An object consisting of `segments` with maximum length of opts.windowLength and `activeSegments` that are valid for the current track timestamp + * @returns An object consisting of `segments` with maximum length of opts.bufferSize * @alpha */ export function useTrackTranscription( @@ -44,10 +44,7 @@ export function useTrackTranscription( ) { const opts = { ...TRACK_TRANSCRIPTION_DEFAULTS, ...options }; const [segments, setSegments] = React.useState>([]); - // const [activeSegments, setActiveSegments] = React.useState>( - // [], - // ); - // const prevActiveSegments = React.useRef([]); + const syncTimestamps = useTrackSyncTime(trackRef); const handleSegmentMessage = (newSegments: TranscriptionSegment[]) => { opts.onTranscription?.(newSegments); @@ -72,20 +69,5 @@ export function useTrackTranscription( }; }, [trackRef && getTrackReferenceId(trackRef), handleSegmentMessage]); - // React.useEffect(() => { - // if (syncTimestamps) { - // const newActiveSegments = getActiveTranscriptionSegments( - // segments, - // syncTimestamps, - // opts.maxAge, - // ); - // // only update active segment array if content actually changed - // if (didActiveSegmentsChange(prevActiveSegments.current, newActiveSegments)) { - // setActiveSegments(newActiveSegments); - // prevActiveSegments.current = newActiveSegments; - // } - // } - // }, [syncTimestamps, segments, opts.maxAge]); - return { segments }; } diff --git a/packages/react/src/hooks/useTranscription.ts b/packages/react/src/hooks/useTranscription.ts index 9f053596f..e1545399a 100644 --- a/packages/react/src/hooks/useTranscription.ts +++ b/packages/react/src/hooks/useTranscription.ts @@ -1,17 +1,30 @@ import * as React from 'react'; import { useTextStream } from './useTextStream'; -export function useTranscription(participantIdentity: string) { - const { textStreams } = useTextStream('lk.chat'); +export interface UseTranscriptionsOptions { + participantIdentities?: string[]; + trackSids?: string[]; +} + +export function useTranscriptions(opts: UseTranscriptionsOptions) { + const { participantIdentities, trackSids } = opts; + const { textStreams } = useTextStream('lk.transcription'); const filteredMessages = React.useMemo( - () => textStreams.filter((stream) => stream.participantInfo.identity === participantIdentity), - [textStreams, participantIdentity], + () => + textStreams + .filter((stream) => + participantIdentities + ? participantIdentities.includes(stream.participantInfo.identity) + : true, + ) + .filter((stream) => + trackSids + ? trackSids.includes(stream.streamInfo.attributes?.['lk.transcribed_track_id'] ?? '') + : true, + ), + [textStreams, participantIdentities, trackSids], ); - const activeTranscription = React.useMemo(() => { - return filteredMessages.at(-1); - }, [filteredMessages]); - - return { activeTranscription: activeTranscription, transcriptions: filteredMessages }; + return filteredMessages; } From 6cf8bd169f6a18c4b00d56be79160a7f19c74b2b Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 27 Mar 2025 17:39:03 +0100 Subject: [PATCH 3/5] cleanup --- packages/core/etc/components-core.api.md | 1 + packages/core/src/components/textStream.ts | 1 - packages/core/src/observables/dataChannel.ts | 3 +- packages/react/etc/components-react.api.md | 11 +++++++ .../participant/ParticipantTile.tsx | 31 ------------------- packages/react/src/hooks/index.ts | 1 + packages/react/src/hooks/useTranscription.ts | 20 ++++++++++-- 7 files changed, 32 insertions(+), 36 deletions(-) diff --git a/packages/core/etc/components-core.api.md b/packages/core/etc/components-core.api.md index 6ba2034d8..9e040a3e8 100644 --- a/packages/core/etc/components-core.api.md +++ b/packages/core/etc/components-core.api.md @@ -159,6 +159,7 @@ export const cssPrefix = "lk"; // @public (undocumented) export const DataTopic: { readonly CHAT: "lk.chat"; + readonly TRANSCRIPTION: "lk.transcription"; }; // @public (undocumented) diff --git a/packages/core/src/components/textStream.ts b/packages/core/src/components/textStream.ts index 1ea99ccd3..9fa124d01 100644 --- a/packages/core/src/components/textStream.ts +++ b/packages/core/src/components/textStream.ts @@ -67,7 +67,6 @@ export function setupTextStream(room: Room, topic: string): Observable { - console.log('accumulatedText', accumulatedText); // Find and update the stream in our array const index = textStreams.findIndex((stream) => stream.streamInfo.id === reader.info.id); if (index !== -1) { diff --git a/packages/core/src/observables/dataChannel.ts b/packages/core/src/observables/dataChannel.ts index cff44e91e..04f6bf9f4 100644 --- a/packages/core/src/observables/dataChannel.ts +++ b/packages/core/src/observables/dataChannel.ts @@ -12,7 +12,8 @@ import { createChatObserver, createDataObserver } from './room'; import { ReceivedChatMessage } from '../components/chat'; export const DataTopic = { - CHAT: 'lk.chat-temp', + CHAT: 'lk.chat', + TRANSCRIPTION: 'lk.transcription', } as const; /** @deprecated */ diff --git a/packages/react/etc/components-react.api.md b/packages/react/etc/components-react.api.md index a070ddf38..d60622cc6 100644 --- a/packages/react/etc/components-react.api.md +++ b/packages/react/etc/components-react.api.md @@ -1213,6 +1213,17 @@ export function useTrackTranscription(trackRef: TrackReferenceOrPlaceholder_4 | // @alpha export function useTrackVolume(trackOrTrackReference?: LocalAudioTrack | RemoteAudioTrack | TrackReference_3, options?: AudioAnalyserOptions): number; +// @beta +export function useTranscriptions(opts?: UseTranscriptionsOptions): TextStreamData_2[]; + +// @beta (undocumented) +export interface UseTranscriptionsOptions { + // (undocumented) + participantIdentities?: string[]; + // (undocumented) + trackSids?: string[]; +} + // @public export function useVisualStableUpdate( trackReferences: TrackReferenceOrPlaceholder_4[], maxItemsOnPage: number, options?: UseVisualStableUpdateOptions): TrackReferenceOrPlaceholder_4[]; diff --git a/packages/react/src/components/participant/ParticipantTile.tsx b/packages/react/src/components/participant/ParticipantTile.tsx index c49eeae3f..5d0706e0b 100644 --- a/packages/react/src/components/participant/ParticipantTile.tsx +++ b/packages/react/src/components/participant/ParticipantTile.tsx @@ -22,7 +22,6 @@ import { VideoTrack } from './VideoTrack'; import { AudioTrack } from './AudioTrack'; import { useParticipantTile } from '../../hooks'; import { useIsEncrypted } from '../../hooks/useIsEncrypted'; -import { useTranscription } from '../../hooks/useTranscription'; /** * The `ParticipantContextIfNeeded` component only creates a `ParticipantContext` @@ -72,7 +71,6 @@ export interface ParticipantTileProps extends React.HTMLAttributes void; } @@ -102,15 +100,12 @@ export const ParticipantTile: ( children, onParticipantClick, disableSpeakingIndicator, - showTranscription = true, ...htmlProps }: ParticipantTileProps, ref, ) { const trackReference = useEnsureTrackRef(trackRef); - const { activeTranscription } = useTranscription(trackReference.participant.identity); - const { elementProps } = useParticipantTile({ htmlProps, disableSpeakingIndicator, @@ -189,32 +184,6 @@ export const ParticipantTile: ( )} - {showTranscription && activeTranscription && ( -
-
- {activeTranscription.text} -
-
- )} diff --git a/packages/react/src/hooks/index.ts b/packages/react/src/hooks/index.ts index 071235ffa..254510a9a 100644 --- a/packages/react/src/hooks/index.ts +++ b/packages/react/src/hooks/index.ts @@ -55,3 +55,4 @@ export * from './useVoiceAssistant'; export * from './useParticipantAttributes'; export * from './useIsRecording'; export * from './useTextStream'; +export * from './useTranscription'; diff --git a/packages/react/src/hooks/useTranscription.ts b/packages/react/src/hooks/useTranscription.ts index e1545399a..0c56da5b8 100644 --- a/packages/react/src/hooks/useTranscription.ts +++ b/packages/react/src/hooks/useTranscription.ts @@ -1,14 +1,28 @@ import * as React from 'react'; import { useTextStream } from './useTextStream'; +import { DataTopic } from '@livekit/components-core'; +/** + * @beta + */ export interface UseTranscriptionsOptions { participantIdentities?: string[]; trackSids?: string[]; } -export function useTranscriptions(opts: UseTranscriptionsOptions) { - const { participantIdentities, trackSids } = opts; - const { textStreams } = useTextStream('lk.transcription'); +/** + * @beta + * useTranscriptions is a hook that returns the transcriptions for the given participant identities and track sids, + * if no options are provided, it will return all transcriptions + * @example + * ```tsx + * const { transcriptions } = useTranscriptions(); + * return
{transcriptions.map((transcription) => transcription.text)}
; + * ``` + */ +export function useTranscriptions(opts?: UseTranscriptionsOptions) { + const { participantIdentities, trackSids } = opts ?? {}; + const { textStreams } = useTextStream(DataTopic.TRANSCRIPTION); const filteredMessages = React.useMemo( () => From df7aad7a463206e5b46e96a916c7ba5b16ee5e53 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Thu, 27 Mar 2025 17:40:25 +0100 Subject: [PATCH 4/5] more cleanup --- packages/react/src/hooks/index.ts | 2 +- .../src/hooks/{useTranscription.ts => useTranscriptions.ts} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename packages/react/src/hooks/{useTranscription.ts => useTranscriptions.ts} (96%) diff --git a/packages/react/src/hooks/index.ts b/packages/react/src/hooks/index.ts index 254510a9a..cdd9ed342 100644 --- a/packages/react/src/hooks/index.ts +++ b/packages/react/src/hooks/index.ts @@ -55,4 +55,4 @@ export * from './useVoiceAssistant'; export * from './useParticipantAttributes'; export * from './useIsRecording'; export * from './useTextStream'; -export * from './useTranscription'; +export * from './useTranscriptions'; diff --git a/packages/react/src/hooks/useTranscription.ts b/packages/react/src/hooks/useTranscriptions.ts similarity index 96% rename from packages/react/src/hooks/useTranscription.ts rename to packages/react/src/hooks/useTranscriptions.ts index 0c56da5b8..a256cbb31 100644 --- a/packages/react/src/hooks/useTranscription.ts +++ b/packages/react/src/hooks/useTranscriptions.ts @@ -16,7 +16,7 @@ export interface UseTranscriptionsOptions { * if no options are provided, it will return all transcriptions * @example * ```tsx - * const { transcriptions } = useTranscriptions(); + * const transcriptions = useTranscriptions(); * return
{transcriptions.map((transcription) => transcription.text)}
; * ``` */ From 867d081928830bb3b79559500a4d1bdf0067227d Mon Sep 17 00:00:00 2001 From: lukasIO Date: Fri, 28 Mar 2025 12:55:49 +0100 Subject: [PATCH 5/5] wip stream compat for transcription segments --- .../react/src/hooks/useTrackTranscription.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/react/src/hooks/useTrackTranscription.ts b/packages/react/src/hooks/useTrackTranscription.ts index 8c917330b..175ced50a 100644 --- a/packages/react/src/hooks/useTrackTranscription.ts +++ b/packages/react/src/hooks/useTrackTranscription.ts @@ -11,6 +11,7 @@ import { import type { TranscriptionSegment } from 'livekit-client'; import * as React from 'react'; import { useTrackSyncTime } from './useTrackSyncTime'; +import { useTranscriptions } from './useTranscriptions'; /** * @alpha @@ -41,6 +42,48 @@ const TRACK_TRANSCRIPTION_DEFAULTS = { export function useTrackTranscription( trackRef: TrackReferenceOrPlaceholder | undefined, options?: TrackTranscriptionOptions, +) { + const legacyTranscription = useLegacyTranscription(trackRef, options); + + const participantIdentities = React.useMemo(() => { + if (!trackRef) { + return []; + } + return [trackRef.participant.identity]; + }, [trackRef]); + const trackSids = React.useMemo(() => { + if (!trackRef) { + return []; + } + return [getTrackReferenceId(trackRef)]; + }, [trackRef]); + const useTranscriptionsOptions = React.useMemo(() => { + return { participantIdentities, trackSids }; + }, [participantIdentities, trackSids]); + const transcriptions = useTranscriptions(useTranscriptionsOptions); + + const streamAsSegments = transcriptions.map((t) => { + const legacySegment: ReceivedTranscriptionSegment = { + id: t.streamInfo.id, + text: t.text, + receivedAt: t.streamInfo.timestamp, + receivedAtMediaTimestamp: 0, + language: '', + startTime: 0, + endTime: 0, + final: false, + firstReceivedTime: 0, + lastReceivedTime: 0, + }; + return legacySegment; + }); + + return streamAsSegments.length > 0 ? { segments: streamAsSegments } : legacyTranscription; +} + +function useLegacyTranscription( + trackRef: TrackReferenceOrPlaceholder | undefined, + options?: TrackTranscriptionOptions, ) { const opts = { ...TRACK_TRANSCRIPTION_DEFAULTS, ...options }; const [segments, setSegments] = React.useState>([]);