diff --git a/packages/core/etc/components-core.api.md b/packages/core/etc/components-core.api.md index 6ba2034d8..9e040a3e8 100644 --- a/packages/core/etc/components-core.api.md +++ b/packages/core/etc/components-core.api.md @@ -159,6 +159,7 @@ export const cssPrefix = "lk"; // @public (undocumented) export const DataTopic: { readonly CHAT: "lk.chat"; + readonly TRANSCRIPTION: "lk.transcription"; }; // @public (undocumented) diff --git a/packages/core/src/components/chat.ts b/packages/core/src/components/chat.ts index 6689f0476..6214b8a6d 100644 --- a/packages/core/src/components/chat.ts +++ b/packages/core/src/components/chat.ts @@ -8,6 +8,7 @@ import { sendMessage, setupDataMessageHandler, } from '../observables/dataChannel'; +import { log } from '../logger'; /** @public */ export type { ChatMessage }; @@ -165,10 +166,14 @@ export function setupChat(room: Room, options?: ChatOptions) { ...chatMsg, ignoreLegacy: serverSupportsDataStreams(), }); - await sendMessage(room.localParticipant, encodedLegacyMsg, { - reliable: true, - topic: legacyTopic, - }); + try { + await sendMessage(room.localParticipant, encodedLegacyMsg, { + reliable: true, + topic: legacyTopic, + }); + } catch (error) { + log.info('could not send message in legacy chat format', error); + } return chatMsg; } finally { isSending$.next(false); diff --git a/packages/core/src/components/textStream.ts b/packages/core/src/components/textStream.ts index 83841ef28..9fa124d01 100644 --- a/packages/core/src/components/textStream.ts +++ b/packages/core/src/components/textStream.ts @@ -97,6 +97,7 @@ export function setupTextStream(room: Room, topic: string): Observable { + room.unregisterTextStreamHandler(topic); textStreamsSubject.complete(); getObservableCache().delete(cacheKey); }); diff --git a/packages/core/src/observables/dataChannel.ts b/packages/core/src/observables/dataChannel.ts index e86d5ff25..04f6bf9f4 100644 --- a/packages/core/src/observables/dataChannel.ts +++ b/packages/core/src/observables/dataChannel.ts @@ -13,6 +13,7 @@ import { ReceivedChatMessage } from '../components/chat'; export const DataTopic = { CHAT: 'lk.chat', + TRANSCRIPTION: 'lk.transcription', } as const; /** @deprecated */ diff --git a/packages/react/etc/components-react.api.md b/packages/react/etc/components-react.api.md index a070ddf38..d60622cc6 100644 --- a/packages/react/etc/components-react.api.md +++ b/packages/react/etc/components-react.api.md @@ -1213,6 +1213,17 @@ export function useTrackTranscription(trackRef: TrackReferenceOrPlaceholder_4 | // @alpha export function useTrackVolume(trackOrTrackReference?: LocalAudioTrack | RemoteAudioTrack | TrackReference_3, options?: AudioAnalyserOptions): number; +// @beta +export function useTranscriptions(opts?: UseTranscriptionsOptions): TextStreamData_2[]; + +// @beta (undocumented) +export interface UseTranscriptionsOptions { + // (undocumented) + participantIdentities?: string[]; + // (undocumented) + trackSids?: string[]; +} + // @public export function useVisualStableUpdate( trackReferences: TrackReferenceOrPlaceholder_4[], maxItemsOnPage: number, options?: UseVisualStableUpdateOptions): TrackReferenceOrPlaceholder_4[]; diff --git a/packages/react/src/hooks/index.ts b/packages/react/src/hooks/index.ts index 071235ffa..cdd9ed342 100644 --- a/packages/react/src/hooks/index.ts +++ b/packages/react/src/hooks/index.ts @@ -55,3 +55,4 @@ export * from './useVoiceAssistant'; export * from './useParticipantAttributes'; export * from './useIsRecording'; export * from './useTextStream'; +export * from './useTranscriptions'; diff --git a/packages/react/src/hooks/useTrackTranscription.ts b/packages/react/src/hooks/useTrackTranscription.ts index 809dd37b4..175ced50a 100644 --- a/packages/react/src/hooks/useTrackTranscription.ts +++ b/packages/react/src/hooks/useTrackTranscription.ts @@ -11,6 +11,7 @@ import { import type { TranscriptionSegment } from 'livekit-client'; import * as React from 'react'; import { useTrackSyncTime } from './useTrackSyncTime'; +import { useTranscriptions } from './useTranscriptions'; /** * @alpha @@ -35,19 +36,58 @@ const TRACK_TRANSCRIPTION_DEFAULTS = { } as const satisfies TrackTranscriptionOptions; /** - * @returns An object consisting of `segments` with maximum length of opts.windowLength and `activeSegments` that are valid for the current track timestamp + * @returns An object consisting of `segments` with maximum length of opts.bufferSize * @alpha */ export function useTrackTranscription( trackRef: TrackReferenceOrPlaceholder | undefined, options?: TrackTranscriptionOptions, +) { + const legacyTranscription = useLegacyTranscription(trackRef, options); + + const participantIdentities = React.useMemo(() => { + if (!trackRef) { + return []; + } + return [trackRef.participant.identity]; + }, [trackRef]); + const trackSids = React.useMemo(() => { + if (!trackRef) { + return []; + } + return [getTrackReferenceId(trackRef)]; + }, [trackRef]); + const useTranscriptionsOptions = React.useMemo(() => { + return { participantIdentities, trackSids }; + }, [participantIdentities, trackSids]); + const transcriptions = useTranscriptions(useTranscriptionsOptions); + + const streamAsSegments = transcriptions.map((t) => { + const legacySegment: ReceivedTranscriptionSegment = { + id: t.streamInfo.id, + text: t.text, + receivedAt: t.streamInfo.timestamp, + receivedAtMediaTimestamp: 0, + language: '', + startTime: 0, + endTime: 0, + final: false, + firstReceivedTime: 0, + lastReceivedTime: 0, + }; + return legacySegment; + }); + + return streamAsSegments.length > 0 ? { segments: streamAsSegments } : legacyTranscription; +} + +function useLegacyTranscription( + trackRef: TrackReferenceOrPlaceholder | undefined, + options?: TrackTranscriptionOptions, ) { const opts = { ...TRACK_TRANSCRIPTION_DEFAULTS, ...options }; const [segments, setSegments] = React.useState>([]); - // const [activeSegments, setActiveSegments] = React.useState>( - // [], - // ); - // const prevActiveSegments = React.useRef([]); + const syncTimestamps = useTrackSyncTime(trackRef); const handleSegmentMessage = (newSegments: TranscriptionSegment[]) => { opts.onTranscription?.(newSegments); @@ -72,20 +112,5 @@ export function useTrackTranscription( }; }, [trackRef && getTrackReferenceId(trackRef), handleSegmentMessage]); - // React.useEffect(() => { - // if (syncTimestamps) { - // const newActiveSegments = getActiveTranscriptionSegments( - // segments, - // syncTimestamps, - // opts.maxAge, - // ); - // // only update active segment array if content actually changed - // if (didActiveSegmentsChange(prevActiveSegments.current, newActiveSegments)) { - // setActiveSegments(newActiveSegments); - // prevActiveSegments.current = newActiveSegments; - // } - // } - // }, [syncTimestamps, segments, opts.maxAge]); - return { segments }; } diff --git a/packages/react/src/hooks/useTranscriptions.ts b/packages/react/src/hooks/useTranscriptions.ts new file mode 100644 index 000000000..a256cbb31 --- /dev/null +++ b/packages/react/src/hooks/useTranscriptions.ts @@ -0,0 +1,44 @@ +import * as React from 'react'; +import { useTextStream } from './useTextStream'; +import { DataTopic } from '@livekit/components-core'; + +/** + * @beta + */ +export interface UseTranscriptionsOptions { + participantIdentities?: string[]; + trackSids?: string[]; +} + +/** + * @beta + * useTranscriptions is a hook that returns the transcriptions for the given participant identities and track sids, + * if no options are provided, it will return all transcriptions + * @example + * ```tsx + * const transcriptions = useTranscriptions(); + * return
{transcriptions.map((transcription) => transcription.text)}
; + * ``` + */ +export function useTranscriptions(opts?: UseTranscriptionsOptions) { + const { participantIdentities, trackSids } = opts ?? {}; + const { textStreams } = useTextStream(DataTopic.TRANSCRIPTION); + + const filteredMessages = React.useMemo( + () => + textStreams + .filter((stream) => + participantIdentities + ? participantIdentities.includes(stream.participantInfo.identity) + : true, + ) + .filter((stream) => + trackSids + ? trackSids.includes(stream.streamInfo.attributes?.['lk.transcribed_track_id'] ?? '') + : true, + ), + [textStreams, participantIdentities, trackSids], + ); + + return filteredMessages; +}