Added Captions button and Captions overlay

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.11.4
+
+### Changes
+
+- Added a new Captions button to the menu bar, which uses the Real-Time Transcriptions feature
+
 ## 0.11.3
 
 ### Changes

README.md

@@ -43,7 +43,9 @@ If you want to use `yarn` to install dependencies, first run the [yarn import](h
 
 Twilio Video has partnered with [Krisp Technologies Inc.](https://krisp.ai/) to add [noise cancellation](https://www.twilio.com/docs/video/noise-cancellation) to the local audio track. This feature is licensed under the [Krisp Plugin for Twilio](https://twilio.github.io/krisp-audio-plugin/LICENSE.html). In order to add this feature to your application, please run `npm run noisecancellation:krisp` immediately after the [previous step](#install-dependencies).
 
-## Install Twilio CLI and RTC Plugin
+### Add Captions (beta)
+
+The on-screen Captions functionality is provided by the [Real-Time Transcriptions for Video(beta)](https://www.twilio.com/docs/video/api/transcriptions) feature. To use captions in this reference application Real-Time Transcriptions must be enabled by default in the [Room settings page](https://console.twilio.com/us1/develop/video/manage/room-settings) on the Twilio Console. Note that usage charges will apply.
 
 ### Install the Twilio CLI
 

package.json

@@ -35,7 +35,7 @@
     "swiper": "^8.1.5",
     "ts-node": "^9.1.1",
     "twilio": "^5.5.2",
-    "twilio-video": "^2.29.0"
+    "twilio-video": "^2.32.1"
   },
   "devDependencies": {
     "@storybook/addon-actions": "^6.5.10",

src/components/Buttons/ToggleCaptionsButton/ToggleCaptionsButton.test.tsx

@@ -0,0 +1,91 @@
+import React from 'react';
+import { shallow } from 'enzyme';
+import ToggleCaptionsButton from './ToggleCaptionsButton';
+import { useAppState } from '../../../state';
+import CaptionsIcon from '../../../icons/CaptionsIcon';
+import CaptionsOffIcon from '../../../icons/CaptionsOffIcon';
+
+jest.mock('../../../state');
+const mockUseAppState = useAppState as jest.Mock<any>;
+
+describe('the ToggleCaptionsButton component', () => {
+  it('should render correctly when captions are enabled', () => {
+    const mockSetIsCaptionsEnabled = jest.fn();
+    mockUseAppState.mockImplementation(() => ({
+      isCaptionsEnabled: true,
+      setIsCaptionsEnabled: mockSetIsCaptionsEnabled,
+    }));
+    const wrapper = shallow(<ToggleCaptionsButton />);
+    const button = wrapper.find('[aria-label="Toggle Captions"]');
+    expect(button.prop('startIcon')).toEqual(<CaptionsIcon />);
+    expect(button.text()).toBe('Hide Captions');
+  });
+
+  it('should render correctly when captions are disabled', () => {
+    const mockSetIsCaptionsEnabled = jest.fn();
+    mockUseAppState.mockImplementation(() => ({
+      isCaptionsEnabled: false,
+      setIsCaptionsEnabled: mockSetIsCaptionsEnabled,
+    }));
+    const wrapper = shallow(<ToggleCaptionsButton />);
+    const button = wrapper.find('[aria-label="Toggle Captions"]');
+    expect(button.prop('startIcon')).toEqual(<CaptionsOffIcon />);
+    expect(button.text()).toBe('Show Captions');
+  });
+
+  it('should call the correct toggle function when clicked', () => {
+    const mockSetIsCaptionsEnabled = jest.fn();
+    mockUseAppState.mockImplementation(() => ({
+      isCaptionsEnabled: false,
+      setIsCaptionsEnabled: mockSetIsCaptionsEnabled,
+    }));
+    const wrapper = shallow(<ToggleCaptionsButton />);
+    const button = wrapper.find('[aria-label="Toggle Captions"]');
+    button.simulate('click');
+    expect(mockSetIsCaptionsEnabled).toHaveBeenCalled();
+  });
+
+  it('should be disabled when disabled prop is true', () => {
+    const mockSetIsCaptionsEnabled = jest.fn();
+    mockUseAppState.mockImplementation(() => ({
+      isCaptionsEnabled: false,
+      setIsCaptionsEnabled: mockSetIsCaptionsEnabled,
+    }));
+    const wrapper = shallow(<ToggleCaptionsButton disabled={true} />);
+    const button = wrapper.find('[aria-label="Toggle Captions"]');
+    expect(button.prop('disabled')).toBe(true);
+  });
+
+  it('should apply className prop', () => {
+    const mockSetIsCaptionsEnabled = jest.fn();
+    mockUseAppState.mockImplementation(() => ({
+      isCaptionsEnabled: false,
+      setIsCaptionsEnabled: mockSetIsCaptionsEnabled,
+    }));
+    const wrapper = shallow(<ToggleCaptionsButton className="test-class" />);
+    const button = wrapper.find('[aria-label="Toggle Captions"]');
+    expect(button.prop('className')).toBe('test-class');
+  });
+
+  it('should show tooltip when captions are disabled', () => {
+    const mockSetIsCaptionsEnabled = jest.fn();
+    mockUseAppState.mockImplementation(() => ({
+      isCaptionsEnabled: false,
+      setIsCaptionsEnabled: mockSetIsCaptionsEnabled,
+    }));
+    const wrapper = shallow(<ToggleCaptionsButton />);
+    const tooltip = wrapper.find('[data-testid="captions-tooltip"]');
+    expect(tooltip.prop('title')).toBe('Requires Real-Time Transcriptions to be enabled in the Twilio Console');
+  });
+
+  it('should not show tooltip when captions are enabled', () => {
+    const mockSetIsCaptionsEnabled = jest.fn();
+    mockUseAppState.mockImplementation(() => ({
+      isCaptionsEnabled: true,
+      setIsCaptionsEnabled: mockSetIsCaptionsEnabled,
+    }));
+    const wrapper = shallow(<ToggleCaptionsButton />);
+    const tooltip = wrapper.find('[data-testid="captions-tooltip"]');
+    expect(tooltip.prop('title')).toBe('');
+  });
+});

src/components/Buttons/ToggleCaptionsButton/ToggleCaptionsButton.tsx

@@ -0,0 +1,33 @@
+import { useCallback } from 'react';
+import Button from '@material-ui/core/Button';
+import Tooltip from '@material-ui/core/Tooltip';
+import CaptionsIcon from '../../../icons/CaptionsIcon';
+import CaptionsOffIcon from '../../../icons/CaptionsOffIcon';
+import { useAppState } from '../../../state';
+
+export default function ToggleCaptionsButton(props: { disabled?: boolean; className?: string }) {
+  const { isCaptionsEnabled, setIsCaptionsEnabled } = useAppState();
+
+  const toggleCaptions = useCallback(() => {
+    setIsCaptionsEnabled(enabled => !enabled);
+  }, [setIsCaptionsEnabled]);
+
+  const tooltipTitle = isCaptionsEnabled ? '' : 'Requires Real-Time Transcriptions to be enabled in the Twilio Console';
+
+  return (
+    <Tooltip title={tooltipTitle} placement="top" data-testid="captions-tooltip">
+      <span>
+        <Button
+          className={props.className}
+          onClick={toggleCaptions}
+          disabled={props.disabled}
+          startIcon={isCaptionsEnabled ? <CaptionsIcon /> : <CaptionsOffIcon />}
+          data-cy="toggle-captions"
+          aria-label="Toggle Captions"
+        >
+          {isCaptionsEnabled ? 'Hide Captions' : 'Show Captions'}
+        </Button>
+      </span>
+    </Tooltip>
+  );
+}

src/components/MenuBar/MenuBar.test.tsx

@@ -3,6 +3,7 @@ import { Button, Grid, Typography } from '@material-ui/core';
 import MenuBar from './MenuBar';
 import { shallow } from 'enzyme';
 import ToggleAudioButton from '../Buttons/ToggleAudioButton/ToggleAudioButton';
+import ToggleCaptionsButton from '../Buttons/ToggleCaptionsButton/ToggleCaptionsButton';
 import ToggleChatButton from '../Buttons/ToggleChatButton/ToggleChatButton';
 import ToggleScreenShareButton from '../Buttons/ToogleScreenShareButton/ToggleScreenShareButton';
 import ToggleVideoButton from '../Buttons/ToggleVideoButton/ToggleVideoButton';
@@ -138,4 +139,9 @@ describe('the MenuBar component', () => {
         .text()
     ).toBe('Test Room | 1 participant');
   });
+
+  it('should render the ToggleCaptionsButton', () => {
+    const wrapper = shallow(<MenuBar />);
+    expect(wrapper.find(ToggleCaptionsButton).exists()).toBe(true);
+  });
 });

src/components/MenuBar/MenuBar.tsx

@@ -13,6 +13,7 @@ import ToggleAudioButton from '../Buttons/ToggleAudioButton/ToggleAudioButton';
 import ToggleChatButton from '../Buttons/ToggleChatButton/ToggleChatButton';
 import ToggleVideoButton from '../Buttons/ToggleVideoButton/ToggleVideoButton';
 import ToggleScreenShareButton from '../Buttons/ToogleScreenShareButton/ToggleScreenShareButton';
+import ToggleCaptionsButton from '../Buttons/ToggleCaptionsButton/ToggleCaptionsButton';
 
 const useStyles = makeStyles((theme: Theme) =>
   createStyles({
@@ -92,6 +93,7 @@ export default function MenuBar() {
             <Grid container justifyContent="center">
               <ToggleAudioButton disabled={isReconnecting} />
               <ToggleVideoButton disabled={isReconnecting} />
+              <ToggleCaptionsButton disabled={isReconnecting} />
               {!isSharingScreen && !isMobile && <ToggleScreenShareButton disabled={isReconnecting} />}
               {process.env.REACT_APP_DISABLE_TWILIO_CONVERSATIONS !== 'true' && <ToggleChatButton />}
               <Hidden smDown>

src/components/Room/Room.tsx

@@ -13,6 +13,7 @@ import { useAppState } from '../../state';
 import useChatContext from '../../hooks/useChatContext/useChatContext';
 import useScreenShareParticipant from '../../hooks/useScreenShareParticipant/useScreenShareParticipant';
 import useVideoContext from '../../hooks/useVideoContext/useVideoContext';
+import TranscriptionsOverlay from '../TranscriptionsOverlay/TranscriptionsOverlay';
 
 const useStyles = makeStyles((theme: Theme) => {
   const totalMobileSidebarHeight = `${theme.sidebarMobileHeight +
@@ -111,6 +112,7 @@ export default function Room() {
 
       <ChatWindow />
       <BackgroundSelectionDialog />
+      <TranscriptionsOverlay />
     </div>
   );
 }

src/components/TranscriptionsOverlay/TranscriptionsOverlay.test.tsx

@@ -0,0 +1,102 @@
+import React from 'react';
+import { shallow } from 'enzyme';
+import TranscriptionsOverlay, { TranscriptionLine } from './TranscriptionsOverlay';
+import { useAppState } from '../../state';
+import useVideoContext from '../../hooks/useVideoContext/useVideoContext';
+import { useTranscriptions } from '../../hooks/useTranscriptions/useTranscriptions';
+
+jest.mock('../../state');
+jest.mock('../../hooks/useVideoContext/useVideoContext');
+jest.mock('../../hooks/useTranscriptions/useTranscriptions');
+
+const mockUseAppState = useAppState as jest.Mock<any>;
+const mockUseVideoContext = useVideoContext as jest.Mock<any>;
+const mockUseTranscriptions = useTranscriptions as jest.Mock<any>;
+
+describe('the TranscriptionsOverlay component', () => {
+  const mockLines: TranscriptionLine[] = [
+    { text: 'Hello world', participant: 'PA..1234', time: 1000 },
+    { text: 'How are you?', participant: 'PA..5678', time: 2000 },
+  ];
+
+  const mockRoom = { state: 'connected' };
+
+  beforeEach(() => {
+    mockUseVideoContext.mockImplementation(() => ({ room: mockRoom }));
+  });
+
+  it('should not render when captions are disabled', () => {
+    mockUseAppState.mockImplementation(() => ({ isCaptionsEnabled: false }));
+    mockUseTranscriptions.mockImplementation(() => ({ lines: mockLines, live: null }));
+    const wrapper = shallow(<TranscriptionsOverlay />);
+    expect(wrapper.isEmptyRender()).toBe(true);
+  });
+
+  it('should not render when not connected to room', () => {
+    mockUseAppState.mockImplementation(() => ({ isCaptionsEnabled: true }));
+    mockUseVideoContext.mockImplementationOnce(() => ({ room: null }));
+    mockUseTranscriptions.mockImplementation(() => ({ lines: mockLines, live: null }));
+    const wrapper = shallow(<TranscriptionsOverlay />);
+    expect(wrapper.isEmptyRender()).toBe(true);
+  });
+
+  it('should not render when no content available', () => {
+    mockUseAppState.mockImplementation(() => ({ isCaptionsEnabled: true }));
+    mockUseTranscriptions.mockImplementation(() => ({ lines: [], live: null }));
+    const wrapper = shallow(<TranscriptionsOverlay />);
+    expect(wrapper.isEmptyRender()).toBe(true);
+  });
+
+  it('should render when captions enabled and content available', () => {
+    mockUseAppState.mockImplementation(() => ({ isCaptionsEnabled: true }));
+    mockUseTranscriptions.mockImplementation(() => ({ lines: mockLines, live: null }));
+    const wrapper = shallow(<TranscriptionsOverlay />);
+    expect(wrapper.exists()).toBe(true);
+    expect(wrapper.find('[role="region"]')).toHaveLength(1);
+  });
+
+  it('should render all provided lines', () => {
+    mockUseAppState.mockImplementation(() => ({ isCaptionsEnabled: true }));
+    mockUseTranscriptions.mockImplementation(() => ({ lines: mockLines, live: null }));
+    const wrapper = shallow(<TranscriptionsOverlay />);
+    expect(wrapper.text()).toContain('PA..1234:');
+    expect(wrapper.text()).toContain('Hello world');
+    expect(wrapper.text()).toContain('PA..5678:');
+    expect(wrapper.text()).toContain('How are you?');
+  });
+
+  it('should render live partial line when provided', () => {
+    const mockLive = { text: 'In progress...', participant: 'PA..9999' };
+    mockUseAppState.mockImplementation(() => ({ isCaptionsEnabled: true }));
+    mockUseTranscriptions.mockImplementation(() => ({ lines: mockLines, live: mockLive }));
+    const wrapper = shallow(<TranscriptionsOverlay />);
+    expect(wrapper.text()).toContain('PA..9999:');
+    expect(wrapper.text()).toContain('In progress...');
+  });
+
+  it('should not render live line when not provided', () => {
+    mockUseAppState.mockImplementation(() => ({ isCaptionsEnabled: true }));
+    mockUseTranscriptions.mockImplementation(() => ({ lines: mockLines, live: null }));
+    const wrapper = shallow(<TranscriptionsOverlay />);
+    expect(wrapper.text()).not.toContain('PA..9999:');
+  });
+
+  it('should limit display to last 5 lines', () => {
+    const manyLines: TranscriptionLine[] = [
+      { text: 'Line 1', participant: 'PA..1111', time: 1000 },
+      { text: 'Line 2', participant: 'PA..2222', time: 2000 },
+      { text: 'Line 3', participant: 'PA..3333', time: 3000 },
+      { text: 'Line 4', participant: 'PA..4444', time: 4000 },
+      { text: 'Line 5', participant: 'PA..5555', time: 5000 },
+      { text: 'Line 6', participant: 'PA..6666', time: 6000 },
+      { text: 'Line 7', participant: 'PA..7777', time: 7000 },
+    ];
+    mockUseAppState.mockImplementation(() => ({ isCaptionsEnabled: true }));
+    mockUseTranscriptions.mockImplementation(() => ({ lines: manyLines, live: null }));
+    const wrapper = shallow(<TranscriptionsOverlay />);
+    expect(wrapper.text()).not.toContain('Line 1');
+    expect(wrapper.text()).not.toContain('Line 2');
+    expect(wrapper.text()).toContain('Line 3');
+    expect(wrapper.text()).toContain('Line 7');
+  });
+});