Twilio with Deepgram

[ketan9712735468]

twilio media websocket routes

elif data['event'] == "media":
  if not has_seen_media:
      app.logger.info("Media message: {}".format(message))
      payload = data['media']['payload']
      chunk = base64.b64decode(payload)
      app.logger.info("Received {} bytes of audio data".format(len(chunk)))
      asyncio.run(transcription_service.send(chunk))
      has_seen_media = True

**TranscriptionService

from deepgram import (
    DeepgramClient,
    DeepgramClientOptions,
    LiveTranscriptionEvents,
    LiveOptions,
)

DEEPGRAM_API_KEY = "api_key"

class TranscriptionService:
    def __init__(self):
        self.dg_connection = None
        self.final_result = ""
        self.speech_final = False
    
    def on_open(self, *args, **kwargs):
        """Handle WebSocket open event"""
        print(f"\n\nOpen Event Triggered with args: {args} and kwargs: {kwargs}\n\n")

    def on_metadata(self, *args, **kwargs):
        """Handle metadata event"""
        print(f"\n\nMetadata Event Triggered with args: {args} and kwargs: {kwargs}\n\n")

    def on_speech_started(self, *args, **kwargs):
        """Handle speech started event"""
        print(f"\n\nSpeech Started Event Triggered with args: {args} and kwargs: {kwargs}\n\n")

    def on_utterance_end(self, *args, **kwargs):
        """Handle utterance end event"""
        print(f"\n\nUtterance End Event Triggered with args: {args} and kwargs: {kwargs}\n\n")

    def on_error(self, *args, **kwargs):
        """Handle error event"""
        print(f"\n\nError Event Triggered with args: {args} and kwargs: {kwargs}\n\n")

    def on_close(self, *args, **kwargs):
        """Handle WebSocket close event"""
        print(f"\n\nClose Event Triggered with args: {args} and kwargs: {kwargs}\n\n")
    
    async def init_deepgram(self):
        """Initialize the Deepgram WebSocket connection"""
        if self.dg_connection:
            print("Deepgram WebSocket already initialized")
            return
        
        try:
            print("Initializing Deepgram WebSocket...")
            deepgram = DeepgramClient(DEEPGRAM_API_KEY)
            self.dg_connection = deepgram.listen.websocket.v("1")

            # Register event handlers
            self.dg_connection.on(LiveTranscriptionEvents.Open, self.on_open)
            self.dg_connection.on(LiveTranscriptionEvents.Transcript, self.on_message)
            self.dg_connection.on(LiveTranscriptionEvents.Metadata, self.on_metadata)
            self.dg_connection.on(LiveTranscriptionEvents.SpeechStarted, self.on_speech_started)
            self.dg_connection.on(LiveTranscriptionEvents.UtteranceEnd, self.on_utterance_end)
            self.dg_connection.on(LiveTranscriptionEvents.Error, self.on_error)
            self.dg_connection.on(LiveTranscriptionEvents.Close, self.on_close)
            
            # Set live options for transcription
            options: LiveOptions = LiveOptions(
                model="nova-2",
                punctuate=True,
                language="en-US",
                encoding="mulaw",
                channels=1,
                sample_rate=8000,
                interim_results=True,
                utterance_end_ms="1000",
                vad_events=True,
                endpointing=200,
            )
            
            self.dg_connection.start(options)

            print("Deepgram WebSocket started successfully")
            return True
        except Exception as e:
            print(f"Error initializing Deepgram: {e}")
            return False

    def on_message(self, result, **kwargs):
        """Process transcription message"""
        alternatives = result.channel.alternatives
        if not alternatives:
            return
        
        text = alternatives[0].transcript.strip()
        is_final = getattr(result, "is_final", False)
        speech_final = getattr(result, "speech_final", False)

        if text:
            print(f"Transcribed: {text}")

        if is_final and text:
            self.final_result += f" {text}"

            if speech_final:
                self.speech_final = True
                print(f"Final Speech Transcription: {self.final_result}")
                self.final_result = ""
            else:
                self.speech_final = False

    async def send(self, chunk):
        """Send audio chunk to Deepgram"""
        if not self.dg_connection:
            await self.init_deepgram()
        if self.dg_connection:
            print(f"Sending chunk of size:>>>>>>>>>>>>>>>> {len(chunk)} bytes")
            self.dg_connection.send(chunk)

I am using deepgram-sdk version 3.8.0(latest) I am trying to get twilio chunk audio to convert into text using this code
I got this error logs from deepgrame

Open Event Triggered with args: (<deepgram.clients.listen.v1.websocket.client.ListenWebSocketClient object at 0x7f890b5f2d70>, OpenResponse(type=<WebSocketEvents.Open: 'Open'>)) and kwargs: {}

Deepgram WebSocket started successfully
Sending chunk of size:>>>>>>>>>>>>>>>> 160 bytes
ConnectionClosed in AbstractSyncWebSocketClient._listening with code 1011: Deepgram did not receive audio data or a text message within the timeout window. See https://dpgr.am/net0001

Error Event Triggered with args: (<deepgram.clients.listen.v1.websocket.client.ListenWebSocketClient object at 0x7f890b5f2d70>, ErrorResponse(description='ConnectionClosed in AbstractSyncWebSocketClient._listening', message='received 1011 (internal error) Deepgram did not receive audio data or a text message within the timeout window. See https://dpgr.am/net0001; then sent 1011 (internal error) Deepgram did not receive audio data or a text message within the timeout window. See https://dpgr.am/net0001', type='ConnectionClosed', variant=None)) and kwargs: {}

send() failed - ConnectionClosed: received 1011 (internal error) Deepgram did not receive audio data or a text message within the timeout window. See https://dpgr.am/net0001; then sent 1011 (internal error) Deepgram did not receive audio data or a text message within the timeout window. See https://dpgr.am/net0001

Close Event Triggered with args: (<deepgram.clients.listen.v1.websocket.client.ListenWebSocketClient object at 0x7f890b5f2d70>, CloseResponse(type=<WebSocketEvents.Close: 'Close'>)) and kwargs: {}

so can you suggest me how can I do that for twilio chunk audio and get fully finalized text

[deepgram-community[bot]]

Thanks for asking your question. Please be sure to reply with as much detail as possible so the community can assist you efficiently.
_{Consider joining our Discord community for more opportunity to engage with your fellow Deepgram users. You can earn points which can be redeemed for cool stuff by being active in our communities!}

[deepgram-community[bot]]

Thanks for asking your question. Please be sure to reply with as much detail as possible so the community can assist you efficiently.
_{Consider joining our Discord community for more opportunity to engage with your fellow Deepgram users. You can earn points which can be redeemed for cool stuff by being active in our communities!}

[deepgram-community[bot]]

Hey there! It looks like you haven't connected your GitHub account to your Deepgram account. You can do this at https://community.deepgram.com - being verified through this process will allow our team to help you in a much more streamlined fashion.

[deepgram-community[bot]]

It looks like we're missing some important information to help debug your issue. Would you mind providing us with the following details in a reply?

• A request ID that triggered your error or issue.

[deepgram-community[bot]]

Hey there! It looks like you haven't connected your GitHub account to your Deepgram account. You can do this at https://community.deepgram.com - being verified through this process will allow our team to help you in a much more streamlined fashion.

[deepgram-community[bot]]

It looks like we're missing some important information to help debug your issue. Would you mind providing us with the following details in a reply?

• The deepgram product you are using (e.g Speech to Text, Agent API)
• A request ID that triggered your error or issue.

[ketan9712735468]

@jkroll-deepgram @jpvajda any sort of assistance would be highly appreciated!

[jkroll-deepgram]

Hi @ketan9712735468, do you have a Deepgram request ID? It sounds like Deepgram is not ultimately receiving the audio you're trying to send.

Have you come across our Twilio guides, Transcribing Twilio Voice Calls in Real-Time with Deepgram and Twilio and Deepgram STT?