openAI TTS sketch

[SaKiEQ]

Howdy,

I cobbled together a openAI TextToSpeech retrieval sketch / function.

I haven't tested this sketch into detail, as its just put together from a larger project I am working on.
I thought I share it so it might help somebody further down the line.

NOTE: you need to register with openAI to obtain an API key - this will not be a free service from openAI !

//***********************************************************************************************
//*
//*       openAI - blocking function to fetch openAI TTS response and play the audio
//*       by SaKiE 2024 - Doha Qatar
//*
//***********************************************************************************************
#pragma once 
#include <HTTPClient.h>
#include <ArduinoJson.h>
#include "AudioTools.h"          // Main AudioTools library
#include "AudioTools/AudioCodecs/CodecMP3Helix.h"
#include "AudioTools/AudioLibs/AudioBoardStream.h"



#define I2S_BCK 26
#define I2S_WS 25
#define I2S_DAT 22


//***************************************************************
//** Defaults openAI
//***************************************************************
const char* model_OpenAI = "tts-1";
const char* voice_OpenAI = "alloy";

const char* apiKey = "*************************";
const char* apiUrl = "https://api.openai.com/v1/audio/speech";


//***************************************************************
//** Defaults wifi
//***************************************************************
// WiFi credentials
const char* ssid = "****************";
const char* password = "****************";

HTTPClient http;

//***************************************************************
//** Objects i2s Pipeline
//***************************************************************
AudioInfo                 info(44100, 2, 16);

AudioBoardStream          i2s(NoBoard);
EncodedAudioStream        decoderTTS(&i2s, new MP3DecoderHelix());  // Decoding stream



void setup() {
        Serial.begin(115200);
        Serial.flush();
        delay(2000);

        //clear the terminal
        for (int i = 0; i < 10; i++) {
              Serial.println();
        }
        Serial.println("\t\tWelcome !");

  //*************************************************************
  //****** Comms
  //*************************************************************
              // Connect to WiFi
              WiFi.begin(ssid, password);
              Serial.print("Connecting to WiFi...");
              while (WiFi.status() != WL_CONNECTED) {
                delay(500);
                Serial.print(".");
              }
              Serial.println("\nWiFi connected.");

  //*************************************************************
  //****** i2s
  //*************************************************************
      //I2S Output Intercafe Config
            Serial.println("\t\tSetting up I2S Speaker Environment");
            auto cfg = i2s.defaultConfig(TX_MODE);
                  cfg.sample_rate = 44100;
                  cfg.bits_per_sample = 16;
                  cfg.channels = 2;
                  cfg.buffer_count = 20;
                  cfg.buffer_size = 512;
                  // Custom I2S output pins
                  cfg.pin_bck = I2S_BCK;
                  cfg.pin_ws = I2S_WS;
                  cfg.pin_data = I2S_DAT;
            i2s.begin(cfg);

      auto config = decoderTTS.defaultConfig();

            //Requires adjustment depending on selected model
            config.sample_rate = 24000;     
            config.bits_per_sample = 16;
            config.channels = 1;

      if (!decoderTTS.begin(config)) {
        Serial.println("Failed to initialize decoder.");
      } else {

        if (fetchOpenAI("I am a wonderful openAI Text To Speech voice !")) {
          Serial.println("Successful openAI TTS-1 retrieval.");
        } else {
          Serial.println("Failed to process openAI TTS requst !");
        }
      }



}


bool fetchOpenAI(const char* requestStr) {
  
  HTTPClient http;

  if (WiFi.status() == WL_CONNECTED) {
    // Prepare the JSON payload
    DynamicJsonDocument jsonDoc(512);
    jsonDoc["model"] = model_OpenAI;
    jsonDoc["input"] = requestStr;
    jsonDoc["voice"] = voice_OpenAI;
    String requestBody;
    serializeJson(jsonDoc, requestBody);

    // Set up the HTTPS request
    http.begin(apiUrl);
    http.addHeader("Authorization", String("Bearer ") + apiKey);
    http.addHeader("Content-Type", "application/json");

    Serial.println("Sending POST request...");
    int httpResponseCode = http.POST(requestBody);

    if (httpResponseCode) {// == 200
      Serial.println("Request successful. Streaming audio...");

      // Use the HTTP response stream directly
      WiFiClient* audioStream = http.getStreamPtr();



      // Stream and play audio
      StreamCopy copier(decoderTTS, *audioStream);

              copier.begin();
              unsigned long lastCopyTime = millis();  // Track the time of last copy to detect stalls
              unsigned long maxIdleTime = 5000;  // Maximum time without data copy before timing out

              Serial.println("Start playing TTS audio...");

              do {
                  if (copier.copy()) {
                      // Reset the last copy time whenever data is successfully copied
                      lastCopyTime = millis();
                  }
                
                  // Check if the stream has been idle too long (without data copied)
                  if (millis() - lastCopyTime > maxIdleTime) {
                      Serial.println("Timeout while copying audio due to inactivity.");
                      break;  // Exit the loop if we timeout
                  }
              } while (copier.available());  // Continue while data is available to copy

              Serial.println("End playing TTS audio.");


      Serial.println("Audio playback complete.");
    } else {
      Serial.printf("HTTP Request failed: %d\n", httpResponseCode);
      return false;
    }

    http.end(); // Close connection
    return true;
  } else {
    Serial.println("WiFi not connected.");
    return false;
  }
}


void loop() {
  yield();
}

[pschatzmann]

Cool,

I suggest to replace

AudioBoardStream i2s(NoBoard);

with

I2SStream i2s;

The AudioBoardStream is mainly intended for codecs that need to be configured via I2C.

[ppzmrm]

Thank you for this -- it is just what I need. Unfortunately, it does something bad to my ESP32. The code builds, and seems to run as far as playing a brief snatch of sound, but then stops playing, claims Audio playback is complete, and crashes the ESP32 with a "corrupt heap" error. Any thoughts as to what is going wrong?

[SaKiEQ]

Howdy

I do not know why your ESP would crash. I not using this version from the above example as I switched to using elevenlabs.com for my current TTS.

as @joba-1 suggest, flushing the JSON should be there by default anyway.
if (! jsonDoc.empty()) {
jsonDoc.clear(); // Removes all elements from the doc
jsonDoc.shrink_to_fit(); // Optionally shrink capacity to fit size
}
Something along this lines after you used the JSON data.

The do-while loop was just used as illustration. In my use cases I use them non-blocking in the main loop.

@pschatzmann I was noticing in other usage cases that the lib helix stops with heap allocation errors, after several calls. Maybe a memory leak ? Or since this are RTOS tasks it takes several cycles for the memory to be released, before I new function call to 'bool fetchOpenAI(const char* requestStr){}' should be made ?

[ppzmrm]

Thank you for getting back to me. I have been having fun with the OpenAI API, linking together STT, ChatGPT and TTS, so I now have a working two-way chat on an ESP32+mic+speaker, complete with conversational memory that makes the interaction quite sophisticated. The only remaining snag is that I currently write the STT output to a SD card file before playing it, so there is rather more of a pause in the conversation than I would like, hence my interest in streaming directly from the openAI TTS API.

[pschatzmann]

Maybe something like this rhasspy example might work

[ppzmrm]

Here is the code I wrote that currently ships the incoming binary stream off to an SD file. This kind of coding is really not my forte (I am only an astronomer!), so I am afraid I am struggling to figure out how to implement instead streaming it straight to the speaker.

`bool TTSOpenAIAPI(String text) {
http.begin(client, serverName);
http.addHeader("Content-Type", "application/json");
http.addHeader("Authorization", String("Bearer ") + openai_api_key);

// Create JSON payload
String payload = "{"model":"tts-1","input":"" + cleantext + "","voice":"" + voice + "","speed":"1.0","response_format":"mp3"}";

int httpResponseCode = http.POST(payload);

if (httpResponseCode == 200) {
File outputFile = SD.open("/answer.mp3", FILE_WRITE);
if (!outputFile) {
Serial.println("Failed to open file for writing.");
http.end();
return false;
}
int bytesWritten = http.writeToStream(&outputFile);
outputFile.close();
http.end();
return true;
} else {
Serial.printf("HTTP POST failed with code %d\n", httpResponseCode);
String responseBody = http.getString();
Serial.println("Response body: " + responseBody);
http.end();
return false;
}
}`

[pschatzmann]

If you want to stick with your http API, then just replace your File with a EncodedAudioStream!

[ppzmrm]

(should say that one of the options it will return is a PCM stream, so in principle there need be very little decoding before passing to the I2S.)

[pschatzmann]

Is it a WAV file or just a PCM Stream ? If it is a PCM Stream, you can just replace the File with an I2SStream:
You can easily double check this with your files: if the content starts with a RIFF then it is a WAV file!

[ppzmrm]

PCM is one of the API's output options (as is WAV). I am sorry I must be being incredible stupid and frustrating to those more competent than me, but I have not been able to do this successfully. Could you bear to show me?