[WIP] chore: get transcript dirctly from python server (instead of node.js …

src/client/game/components/ai-bot/gpt.ts

@@ -1,13 +1,15 @@
 import { Message } from './types';
+const audioUrl = 'http://localhost:5000/audio';
+const gptUrl = '/api/nlp/gpt';
 
 type ReplyResponse = {
   message: string;
   data: string;
 };
 
-export async function getReply(messages: Message[]): Promise<string> {
+export const getReply = async (messages: Message[]): Promise<string> => {
   try {
-    const response = await fetch('/api/nlp/gpt', {
+    const response = await fetch(gptUrl, {
       method: 'POST',
       body: JSON.stringify({ messages }),
       headers: {
@@ -26,4 +28,35 @@ export async function getReply(messages: Message[]): Promise<string> {
 
     return '';
   }
+};
+
+export const getAudio = async (text: string): Promise<Blob | null> => {
+  try {
+    const response = await fetch(audioUrl, {
+      method: 'POST',
+      body: JSON.stringify({ text }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    const result = await response.blob();
+
+    if (response.status >= 300) {
+      throw new Error(response.statusText);
+    }
+
+    return result;
+  } catch (err) {
+    console.error(err);
+
+    return null;
+  }
+};
+
+export function replayBlob(blob: Blob) {
+  console.log('replayBlob', blob);
+  const blobURL = window.URL.createObjectURL(blob);
+  const audio = new Audio(blobURL);
+  audio.play();
 }

src/client/game/components/ai-bot/index.ts

@@ -1,9 +1,10 @@
-import { getReply } from './gpt';
+import { getReply, getAudio, replayBlob } from './gpt';
 import SpeechRecognition from './speech-recognition/main';
 import SpeechSynthesiser from './speech-synthesis';
 import { Author, Message } from './types';
 
-const recognition = new SpeechRecognition('/api/nlp/transcript');
+const transcriptUrl = 'http://localhost:5000/transcript';
+const recognition = new SpeechRecognition(transcriptUrl);
 const speechSynthesiser = new SpeechSynthesiser();
 
 recognition.continuous = false;
@@ -33,21 +34,32 @@ recognition.onresult = async event => {
   });
   logMessages();
 
-  const reply = await getReply(state.messages);
+  if (!transcript) {
+    restartRecognition();
+    return;
+  }
 
-  if (transcript && reply) {
-    state.messages.push({
-      text: reply,
-      author: Author.Bot,
-      isBot: true,
-    });
-    logMessages();
+  const reply = await getReply(state.messages);
 
-    speechSynthesiser.speak(reply);
-    speechSynthesiser.onend = restartRecognition;
-  } else {
+  if (!reply) {
     restartRecognition();
+    return;
   }
+
+  state.messages.push({
+    text: reply,
+    author: Author.Bot,
+    isBot: true,
+  });
+  logMessages();
+
+  speechSynthesiser.speak(reply);
+  speechSynthesiser.onend = restartRecognition;
+
+  // TODO implement receiving audio from API insetad of using speechSynthesiser
+  // const blob = await getAudio(reply);
+  // replayBlob(blob);
+  // setTimeout(restartRecognition, 3000);
 };
 
 recognition.onerror = event => {

src/server/routes/nlp.ts

@@ -6,7 +6,7 @@ import path from 'path';
 import multer from 'multer';
 
 import { getReply } from '../../services/nlp/gpt';
-import { getTranscript } from '../../services/nlp/transcription';
+// import { getTranscript } from '../../services/nlp/transcription';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -32,33 +32,32 @@ const storage = multer.diskStorage({
 
 const upload = multer({ storage: storage });
 
-router
-  .post('/gpt', jsonParser, async (req, response) => {
-    const { messages } = req.body;
+router.post('/gpt', jsonParser, async (req, response) => {
+  const { messages } = req.body;
 
-    try {
-      const reply = await getReply(messages);
+  try {
+    const reply = await getReply(messages);
 
-      return response.status(200).json({ message: 'success', data: reply });
-    } catch (error) {
-      console.log(error.message);
-      return response.status(400).json({ message: error.message });
-    }
-  })
-  .post(
-    '/transcript',
-    upload.single('sample'),
-    async function (req: any, response, next) {
-      try {
-        const data = await getTranscript(req.file.path);
-        fs.unlinkSync(req.file.path);
-
-        return response.status(200).json({ message: 'success', data });
-      } catch (error) {
-        console.log(error.message);
-        return response.status(400).json({ message: error.message });
-      }
-    }
-  );
+    return response.status(200).json({ message: 'success', data: reply });
+  } catch (error) {
+    console.log(error.message);
+    return response.status(400).json({ message: error.message });
+  }
+});
+// .post(
+//   '/transcript',
+//   upload.single('sample'),
+//   async function (req: any, response, next) {
+//     try {
+//       const data = await getTranscript(req.file.path);
+//       fs.unlinkSync(req.file.path);
+
+//       return response.status(200).json({ message: 'success', data });
+//     } catch (error) {
+//       console.log(error.message);
+//       return response.status(400).json({ message: error.message });
+//     }
+//   }
+// );
 
 export default router;

src/services/nlp/transcription.ts

@@ -1,17 +1,17 @@
-import axios from 'axios';
+// import axios from 'axios';
 
-const recognizeUrl = 'http://localhost:5000/transcript';
+// const recognizeUrl = 'http://localhost:5000/transcript';
 
-export async function getTranscript(path) {
-  try {
-    const response = await axios.post(recognizeUrl, {
-      path,
-    });
+// export async function getTranscript(path) {
+//   try {
+//     const response = await axios.post(recognizeUrl, {
+//       path,
+//     });
 
-    return response.data.data;
-  } catch (error) {
-    console.log(error.message);
+//     return response.data.data;
+//   } catch (error) {
+//     console.log(error.message);
 
-    return error;
-  }
-}
+//     return error;
+//   }
+// }