From 3b6d860f5dddc4319443f6bf6e01bb11b62d90cc Mon Sep 17 00:00:00 2001 From: maksim Date: Tue, 27 Sep 2022 21:26:31 +0200 Subject: [PATCH] chore: get transcript dirctly from python server (instead of node.js server) --- src/client/game/components/ai-bot/gpt.ts | 37 ++++++++++++++- src/client/game/components/ai-bot/index.ts | 38 ++++++++++------ src/server/routes/nlp.ts | 53 +++++++++++----------- src/services/nlp/transcription.ts | 26 +++++------ 4 files changed, 99 insertions(+), 55 deletions(-) diff --git a/src/client/game/components/ai-bot/gpt.ts b/src/client/game/components/ai-bot/gpt.ts index cb9dd77..c1b503a 100644 --- a/src/client/game/components/ai-bot/gpt.ts +++ b/src/client/game/components/ai-bot/gpt.ts @@ -1,13 +1,15 @@ import { Message } from './types'; +const audioUrl = 'http://localhost:5000/audio'; +const gptUrl = '/api/nlp/gpt'; type ReplyResponse = { message: string; data: string; }; -export async function getReply(messages: Message[]): Promise { +export const getReply = async (messages: Message[]): Promise => { try { - const response = await fetch('/api/nlp/gpt', { + const response = await fetch(gptUrl, { method: 'POST', body: JSON.stringify({ messages }), headers: { @@ -26,4 +28,35 @@ export async function getReply(messages: Message[]): Promise { return ''; } +}; + +export const getAudio = async (text: string): Promise => { + try { + const response = await fetch(audioUrl, { + method: 'POST', + body: JSON.stringify({ text }), + headers: { + 'Content-Type': 'application/json', + }, + }); + + const result = await response.blob(); + + if (response.status >= 300) { + throw new Error(response.statusText); + } + + return result; + } catch (err) { + console.error(err); + + return null; + } +}; + +export function replayBlob(blob: Blob) { + console.log('replayBlob', blob); + const blobURL = window.URL.createObjectURL(blob); + const audio = new Audio(blobURL); + audio.play(); } diff --git a/src/client/game/components/ai-bot/index.ts b/src/client/game/components/ai-bot/index.ts index dabdfa3..21db4b5 100644 --- a/src/client/game/components/ai-bot/index.ts +++ b/src/client/game/components/ai-bot/index.ts @@ -1,9 +1,10 @@ -import { getReply } from './gpt'; +import { getReply, getAudio, replayBlob } from './gpt'; import SpeechRecognition from './speech-recognition/main'; import SpeechSynthesiser from './speech-synthesis'; import { Author, Message } from './types'; -const recognition = new SpeechRecognition('/api/nlp/transcript'); +const transcriptUrl = 'http://localhost:5000/transcript'; +const recognition = new SpeechRecognition(transcriptUrl); const speechSynthesiser = new SpeechSynthesiser(); recognition.continuous = false; @@ -33,21 +34,32 @@ recognition.onresult = async event => { }); logMessages(); - const reply = await getReply(state.messages); + if (!transcript) { + restartRecognition(); + return; + } - if (transcript && reply) { - state.messages.push({ - text: reply, - author: Author.Bot, - isBot: true, - }); - logMessages(); + const reply = await getReply(state.messages); - speechSynthesiser.speak(reply); - speechSynthesiser.onend = restartRecognition; - } else { + if (!reply) { restartRecognition(); + return; } + + state.messages.push({ + text: reply, + author: Author.Bot, + isBot: true, + }); + logMessages(); + + speechSynthesiser.speak(reply); + speechSynthesiser.onend = restartRecognition; + + // TODO implement receiving audio from API insetad of using speechSynthesiser + // const blob = await getAudio(reply); + // replayBlob(blob); + // setTimeout(restartRecognition, 3000); }; recognition.onerror = event => { diff --git a/src/server/routes/nlp.ts b/src/server/routes/nlp.ts index 851d254..199b6c0 100644 --- a/src/server/routes/nlp.ts +++ b/src/server/routes/nlp.ts @@ -6,7 +6,7 @@ import path from 'path'; import multer from 'multer'; import { getReply } from '../../services/nlp/gpt'; -import { getTranscript } from '../../services/nlp/transcription'; +// import { getTranscript } from '../../services/nlp/transcription'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -32,33 +32,32 @@ const storage = multer.diskStorage({ const upload = multer({ storage: storage }); -router - .post('/gpt', jsonParser, async (req, response) => { - const { messages } = req.body; +router.post('/gpt', jsonParser, async (req, response) => { + const { messages } = req.body; - try { - const reply = await getReply(messages); + try { + const reply = await getReply(messages); - return response.status(200).json({ message: 'success', data: reply }); - } catch (error) { - console.log(error.message); - return response.status(400).json({ message: error.message }); - } - }) - .post( - '/transcript', - upload.single('sample'), - async function (req: any, response, next) { - try { - const data = await getTranscript(req.file.path); - fs.unlinkSync(req.file.path); - - return response.status(200).json({ message: 'success', data }); - } catch (error) { - console.log(error.message); - return response.status(400).json({ message: error.message }); - } - } - ); + return response.status(200).json({ message: 'success', data: reply }); + } catch (error) { + console.log(error.message); + return response.status(400).json({ message: error.message }); + } +}); +// .post( +// '/transcript', +// upload.single('sample'), +// async function (req: any, response, next) { +// try { +// const data = await getTranscript(req.file.path); +// fs.unlinkSync(req.file.path); + +// return response.status(200).json({ message: 'success', data }); +// } catch (error) { +// console.log(error.message); +// return response.status(400).json({ message: error.message }); +// } +// } +// ); export default router; diff --git a/src/services/nlp/transcription.ts b/src/services/nlp/transcription.ts index 3549697..433de83 100644 --- a/src/services/nlp/transcription.ts +++ b/src/services/nlp/transcription.ts @@ -1,17 +1,17 @@ -import axios from 'axios'; +// import axios from 'axios'; -const recognizeUrl = 'http://localhost:5000/transcript'; +// const recognizeUrl = 'http://localhost:5000/transcript'; -export async function getTranscript(path) { - try { - const response = await axios.post(recognizeUrl, { - path, - }); +// export async function getTranscript(path) { +// try { +// const response = await axios.post(recognizeUrl, { +// path, +// }); - return response.data.data; - } catch (error) { - console.log(error.message); +// return response.data.data; +// } catch (error) { +// console.log(error.message); - return error; - } -} +// return error; +// } +// }