From 03cb4808b511f0e3584aaf9775ab66b456ac10e4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 2 Jul 2025 07:16:41 +0000 Subject: [PATCH 1/3] Initial plan From 0d5022bd9c639192fa2c833fc70f22bab222506e Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 2 Jul 2025 07:19:37 +0000 Subject: [PATCH 2/3] Initial assessment - planning dynamic voice changing support implementation Co-authored-by: basvandorst <1196963+basvandorst@users.noreply.github.com> --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index f62669b..22228ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,11 +1,11 @@ { - "name": "real-time-gpt4o-video-chat", + "name": "real-time-gpt4o-video-voice-chat", "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "real-time-gpt4o-video-chat", + "name": "real-time-gpt4o-video-voice-chat", "version": "1.0.0", "license": "MIT", "dependencies": { From 64b0fbb14b8594334fcaf7a415a5de175ad49fd5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 2 Jul 2025 07:23:38 +0000 Subject: [PATCH 3/3] Implement dynamic voice changing support with consolidated session updates Co-authored-by: basvandorst <1196963+basvandorst@users.noreply.github.com> --- client/app.js | 48 +++++++- package-lock.json | 276 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 3 + 3 files changed, 324 insertions(+), 3 deletions(-) diff --git a/client/app.js b/client/app.js index cef4299..f39d8df 100644 --- a/client/app.js +++ b/client/app.js @@ -90,8 +90,51 @@ class DemoApp { text: `Hello, what can i do for you?` } ]); - this.client.updateSession({ instructions: OPENAI_INSTRUCTIONS }); - this.client.updateSession({ input_audio_transcription: { model: 'whisper-1' } }); + await this.client.updateSession({ + instructions: OPENAI_INSTRUCTIONS, + input_audio_transcription: { model: 'whisper-1' }, + voice: 'shimmer' + }); + + // Voice control tools + this.client.addTool({ + 'name': 'change_voice', + 'description': 'Changes the AI voice to a different option', + 'parameters': { + 'type': 'object', + 'properties': { + 'voice': { + 'type': 'string', + 'enum': ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer'], + 'description': 'The voice to switch to' + } + }, + 'required': ['voice'] + } + }, async ({ voice }) => { + await this.client.updateSession({ voice }); + return { + success: true, + message: `Voice changed to ${voice}`, + current_voice: voice + }; + }); + + this.client.addTool({ + 'name': 'get_current_voice', + 'description': 'Returns the currently active voice', + 'parameters': { + 'type': 'object', + 'properties': {} + } + }, async () => { + const session = await this.client.getSession(); + return { + current_voice: session.voice || 'default' + }; + }); + + // Camera reference tool this.client.addTool({ 'name': 'detect_reference_question', 'description': 'Returns the answers to context-dependent questions that refer to previously mentioned or visually present subjects (e.g., \'What is this?\', \'How do I solve this?\')', @@ -151,7 +194,6 @@ class DemoApp { }; } }); - } async startRecording() { diff --git a/package-lock.json b/package-lock.json index 22228ba..2042d0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "@openai/realtime-api-beta": "github:openai/openai-realtime-api-beta", + "@parcel/watcher-linux-x64-glibc": "*", "axios": "^1.7.7", "cors": "^2.8.5", "express": "^4.21.1", @@ -18,6 +19,9 @@ "devDependencies": { "nodemon": "^3.1.7", "parcel": "^2.12.0" + }, + "optionalDependencies": { + "@parcel/watcher-linux-x64-glibc": "^2.5.1" } }, "node_modules/@babel/code-frame": { @@ -1430,6 +1434,27 @@ "@parcel/watcher-win32-x64": "2.5.0" } }, + "node_modules/@parcel/watcher-android-arm64": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-android-arm64/-/watcher-android-arm64-2.5.0.tgz", + "integrity": "sha512-qlX4eS28bUcQCdribHkg/herLe+0A9RyYC+mm2PXpncit8z5b3nSqGVzMNR3CmtAOgRutiZ02eIJJgP/b1iEFQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, "node_modules/@parcel/watcher-darwin-arm64": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-arm64/-/watcher-darwin-arm64-2.5.0.tgz", @@ -1451,6 +1476,257 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/@parcel/watcher-darwin-x64": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-x64/-/watcher-darwin-x64-2.5.0.tgz", + "integrity": "sha512-9rhlwd78saKf18fT869/poydQK8YqlU26TMiNg7AIu7eBp9adqbJZqmdFOsbZ5cnLp5XvRo9wcFmNHgHdWaGYA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-freebsd-x64": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-freebsd-x64/-/watcher-freebsd-x64-2.5.0.tgz", + "integrity": "sha512-syvfhZzyM8kErg3VF0xpV8dixJ+RzbUaaGaeb7uDuz0D3FK97/mZ5AJQ3XNnDsXX7KkFNtyQyFrXZzQIcN49Tw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm-glibc": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-glibc/-/watcher-linux-arm-glibc-2.5.0.tgz", + "integrity": "sha512-0VQY1K35DQET3dVYWpOaPFecqOT9dbuCfzjxoQyif1Wc574t3kOSkKevULddcR9znz1TcklCE7Ht6NIxjvTqLA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm-musl": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-musl/-/watcher-linux-arm-musl-2.5.0.tgz", + "integrity": "sha512-6uHywSIzz8+vi2lAzFeltnYbdHsDm3iIB57d4g5oaB9vKwjb6N6dRIgZMujw4nm5r6v9/BQH0noq6DzHrqr2pA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm64-glibc": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-glibc/-/watcher-linux-arm64-glibc-2.5.0.tgz", + "integrity": "sha512-BfNjXwZKxBy4WibDb/LDCriWSKLz+jJRL3cM/DllnHH5QUyoiUNEp3GmL80ZqxeumoADfCCP19+qiYiC8gUBjA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-arm64-musl": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-musl/-/watcher-linux-arm64-musl-2.5.0.tgz", + "integrity": "sha512-S1qARKOphxfiBEkwLUbHjCY9BWPdWnW9j7f7Hb2jPplu8UZ3nes7zpPOW9bkLbHRvWM0WDTsjdOTUgW0xLBN1Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-x64-glibc": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-glibc/-/watcher-linux-x64-glibc-2.5.1.tgz", + "integrity": "sha512-GcESn8NZySmfwlTsIur+49yDqSny2IhPeZfXunQi48DMugKeZ7uy1FX83pO0X22sHntJ4Ub+9k34XQCX+oHt2A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-linux-x64-musl": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-musl/-/watcher-linux-x64-musl-2.5.0.tgz", + "integrity": "sha512-iqOC+GoTDoFyk/VYSFHwjHhYrk8bljW6zOhPuhi5t9ulqiYq1togGJB5e3PwYVFFfeVgc6pbz3JdQyDoBszVaA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-win32-arm64": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-arm64/-/watcher-win32-arm64-2.5.0.tgz", + "integrity": "sha512-twtft1d+JRNkM5YbmexfcH/N4znDtjgysFaV9zvZmmJezQsKpkfLYJ+JFV3uygugK6AtIM2oADPkB2AdhBrNig==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-win32-ia32": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-ia32/-/watcher-win32-ia32-2.5.0.tgz", + "integrity": "sha512-+rgpsNRKwo8A53elqbbHXdOMtY/tAtTzManTWShB5Kk54N8Q9mzNWV7tV+IbGueCbcj826MfWGU3mprWtuf1TA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher-win32-x64": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-x64/-/watcher-win32-x64-2.5.0.tgz", + "integrity": "sha512-lPrxve92zEHdgeff3aiu4gDOIt4u7sJYha6wbdEZDCDUhtjTsOMiaJzG5lMY4GkWH8p0fMmO2Ppq5G5XXG+DQw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/@parcel/watcher/node_modules/@parcel/watcher-linux-x64-glibc": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-glibc/-/watcher-linux-x64-glibc-2.5.0.tgz", + "integrity": "sha512-d9AOkusyXARkFD66S6zlGXyzx5RvY+chTP9Jp0ypSTC9d4lzyRs9ovGf/80VCxjKddcUvnsGwCHWuF2EoPgWjw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, "node_modules/@parcel/workers": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/@parcel/workers/-/workers-2.12.0.tgz", diff --git a/package.json b/package.json index 646d2d5..0e4ca92 100644 --- a/package.json +++ b/package.json @@ -19,5 +19,8 @@ "devDependencies": { "nodemon": "^3.1.7", "parcel": "^2.12.0" + }, + "optionalDependencies": { + "@parcel/watcher-linux-x64-glibc": "^2.5.1" } }