diff --git a/README.md b/README.md index df0593612..43fb80e90 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,37 @@ Project consists of: - communication layer for processing audio in and out - a boilerplate view for starting to build your apps and view logs +## Handling Context with Incomplete Turns + +When working with the Live API, you might want to send contextual text data while continuing to stream real-time input. This is useful when you want to provide additional context to the model without ending the current conversation turn. + +### Example: Sending Context Without Completing a Turn + +```typescript +// Send context without completing the turn +client.sendContext([{ text: "The user is looking at a laptop." }]); + +// Continue with real-time input (will force complete the turn if needed) +client.sendRealtimeInput([ + { + mimeType: "audio/pcm;rate=16000", + data: audioData, + } +]); + +// Check if there's an incomplete turn +if (client.hasIncompleteTurn()) { + // Manually complete a turn if needed + client.completeTurn(); +} +``` + +### Differences Between send(), sendContext(), and sendRealtimeInput() + +- `send(parts, turnComplete = true)`: Standard method to send text data. By default completes the turn. +- `sendContext(parts)`: A convenience method that calls `send(parts, false)`. Used when providing context without completing the turn. +- `sendRealtimeInput(chunks, completeTurn = true)`: Sends real-time media data. The `completeTurn` parameter determines if any incomplete turns should be automatically completed first. + ## Available Scripts In the project directory, you can run: diff --git a/src/components/control-tray/ControlTray.tsx b/src/components/control-tray/ControlTray.tsx index 05085f508..8e0d87342 100644 --- a/src/components/control-tray/ControlTray.tsx +++ b/src/components/control-tray/ControlTray.tsx @@ -97,7 +97,7 @@ function ControlTray({ mimeType: "audio/pcm;rate=16000", data: base64, }, - ]); + ], true); }; if (connected && !muted && audioRecorder) { audioRecorder.on("data", onData).on("volume", setInVolume).start(); @@ -131,7 +131,7 @@ function ControlTray({ ctx.drawImage(videoRef.current, 0, 0, canvas.width, canvas.height); const base64 = canvas.toDataURL("image/jpeg", 1.0); const data = base64.slice(base64.indexOf(",") + 1, Infinity); - client.sendRealtimeInput([{ mimeType: "image/jpeg", data }]); + client.sendRealtimeInput([{ mimeType: "image/jpeg", data }], true); } if (connected) { timeoutId = window.setTimeout(sendVideoFrame, 1000 / 0.5); diff --git a/src/components/side-panel/SidePanel.tsx b/src/components/side-panel/SidePanel.tsx index 26aef5943..0dd72a066 100644 --- a/src/components/side-panel/SidePanel.tsx +++ b/src/components/side-panel/SidePanel.tsx @@ -42,6 +42,7 @@ export default function SidePanel() { value: string; label: string; } | null>(null); + const [completeTurn, setCompleteTurn] = useState(true); const inputRef = useRef(null); //scroll the log to the bottom when new logs come in @@ -127,6 +128,25 @@ export default function SidePanel() { />
+
+ + {client.hasIncompleteTurn() && ( + + )} +