feat(ai): add support for the Live API (#9224)

Author: dlarocque

.changeset/blue-pets-sin.md

@@ -0,0 +1,6 @@
+---
+'firebase': minor
+'@firebase/ai': minor
+---
+
+Add support for the Gemini Live API.

common/api-review/ai.api.md

@@ -41,6 +41,7 @@ export const AIErrorCode: {
     readonly REQUEST_ERROR: "request-error";
     readonly RESPONSE_ERROR: "response-error";
     readonly FETCH_ERROR: "fetch-error";
+    readonly SESSION_CLOSED: "session-closed";
     readonly INVALID_CONTENT: "invalid-content";
     readonly API_NOT_ENABLED: "api-not-enabled";
     readonly INVALID_SCHEMA: "invalid-schema";
@@ -94,6 +95,11 @@ export class ArraySchema extends Schema {
     toJSON(): SchemaRequest;
 }
 
+// @beta
+export interface AudioConversationController {
+    stop: () => Promise<void>;
+}
+
 // @public
 export abstract class Backend {
     protected constructor(type: BackendType);
@@ -290,6 +296,7 @@ export type FinishReason = (typeof FinishReason)[keyof typeof FinishReason];
 export interface FunctionCall {
     // (undocumented)
     args: object;
+    id?: string;
     // (undocumented)
     name: string;
 }
@@ -342,6 +349,7 @@ export interface FunctionDeclarationsTool {
 
 // @public
 export interface FunctionResponse {
+    id?: string;
     // (undocumented)
     name: string;
     // (undocumented)
@@ -480,6 +488,9 @@ export function getGenerativeModel(ai: AI, modelParams: ModelParams | HybridPara
 // @beta
 export function getImagenModel(ai: AI, modelParams: ImagenModelParams, requestOptions?: RequestOptions): ImagenModel;
 
+// @beta
+export function getLiveGenerativeModel(ai: AI, modelParams: LiveModelParams): LiveGenerativeModel;
+
 // @public
 export class GoogleAIBackend extends Backend {
     constructor();
@@ -813,6 +824,96 @@ export interface LanguageModelPromptOptions {
     responseConstraint?: object;
 }
 
+// @beta
+export interface LiveGenerationConfig {
+    frequencyPenalty?: number;
+    maxOutputTokens?: number;
+    presencePenalty?: number;
+    responseModalities?: ResponseModality[];
+    speechConfig?: SpeechConfig;
+    temperature?: number;
+    topK?: number;
+    topP?: number;
+}
+
+// @beta
+export class LiveGenerativeModel extends AIModel {
+    // Warning: (ae-forgotten-export) The symbol "WebSocketHandler" needs to be exported by the entry point index.d.ts
+    //
+    // @internal
+    constructor(ai: AI, modelParams: LiveModelParams,
+    _webSocketHandler: WebSocketHandler);
+    connect(): Promise<LiveSession>;
+    // (undocumented)
+    generationConfig: LiveGenerationConfig;
+    // (undocumented)
+    systemInstruction?: Content;
+    // (undocumented)
+    toolConfig?: ToolConfig;
+    // (undocumented)
+    tools?: Tool[];
+    }
+
+// @beta
+export interface LiveModelParams {
+    // (undocumented)
+    generationConfig?: LiveGenerationConfig;
+    // (undocumented)
+    model: string;
+    // (undocumented)
+    systemInstruction?: string | Part | Content;
+    // (undocumented)
+    toolConfig?: ToolConfig;
+    // (undocumented)
+    tools?: Tool[];
+}
+
+// @beta
+export const LiveResponseType: {
+    SERVER_CONTENT: string;
+    TOOL_CALL: string;
+    TOOL_CALL_CANCELLATION: string;
+};
+
+// @beta
+export type LiveResponseType = (typeof LiveResponseType)[keyof typeof LiveResponseType];
+
+// @beta
+export interface LiveServerContent {
+    interrupted?: boolean;
+    modelTurn?: Content;
+    turnComplete?: boolean;
+    // (undocumented)
+    type: 'serverContent';
+}
+
+// @beta
+export interface LiveServerToolCall {
+    functionCalls: FunctionCall[];
+    // (undocumented)
+    type: 'toolCall';
+}
+
+// @beta
+export interface LiveServerToolCallCancellation {
+    functionIds: string[];
+    // (undocumented)
+    type: 'toolCallCancellation';
+}
+
+// @beta
+export class LiveSession {
+    // @internal
+    constructor(webSocketHandler: WebSocketHandler, serverMessages: AsyncGenerator<unknown>);
+    close(): Promise<void>;
+    inConversation: boolean;
+    isClosed: boolean;
+    receive(): AsyncGenerator<LiveServerContent | LiveServerToolCall | LiveServerToolCallCancellation>;
+    send(request: string | Array<string | Part>, turnComplete?: boolean): Promise<void>;
+    sendMediaChunks(mediaChunks: GenerativeContentBlob[]): Promise<void>;
+    sendMediaStream(mediaChunkStream: ReadableStream<GenerativeContentBlob>): Promise<void>;
+    }
+
 // @public
 export const Modality: {
     readonly MODALITY_UNSPECIFIED: "MODALITY_UNSPECIFIED";
@@ -885,6 +986,11 @@ export type Part = TextPart | InlineDataPart | FunctionCallPart | FunctionRespon
 // @public
 export const POSSIBLE_ROLES: readonly ["user", "model", "function", "system"];
 
+// @beta
+export interface PrebuiltVoiceConfig {
+    voiceName?: string;
+}
+
 // @public
 export interface PromptFeedback {
     // (undocumented)
@@ -904,6 +1010,7 @@ export interface RequestOptions {
 export const ResponseModality: {
     readonly TEXT: "TEXT";
     readonly IMAGE: "IMAGE";
+    readonly AUDIO: "AUDIO";
 };
 
 // @beta
@@ -1048,6 +1155,19 @@ export interface Segment {
     text: string;
 }
 
+// @beta
+export interface SpeechConfig {
+    voiceConfig?: VoiceConfig;
+}
+
+// @beta
+export function startAudioConversation(liveSession: LiveSession, options?: StartAudioConversationOptions): Promise<AudioConversationController>;
+
+// @beta
+export interface StartAudioConversationOptions {
+    functionCallingHandler?: (functionCalls: LiveServerToolCall['functionCalls']) => Promise<Part>;
+}
+
 // @public
 export interface StartChatParams extends BaseParams {
     // (undocumented)
@@ -1130,6 +1250,11 @@ export interface VideoMetadata {
     startOffset: string;
 }
 
+// @beta
+export interface VoiceConfig {
+    prebuiltVoiceConfig?: PrebuiltVoiceConfig;
+}
+
 // @public (undocumented)
 export interface WebAttribution {
     // (undocumented)

docs-devsite/_toc.yaml

@@ -16,6 +16,8 @@ toc:
     path: /docs/reference/js/ai.anyofschema.md
   - title: ArraySchema
     path: /docs/reference/js/ai.arrayschema.md
+  - title: AudioConversationController
+    path: /docs/reference/js/ai.audioconversationcontroller.md
   - title: Backend
     path: /docs/reference/js/ai.backend.md
   - title: BaseParams
@@ -124,6 +126,20 @@ toc:
     path: /docs/reference/js/ai.languagemodelmessagecontent.md
   - title: LanguageModelPromptOptions
     path: /docs/reference/js/ai.languagemodelpromptoptions.md
+  - title: LiveGenerationConfig
+    path: /docs/reference/js/ai.livegenerationconfig.md
+  - title: LiveGenerativeModel
+    path: /docs/reference/js/ai.livegenerativemodel.md
+  - title: LiveModelParams
+    path: /docs/reference/js/ai.livemodelparams.md
+  - title: LiveServerContent
+    path: /docs/reference/js/ai.liveservercontent.md
+  - title: LiveServerToolCall
+    path: /docs/reference/js/ai.liveservertoolcall.md
+  - title: LiveServerToolCallCancellation
+    path: /docs/reference/js/ai.liveservertoolcallcancellation.md
+  - title: LiveSession
+    path: /docs/reference/js/ai.livesession.md
   - title: ModalityTokenCount
     path: /docs/reference/js/ai.modalitytokencount.md
   - title: ModelParams
@@ -136,6 +152,8 @@ toc:
     path: /docs/reference/js/ai.objectschemarequest.md
   - title: OnDeviceParams
     path: /docs/reference/js/ai.ondeviceparams.md
+  - title: PrebuiltVoiceConfig
+    path: /docs/reference/js/ai.prebuiltvoiceconfig.md
   - title: PromptFeedback
     path: /docs/reference/js/ai.promptfeedback.md
   - title: RequestOptions
@@ -160,6 +178,10 @@ toc:
     path: /docs/reference/js/ai.searchentrypoint.md
   - title: Segment
     path: /docs/reference/js/ai.segment.md
+  - title: SpeechConfig
+    path: /docs/reference/js/ai.speechconfig.md
+  - title: StartAudioConversationOptions
+    path: /docs/reference/js/ai.startaudioconversationoptions.md
   - title: StartChatParams
     path: /docs/reference/js/ai.startchatparams.md
   - title: StringSchema
@@ -176,6 +198,8 @@ toc:
     path: /docs/reference/js/ai.vertexaibackend.md
   - title: VideoMetadata
     path: /docs/reference/js/ai.videometadata.md
+  - title: VoiceConfig
+    path: /docs/reference/js/ai.voiceconfig.md
   - title: WebAttribution
     path: /docs/reference/js/ai.webattribution.md
   - title: WebGroundingChunk

docs-devsite/ai.audioconversationcontroller.md

@@ -0,0 +1,41 @@
+Project: /docs/reference/js/_project.yaml
+Book: /docs/reference/_book.yaml
+page_type: reference
+
+{% comment %}
+DO NOT EDIT THIS FILE!
+This is generated by the JS SDK team, and any local changes will be
+overwritten. Changes should be made in the source code at
+https://github.com/firebase/firebase-js-sdk
+{% endcomment %}
+
+# AudioConversationController interface
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+A controller for managing an active audio conversation.
+
+<b>Signature:</b>
+
+```typescript
+export interface AudioConversationController 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [stop](./ai.audioconversationcontroller.md#audioconversationcontrollerstop) | () =&gt; Promise&lt;void&gt; | <b><i>(Public Preview)</i></b> Stops the audio conversation, closes the microphone connection, and cleans up resources. Returns a promise that resolves when cleanup is complete. |
+
+## AudioConversationController.stop
+
+> This API is provided as a preview for developers and may change based on feedback that we receive. Do not use this API in a production environment.
+> 
+
+Stops the audio conversation, closes the microphone connection, and cleans up resources. Returns a promise that resolves when cleanup is complete.
+
+<b>Signature:</b>
+
+```typescript
+stop: () => Promise<void>;
+```

docs-devsite/ai.functioncall.md

@@ -23,6 +23,7 @@ export interface FunctionCall
 |  Property | Type | Description |
 |  --- | --- | --- |
 |  [args](./ai.functioncall.md#functioncallargs) | object |  |
+|  [id](./ai.functioncall.md#functioncallid) | string | The id of the function call. This must be sent back in the associated [FunctionResponse](./ai.functionresponse.md#functionresponse_interface)<!-- -->. |
 |  [name](./ai.functioncall.md#functioncallname) | string |  |
 
 ## FunctionCall.args
@@ -33,6 +34,18 @@ export interface FunctionCall
 args: object;
 ```
 
+## FunctionCall.id
+
+The id of the function call. This must be sent back in the associated [FunctionResponse](./ai.functionresponse.md#functionresponse_interface)<!-- -->.
+
+This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be `undefined`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+id?: string;
+```
+
 ## FunctionCall.name
 
 <b>Signature:</b>

docs-devsite/ai.functionresponse.md

@@ -22,9 +22,22 @@ export interface FunctionResponse
 
 |  Property | Type | Description |
 |  --- | --- | --- |
+|  [id](./ai.functionresponse.md#functionresponseid) | string | The id of the [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->. |
 |  [name](./ai.functionresponse.md#functionresponsename) | string |  |
 |  [response](./ai.functionresponse.md#functionresponseresponse) | object |  |
 
+## FunctionResponse.id
+
+The id of the [FunctionCall](./ai.functioncall.md#functioncall_interface)<!-- -->.
+
+This property is only supported in the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->). When using the Gemini Developer API ([GoogleAIBackend](./ai.googleaibackend.md#googleaibackend_class)<!-- -->), this property will be `undefined`<!-- -->.
+
+<b>Signature:</b>
+
+```typescript
+id?: string;
+```
+
 ## FunctionResponse.name
 
 <b>Signature:</b>