Skip to content

BIDI mini refactoring and new additions #7267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -113,13 +113,17 @@ internal constructor(
val receivedJson = JSON.parseToJsonElement(receivedJsonStr)

return if (receivedJson is JsonObject && "setupComplete" in receivedJson) {
LiveSession(session = webSession, blockingDispatcher = blockingDispatcher)
LiveSession(
session = webSession,
blockingDispatcher = blockingDispatcher,
firebaseApp = controller.firebaseApp
)
} else {
webSession.close()
throw ServiceConnectionHandshakeFailedException("Unable to connect to the server")
}
} catch (e: ClosedReceiveChannelException) {
throw ServiceConnectionHandshakeFailedException("Channel was closed by the server", e)
throw ServiceConnectionHandshakeFailedException("Error: Too many concurrent live requests", e)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is that always the case? Is there any other reason why the handshake may fail?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From what i have seen it's always the case.. We could ask the proxy to return better error message.

}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ internal constructor(
private val requestOptions: RequestOptions,
httpEngine: HttpClientEngine,
private val apiClient: String,
private val firebaseApp: FirebaseApp,
internal val firebaseApp: FirebaseApp,
private val appVersion: Int = 0,
private val googleAppId: String,
private val headerProvider: HeaderProvider?,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import com.google.firebase.ai.type.MediaData
import com.google.firebase.ai.type.PublicPreviewAPI
import com.google.firebase.ai.type.SessionAlreadyReceivingException
import io.ktor.websocket.close
import kotlinx.coroutines.isActive
import kotlinx.coroutines.reactive.asPublisher
import org.reactivestreams.Publisher

Expand All @@ -51,6 +52,12 @@ public abstract class LiveSessionFutures internal constructor() {
functionCallHandler: ((FunctionCallPart) -> FunctionResponsePart)?
): ListenableFuture<Unit>

/** Indicates whether the underlying websocket connection is active. */
public abstract fun isActive(): ListenableFuture<Boolean>

/** Indicates whether an audio conversation is being used for this session object. */
public abstract fun isAudioConversationActive(): ListenableFuture<Boolean>

/**
* Starts an audio conversation with the model, which can only be stopped using
* [stopAudioConversation].
Expand Down Expand Up @@ -169,6 +176,11 @@ public abstract class LiveSessionFutures internal constructor() {
override fun startAudioConversation() =
SuspendToFutureAdapter.launchFuture { session.startAudioConversation() }

override fun isActive() = SuspendToFutureAdapter.launchFuture { session.isActive() }

override fun isAudioConversationActive() =
SuspendToFutureAdapter.launchFuture { session.isAudioConversationActive() }

override fun stopAudioConversation() =
SuspendToFutureAdapter.launchFuture { session.stopAudioConversation() }

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,10 @@ public class AudioRecordInitializationFailedException(message: String) :
public class ServiceConnectionHandshakeFailedException(message: String, cause: Throwable? = null) :
FirebaseAIException(message, cause)

/** The request is missing a permission that is required to perform the requested operation. */
public class PermissionMissingException(message: String, cause: Throwable? = null) :
FirebaseAIException(message, cause)

/** Catch all case for exceptions not explicitly expected. */
public class UnknownException internal constructor(message: String, cause: Throwable? = null) :
FirebaseAIException(message, cause)
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,13 @@
package com.google.firebase.ai.type

import android.Manifest.permission.RECORD_AUDIO
import android.content.pm.PackageManager
import android.media.AudioFormat
import android.media.AudioTrack
import android.util.Log
import androidx.annotation.RequiresPermission
import androidx.core.content.ContextCompat
import com.google.firebase.FirebaseApp
import com.google.firebase.ai.common.JSON
import com.google.firebase.ai.common.util.CancelledCoroutineScope
import com.google.firebase.ai.common.util.accumulateUntil
Expand Down Expand Up @@ -58,7 +61,8 @@ public class LiveSession
internal constructor(
private val session: ClientWebSocketSession,
@Blocking private val blockingDispatcher: CoroutineContext,
private var audioHelper: AudioHelper? = null
private var audioHelper: AudioHelper? = null,
private val firebaseApp: FirebaseApp,
) {
/**
* Coroutine scope that we batch data on for [startAudioConversation].
Expand Down Expand Up @@ -93,6 +97,14 @@ internal constructor(
public suspend fun startAudioConversation(
functionCallHandler: ((FunctionCallPart) -> FunctionResponsePart)? = null
) {

val context = firebaseApp.applicationContext
if (
ContextCompat.checkSelfPermission(context, RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED
) {
throw PermissionMissingException("Audio access not provided by the user")
}

FirebaseAIException.catchAsync {
if (scope.isActive) {
Log.w(
Expand Down Expand Up @@ -131,6 +143,12 @@ internal constructor(
}
}

/** Indicates whether the underlying websocket connection is active. */
public fun isActive(): Boolean = session.isActive

/** Indicates whether an audio conversation is being used for this session object. */
public fun isAudioConversationActive(): Boolean = (audioHelper != null)

/**
* Receives responses from the model for both streaming and standard requests.
*
Expand Down
Loading