diff --git a/.changeset/odd-zoos-dance.md b/.changeset/odd-zoos-dance.md new file mode 100644 index 000000000..64fd3f70c --- /dev/null +++ b/.changeset/odd-zoos-dance.md @@ -0,0 +1,7 @@ +--- +"client-sdk-android": minor +--- + +Default to scaling and cropping camera output to fit desired dimensions + +* This behavior may be turned off through the `VideoCaptureParams.adaptOutputToDimensions` diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt index 8d813750e..25484674f 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt @@ -32,6 +32,7 @@ import io.livekit.android.room.track.video.CameraCapturerUtils.findCamera import io.livekit.android.room.track.video.CameraCapturerUtils.getCameraPosition import io.livekit.android.room.track.video.CameraCapturerWithSize import io.livekit.android.room.track.video.CaptureDispatchObserver +import io.livekit.android.room.track.video.ScaleCropVideoProcessor import io.livekit.android.room.track.video.VideoCapturerWithSize import io.livekit.android.room.util.EncodingUtils import io.livekit.android.util.FlowObservable @@ -473,11 +474,22 @@ constructor( videoProcessor: VideoProcessor? = null, ): LocalVideoTrack { val source = peerConnectionFactory.createVideoSource(options.isScreencast) - source.setVideoProcessor(videoProcessor) + + val finalVideoProcessor = if (options.captureParams.adaptOutputToDimensions) { + ScaleCropVideoProcessor( + targetWidth = options.captureParams.width, + targetHeight = options.captureParams.height, + ).apply { + childVideoProcessor = videoProcessor + } + } else { + videoProcessor + } + source.setVideoProcessor(finalVideoProcessor) val surfaceTextureHelper = SurfaceTextureHelper.create("VideoCaptureThread", rootEglBase.eglBaseContext) - // Dispatch raw frames to local renderer only if not using a VideoProcessor. + // Dispatch raw frames to local renderer only if not using a user-provided VideoProcessor. val dispatchObserver = if (videoProcessor == null) { CaptureDispatchObserver().apply { registerObserver(source.capturerObserver) diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrackOptions.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrackOptions.kt index 09c2d8320..bc8c1d246 100644 --- a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrackOptions.kt +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrackOptions.kt @@ -29,10 +29,27 @@ data class LocalVideoTrackOptions( val captureParams: VideoCaptureParameter = VideoPreset169.H720.capture, ) -data class VideoCaptureParameter( +data class VideoCaptureParameter +@JvmOverloads +constructor( + /** + * Desired width. + */ val width: Int, + /** + * Desired height. + */ val height: Int, + /** + * Capture frame rate. + */ val maxFps: Int, + /** + * Sometimes the capturer may not support the exact desired dimensions requested. + * If this is enabled, it will scale down and crop the captured frames to the + * same aspect ratio as [width]:[height]. + */ + val adaptOutputToDimensions: Boolean = true, ) data class VideoEncoding( @@ -213,7 +230,7 @@ enum class ScreenSharePresets( * Uses the original resolution without resizing. */ ORIGINAL( - VideoCaptureParameter(0, 0, 30), + VideoCaptureParameter(0, 0, 30, adaptOutputToDimensions = false), VideoEncoding(7_000_000, 30), ) } diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ChainVideoProcessor.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ChainVideoProcessor.kt new file mode 100644 index 000000000..96822b93a --- /dev/null +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ChainVideoProcessor.kt @@ -0,0 +1,71 @@ +/* + * Copyright 2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.track.video + +import androidx.annotation.CallSuper +import livekit.org.webrtc.VideoFrame +import livekit.org.webrtc.VideoProcessor +import livekit.org.webrtc.VideoSink + +/** + * A VideoProcessor that can be chained together. + * + * Child classes should propagate frames down to the + * next link through [continueChain]. + */ +abstract class ChainVideoProcessor : VideoProcessor { + + /** + * The video sink where frames that have been completely processed are sent. + */ + var videoSink: VideoSink? = null + private set + + /** + * The next link in the chain to feed frames to. + * + * Setting [childVideoProcessor] to null will mean that this is object + * the end of the chain, and processed frames are ready to be published. + */ + var childVideoProcessor: VideoProcessor? = null + set(value) { + value?.setSink(videoSink) + field = value + } + + @CallSuper + override fun onCapturerStarted(started: Boolean) { + childVideoProcessor?.onCapturerStarted(started) + } + + @CallSuper + override fun onCapturerStopped() { + childVideoProcessor?.onCapturerStopped() + } + + final override fun setSink(videoSink: VideoSink?) { + childVideoProcessor?.setSink(videoSink) + this.videoSink = videoSink + } + + /** + * A utility method to pass the frame down to the next link in the chain. + */ + protected fun continueChain(frame: VideoFrame) { + childVideoProcessor?.onFrameCaptured(frame) ?: videoSink?.onFrame(frame) + } +} diff --git a/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt new file mode 100644 index 000000000..7b55366ee --- /dev/null +++ b/livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt @@ -0,0 +1,96 @@ +/* + * Copyright 2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.android.room.track.video + +import livekit.org.webrtc.VideoFrame +import kotlin.math.max +import kotlin.math.roundToInt + +/** + * A video processor that scales down and crops to match + * the target dimensions and aspect ratio. + * + * If the frames are smaller than the target dimensions, + * upscaling will not occur, instead only cropping to match + * the aspect ratio. + */ +class ScaleCropVideoProcessor( + var targetWidth: Int, + var targetHeight: Int, +) : ChainVideoProcessor() { + + override fun onFrameCaptured(frame: VideoFrame) { + if (frame.rotatedWidth == targetWidth && frame.rotatedHeight == targetHeight) { + // already the perfect size, just pass along the frame. + continueChain(frame) + return + } + + val width = frame.buffer.width + val height = frame.buffer.height + // Ensure target dimensions don't exceed source dimensions + val scaleWidth: Int + val scaleHeight: Int + + if (targetWidth > width || targetHeight > height) { + // Calculate scale factor to fit within source dimensions + val widthScale = targetWidth.toDouble() / width + val heightScale = targetHeight.toDouble() / height + val scale = max(widthScale, heightScale) + + // Apply scale to target dimensions + scaleWidth = (targetWidth / scale).roundToInt() + scaleHeight = (targetHeight / scale).roundToInt() + } else { + scaleWidth = targetWidth + scaleHeight = targetHeight + } + + val sourceRatio = width.toDouble() / height + val targetRatio = scaleWidth.toDouble() / scaleHeight + + val cropWidth: Int + val cropHeight: Int + + // Calculate crop dimension + if (sourceRatio > targetRatio) { + // source is wider, crop height + cropHeight = height + cropWidth = (height * targetRatio).roundToInt() + } else { + // source is taller, crop width + cropWidth = width + cropHeight = (width / targetRatio).roundToInt() + } + + // Calculate center offsets + val offsetX = (width - cropWidth) / 2 + val offsetY = (height - cropHeight) / 2 + val newBuffer = frame.buffer.cropAndScale( + offsetX, + offsetY, + cropWidth, + cropHeight, + scaleWidth, + scaleHeight, + ) + + val croppedFrame = VideoFrame(newBuffer, frame.rotation, frame.timestampNs) + continueChain(croppedFrame) + croppedFrame.release() + } +}