Default to scaling and cropping camera output to fit desired dimensions

.changeset/odd-zoos-dance.md

@@ -0,0 +1,7 @@
+---
+"client-sdk-android": minor
+---
+
+Default to scaling and cropping camera output to fit desired dimensions
+
+* This behavior may be turned off through the `VideoCaptureParams.adaptOutputToDimensions`

livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrack.kt

@@ -32,6 +32,7 @@ import io.livekit.android.room.track.video.CameraCapturerUtils.findCamera
 import io.livekit.android.room.track.video.CameraCapturerUtils.getCameraPosition
 import io.livekit.android.room.track.video.CameraCapturerWithSize
 import io.livekit.android.room.track.video.CaptureDispatchObserver
+import io.livekit.android.room.track.video.ScaleCropVideoProcessor
 import io.livekit.android.room.track.video.VideoCapturerWithSize
 import io.livekit.android.room.util.EncodingUtils
 import io.livekit.android.util.FlowObservable
@@ -473,11 +474,22 @@ constructor(
             videoProcessor: VideoProcessor? = null,
         ): LocalVideoTrack {
             val source = peerConnectionFactory.createVideoSource(options.isScreencast)
-            source.setVideoProcessor(videoProcessor)
+
+            val finalVideoProcessor = if (options.captureParams.adaptOutputToDimensions) {
+                ScaleCropVideoProcessor(
+                    targetWidth = options.captureParams.width,
+                    targetHeight = options.captureParams.height,
+                ).apply {
+                    childVideoProcessor = videoProcessor
+                }
+            } else {
+                videoProcessor
+            }
+            source.setVideoProcessor(finalVideoProcessor)
 
             val surfaceTextureHelper = SurfaceTextureHelper.create("VideoCaptureThread", rootEglBase.eglBaseContext)
 
-            // Dispatch raw frames to local renderer only if not using a VideoProcessor.
+            // Dispatch raw frames to local renderer only if not using a user-provided VideoProcessor.
             val dispatchObserver = if (videoProcessor == null) {
                 CaptureDispatchObserver().apply {
                     registerObserver(source.capturerObserver)

livekit-android-sdk/src/main/java/io/livekit/android/room/track/LocalVideoTrackOptions.kt

@@ -29,10 +29,27 @@ data class LocalVideoTrackOptions(
     val captureParams: VideoCaptureParameter = VideoPreset169.H720.capture,
 )
 
-data class VideoCaptureParameter(
+data class VideoCaptureParameter
+@JvmOverloads
+constructor(
+    /**
+     * Desired width.
+     */
     val width: Int,
+    /**
+     * Desired height.
+     */
     val height: Int,
+    /**
+     * Capture frame rate.
+     */
     val maxFps: Int,
+    /**
+     * Sometimes the capturer may not support the exact desired dimensions requested.
+     * If this is enabled, it will scale down and crop the captured frames to the
+     * same aspect ratio as [width]:[height].
+     */
+    val adaptOutputToDimensions: Boolean = true,
 )
 
 data class VideoEncoding(
@@ -213,7 +230,7 @@ enum class ScreenSharePresets(
      * Uses the original resolution without resizing.
      */
     ORIGINAL(
-        VideoCaptureParameter(0, 0, 30),
+        VideoCaptureParameter(0, 0, 30, adaptOutputToDimensions = false),
         VideoEncoding(7_000_000, 30),
     )
 }

livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ChainVideoProcessor.kt

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.room.track.video
+
+import androidx.annotation.CallSuper
+import livekit.org.webrtc.VideoFrame
+import livekit.org.webrtc.VideoProcessor
+import livekit.org.webrtc.VideoSink
+
+/**
+ * A VideoProcessor that can be chained together.
+ *
+ * Child classes should propagate frames down to the
+ * next link through [continueChain].
+ */
+abstract class ChainVideoProcessor : VideoProcessor {
+
+    /**
+     * The video sink where frames that have been completely processed are sent.
+     */
+    var videoSink: VideoSink? = null
+        private set
+
+    /**
+     * The next link in the chain to feed frames to.
+     *
+     * Setting [childVideoProcessor] to null will mean that this is object
+     * the end of the chain, and processed frames are ready to be published.
+     */
+    var childVideoProcessor: VideoProcessor? = null
+        set(value) {
+            value?.setSink(videoSink)
+            field = value
+        }
+
+    @CallSuper
+    override fun onCapturerStarted(started: Boolean) {
+        childVideoProcessor?.onCapturerStarted(started)
+    }
+
+    @CallSuper
+    override fun onCapturerStopped() {
+        childVideoProcessor?.onCapturerStopped()
+    }
+
+    final override fun setSink(videoSink: VideoSink?) {
+        childVideoProcessor?.setSink(videoSink)
+        this.videoSink = videoSink
+    }
+
+    /**
+     * A utility method to pass the frame down to the next link in the chain.
+     */
+    protected fun continueChain(frame: VideoFrame) {
+        childVideoProcessor?.onFrameCaptured(frame) ?: videoSink?.onFrame(frame)
+    }
+}

livekit-android-sdk/src/main/java/io/livekit/android/room/track/video/ScaleCropVideoProcessor.kt

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2024 LiveKit, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.livekit.android.room.track.video
+
+import livekit.org.webrtc.VideoFrame
+import kotlin.math.max
+import kotlin.math.roundToInt
+
+/**
+ * A video processor that scales down and crops to match
+ * the target dimensions and aspect ratio.
+ *
+ * If the frames are smaller than the target dimensions,
+ * upscaling will not occur, instead only cropping to match
+ * the aspect ratio.
+ */
+class ScaleCropVideoProcessor(
+    var targetWidth: Int,
+    var targetHeight: Int,
+) : ChainVideoProcessor() {
+
+    override fun onFrameCaptured(frame: VideoFrame) {
+        if (frame.rotatedWidth == targetWidth && frame.rotatedHeight == targetHeight) {
+            // already the perfect size, just pass along the frame.
+            continueChain(frame)
+            return
+        }
+
+        val width = frame.buffer.width
+        val height = frame.buffer.height
+        // Ensure target dimensions don't exceed source dimensions
+        val scaleWidth: Int
+        val scaleHeight: Int
+
+        if (targetWidth > width || targetHeight > height) {
+            // Calculate scale factor to fit within source dimensions
+            val widthScale = targetWidth.toDouble() / width
+            val heightScale = targetHeight.toDouble() / height
+            val scale = max(widthScale, heightScale)
+
+            // Apply scale to target dimensions
+            scaleWidth = (targetWidth / scale).roundToInt()
+            scaleHeight = (targetHeight / scale).roundToInt()
+        } else {
+            scaleWidth = targetWidth
+            scaleHeight = targetHeight
+        }
+
+        val sourceRatio = width.toDouble() / height
+        val targetRatio = scaleWidth.toDouble() / scaleHeight
+
+        val cropWidth: Int
+        val cropHeight: Int
+
+        // Calculate crop dimension
+        if (sourceRatio > targetRatio) {
+            // source is wider, crop height
+            cropHeight = height
+            cropWidth = (height * targetRatio).roundToInt()
+        } else {
+            // source is taller, crop width
+            cropWidth = width
+            cropHeight = (width / targetRatio).roundToInt()
+        }
+
+        // Calculate center offsets
+        val offsetX = (width - cropWidth) / 2
+        val offsetY = (height - cropHeight) / 2
+        val newBuffer = frame.buffer.cropAndScale(
+            offsetX,
+            offsetY,
+            cropWidth,
+            cropHeight,
+            scaleWidth,
+            scaleHeight,
+        )
+
+        val croppedFrame = VideoFrame(newBuffer, frame.rotation, frame.timestampNs)
+        continueChain(croppedFrame)
+        croppedFrame.release()
+    }
+}