Implement an FfmpegVideoRenderer

.gitignore

@@ -69,6 +69,8 @@ extensions/flac/src/main/jni/flac
 
 # FFmpeg extension
 extensions/ffmpeg/src/main/jni/ffmpeg
+extensions/ffmpegvideo/.cxx
+extensions/ffmpegvideo/src/main/jni/include
 
 # Cronet extension
 extensions/cronet/jniLibs/*

core_settings.gradle

@@ -29,6 +29,7 @@ include modulePrefix + 'testutils'
 include modulePrefix + 'testdata'
 include modulePrefix + 'extension-av1'
 include modulePrefix + 'extension-ffmpeg'
+include modulePrefix + 'extension-ffmpegvideo'
 include modulePrefix + 'extension-flac'
 include modulePrefix + 'extension-gvr'
 include modulePrefix + 'extension-ima'
@@ -55,6 +56,7 @@ project(modulePrefix + 'testutils').projectDir = new File(rootDir, 'testutils')
 project(modulePrefix + 'testdata').projectDir = new File(rootDir, 'testdata')
 project(modulePrefix + 'extension-av1').projectDir = new File(rootDir, 'extensions/av1')
 project(modulePrefix + 'extension-ffmpeg').projectDir = new File(rootDir, 'extensions/ffmpeg')
+project(modulePrefix + 'extension-ffmpegvideo').projectDir = new File(rootDir, 'extensions/ffmpegvideo')
 project(modulePrefix + 'extension-flac').projectDir = new File(rootDir, 'extensions/flac')
 project(modulePrefix + 'extension-gvr').projectDir = new File(rootDir, 'extensions/gvr')
 project(modulePrefix + 'extension-ima').projectDir = new File(rootDir, 'extensions/ima')

extensions/ffmpegvideo/README.md

@@ -0,0 +1,86 @@
+# ExoPlayer FFmpeg extension #
+
+The Ffmpeg extension provides `FfmpegAudioRenderer` and `FfmpegVideoRenderer`, which uses FFmpeg 
+native library to decode videos.
+
+***This extension is currently in its very infancy and is under development.***
+
+***Whats working?***
+video supported codec: only H.264
+audio supported codec: same as original extension
+supported surface type: video_decoder_gl_surface_view
+
+***On Plan:***
+- [ ] Support other surface types
+- [ ] Organize the code
+- [ ] Fix possible issues
+- [ ] Video Decoder support Format.rotationDegrees
+- [ ] Support other codecs
+
+
+## License note ##
+
+Please note that whilst the code in this repository is licensed under
+[Apache 2.0][], using this extension also requires building and including one or
+more external libraries as described below. These are licensed separately.
+
+[Apache 2.0]: https://github.com/google/ExoPlayer/blob/release-v2/LICENSE
+
+## Build instructions (Linux, macOS) ##
+
+To use this extension you need to clone the ExoPlayer repository and depend on
+its modules locally. Instructions for doing this can be found in ExoPlayer's
+[top level README][].
+
+I provided the compiled FFmpeg [*.so files][] and [header files][]. You need to copy 
+the .so files to the `src/main/libs` directory and the header files to 
+the `src/main/jni/include` directory. Of course you can also compile it yourself.
+
+
+## Using the extension ##
+
+Like av1 extension, pass `EXTENSION_RENDERER_MODE_PREFER`, use `FFmpegRenderersFactory` 
+instead of `DefaultRenderersFactory` to create `FfmpegVideoRenderer` and `FfmpegAudioRenderer`. 
+Then you can observe the related logs of `EventLogger#decoderInitialized` in logcat 
+to determine whether the ffmpeg extension is used correctly.
+
+## Using the extension in the demo application ##
+
+To try out playback using the extension in the [demo application][], see
+[enabling extension decoders][].
+
+use `FFmpegRenderersFactory` instead of `DefaultRenderersFactory`.
+
+[demo application]: https://exoplayer.dev/demo-application.html
+[enabling extension decoders]: https://exoplayer.dev/demo-application.html#enabling-extension-decoders
+
+## Rendering options ##
+
+There are two possibilities for rendering the output `Libgav1VideoRenderer`
+gets from the libgav1 decoder:
+
+* GL rendering using GL shader for color space conversion
+  * If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by
+    setting `surface_type` of `PlayerView` to be
+    `video_decoder_gl_surface_view`.
+  * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message
+    of type `C.MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER` with an instance of
+    `VideoDecoderOutputBufferRenderer` as its object.
+
+* Native rendering using `ANativeWindow`
+  * If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled
+    by default.
+  * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message of
+    type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object.
+
+Note: Although the default option uses `ANativeWindow`, based on our testing the
+GL rendering mode has better performance, so should be preferred
+
+## Links ##
+
+* [Javadoc][]: Classes matching `com.google.android.exoplayer2.ext.av1.*`
+  belong to this module.
+
+[Javadoc]: https://exoplayer.dev/doc/reference/index.html
+[*.so files]: https://drive.google.com/open?id=14v4tz5L_jU7di3xWrY-uhuS7K5mcwj3g
+[header files]: https://drive.google.com/open?id=1dDZ9R4cLPpgcHOCoUpClrOlqnGL2UTSr

extensions/ffmpegvideo/build.gradle

@@ -0,0 +1,90 @@
+// Copyright (C) 2019 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+apply from: '../../constants.gradle'
+apply plugin: 'com.android.library'
+
+android {
+    compileSdkVersion project.ext.compileSdkVersion
+
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+
+    defaultConfig {
+        minSdkVersion project.ext.minSdkVersion
+        targetSdkVersion project.ext.targetSdkVersion
+        consumerProguardFiles 'proguard-rules.txt'
+
+        externalNativeBuild {
+            cmake {
+                // Debug CMake build type causes video frames to drop,
+                // so native library should always use Release build type.
+                arguments "-DCMAKE_BUILD_TYPE=Release"
+                targets "ffmpegJNI"
+            }
+        }
+    }
+
+    externalNativeBuild {
+        cmake {
+            version '3.10.2'
+            path "src/main/jni/CMakeLists.txt"
+        }
+    }
+
+    buildTypes {
+        debug {
+            ndk {
+                abiFilters 'arm64-v8a'/*, 'x86_64'*/
+            }
+        }
+    }
+
+    // This option resolves the problem of finding libgav1JNI.so
+    // on multiple paths. The first one found is picked.
+    packagingOptions {
+        pickFirst 'lib/arm64-v8a/libffmpegJNI.so'
+        pickFirst 'lib/armeabi-v7a/libffmpegJNI.so'
+        pickFirst 'lib/x86/libffmpegJNI.so'
+        pickFirst 'lib/x86_64/libffmpegJNI.so'
+    }
+
+    sourceSets.main {
+        // As native JNI library build is invoked from gradle, this is
+        // not needed. However, it exposes the built library and keeps
+        // consistency with the other extensions.
+        jniLibs.srcDir 'src/main/libs'
+    }
+}
+
+// Configure the native build only if libgav1 is present, to avoid gradle sync
+// failures if libgav1 hasn't been checked out according to the README and CMake
+// isn't installed.
+//if (project.file('src/main/jni/libffmpeg').exists()) {
+//    android.externalNativeBuild.cmake.path = 'src/main/jni/CMakeLists.txt'
+//    android.externalNativeBuild.cmake.version = '3.7.1+'
+//}
+
+dependencies {
+    implementation project(modulePrefix + 'library-core')
+    implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion
+    compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion
+    compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion
+}
+
+ext {
+    javadocTitle = 'FFmpeg video extension'
+}
+apply from: '../../javadoc_library.gradle'

extensions/ffmpegvideo/proguard-rules.txt

@@ -0,0 +1,7 @@
+# Proguard rules specific to the AV1 extension.
+
+# This prevents the names of native methods from being obfuscated.
+-keepclasseswithmembernames class * {
+    native <methods>;
+}
+

extensions/ffmpegvideo/src/main/AndroidManifest.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2019 The Android Open Source Project
+
+     Licensed under the Apache License, Version 2.0 (the "License");
+     you may not use this file except in compliance with the License.
+     You may obtain a copy of the License at
+
+          http://www.apache.org/licenses/LICENSE-2.0
+
+     Unless required by applicable law or agreed to in writing, software
+     distributed under the License is distributed on an "AS IS" BASIS,
+     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+     See the License for the specific language governing permissions and
+     limitations under the License.
+-->
+
+<manifest package="com.google.android.exoplayer2.ext.ffmpeg"/>

extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java

@@ -0,0 +1,137 @@
+package com.google.android.exoplayer2.ext.ffmpeg;
+
+import android.content.Context;
+import android.os.Handler;
+import com.google.android.exoplayer2.DefaultRenderersFactory;
+import com.google.android.exoplayer2.Renderer;
+import com.google.android.exoplayer2.audio.AudioCapabilities;
+import com.google.android.exoplayer2.audio.AudioProcessor;
+import com.google.android.exoplayer2.audio.AudioRendererEventListener;
+import com.google.android.exoplayer2.audio.DefaultAudioSink;
+import com.google.android.exoplayer2.audio.MediaCodecAudioRenderer;
+import com.google.android.exoplayer2.mediacodec.MediaCodecSelector;
+import com.google.android.exoplayer2.util.Log;
+import com.google.android.exoplayer2.video.MediaCodecVideoRenderer;
+import com.google.android.exoplayer2.video.VideoRendererEventListener;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+
+public class FFmpegRenderersFactory extends DefaultRenderersFactory {
+
+  private static final String TAG = "FFmpegRenderersFactory";
+
+  public FFmpegRenderersFactory(Context context) {
+    super(context);
+  }
+
+  @Override
+  protected void buildVideoRenderers(
+      Context context,
+      @ExtensionRendererMode int extensionRendererMode,
+      MediaCodecSelector mediaCodecSelector,
+      boolean enableDecoderFallback,
+      Handler eventHandler,
+      VideoRendererEventListener eventListener,
+      long allowedVideoJoiningTimeMs,
+      ArrayList<Renderer> out) {
+    MediaCodecVideoRenderer videoRenderer =
+        new MediaCodecVideoRenderer(
+            context,
+            mediaCodecSelector,
+            allowedVideoJoiningTimeMs,
+            enableDecoderFallback,
+            eventHandler,
+            eventListener,
+            MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
+    out.add(videoRenderer);
+
+    if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) {
+      return;
+    }
+    int extensionRendererIndex = out.size();
+    if (extensionRendererMode == EXTENSION_RENDERER_MODE_PREFER) {
+      extensionRendererIndex--;
+    }
+
+    try {
+      // Full class names used for constructor args so the LINT rule triggers if any of them move.
+      // LINT.IfChange
+      Class<?> clazz = Class
+          .forName("com.google.android.exoplayer2.ext.ffmpeg.FfmpegVideoRenderer");
+      Constructor<?> constructor =
+          clazz.getConstructor(
+              long.class,
+              Handler.class,
+              com.google.android.exoplayer2.video.VideoRendererEventListener.class,
+              int.class);
+      // LINT.ThenChange(../../../../../../../proguard-rules.txt)
+      Renderer renderer =
+          (Renderer)
+              constructor.newInstance(
+                  allowedVideoJoiningTimeMs,
+                  eventHandler,
+                  eventListener,
+                  MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY);
+      out.add(extensionRendererIndex++, renderer);
+      Log.i(TAG, "Loaded FfmpegVideoRenderer.");
+    } catch (ClassNotFoundException e) {
+      // Expected if the app was built without the extension.
+    } catch (Exception e) {
+      // The extension is present, but instantiation failed.
+      throw new RuntimeException("Error instantiating Ffmpeg extension", e);
+    }
+
+  }
+
+  @Override
+  protected void buildAudioRenderers(
+      Context context,
+      int extensionRendererMode,
+      MediaCodecSelector mediaCodecSelector,
+      boolean enableDecoderFallback,
+      AudioProcessor[] audioProcessors,
+      Handler eventHandler,
+      AudioRendererEventListener eventListener,
+      ArrayList<Renderer> out) {
+    MediaCodecAudioRenderer audioRenderer =
+        new MediaCodecAudioRenderer(
+            context,
+            mediaCodecSelector,
+            enableDecoderFallback,
+            eventHandler,
+            eventListener,
+            new DefaultAudioSink(AudioCapabilities.getCapabilities(context), audioProcessors));
+    out.add(audioRenderer);
+
+    if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) {
+      return;
+    }
+    int extensionRendererIndex = out.size();
+    if (extensionRendererMode == EXTENSION_RENDERER_MODE_PREFER) {
+      extensionRendererIndex--;
+    }
+
+    try {
+      // Full class names used for constructor args so the LINT rule triggers if any of them move.
+      // LINT.IfChange
+      Class<?> clazz =
+          Class.forName("com.google.android.exoplayer2.ext.ffmpeg.FfmpegAudioRenderer");
+      Constructor<?> constructor =
+          clazz.getConstructor(
+              android.os.Handler.class,
+              com.google.android.exoplayer2.audio.AudioRendererEventListener.class,
+              com.google.android.exoplayer2.audio.AudioProcessor[].class);
+      // LINT.ThenChange(../../../../../../../proguard-rules.txt)
+      Renderer renderer =
+          (Renderer) constructor.newInstance(eventHandler, eventListener, audioProcessors);
+      out.add(extensionRendererIndex++, renderer);
+      Log.i(TAG, "Loaded FfmpegAudioRenderer.");
+    } catch (ClassNotFoundException e) {
+      // Expected if the app was built without the extension.
+    } catch (Exception e) {
+      // The extension is present, but instantiation failed.
+      throw new RuntimeException("Error instantiating FFmpeg extension", e);
+    }
+  }
+
+}