diff --git a/.gitignore b/.gitignore index cb4cfaada1c..4c85a0f69f7 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,8 @@ extensions/flac/src/main/jni/flac # FFmpeg extension extensions/ffmpeg/src/main/jni/ffmpeg +extensions/ffmpegvideo/.cxx +extensions/ffmpegvideo/src/main/jni/include # Cronet extension extensions/cronet/jniLibs/* diff --git a/core_settings.gradle b/core_settings.gradle index ac569331556..3f7e9956726 100644 --- a/core_settings.gradle +++ b/core_settings.gradle @@ -29,6 +29,7 @@ include modulePrefix + 'testutils' include modulePrefix + 'testdata' include modulePrefix + 'extension-av1' include modulePrefix + 'extension-ffmpeg' +include modulePrefix + 'extension-ffmpegvideo' include modulePrefix + 'extension-flac' include modulePrefix + 'extension-gvr' include modulePrefix + 'extension-ima' @@ -55,6 +56,7 @@ project(modulePrefix + 'testutils').projectDir = new File(rootDir, 'testutils') project(modulePrefix + 'testdata').projectDir = new File(rootDir, 'testdata') project(modulePrefix + 'extension-av1').projectDir = new File(rootDir, 'extensions/av1') project(modulePrefix + 'extension-ffmpeg').projectDir = new File(rootDir, 'extensions/ffmpeg') +project(modulePrefix + 'extension-ffmpegvideo').projectDir = new File(rootDir, 'extensions/ffmpegvideo') project(modulePrefix + 'extension-flac').projectDir = new File(rootDir, 'extensions/flac') project(modulePrefix + 'extension-gvr').projectDir = new File(rootDir, 'extensions/gvr') project(modulePrefix + 'extension-ima').projectDir = new File(rootDir, 'extensions/ima') diff --git a/extensions/ffmpegvideo/README.md b/extensions/ffmpegvideo/README.md new file mode 100644 index 00000000000..7cdab350ae9 --- /dev/null +++ b/extensions/ffmpegvideo/README.md @@ -0,0 +1,86 @@ +# ExoPlayer FFmpeg extension # + +The Ffmpeg extension provides `FfmpegAudioRenderer` and `FfmpegVideoRenderer`, which uses FFmpeg +native library to decode videos. + +***This extension is currently in its very infancy and is under development.*** + +***Whats working?*** +video supported codec: only H.264 +audio supported codec: same as original extension +supported surface type: video_decoder_gl_surface_view + +***On Plan:*** +- [ ] Support other surface types +- [ ] Organize the code +- [ ] Fix possible issues +- [ ] Video Decoder support Format.rotationDegrees +- [ ] Support other codecs + + +## License note ## + +Please note that whilst the code in this repository is licensed under +[Apache 2.0][], using this extension also requires building and including one or +more external libraries as described below. These are licensed separately. + +[Apache 2.0]: https://github.com/google/ExoPlayer/blob/release-v2/LICENSE + +## Build instructions (Linux, macOS) ## + +To use this extension you need to clone the ExoPlayer repository and depend on +its modules locally. Instructions for doing this can be found in ExoPlayer's +[top level README][]. + +I provided the compiled FFmpeg [*.so files][] and [header files][]. You need to copy +the .so files to the `src/main/libs` directory and the header files to +the `src/main/jni/include` directory. Of course you can also compile it yourself. + + +## Using the extension ## + +Like av1 extension, pass `EXTENSION_RENDERER_MODE_PREFER`, use `FFmpegRenderersFactory` +instead of `DefaultRenderersFactory` to create `FfmpegVideoRenderer` and `FfmpegAudioRenderer`. +Then you can observe the related logs of `EventLogger#decoderInitialized` in logcat +to determine whether the ffmpeg extension is used correctly. + +## Using the extension in the demo application ## + +To try out playback using the extension in the [demo application][], see +[enabling extension decoders][]. + +use `FFmpegRenderersFactory` instead of `DefaultRenderersFactory`. + +[demo application]: https://exoplayer.dev/demo-application.html +[enabling extension decoders]: https://exoplayer.dev/demo-application.html#enabling-extension-decoders + +## Rendering options ## + +There are two possibilities for rendering the output `Libgav1VideoRenderer` +gets from the libgav1 decoder: + +* GL rendering using GL shader for color space conversion + * If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by + setting `surface_type` of `PlayerView` to be + `video_decoder_gl_surface_view`. + * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message + of type `C.MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER` with an instance of + `VideoDecoderOutputBufferRenderer` as its object. + +* Native rendering using `ANativeWindow` + * If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled + by default. + * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message of + type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object. + +Note: Although the default option uses `ANativeWindow`, based on our testing the +GL rendering mode has better performance, so should be preferred + +## Links ## + +* [Javadoc][]: Classes matching `com.google.android.exoplayer2.ext.av1.*` + belong to this module. + +[Javadoc]: https://exoplayer.dev/doc/reference/index.html +[*.so files]: https://drive.google.com/open?id=14v4tz5L_jU7di3xWrY-uhuS7K5mcwj3g +[header files]: https://drive.google.com/open?id=1dDZ9R4cLPpgcHOCoUpClrOlqnGL2UTSr diff --git a/extensions/ffmpegvideo/build.gradle b/extensions/ffmpegvideo/build.gradle new file mode 100644 index 00000000000..40278a439ef --- /dev/null +++ b/extensions/ffmpegvideo/build.gradle @@ -0,0 +1,90 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +apply from: '../../constants.gradle' +apply plugin: 'com.android.library' + +android { + compileSdkVersion project.ext.compileSdkVersion + + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + + defaultConfig { + minSdkVersion project.ext.minSdkVersion + targetSdkVersion project.ext.targetSdkVersion + consumerProguardFiles 'proguard-rules.txt' + + externalNativeBuild { + cmake { + // Debug CMake build type causes video frames to drop, + // so native library should always use Release build type. + arguments "-DCMAKE_BUILD_TYPE=Release" + targets "ffmpegJNI" + } + } + } + + externalNativeBuild { + cmake { + version '3.10.2' + path "src/main/jni/CMakeLists.txt" + } + } + + buildTypes { + debug { + ndk { + abiFilters 'arm64-v8a'/*, 'x86_64'*/ + } + } + } + + // This option resolves the problem of finding libgav1JNI.so + // on multiple paths. The first one found is picked. + packagingOptions { + pickFirst 'lib/arm64-v8a/libffmpegJNI.so' + pickFirst 'lib/armeabi-v7a/libffmpegJNI.so' + pickFirst 'lib/x86/libffmpegJNI.so' + pickFirst 'lib/x86_64/libffmpegJNI.so' + } + + sourceSets.main { + // As native JNI library build is invoked from gradle, this is + // not needed. However, it exposes the built library and keeps + // consistency with the other extensions. + jniLibs.srcDir 'src/main/libs' + } +} + +// Configure the native build only if libgav1 is present, to avoid gradle sync +// failures if libgav1 hasn't been checked out according to the README and CMake +// isn't installed. +//if (project.file('src/main/jni/libffmpeg').exists()) { +// android.externalNativeBuild.cmake.path = 'src/main/jni/CMakeLists.txt' +// android.externalNativeBuild.cmake.version = '3.7.1+' +//} + +dependencies { + implementation project(modulePrefix + 'library-core') + implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion + compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion + compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion +} + +ext { + javadocTitle = 'FFmpeg video extension' +} +apply from: '../../javadoc_library.gradle' diff --git a/extensions/ffmpegvideo/proguard-rules.txt b/extensions/ffmpegvideo/proguard-rules.txt new file mode 100644 index 00000000000..9d73f7e2b58 --- /dev/null +++ b/extensions/ffmpegvideo/proguard-rules.txt @@ -0,0 +1,7 @@ +# Proguard rules specific to the AV1 extension. + +# This prevents the names of native methods from being obfuscated. +-keepclasseswithmembernames class * { + native ; +} + diff --git a/extensions/ffmpegvideo/src/main/AndroidManifest.xml b/extensions/ffmpegvideo/src/main/AndroidManifest.xml new file mode 100644 index 00000000000..d53bca4ca22 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/AndroidManifest.xml @@ -0,0 +1,17 @@ + + + + diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java new file mode 100644 index 00000000000..229589dff23 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java @@ -0,0 +1,137 @@ +package com.google.android.exoplayer2.ext.ffmpeg; + +import android.content.Context; +import android.os.Handler; +import com.google.android.exoplayer2.DefaultRenderersFactory; +import com.google.android.exoplayer2.Renderer; +import com.google.android.exoplayer2.audio.AudioCapabilities; +import com.google.android.exoplayer2.audio.AudioProcessor; +import com.google.android.exoplayer2.audio.AudioRendererEventListener; +import com.google.android.exoplayer2.audio.DefaultAudioSink; +import com.google.android.exoplayer2.audio.MediaCodecAudioRenderer; +import com.google.android.exoplayer2.mediacodec.MediaCodecSelector; +import com.google.android.exoplayer2.util.Log; +import com.google.android.exoplayer2.video.MediaCodecVideoRenderer; +import com.google.android.exoplayer2.video.VideoRendererEventListener; +import java.lang.reflect.Constructor; +import java.util.ArrayList; + +public class FFmpegRenderersFactory extends DefaultRenderersFactory { + + private static final String TAG = "FFmpegRenderersFactory"; + + public FFmpegRenderersFactory(Context context) { + super(context); + } + + @Override + protected void buildVideoRenderers( + Context context, + @ExtensionRendererMode int extensionRendererMode, + MediaCodecSelector mediaCodecSelector, + boolean enableDecoderFallback, + Handler eventHandler, + VideoRendererEventListener eventListener, + long allowedVideoJoiningTimeMs, + ArrayList out) { + MediaCodecVideoRenderer videoRenderer = + new MediaCodecVideoRenderer( + context, + mediaCodecSelector, + allowedVideoJoiningTimeMs, + enableDecoderFallback, + eventHandler, + eventListener, + MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); + out.add(videoRenderer); + + if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { + return; + } + int extensionRendererIndex = out.size(); + if (extensionRendererMode == EXTENSION_RENDERER_MODE_PREFER) { + extensionRendererIndex--; + } + + try { + // Full class names used for constructor args so the LINT rule triggers if any of them move. + // LINT.IfChange + Class clazz = Class + .forName("com.google.android.exoplayer2.ext.ffmpeg.FfmpegVideoRenderer"); + Constructor constructor = + clazz.getConstructor( + long.class, + Handler.class, + com.google.android.exoplayer2.video.VideoRendererEventListener.class, + int.class); + // LINT.ThenChange(../../../../../../../proguard-rules.txt) + Renderer renderer = + (Renderer) + constructor.newInstance( + allowedVideoJoiningTimeMs, + eventHandler, + eventListener, + MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); + out.add(extensionRendererIndex++, renderer); + Log.i(TAG, "Loaded FfmpegVideoRenderer."); + } catch (ClassNotFoundException e) { + // Expected if the app was built without the extension. + } catch (Exception e) { + // The extension is present, but instantiation failed. + throw new RuntimeException("Error instantiating Ffmpeg extension", e); + } + + } + + @Override + protected void buildAudioRenderers( + Context context, + int extensionRendererMode, + MediaCodecSelector mediaCodecSelector, + boolean enableDecoderFallback, + AudioProcessor[] audioProcessors, + Handler eventHandler, + AudioRendererEventListener eventListener, + ArrayList out) { + MediaCodecAudioRenderer audioRenderer = + new MediaCodecAudioRenderer( + context, + mediaCodecSelector, + enableDecoderFallback, + eventHandler, + eventListener, + new DefaultAudioSink(AudioCapabilities.getCapabilities(context), audioProcessors)); + out.add(audioRenderer); + + if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { + return; + } + int extensionRendererIndex = out.size(); + if (extensionRendererMode == EXTENSION_RENDERER_MODE_PREFER) { + extensionRendererIndex--; + } + + try { + // Full class names used for constructor args so the LINT rule triggers if any of them move. + // LINT.IfChange + Class clazz = + Class.forName("com.google.android.exoplayer2.ext.ffmpeg.FfmpegAudioRenderer"); + Constructor constructor = + clazz.getConstructor( + android.os.Handler.class, + com.google.android.exoplayer2.audio.AudioRendererEventListener.class, + com.google.android.exoplayer2.audio.AudioProcessor[].class); + // LINT.ThenChange(../../../../../../../proguard-rules.txt) + Renderer renderer = + (Renderer) constructor.newInstance(eventHandler, eventListener, audioProcessors); + out.add(extensionRendererIndex++, renderer); + Log.i(TAG, "Loaded FfmpegAudioRenderer."); + } catch (ClassNotFoundException e) { + // Expected if the app was built without the extension. + } catch (Exception e) { + // The extension is present, but instantiation failed. + throw new RuntimeException("Error instantiating FFmpeg extension", e); + } + } + +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java new file mode 100644 index 00000000000..f0ac07b8aa7 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java @@ -0,0 +1,230 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.decoder.SimpleDecoder; +import com.google.android.exoplayer2.decoder.SimpleOutputBuffer; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.android.exoplayer2.util.ParsableByteArray; +import com.google.android.exoplayer2.util.Util; +import java.nio.ByteBuffer; +import java.util.List; + +/** + * FFmpeg audio decoder. + */ +/* package */ final class FfmpegAudioDecoder extends + SimpleDecoder { + + // Output buffer sizes when decoding PCM mu-law streams, which is the maximum FFmpeg outputs. + private static final int OUTPUT_BUFFER_SIZE_16BIT = 65536; + private static final int OUTPUT_BUFFER_SIZE_32BIT = OUTPUT_BUFFER_SIZE_16BIT * 2; + + // Error codes matching ffmpeg_audio_jni.cc. + private static final int DECODER_ERROR_INVALID_DATA = -1; + private static final int DECODER_ERROR_OTHER = -2; + + private final String codecName; + @Nullable private final byte[] extraData; + private final @C.Encoding int encoding; + private final int outputBufferSize; + + private long nativeContext; // May be reassigned on resetting the codec. + private boolean hasOutputFormat; + private volatile int channelCount; + private volatile int sampleRate; + + public FfmpegAudioDecoder( + int numInputBuffers, + int numOutputBuffers, + int initialInputBufferSize, + Format format, + boolean outputFloat) + throws FfmpegAudioDecoderException { + super(new DecoderInputBuffer[numInputBuffers], new SimpleOutputBuffer[numOutputBuffers]); + if (!FfmpegLibrary.isAvailable()) { + throw new FfmpegAudioDecoderException("Failed to load decoder native libraries."); + } + Assertions.checkNotNull(format.sampleMimeType); + codecName = + Assertions.checkNotNull(FfmpegLibrary.getAudioCodecName(format.sampleMimeType)); + extraData = getExtraData(format.sampleMimeType, format.initializationData); + encoding = outputFloat ? C.ENCODING_PCM_FLOAT : C.ENCODING_PCM_16BIT; + outputBufferSize = outputFloat ? OUTPUT_BUFFER_SIZE_32BIT : OUTPUT_BUFFER_SIZE_16BIT; + nativeContext = + ffmpegInitialize(codecName, extraData, outputFloat, format.sampleRate, format.channelCount); + if (nativeContext == 0) { + throw new FfmpegAudioDecoderException("Initialization failed."); + } + setInitialInputBufferSize(initialInputBufferSize); + } + + @Override + public String getName() { + return "ffmpeg" + FfmpegLibrary.getVersion() + "-" + codecName; + } + + @Override + protected DecoderInputBuffer createInputBuffer() { + return new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); + } + + @Override + protected SimpleOutputBuffer createOutputBuffer() { + return new SimpleOutputBuffer(this); + } + + @Override + protected FfmpegAudioDecoderException createUnexpectedDecodeException(Throwable error) { + return new FfmpegAudioDecoderException("Unexpected decode error", error); + } + + @Override + protected @Nullable FfmpegAudioDecoderException decode( + DecoderInputBuffer inputBuffer, SimpleOutputBuffer outputBuffer, boolean reset) { + if (reset) { + nativeContext = ffmpegReset(nativeContext, extraData); + if (nativeContext == 0) { + return new FfmpegAudioDecoderException("Error resetting (see logcat)."); + } + } + ByteBuffer inputData = Util.castNonNull(inputBuffer.data); + int inputSize = inputData.limit(); + ByteBuffer outputData = outputBuffer.init(inputBuffer.timeUs, outputBufferSize); + int result = ffmpegDecode(nativeContext, inputData, inputSize, outputData, outputBufferSize); + if (result == DECODER_ERROR_INVALID_DATA) { + // Treat invalid data errors as non-fatal to match the behavior of MediaCodec. No output will + // be produced for this buffer, so mark it as decode-only to ensure that the audio sink's + // position is reset when more audio is produced. + outputBuffer.setFlags(C.BUFFER_FLAG_DECODE_ONLY); + return null; + } else if (result == DECODER_ERROR_OTHER) { + return new FfmpegAudioDecoderException("Error decoding (see logcat)."); + } + if (!hasOutputFormat) { + channelCount = ffmpegGetChannelCount(nativeContext); + sampleRate = ffmpegGetSampleRate(nativeContext); + if (sampleRate == 0 && "alac".equals(codecName)) { + Assertions.checkNotNull(extraData); + // ALAC decoder did not set the sample rate in earlier versions of FFMPEG. + // See https://trac.ffmpeg.org/ticket/6096 + ParsableByteArray parsableExtraData = new ParsableByteArray(extraData); + parsableExtraData.setPosition(extraData.length - 4); + sampleRate = parsableExtraData.readUnsignedIntToInt(); + } + hasOutputFormat = true; + } + outputData.position(0); + outputData.limit(result); + return null; + } + + @Override + public void release() { + super.release(); + ffmpegRelease(nativeContext); + nativeContext = 0; + } + + /** Returns the channel count of output audio. */ + public int getChannelCount() { + return channelCount; + } + + /** Returns the sample rate of output audio. */ + public int getSampleRate() { + return sampleRate; + } + + /** + * Returns the encoding of output audio. + */ + public @C.Encoding int getEncoding() { + return encoding; + } + + /** + * Returns FFmpeg-compatible codec-specific initialization data ("extra data"), or {@code null} if + * not required. + */ + private static @Nullable byte[] getExtraData(String mimeType, List initializationData) { + switch (mimeType) { + case MimeTypes.AUDIO_AAC: + case MimeTypes.AUDIO_OPUS: + return initializationData.get(0); + case MimeTypes.AUDIO_ALAC: + return getAlacExtraData(initializationData); + case MimeTypes.AUDIO_VORBIS: + return getVorbisExtraData(initializationData); + default: + // Other codecs do not require extra data. + return null; + } + } + + private static byte[] getAlacExtraData(List initializationData) { + // FFmpeg's ALAC decoder expects an ALAC atom, which contains the ALAC "magic cookie", as extra + // data. initializationData[0] contains only the magic cookie, and so we need to package it into + // an ALAC atom. See: + // https://ffmpeg.org/doxygen/0.6/alac_8c.html + // https://github.com/macosforge/alac/blob/master/ALACMagicCookieDescription.txt + byte[] magicCookie = initializationData.get(0); + int alacAtomLength = 12 + magicCookie.length; + ByteBuffer alacAtom = ByteBuffer.allocate(alacAtomLength); + alacAtom.putInt(alacAtomLength); + alacAtom.putInt(0x616c6163); // type=alac + alacAtom.putInt(0); // version=0, flags=0 + alacAtom.put(magicCookie, /* offset= */ 0, magicCookie.length); + return alacAtom.array(); + } + + private static byte[] getVorbisExtraData(List initializationData) { + byte[] header0 = initializationData.get(0); + byte[] header1 = initializationData.get(1); + byte[] extraData = new byte[header0.length + header1.length + 6]; + extraData[0] = (byte) (header0.length >> 8); + extraData[1] = (byte) (header0.length & 0xFF); + System.arraycopy(header0, 0, extraData, 2, header0.length); + extraData[header0.length + 2] = 0; + extraData[header0.length + 3] = 0; + extraData[header0.length + 4] = (byte) (header1.length >> 8); + extraData[header0.length + 5] = (byte) (header1.length & 0xFF); + System.arraycopy(header1, 0, extraData, header0.length + 6, header1.length); + return extraData; + } + + private native long ffmpegInitialize( + String codecName, + @Nullable byte[] extraData, + boolean outputFloat, + int rawSampleRate, + int rawChannelCount); + + private native int ffmpegDecode(long context, ByteBuffer inputData, int inputSize, + ByteBuffer outputData, int outputSize); + private native int ffmpegGetChannelCount(long context); + private native int ffmpegGetSampleRate(long context); + + private native long ffmpegReset(long context, @Nullable byte[] extraData); + + private native void ffmpegRelease(long context); + +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoderException.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoderException.java new file mode 100644 index 00000000000..82aac5a9ef4 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoderException.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import com.google.android.exoplayer2.audio.AudioDecoderException; + +/** + * Thrown when an FFmpeg decoder error occurs. + */ +public final class FfmpegAudioDecoderException extends AudioDecoderException { + + /* package */ FfmpegAudioDecoderException(String message) { + super(message); + } + + /* package */ FfmpegAudioDecoderException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java new file mode 100644 index 00000000000..726a367160b --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import android.os.Handler; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.audio.AudioProcessor; +import com.google.android.exoplayer2.audio.AudioRendererEventListener; +import com.google.android.exoplayer2.audio.AudioSink; +import com.google.android.exoplayer2.audio.DefaultAudioSink; +import com.google.android.exoplayer2.audio.SimpleDecoderAudioRenderer; +import com.google.android.exoplayer2.drm.ExoMediaCrypto; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.MimeTypes; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Decodes and renders audio using FFmpeg. + */ +public final class FfmpegAudioRenderer extends SimpleDecoderAudioRenderer { + + /** The number of input and output buffers. */ + private static final int NUM_BUFFERS = 16; + /** The default input buffer size. */ + private static final int DEFAULT_INPUT_BUFFER_SIZE = 960 * 6; + + private final boolean enableFloatOutput; + + private @MonotonicNonNull FfmpegAudioDecoder decoder; + + public FfmpegAudioRenderer() { + this(/* eventHandler= */ null, /* eventListener= */ null); + } + + /** + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param audioProcessors Optional {@link AudioProcessor}s that will process audio before output. + */ + public FfmpegAudioRenderer( + @Nullable Handler eventHandler, + @Nullable AudioRendererEventListener eventListener, + AudioProcessor... audioProcessors) { + this( + eventHandler, + eventListener, + new DefaultAudioSink(/* audioCapabilities= */ null, audioProcessors), + /* enableFloatOutput= */ false); + } + + /** + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param audioSink The sink to which audio will be output. + * @param enableFloatOutput Whether to enable 32-bit float audio format, if supported on the + * device/build and if the input format may have bit depth higher than 16-bit. When using + * 32-bit float output, any audio processing will be disabled, including playback speed/pitch + * adjustment. + */ + public FfmpegAudioRenderer( + @Nullable Handler eventHandler, + @Nullable AudioRendererEventListener eventListener, + AudioSink audioSink, + boolean enableFloatOutput) { + super( + eventHandler, + eventListener, + audioSink); + this.enableFloatOutput = enableFloatOutput; + } + + @Override + @FormatSupport + protected int supportsFormatInternal(Format format) { + Assertions.checkNotNull(format.sampleMimeType); + if (!FfmpegLibrary.isAvailable()) { + return FORMAT_UNSUPPORTED_TYPE; + } else if (!FfmpegLibrary.supportsAudioFormat(format.sampleMimeType) || !isOutputSupported(format)) { + return FORMAT_UNSUPPORTED_SUBTYPE; + } else if (format.drmInitData != null && format.exoMediaCryptoType == null) { + return FORMAT_UNSUPPORTED_DRM; + } else { + return FORMAT_HANDLED; + } + } + + @Override + @AdaptiveSupport + public final int supportsMixedMimeTypeAdaptation() { + return ADAPTIVE_NOT_SEAMLESS; + } + + @Override + protected FfmpegAudioDecoder createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) + throws FfmpegAudioDecoderException { + int initialInputBufferSize = + format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; + decoder = + new FfmpegAudioDecoder( + NUM_BUFFERS, NUM_BUFFERS, initialInputBufferSize, format, shouldUseFloatOutput(format)); + return decoder; + } + + @Override + public Format getOutputFormat() { + Assertions.checkNotNull(decoder); + return new Format.Builder() + .setSampleMimeType(MimeTypes.AUDIO_RAW) + .setChannelCount(decoder.getChannelCount()) + .setSampleRate(decoder.getSampleRate()) + .setPcmEncoding(decoder.getEncoding()) + .build(); + } + + private boolean isOutputSupported(Format inputFormat) { + return shouldUseFloatOutput(inputFormat) + || supportsOutput(inputFormat.channelCount, C.ENCODING_PCM_16BIT); + } + + private boolean shouldUseFloatOutput(Format inputFormat) { + Assertions.checkNotNull(inputFormat.sampleMimeType); + if (!enableFloatOutput || !supportsOutput(inputFormat.channelCount, C.ENCODING_PCM_FLOAT)) { + return false; + } + switch (inputFormat.sampleMimeType) { + case MimeTypes.AUDIO_RAW: + // For raw audio, output in 32-bit float encoding if the bit depth is > 16-bit. + return inputFormat.pcmEncoding == C.ENCODING_PCM_24BIT + || inputFormat.pcmEncoding == C.ENCODING_PCM_32BIT + || inputFormat.pcmEncoding == C.ENCODING_PCM_FLOAT; + case MimeTypes.AUDIO_AC3: + // AC-3 is always 16-bit, so there is no point outputting in 32-bit float encoding. + return false; + default: + // For all other formats, assume that it's worth using 32-bit float encoding. + return true; + } + } + +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java new file mode 100644 index 00000000000..6bcbf3693f7 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.ExoPlayerLibraryInfo; +import com.google.android.exoplayer2.util.LibraryLoader; +import com.google.android.exoplayer2.util.Log; +import com.google.android.exoplayer2.util.MimeTypes; + +/** + * Configures and queries the underlying native library. + */ +public final class FfmpegLibrary { + + static { + ExoPlayerLibraryInfo.registerModule("goog.exo.ffmpegvideo"); + } + + private static final String TAG = "FfmpegLibrary"; + + private static final LibraryLoader LOADER = + new LibraryLoader("ffmpeg", "ffmpegJNI"); + + private FfmpegLibrary() {} + + /** + * Override the names of the FFmpeg native libraries. If an application wishes to call this + * method, it must do so before calling any other method defined by this class, and before + * instantiating a {@link FfmpegAudioRenderer} instance. + * + * @param libraries The names of the FFmpeg native libraries. + */ + public static void setLibraries(String... libraries) { + LOADER.setLibraries(libraries); + } + + /** + * Returns whether the underlying library is available, loading it if necessary. + */ + public static boolean isAvailable() { + return LOADER.isAvailable(); + } + + /** Returns the version of the underlying library if available, or null otherwise. */ + public static @Nullable String getVersion() { + return isAvailable() ? ffmpegGetVersion() : null; + } + + /** + * Returns whether the underlying library supports the specified MIME type. + * + * @param mimeType The MIME type to check. + */ + public static boolean supportsAudioFormat(String mimeType) { + if (!isAvailable()) { + return false; + } + String codecName = getAudioCodecName(mimeType); + if (codecName == null) { + return false; + } + if (!ffmpegHasDecoder(codecName)) { + Log.w(TAG, "No " + codecName + " decoder available. Check the FFmpeg build configuration."); + return false; + } + return true; + } + + /** + * Returns whether the underlying library supports the specified MIME type. + * + * @param mimeType The MIME type to check. + */ + public static boolean supportsVideoFormat(String mimeType) { + if (!isAvailable()) { + return false; + } + String codecName = getVideoCodecName(mimeType); + if (codecName == null) { + return false; + } + if (!ffmpegHasDecoder(codecName)) { + Log.w(TAG, "No " + codecName + " decoder available. Check the FFmpeg build configuration."); + return false; + } + return true; + } + + /** + * Returns the name of the FFmpeg decoder that could be used to decode the format, or {@code null} + * if it's unsupported. + */ + /* package */ static @Nullable String getAudioCodecName(String mimeType) { + switch (mimeType) { + case MimeTypes.AUDIO_AAC: + return "aac"; + case MimeTypes.AUDIO_MPEG: + case MimeTypes.AUDIO_MPEG_L1: + case MimeTypes.AUDIO_MPEG_L2: + return "mp3"; + case MimeTypes.AUDIO_AC3: + return "ac3"; + case MimeTypes.AUDIO_E_AC3: + case MimeTypes.AUDIO_E_AC3_JOC: + return "eac3"; + case MimeTypes.AUDIO_TRUEHD: + return "truehd"; + case MimeTypes.AUDIO_DTS: + case MimeTypes.AUDIO_DTS_HD: + return "dca"; + case MimeTypes.AUDIO_VORBIS: + return "vorbis"; + case MimeTypes.AUDIO_OPUS: + return "opus"; + case MimeTypes.AUDIO_AMR_NB: + return "amrnb"; + case MimeTypes.AUDIO_AMR_WB: + return "amrwb"; + case MimeTypes.AUDIO_FLAC: + return "flac"; + case MimeTypes.AUDIO_ALAC: + return "alac"; + case MimeTypes.AUDIO_MLAW: + return "pcm_mulaw"; + case MimeTypes.AUDIO_ALAW: + return "pcm_alaw"; + default: + return null; + } + } + + /** + * Returns the name of the FFmpeg decoder that could be used to decode the format, or {@code null} + * if it's unsupported. + */ + /* package */ static @Nullable String getVideoCodecName(String mimeType) { + switch (mimeType) { + case MimeTypes.VIDEO_H264: + return "h264"; + case MimeTypes.VIDEO_H265: + return "hevc"; + default: + return null; + } + } + + private static native String ffmpegGetVersion(); + private static native boolean ffmpegHasDecoder(String codecName); + +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java new file mode 100644 index 00000000000..874299453f3 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java @@ -0,0 +1,258 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import android.util.Log; +import android.view.Surface; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.decoder.SimpleDecoder; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.VideoDecoderInputBuffer; +import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer; +import java.nio.ByteBuffer; +import java.util.List; + +/** + * Ffmpeg Video decoder. + */ +/* package */ final class FfmpegVideoDecoder + extends + SimpleDecoder { + + // Error codes matching ffmpeg_video_jni.cc. + private static final int DECODER_ERROR_INVALID_DATA = -1; + private static final int DECODER_ERROR_OTHER = -2; + private static final int DECODER_ERROR_READ_FRAME = -3; + private static final int DECODER_ERROR_SEND_PACKET = -4; + + private final String codecName; + private long nativeContext; + @Nullable private final byte[] extraData; + private Format format; + + @C.VideoOutputMode private volatile int outputMode; + + /** + * Creates a Ffmpeg video Decoder. + * + * @param numInputBuffers Number of input buffers. + * @param numOutputBuffers Number of output buffers. + * @param initialInputBufferSize The initial size of each input buffer, in bytes. + * @param threads Number of threads libgav1 will use to decode. + * @throws FfmpegVideoDecoderException Thrown if an exception occurs when initializing the + * decoder. + */ + public FfmpegVideoDecoder( + int numInputBuffers, int numOutputBuffers, int initialInputBufferSize, int threads, Format format) + throws FfmpegVideoDecoderException { + super( + new VideoDecoderInputBuffer[numInputBuffers], + new VideoDecoderOutputBuffer[numOutputBuffers]); + if (!FfmpegLibrary.isAvailable()) { + throw new FfmpegVideoDecoderException("Failed to load decoder native library."); + } + codecName = Assertions.checkNotNull(FfmpegLibrary.getVideoCodecName(format.sampleMimeType)); + extraData = getExtraData(format.sampleMimeType, format.initializationData); + this.format = format; + nativeContext = ffmpegInitialize(codecName, extraData, threads); + if (nativeContext == 0) { + throw new FfmpegVideoDecoderException("Failed to initialize decoder."); + } + setInitialInputBufferSize(initialInputBufferSize); + } + + /** + * Returns FFmpeg-compatible codec-specific initialization data ("extra data"), or {@code null} if + * not required. + */ + @Nullable + private static byte[] getExtraData(String mimeType, List initializationData) { + switch (mimeType) { + case MimeTypes.VIDEO_H264: + byte[] sps = initializationData.get(0); + byte[] pps = initializationData.get(1); + byte[] extraData = new byte[sps.length + pps.length]; + System.arraycopy(sps, 0, extraData, 0, sps.length); + System.arraycopy(pps, 0, extraData, sps.length, pps.length); + return extraData; + case MimeTypes.VIDEO_H265: + return initializationData.get(0); + default: + // Other codecs do not require extra data. + return null; + } + } + + @Override + public String getName() { + return "ffmpeg" + FfmpegLibrary.getVersion() + "-" + codecName; + } + + /** + * Sets the output mode for frames rendered by the decoder. + * + * @param outputMode The output mode. + */ + public void setOutputMode(@C.VideoOutputMode int outputMode) { + this.outputMode = outputMode; + } + + @Override + protected VideoDecoderInputBuffer createInputBuffer() { + return new VideoDecoderInputBuffer(); + } + + @Override + protected VideoDecoderOutputBuffer createOutputBuffer() { + return new VideoDecoderOutputBuffer(this::releaseOutputBuffer); + } + + @Override + @Nullable + protected FfmpegVideoDecoderException decode( + VideoDecoderInputBuffer inputBuffer, VideoDecoderOutputBuffer outputBuffer, boolean reset) { + if (reset) { + nativeContext = ffmpegReset(nativeContext); + if (nativeContext == 0) { + return new FfmpegVideoDecoderException("Error resetting (see logcat)."); + } + } + + // send packet + ByteBuffer inputData = Util.castNonNull(inputBuffer.data); + int inputSize = inputData.limit(); + // enqueue origin data + boolean needSendAgain = false; + int sendPacketResult = ffmpegSendPacket(nativeContext, inputData, inputSize, + inputBuffer.timeUs); + if (sendPacketResult == DECODER_ERROR_INVALID_DATA) { + outputBuffer.setFlags(C.BUFFER_FLAG_DECODE_ONLY); + return null; + } else if (sendPacketResult == DECODER_ERROR_READ_FRAME) { + // need read frame + needSendAgain = true; + } else if (sendPacketResult == DECODER_ERROR_OTHER) { + return new FfmpegVideoDecoderException("ffmpegDecode error: (see logcat)"); + } + + // receive frame + boolean decodeOnly = inputBuffer.isDecodeOnly(); + // We need to dequeue the decoded frame from the decoder even when the input data is + // decode-only. + int getFrameResult = ffmpegReceiveFrame(nativeContext, outputMode, outputBuffer, decodeOnly); + if (getFrameResult == DECODER_ERROR_SEND_PACKET) { + return null; + } else if (getFrameResult == DECODER_ERROR_OTHER) { + return new FfmpegVideoDecoderException("ffmpegDecode error: (see logcat)"); + } + + if (getFrameResult == DECODER_ERROR_INVALID_DATA) { + outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY); + } + + if (!decodeOnly) { + outputBuffer.colorInfo = inputBuffer.colorInfo; + } + + if (needSendAgain) { + Log.e("ffmpeg_jni", "timeUs=" + inputBuffer.timeUs + ", " + "nendSendAagin"); + } + + return null; + } + + @Override + protected FfmpegVideoDecoderException createUnexpectedDecodeException(Throwable error) { + return new FfmpegVideoDecoderException("Unexpected decode error", error); + } + + @Override + public void release() { + super.release(); + ffmpegRelease(nativeContext); + nativeContext = 0; + } + + @Override + protected void releaseOutputBuffer(VideoDecoderOutputBuffer buffer) { + // Decode only frames do not acquire a reference on the internal decoder buffer and thus do not + // require a call to vpxReleaseFrame. +// if (outputMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && !buffer.isDecodeOnly()) { +// gav1ReleaseFrame(nativeContext, buffer); +// } + super.releaseOutputBuffer(buffer); + } + + /** + * Renders output buffer to the given surface. Must only be called when in {@link + * C#VIDEO_OUTPUT_MODE_SURFACE_YUV} mode. + * + * @param outputBuffer Output buffer. + * @param surface Output surface. + * @throws FfmpegVideoDecoderException Thrown if called with invalid output mode or frame + * rendering fails. + */ + public void renderToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface) + throws FfmpegVideoDecoderException { + if (outputBuffer.mode != C.VIDEO_OUTPUT_MODE_SURFACE_YUV) { + throw new FfmpegVideoDecoderException("Invalid output mode."); + } + if (ffmpegRenderFrame( + nativeContext, surface, + outputBuffer, outputBuffer.width, outputBuffer.height) == DECODER_ERROR_OTHER) { + throw new FfmpegVideoDecoderException( + "Buffer render error: "); + } + } + + private native long ffmpegInitialize(String codecName, @Nullable byte[] extraData, int threads); + + private native long ffmpegReset(long context); + + private native void ffmpegRelease(long context); + + private native int ffmpegRenderFrame( + long context, Surface surface, VideoDecoderOutputBuffer outputBuffer, + int displayedWidth, + int displayedHeight); + + /** + * Decodes the encoded data passed. + * + * @param context Decoder context. + * @param encodedData Encoded data. + * @param length Length of the data buffer. + * @return 0 if successful, {@link #DECODER_ERROR_OTHER} if an error occurred. + */ + private native int ffmpegSendPacket(long context, ByteBuffer encodedData, int length, + long inputTime); + + /** + * Gets the decoded frame. + * + * @param context Decoder context. + * @param outputBuffer Output buffer for the decoded frame. + * @return 0 if successful, {@link #DECODER_ERROR_INVALID_DATA} if successful but the frame is + * decode-only, {@link #DECODER_ERROR_OTHER} if an error occurred. + */ + private native int ffmpegReceiveFrame( + long context, int outputMode, VideoDecoderOutputBuffer outputBuffer, boolean decodeOnly); + +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoderException.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoderException.java new file mode 100644 index 00000000000..164c183ea44 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoderException.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import com.google.android.exoplayer2.video.VideoDecoderException; + +/** Thrown when a libgav1 decoder error occurs. */ +public final class FfmpegVideoDecoderException extends VideoDecoderException { + + /* package */ FfmpegVideoDecoderException(String message) { + super(message); + } + + /* package */ FfmpegVideoDecoderException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java new file mode 100644 index 00000000000..770a6a5fd21 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import static java.lang.Runtime.getRuntime; + +import android.os.Handler; +import android.view.Surface; +import androidx.annotation.Keep; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.ExoPlaybackException; +import com.google.android.exoplayer2.ExoPlayer; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.PlayerMessage.Target; +import com.google.android.exoplayer2.RendererCapabilities; +import com.google.android.exoplayer2.decoder.SimpleDecoder; +import com.google.android.exoplayer2.drm.ExoMediaCrypto; +import com.google.android.exoplayer2.util.TraceUtil; +import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.SimpleDecoderVideoRenderer; +import com.google.android.exoplayer2.video.VideoDecoderException; +import com.google.android.exoplayer2.video.VideoDecoderInputBuffer; +import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer; +import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer; +import com.google.android.exoplayer2.video.VideoRendererEventListener; + +/** + * Decodes and renders video using libgav1 decoder. + * + *

This renderer accepts the following messages sent via {@link ExoPlayer#createMessage(Target)} + * on the playback thread: + * + *

    + *
  • Message with type {@link #MSG_SET_SURFACE} to set the output surface. The message payload + * should be the target {@link Surface}, or null. + *
  • Message with type {@link #MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER} to set the output + * buffer renderer. The message payload should be the target {@link + * VideoDecoderOutputBufferRenderer}, or null. + *
+ */ +@Keep +public class FfmpegVideoRenderer extends SimpleDecoderVideoRenderer { + + private static final int DEFAULT_NUM_OF_INPUT_BUFFERS = 4; + private static final int DEFAULT_NUM_OF_OUTPUT_BUFFERS = 4; + /* Default size based on 720p resolution video compressed by a factor of two. */ + private static final int DEFAULT_INPUT_BUFFER_SIZE = + Util.ceilDivide(1280, 64) * Util.ceilDivide(720, 64) * (64 * 64 * 3 / 2) / 2; + + /** The number of input buffers. */ + private final int numInputBuffers; + /** + * The number of output buffers. The renderer may limit the minimum possible value due to + * requiring multiple output buffers to be dequeued at a time for it to make progress. + */ + private final int numOutputBuffers; + + private final int threads; + + @Nullable private FfmpegVideoDecoder decoder; + + /** + * Creates a Libgav1VideoRenderer. + * + * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer + * can attempt to seamlessly join an ongoing playback. + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between + * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + */ + public FfmpegVideoRenderer( + long allowedJoiningTimeMs, + @Nullable Handler eventHandler, + @Nullable VideoRendererEventListener eventListener, + int maxDroppedFramesToNotify) { + this( + allowedJoiningTimeMs, + eventHandler, + eventListener, + maxDroppedFramesToNotify, + /* threads= */ getRuntime().availableProcessors(), + DEFAULT_NUM_OF_INPUT_BUFFERS, + DEFAULT_NUM_OF_OUTPUT_BUFFERS); + } + + /** + * Creates a Libgav1VideoRenderer. + * + * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer + * can attempt to seamlessly join an ongoing playback. + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between + * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param threads Number of threads libgav1 will use to decode. + * @param numInputBuffers Number of input buffers. + * @param numOutputBuffers Number of output buffers. + */ + public FfmpegVideoRenderer( + long allowedJoiningTimeMs, + @Nullable Handler eventHandler, + @Nullable VideoRendererEventListener eventListener, + int maxDroppedFramesToNotify, + int threads, + int numInputBuffers, + int numOutputBuffers) { + super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify); + this.threads = threads; + this.numInputBuffers = numInputBuffers; + this.numOutputBuffers = numOutputBuffers; + } + + @Override + @Capabilities + public final int supportsFormat(Format format) { + if (!FfmpegLibrary.isAvailable() + || !FfmpegLibrary.supportsVideoFormat(format.sampleMimeType)) { + return RendererCapabilities.create(FORMAT_UNSUPPORTED_TYPE); + } + if (format.drmInitData != null && format.exoMediaCryptoType == null) { + return RendererCapabilities.create(FORMAT_UNSUPPORTED_DRM); + } + return RendererCapabilities.create(FORMAT_HANDLED, ADAPTIVE_SEAMLESS, TUNNELING_NOT_SUPPORTED); + } + + @Override + protected SimpleDecoder< + VideoDecoderInputBuffer, + ? extends VideoDecoderOutputBuffer, + ? extends VideoDecoderException> + createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) + throws VideoDecoderException { + TraceUtil.beginSection("createGav1Decoder"); + int initialInputBufferSize = + format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; + FfmpegVideoDecoder decoder = + new FfmpegVideoDecoder(numInputBuffers, numOutputBuffers, initialInputBufferSize, threads, format); + this.decoder = decoder; + TraceUtil.endSection(); + return decoder; + } + + @Override + protected void renderOutputBufferToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface) + throws FfmpegVideoDecoderException { + if (decoder == null) { + throw new FfmpegVideoDecoderException( + "Failed to render output buffer to surface: decoder is not initialized."); + } + decoder.renderToSurface(outputBuffer, surface); + outputBuffer.release(); + } + + @Override + protected void setDecoderOutputMode(@C.VideoOutputMode int outputMode) { + if (decoder != null) { + decoder.setOutputMode(outputMode); + } + } + + // PlayerMessage.Target implementation. + + @Override + public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException { + if (messageType == MSG_SET_SURFACE) { + setOutputSurface((Surface) message); + } else if (messageType == MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER) { + setOutputBufferRenderer((VideoDecoderOutputBufferRenderer) message); + } else { + super.handleMessage(messageType, message); + } + } +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/package-info.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/package-info.java new file mode 100644 index 00000000000..a9fedb19cb6 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@NonNullApi +package com.google.android.exoplayer2.ext.ffmpeg; + +import com.google.android.exoplayer2.util.NonNullApi; diff --git a/extensions/ffmpegvideo/src/main/jni/CMakeLists.txt b/extensions/ffmpegvideo/src/main/jni/CMakeLists.txt new file mode 100644 index 00000000000..de6720083f2 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/jni/CMakeLists.txt @@ -0,0 +1,65 @@ +# libgav1JNI requires modern CMake. +cmake_minimum_required(VERSION 3.7.1 FATAL_ERROR) + +# libgav1JNI requires C++11. +set(CMAKE_CXX_STANDARD 11) + +project(libffmpegJNI C CXX) + +# Devices using armeabi-v7a are not required to support +# Neon which is why Neon is disabled by default for +# armeabi-v7a build. This flag enables it. +if(${ANDROID_ABI} MATCHES "armeabi-v7a") + add_compile_options("-mfpu=neon") + add_compile_options("-fPIC") +endif() + +set(libgav1_jni_root "${CMAKE_CURRENT_SOURCE_DIR}") +set(libgav1_jni_build "${CMAKE_BINARY_DIR}") +set(libgav1_jni_output_directory + ${libgav1_jni_root}/../libs/${ANDROID_ABI}/) + +#set(libgav1_root "${libgav1_jni_root}/libgav1") +#set(libgav1_build "${libgav1_jni_build}/libgav1") + +#set(cpu_features_root "${libgav1_jni_root}/cpu_features") +#set(cpu_features_build "${libgav1_jni_build}/cpu_features") + +# Build cpu_features library. +#add_subdirectory("${cpu_features_root}" +# "${cpu_features_build}" +# EXCLUDE_FROM_ALL) + +# Build libgav1. +#add_subdirectory("${libgav1_root}" +# "${libgav1_build}" +# EXCLUDE_FROM_ALL) +add_library(ffmpeg + SHARED + IMPORTED) +set_target_properties(ffmpeg PROPERTIES + IMPORTED_LOCATION + ${libgav1_jni_output_directory}/libffmpeg.so) + +# Build libgav1JNI. +add_library(ffmpegJNI + SHARED + ffmpeg_audio_jni.cc + ffmpeg_video_jni.cc) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) + +# Locate NDK log library. +find_library(android_log_lib log) + +# Link libgav1JNI against used libraries. +target_link_libraries(ffmpegJNI + PRIVATE android + PRIVATE ffmpeg +# PRIVATE cpu_features +# PRIVATE libgav1_static + PRIVATE ${android_log_lib}) + +# Specify output directory for libgav1JNI. +set_target_properties(ffmpegJNI PROPERTIES + LIBRARY_OUTPUT_DIRECTORY + ${libgav1_jni_output_directory}) diff --git a/extensions/ffmpegvideo/src/main/jni/ffmpeg_audio_jni.cc b/extensions/ffmpegvideo/src/main/jni/ffmpeg_audio_jni.cc new file mode 100644 index 00000000000..89dc2d3de06 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/jni/ffmpeg_audio_jni.cc @@ -0,0 +1,360 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +extern "C" { +#ifdef __cplusplus +#define __STDC_CONSTANT_MACROS +#ifdef _STDINT_H +#undef _STDINT_H +#endif +#include +#endif +#include +#include +#include +#include +#include +} + +#define LOG_TAG "ffmpeg_jni" +#define LOGE(...) ((void)__android_log_print(ANDROID_LOG_ERROR, LOG_TAG, \ + __VA_ARGS__)) + +#define DECODER_FUNC(RETURN_TYPE, NAME, ...) \ + extern "C" { \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegAudioDecoder_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\ + } \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegAudioDecoder_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ + +#define LIBRARY_FUNC(RETURN_TYPE, NAME, ...) \ + extern "C" { \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegLibrary_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\ + } \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegLibrary_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ + +#define ERROR_STRING_BUFFER_LENGTH 256 + +// Output format corresponding to AudioFormat.ENCODING_PCM_16BIT. +static const AVSampleFormat OUTPUT_FORMAT_PCM_16BIT = AV_SAMPLE_FMT_S16; +// Output format corresponding to AudioFormat.ENCODING_PCM_FLOAT. +static const AVSampleFormat OUTPUT_FORMAT_PCM_FLOAT = AV_SAMPLE_FMT_FLT; + +// Error codes matching FfmpegAudioDecoder.java. +static const int DECODER_ERROR_INVALID_DATA = -1; +static const int DECODER_ERROR_OTHER = -2; + +/** + * Returns the AVCodec with the specified name, or NULL if it is not available. + */ +AVCodec *getCodecByName(JNIEnv* env, jstring codecName); + +/** + * Allocates and opens a new AVCodecContext for the specified codec, passing the + * provided extraData as initialization data for the decoder if it is non-NULL. + * Returns the created context. + */ +AVCodecContext *createContext(JNIEnv *env, AVCodec *codec, jbyteArray extraData, + jboolean outputFloat, jint rawSampleRate, + jint rawChannelCount); + +/** + * Decodes the packet into the output buffer, returning the number of bytes + * written, or a negative DECODER_ERROR constant value in the case of an error. + */ +int decodePacket(AVCodecContext *context, AVPacket *packet, + uint8_t *outputBuffer, int outputSize); + +/** + * Outputs a log message describing the avcodec error number. + */ +void logError(const char *functionName, int errorNumber); + +/** + * Releases the specified context. + */ +void releaseContext(AVCodecContext *context); + +jint JNI_OnLoad(JavaVM *vm, void *reserved) { + JNIEnv *env; + if (vm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6) != JNI_OK) { + return -1; + } + avcodec_register_all(); + return JNI_VERSION_1_6; +} + +LIBRARY_FUNC(jstring, ffmpegGetVersion) { + return env->NewStringUTF(LIBAVCODEC_IDENT); +} + +LIBRARY_FUNC(jboolean, ffmpegHasDecoder, jstring codecName) { + return getCodecByName(env, codecName) != NULL; +} + +DECODER_FUNC(jlong, ffmpegInitialize, jstring codecName, jbyteArray extraData, + jboolean outputFloat, jint rawSampleRate, jint rawChannelCount) { + AVCodec *codec = getCodecByName(env, codecName); + if (!codec) { + LOGE("Codec not found."); + return 0L; + } + return (jlong)createContext(env, codec, extraData, outputFloat, rawSampleRate, + rawChannelCount); +} + +DECODER_FUNC(jint, ffmpegDecode, jlong context, jobject inputData, + jint inputSize, jobject outputData, jint outputSize) { + if (!context) { + LOGE("Context must be non-NULL."); + return -1; + } + if (!inputData || !outputData) { + LOGE("Input and output buffers must be non-NULL."); + return -1; + } + if (inputSize < 0) { + LOGE("Invalid input buffer size: %d.", inputSize); + return -1; + } + if (outputSize < 0) { + LOGE("Invalid output buffer length: %d", outputSize); + return -1; + } + uint8_t *inputBuffer = (uint8_t *) env->GetDirectBufferAddress(inputData); + uint8_t *outputBuffer = (uint8_t *) env->GetDirectBufferAddress(outputData); + AVPacket packet; + av_init_packet(&packet); + packet.data = inputBuffer; + packet.size = inputSize; + return decodePacket((AVCodecContext *) context, &packet, outputBuffer, + outputSize); +} + +DECODER_FUNC(jint, ffmpegGetChannelCount, jlong context) { + if (!context) { + LOGE("Context must be non-NULL."); + return -1; + } + return ((AVCodecContext *) context)->channels; +} + +DECODER_FUNC(jint, ffmpegGetSampleRate, jlong context) { + if (!context) { + LOGE("Context must be non-NULL."); + return -1; + } + return ((AVCodecContext *) context)->sample_rate; +} + +DECODER_FUNC(jlong, ffmpegReset, jlong jContext, jbyteArray extraData) { + AVCodecContext *context = (AVCodecContext *) jContext; + if (!context) { + LOGE("Tried to reset without a context."); + return 0L; + } + + AVCodecID codecId = context->codec_id; + if (codecId == AV_CODEC_ID_TRUEHD) { + // Release and recreate the context if the codec is TrueHD. + // TODO: Figure out why flushing doesn't work for this codec. + releaseContext(context); + AVCodec *codec = avcodec_find_decoder(codecId); + if (!codec) { + LOGE("Unexpected error finding codec %d.", codecId); + return 0L; + } + jboolean outputFloat = + (jboolean)(context->request_sample_fmt == OUTPUT_FORMAT_PCM_FLOAT); + return (jlong)createContext(env, codec, extraData, outputFloat, + /* rawSampleRate= */ -1, + /* rawChannelCount= */ -1); + } + + avcodec_flush_buffers(context); + return (jlong) context; +} + +DECODER_FUNC(void, ffmpegRelease, jlong context) { + if (context) { + releaseContext((AVCodecContext *) context); + } +} + +AVCodec *getCodecByName(JNIEnv* env, jstring codecName) { + if (!codecName) { + return NULL; + } + const char *codecNameChars = env->GetStringUTFChars(codecName, NULL); + AVCodec *codec = avcodec_find_decoder_by_name(codecNameChars); + env->ReleaseStringUTFChars(codecName, codecNameChars); + return codec; +} + +AVCodecContext *createContext(JNIEnv *env, AVCodec *codec, jbyteArray extraData, + jboolean outputFloat, jint rawSampleRate, + jint rawChannelCount) { + AVCodecContext *context = avcodec_alloc_context3(codec); + if (!context) { + LOGE("Failed to allocate context."); + return NULL; + } + context->request_sample_fmt = + outputFloat ? OUTPUT_FORMAT_PCM_FLOAT : OUTPUT_FORMAT_PCM_16BIT; + if (extraData) { + jsize size = env->GetArrayLength(extraData); + context->extradata_size = size; + context->extradata = + (uint8_t *) av_malloc(size + AV_INPUT_BUFFER_PADDING_SIZE); + if (!context->extradata) { + LOGE("Failed to allocate extradata."); + releaseContext(context); + return NULL; + } + env->GetByteArrayRegion(extraData, 0, size, (jbyte *) context->extradata); + } + if (context->codec_id == AV_CODEC_ID_PCM_MULAW || + context->codec_id == AV_CODEC_ID_PCM_ALAW) { + context->sample_rate = rawSampleRate; + context->channels = rawChannelCount; + context->channel_layout = av_get_default_channel_layout(rawChannelCount); + } + context->err_recognition = AV_EF_IGNORE_ERR; + int result = avcodec_open2(context, codec, NULL); + if (result < 0) { + logError("avcodec_open2", result); + releaseContext(context); + return NULL; + } + return context; +} + +int decodePacket(AVCodecContext *context, AVPacket *packet, + uint8_t *outputBuffer, int outputSize) { + int result = 0; + // Queue input data. + result = avcodec_send_packet(context, packet); + if (result) { + logError("avcodec_send_packet", result); + return result == AVERROR_INVALIDDATA ? DECODER_ERROR_INVALID_DATA + : DECODER_ERROR_OTHER; + } + + // Dequeue output data until it runs out. + int outSize = 0; + while (true) { + AVFrame *frame = av_frame_alloc(); + if (!frame) { + LOGE("Failed to allocate output frame."); + return -1; + } + result = avcodec_receive_frame(context, frame); + if (result) { + av_frame_free(&frame); + if (result == AVERROR(EAGAIN)) { + break; + } + logError("avcodec_receive_frame", result); + return result; + } + + // Resample output. + AVSampleFormat sampleFormat = context->sample_fmt; + int channelCount = context->channels; + int channelLayout = context->channel_layout; + int sampleRate = context->sample_rate; + int sampleCount = frame->nb_samples; + int dataSize = av_samples_get_buffer_size(NULL, channelCount, sampleCount, + sampleFormat, 1); + SwrContext *resampleContext; + if (context->opaque) { + resampleContext = (SwrContext *)context->opaque; + } else { + resampleContext = swr_alloc(); + av_opt_set_int(resampleContext, "in_channel_layout", channelLayout, 0); + av_opt_set_int(resampleContext, "out_channel_layout", channelLayout, 0); + av_opt_set_int(resampleContext, "in_sample_rate", sampleRate, 0); + av_opt_set_int(resampleContext, "out_sample_rate", sampleRate, 0); + av_opt_set_int(resampleContext, "in_sample_fmt", sampleFormat, 0); + // The output format is always the requested format. + av_opt_set_int(resampleContext, "out_sample_fmt", + context->request_sample_fmt, 0); + result = swr_init(resampleContext); + if (result < 0) { + logError("swr_init", result); + av_frame_free(&frame); + return -1; + } + context->opaque = resampleContext; + } + int inSampleSize = av_get_bytes_per_sample(sampleFormat); + int outSampleSize = av_get_bytes_per_sample(context->request_sample_fmt); + int outSamples = swr_get_out_samples(resampleContext, sampleCount); + int bufferOutSize = outSampleSize * channelCount * outSamples; + if (outSize + bufferOutSize > outputSize) { + LOGE("Output buffer size (%d) too small for output data (%d).", + outputSize, outSize + bufferOutSize); + av_frame_free(&frame); + return -1; + } + result = swr_convert(resampleContext, &outputBuffer, bufferOutSize, + (const uint8_t **)frame->data, frame->nb_samples); + av_frame_free(&frame); + if (result < 0) { + logError("swr_convert", result); + return result; + } + int available = swr_get_out_samples(resampleContext, 0); + if (available != 0) { + LOGE("Expected no samples remaining after resampling, but found %d.", + available); + return -1; + } + outputBuffer += bufferOutSize; + outSize += bufferOutSize; + } + return outSize; +} + +void logError(const char *functionName, int errorNumber) { + char *buffer = (char *) malloc(ERROR_STRING_BUFFER_LENGTH * sizeof(char)); + av_strerror(errorNumber, buffer, ERROR_STRING_BUFFER_LENGTH); + LOGE("Error in %s: %s", functionName, buffer); + free(buffer); +} + +void releaseContext(AVCodecContext *context) { + if (!context) { + return; + } + SwrContext *swrContext; + if ((swrContext = (SwrContext *)context->opaque)) { + swr_free(&swrContext); + context->opaque = NULL; + } + avcodec_free_context(&context); +} + diff --git a/extensions/ffmpegvideo/src/main/jni/ffmpeg_video_jni.cc b/extensions/ffmpegvideo/src/main/jni/ffmpeg_video_jni.cc new file mode 100644 index 00000000000..2cb5a1c16d9 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/jni/ffmpeg_video_jni.cc @@ -0,0 +1,436 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include + +extern "C" { +#ifdef __cplusplus +#define __STDC_CONSTANT_MACROS +#ifdef _STDINT_H +#undef _STDINT_H +#endif + +#endif +#include +#include +#include +#include +#include +} + +#define LOG_TAG "ffmpeg_jni" +#define LOGE(...) ((void)__android_log_print(ANDROID_LOG_ERROR, LOG_TAG, \ + __VA_ARGS__)) + +#define DECODER_FUNC(RETURN_TYPE, NAME, ...) \ + extern "C" { \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegVideoDecoder_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\ + } \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegVideoDecoder_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ + +#define ERROR_STRING_BUFFER_LENGTH 256 + + +namespace { +// Error codes matching FfmpegAudioDecoder.java. +const int DECODER_SUCCESS = 0; +const int DECODER_ERROR_INVALID_DATA = -1; +const int DECODER_ERROR_OTHER = -2; +const int DECODER_ERROR_READ_FRAME = -3; +const int DECODER_ERROR_SEND_PACKET = -4; + +// YUV plane indices. +const int kPlaneY = 0; +const int kPlaneU = 1; +const int kPlaneV = 2; +const int kMaxPlanes = 3; + +// Android YUV format. See: +// https://developer.android.com/reference/android/graphics/ImageFormat.html#YV12. +const int kImageFormatYV12 = 0x32315659; + +// LINT.IfChange +// Output modes. +const int kOutputModeYuv = 0; +const int kOutputModeSurfaceYuv = 1; +// LINT.ThenChange(../../../../../library/core/src/main/java/com/google/android/exoplayer2/C.java) + +// LINT.IfChange +const int kColorSpaceUnknown = 0; +// LINT.ThenChange(../../../../../library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderOutputBuffer.java) + +struct JniContext { + ~JniContext() { + if (native_window) { + ANativeWindow_release(native_window); + } + } + + bool MaybeAcquireNativeWindow(JNIEnv *env, jobject new_surface) { + if (surface == new_surface) { + return true; + } + if (native_window) { + ANativeWindow_release(native_window); + } + native_window_width = 0; + native_window_height = 0; + native_window = ANativeWindow_fromSurface(env, new_surface); + if (native_window == nullptr) { + LOGE("kJniStatusANativeWindowError"); + surface = nullptr; + return false; + } + surface = new_surface; + return true; + } + + jfieldID data_field; + jfieldID yuvPlanes_field; + jfieldID yuvStrides_field; + jmethodID init_for_private_frame_method; + jmethodID init_for_yuv_frame_method; + jmethodID init_method; + + AVCodecContext *codecContext; + + ANativeWindow *native_window = nullptr; + jobject surface = nullptr; + int native_window_width = 0; + int native_window_height = 0; +}; + + +AVCodec *getCodecByName(JNIEnv *env, jstring codecName) { + if (!codecName) { + return NULL; + } + const char *codecNameChars = env->GetStringUTFChars(codecName, NULL); + AVCodec *codec = avcodec_find_decoder_by_name(codecNameChars); + env->ReleaseStringUTFChars(codecName, codecNameChars); + return codec; +} + +void logError(const char *functionName, int errorNumber) { + char *buffer = (char *) malloc(ERROR_STRING_BUFFER_LENGTH * sizeof(char)); + av_strerror(errorNumber, buffer, ERROR_STRING_BUFFER_LENGTH); + LOGE("Error in %s: %s", functionName, buffer); + free(buffer); +} + +void releaseContext(AVCodecContext *context) { + if (!context) { + return; + } + + avcodec_free_context(&context); +} + +JniContext *createContext(JNIEnv *env, + AVCodec *codec, + jbyteArray extraData, + jint threads) { + JniContext *jniContext = new(std::nothrow) JniContext(); + + AVCodecContext *codecContext = avcodec_alloc_context3(codec); + if (!codecContext) { + LOGE("Failed to allocate context."); + return NULL; + } + + if (extraData) { + jsize size = env->GetArrayLength(extraData); + codecContext->extradata_size = size; + codecContext->extradata = + (uint8_t *) av_malloc(size + AV_INPUT_BUFFER_PADDING_SIZE); + if (!codecContext->extradata) { + LOGE("Failed to allocate extradata."); + releaseContext(codecContext); + return NULL; + } + env->GetByteArrayRegion(extraData, 0, size, (jbyte *) codecContext->extradata); + } + + codecContext->thread_count = threads; + codecContext->err_recognition = AV_EF_IGNORE_ERR; + int result = avcodec_open2(codecContext, codec, NULL); + if (result < 0) { + logError("avcodec_open2", result); + releaseContext(codecContext); + return NULL; + } + + jniContext->codecContext = codecContext; + + // Populate JNI References. + const jclass outputBufferClass = env->FindClass( + "com/google/android/exoplayer2/video/VideoDecoderOutputBuffer"); + jniContext->data_field = env->GetFieldID(outputBufferClass, "data", "Ljava/nio/ByteBuffer;"); + jniContext->yuvPlanes_field = + env->GetFieldID(outputBufferClass, "yuvPlanes", "[Ljava/nio/ByteBuffer;"); + jniContext->yuvStrides_field = env->GetFieldID(outputBufferClass, "yuvStrides", "[I"); + jniContext->init_for_private_frame_method = + env->GetMethodID(outputBufferClass, "initForPrivateFrame", "(II)V"); + jniContext->init_for_yuv_frame_method = + env->GetMethodID(outputBufferClass, "initForYuvFrame", "(IIIII)Z"); + jniContext->init_method = + env->GetMethodID(outputBufferClass, "init", "(JILjava/nio/ByteBuffer;)V"); + + return jniContext; +} + +void CopyPlane(const uint8_t *source, int source_stride, uint8_t *destination, + int destination_stride, int width, int height) { + while (height--) { + std::memcpy(destination, source, width); + source += source_stride; + destination += destination_stride; + } +} + +constexpr int AlignTo16(int value) { return (value + 15) & (~15); } + +} + +DECODER_FUNC(jlong, ffmpegInitialize, jstring codecName, jbyteArray extraData, jint threads) { + AVCodec *codec = getCodecByName(env, codecName); + if (!codec) { + LOGE("Codec not found."); + return 0L; + } + + return (jlong) createContext(env, codec, extraData, threads); +} + +DECODER_FUNC(jlong, ffmpegReset, jlong jContext) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *context = jniContext->codecContext; + if (!context) { + LOGE("Tried to reset without a context."); + return 0L; + } + +// LOGE("avcodec_flush_buffers"); + avcodec_flush_buffers(context); + return (jlong) jniContext; +} + +DECODER_FUNC(void, ffmpegRelease, jlong jContext) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *context = jniContext->codecContext; + if (context) { + releaseContext(context); + } +} + + +DECODER_FUNC(jint, ffmpegSendPacket, jlong jContext, jobject encodedData, + jint length, jlong inputTimeUs) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *avContext = jniContext->codecContext; + + uint8_t *inputBuffer = (uint8_t *) env->GetDirectBufferAddress(encodedData); + AVPacket packet; + av_init_packet(&packet); + packet.data = inputBuffer; + packet.size = length; + packet.pts = inputTimeUs; + + int result = 0; + // Queue input data. + result = avcodec_send_packet(avContext, &packet); + if (result) { + logError("avcodec_send_packet", result); + if (result == AVERROR_INVALIDDATA) { + // need more data + return DECODER_ERROR_INVALID_DATA; + } else if (result == AVERROR(EAGAIN)) { + // need read frame + return DECODER_ERROR_READ_FRAME; + } else { + return DECODER_ERROR_OTHER; + } + } + return result; +} + +DECODER_FUNC(jint, ffmpegReceiveFrame, jlong jContext, jint outputMode, jobject jOutputBuffer, + jboolean decodeOnly) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *avContext = jniContext->codecContext; + int result = 0; + + AVFrame *frame = av_frame_alloc(); + if (!frame) { + LOGE("Failed to allocate output frame."); + return DECODER_ERROR_OTHER; + } + result = avcodec_receive_frame(avContext, frame); + + // fail + if (decodeOnly || result == AVERROR(EAGAIN)) { + // This is not an error. The input data was decode-only or no displayable + // frames are available. + av_frame_free(&frame); + return DECODER_ERROR_INVALID_DATA; + } + if (result) { + av_frame_free(&frame); + logError("avcodec_receive_frame", result); + return DECODER_ERROR_OTHER; + } + + // success + // init time and mode + env->CallVoidMethod(jOutputBuffer, jniContext->init_method, frame->pts, outputMode, nullptr); + + // init data + const jboolean init_result = env->CallBooleanMethod( + jOutputBuffer, jniContext->init_for_yuv_frame_method, + frame->width, + frame->height, + frame->linesize[0], frame->linesize[1], + 0); + if (env->ExceptionCheck()) { + // Exception is thrown in Java when returning from the native call. + return DECODER_ERROR_OTHER; + } + if (!init_result) { + return DECODER_ERROR_OTHER; + } + + const jobject data_object = env->GetObjectField(jOutputBuffer, jniContext->data_field); + jbyte *data = reinterpret_cast(env->GetDirectBufferAddress(data_object)); + const int32_t uvHeight = (frame->height + 1) / 2; + const uint64_t yLength = frame->linesize[0] * frame->height; + const uint64_t uvLength = frame->linesize[1] * uvHeight; + + // todo rotate YUV data + + memcpy(data, frame->data[0], yLength); + memcpy(data + yLength, frame->data[1], uvLength); + memcpy(data + yLength + uvLength, frame->data[2], uvLength); + + av_frame_free(&frame); + + return result; +} + +DECODER_FUNC(jint, ffmpegRenderFrame, jlong jContext, jobject jSurface, + jobject jOutputBuffer, jint displayedWidth, jint displayedHeight) { + JniContext *const jniContext = reinterpret_cast(jContext); + if (!jniContext->MaybeAcquireNativeWindow(env, jSurface)) { + return DECODER_ERROR_OTHER; + } + + if (jniContext->native_window_width != displayedWidth || + jniContext->native_window_height != displayedHeight) { + if (ANativeWindow_setBuffersGeometry( + jniContext->native_window, + displayedWidth, + displayedHeight, + kImageFormatYV12)) { + LOGE("kJniStatusANativeWindowError"); + return DECODER_ERROR_OTHER; + } + jniContext->native_window_width = displayedWidth; + jniContext->native_window_height = displayedHeight; + } + + ANativeWindow_Buffer native_window_buffer; + if (ANativeWindow_lock(jniContext->native_window, &native_window_buffer, + /*inOutDirtyBounds=*/nullptr) || + native_window_buffer.bits == nullptr) { + LOGE("kJniStatusANativeWindowError"); + return DECODER_ERROR_OTHER; + } + + jobject yuvPlanes_object = env->GetObjectField(jOutputBuffer, jniContext->yuvPlanes_field); + jobjectArray yuvPlanes_array = static_cast(yuvPlanes_object); + jobject yuvPlanesY = env->GetObjectArrayElement(yuvPlanes_array, kPlaneY); + jobject yuvPlanesU = env->GetObjectArrayElement(yuvPlanes_array, kPlaneU); + jobject yuvPlanesV = env->GetObjectArrayElement(yuvPlanes_array, kPlaneV); + jbyte *planeY = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesY)); + jbyte *planeU = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesU)); + jbyte *planeV = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesV)); + + jobject yuvStrides_object = env->GetObjectField(jOutputBuffer, jniContext->yuvStrides_field); + jintArray *yuvStrides_array = reinterpret_cast(&yuvStrides_object); + + int *yuvStrides = env->GetIntArrayElements(*yuvStrides_array, NULL); + int strideY = yuvStrides[kPlaneY]; + int strideU = yuvStrides[kPlaneU]; + int strideV = yuvStrides[kPlaneV]; + + // Y plane + CopyPlane(reinterpret_cast(planeY), + strideY, + reinterpret_cast(native_window_buffer.bits), + native_window_buffer.stride, + displayedWidth, + displayedHeight); + + const int y_plane_size = + native_window_buffer.stride * native_window_buffer.height; + const int32_t native_window_buffer_uv_height = + (native_window_buffer.height + 1) / 2; + const int native_window_buffer_uv_stride = + AlignTo16(native_window_buffer.stride / 2); + + // TODO(b/140606738): Handle monochrome videos. + + // V plane + // Since the format for ANativeWindow is YV12, V plane is being processed + // before U plane. + const int v_plane_height = std::min(native_window_buffer_uv_height, + displayedHeight); + CopyPlane( + reinterpret_cast(planeV), + strideV, + reinterpret_cast(native_window_buffer.bits) + y_plane_size, + native_window_buffer_uv_stride, displayedWidth, + v_plane_height); + + const int v_plane_size = v_plane_height * native_window_buffer_uv_stride; + + // U plane + CopyPlane( + reinterpret_cast(planeU), + strideU, + reinterpret_cast(native_window_buffer.bits) + + y_plane_size + v_plane_size, + native_window_buffer_uv_stride, displayedWidth, + std::min(native_window_buffer_uv_height, + displayedHeight)); + + + env->ReleaseIntArrayElements(*yuvStrides_array, yuvStrides, 0); + + if (ANativeWindow_unlockAndPost(jniContext->native_window)) { + LOGE("kJniStatusANativeWindowError"); + return DECODER_ERROR_OTHER; + } + + return DECODER_SUCCESS; +}