fix(ios): video texture support

Author: triniwiz

CanvasNative.podspec

@@ -2,7 +2,7 @@ Pod::Spec.new do |s|
 
     s.name         = "CanvasNative"
 
-    s.version      = "0.9.10"
+    s.version      = "0.9.11"
 
     s.summary      = "A Canvas library"
 

packages/canvas-media/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@nativescript/canvas-media",
-	"version": "0.6.0",
+	"version": "0.7.0",
 	"description": "Canvas media",
 	"main": "index",
 	"typings": "index.d.ts",

packages/canvas-media/video/index.ios.ts

@@ -1,6 +1,7 @@
 import { controlsProperty, VideoBase, playsinlineProperty, mutedProperty, srcProperty, currentTimeProperty } from './common';
 import { Source } from '../common';
 import { knownFolders, path } from '@nativescript/core';
+declare const Utils;
 @NativeClass()
 class NativeObject extends NSObject {
 	_owner: WeakRef<Video>;
@@ -26,7 +27,9 @@ class NativeObject extends NSObject {
 
 				*/
 				} else if (owner._player.currentItem.status === AVPlayerItemStatus.ReadyToPlay) {
-					console.log('ready');
+					if (!owner._videoSize) {
+						owner._videoSize = owner._asset.tracksWithMediaType(AVMediaTypeVideo)?.[0].naturalSize ?? undefined;
+					}
 				}
 			}
 		} else if (path === 'loadedTimeRanges') {
@@ -83,6 +86,7 @@ export class Video extends VideoBase {
 	_fps: number;
 	_ctx: any;
 	_asset: AVURLAsset;
+	_videoSize: any;
 	get _player() {
 		return this.#player;
 	}
@@ -125,60 +129,39 @@ export class Video extends VideoBase {
 		}
 		if (this._assetOutput) {
 			try {
-				const currentTime = this.#player.currentTime();
-				if (!this._assetOutput.hasNewPixelBufferForItemTime(currentTime)) {
-					return;
-				}
-				const sampleBuffer = this._assetOutput.copyPixelBufferForItemTimeItemTimeForDisplay(currentTime, null);
-				if (sampleBuffer !== 0) {
-					/*	const currentSampleTime = CMSampleBufferGetOutputPresentationTimeStamp(sampleBuffer);
-								const differenceFromLastFrame = CMTimeSubtract(currentSampleTime, this['previousFrameTime']);
-								const currentActualTime = CFAbsoluteTimeGetCurrent();
+					// _ player: AVPlayer, _ output: AVPlayerItemVideoOutput,_ videoSize: CGSize
+					Utils.drawFrame(this.#player, this._assetOutput, this._videoSize);
+					/*
+					const currentTime = this.#player.currentTime();
+					if (!this._assetOutput.hasNewPixelBufferForItemTime(currentTime)) {
+						return;
+					}
+					const pixel_buffer = this._assetOutput.copyPixelBufferForItemTimeItemTimeForDisplay(currentTime, null);
+					if (pixel_buffer !== 0) {
+						CVPixelBufferLockBaseAddress(pixel_buffer, 0);
+						const bytesPerRow = CVPixelBufferGetBytesPerRow(pixel_buffer);
+						const line_base = CVPixelBufferGetBaseAddress(pixel_buffer) as any;
 	
-								const frameTimeDifference = CMTimeGetSeconds(differenceFromLastFrame);
-								const actualTimeDifference = currentActualTime - this['previousActualFrameTime'];
-								
-			
-								if (frameTimeDifference > actualTimeDifference) {
-									usleep(1000000.0 * (frameTimeDifference - actualTimeDifference));
-								}
+						if (bytesPerRow / BYTES_PER_TEXEL === this._videoSize.width) {
+							glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, this._videoSize.width, this._videoSize.height, 0, GL_BGRA, GL_UNSIGNED_BYTE, line_base);
+						} else {
+							glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, this._videoSize.width, this._videoSize.height, 0, GL_BGRA, GL_UNSIGNED_BYTE, null);
+							for (let i = 0; i < this._videoSize.height; ++i) {
+								glTexSubImage2D(GL_TEXTURE_2D, 0, 0, i, this._videoSize.width, 1, GL_BGRA, GL_UNSIGNED_BYTE, line_base.add(i * bytesPerRow));
+							}
+						}
 	
-								this['previousFrameTime'] = currentSampleTime;
-								this['previousActualFrameTime'] = CFAbsoluteTimeGetCurrent();
-								*/
-
-					//const pixel_buffer = CMSampleBufferGetImageBuffer(sampleBuffer);
-
-					//const startTime = CFAbsoluteTimeGetCurrent();
-
-					const GL_TEXTURE_2D = 3553;
-					const GL_RGBA = 6408;
-					const GL_BGRA_EXT = 32993;
-					const GL_UNSIGNED_BYTE = 5121;
-
-					//const startTime = CFAbsoluteTimeGetCurrent();
-					CVPixelBufferLockBaseAddress(sampleBuffer, 0);
-					//const bpr = CVPixelBufferGetBytesPerRow(pixel_buffer);
-					const width = CVPixelBufferGetBytesPerRow(sampleBuffer) / 4;
-					const height = CVPixelBufferGetHeight(sampleBuffer);
-					const line_base = CVPixelBufferGetBaseAddress(sampleBuffer);
-					glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_BGRA_EXT, GL_UNSIGNED_BYTE, line_base);
-
-					CVPixelBufferUnlockBaseAddress(sampleBuffer, 0);
-					// CMSampleBufferInvalidate(sampleBuffer);
-
-					/*const currentFrameTime = CFAbsoluteTimeGetCurrent() - startTime;
-							console.log('fps', currentFrameTime * 1000);
-							*/
-					//this._previousTS = currentTS;
-					// may not need to release
-					/*
-								 Unlike regular Core Foundation objects, toll-free bridged types are automatically memory managed by NativeScript,
-								  so there is no need to retain or release them using CFRetain and CFRelease.
-								*/
-					// https://docs.nativescript.org/core-concepts/ios-runtime/marshalling-overview#corefoundation-objects
-					//CFRelease(sampleBuffer);
-				}
+						CVPixelBufferUnlockBaseAddress(pixel_buffer, 0);
+						// may not need to release
+						
+								//	 Unlike regular Core Foundation objects, toll-free bridged types are automatically memory managed by NativeScript,
+								//	  so there is no need to retain or release them using CFRetain and CFRelease.
+									
+						// https://docs.nativescript.org/core-concepts/ios-runtime/marshalling-overview#corefoundation-objects
+						//CFRelease(sampleBuffer);
+					}
+	
+					*/
 			} catch (e) {
 				console.log('getCurrentFrame error:', e);
 			}
@@ -305,6 +288,8 @@ export class Video extends VideoBase {
 			this._asset = AVURLAsset.assetWithURL(url);
 			const keys = ['tracks', 'duration'];
 			this._asset.loadValuesAsynchronouslyForKeysCompletionHandler(keys, () => {
+				this._videoSize = this._asset.tracksWithMediaType(AVMediaTypeVideo)?.[0].naturalSize ?? undefined;
+
 				const fps = this._asset.tracks.firstObject?.nominalFrameRate ?? 30;
 
 				const _interval = CMTimeMake(1, fps);

packages/canvas/platforms/ios/Podfile

@@ -1,6 +1,6 @@
 use_frameworks!
 platform :ios, '11.0'
-pod 'CanvasNative' , '~> 0.9.10'
+pod 'CanvasNative' , '~> 0.9.11'
 #pod 'CanvasNative', :path => "$(SRCROOT)/../../../../../"
 
 

packages/canvas/src-native/canvas-ios/CanvasNative.xcodeproj/project.xcworkspace/xcuserdata/triniwiz.xcuserdatad/UserInterfaceState.xcuserstate

@@ -6,29 +6,29 @@
 	<array>
 		<dict>
 			<key>LibraryIdentifier</key>
-			<string>ios-x86_64-simulator</string>
+			<string>ios-arm64</string>
 			<key>LibraryPath</key>
 			<string>CanvasNative.framework</string>
 			<key>SupportedArchitectures</key>
 			<array>
-				<string>x86_64</string>
+				<string>arm64</string>
 			</array>
 			<key>SupportedPlatform</key>
 			<string>ios</string>
-			<key>SupportedPlatformVariant</key>
-			<string>simulator</string>
 		</dict>
 		<dict>
 			<key>LibraryIdentifier</key>
-			<string>ios-arm64</string>
+			<string>ios-x86_64-simulator</string>
 			<key>LibraryPath</key>
 			<string>CanvasNative.framework</string>
 			<key>SupportedArchitectures</key>
 			<array>
-				<string>arm64</string>
+				<string>x86_64</string>
 			</array>
 			<key>SupportedPlatform</key>
 			<string>ios</string>
+			<key>SupportedPlatformVariant</key>
+			<string>simulator</string>
 		</dict>
 	</array>
 	<key>CFBundlePackageType</key>

packages/canvas/src-native/canvas-ios/CanvasNative/Dist/CanvasNative.framework.dSYM.zip

@@ -1240,6 +1240,7 @@ SWIFT_CLASS_NAMED("TNSWebGL2RenderingContext")
 - (int32_t)getFragDataLocation:(uint32_t)program :(NSString * _Nonnull)name SWIFT_WARN_UNUSED_RESULT;
 - (id _Nonnull)getIndexedParameter:(uint32_t)target :(uint32_t)index SWIFT_WARN_UNUSED_RESULT;
 - (id _Nonnull)getInternalformatParameter:(uint32_t)target :(uint32_t)internalformat :(uint32_t)pname SWIFT_WARN_UNUSED_RESULT;
+- (id _Nullable)getParameter:(uint32_t)pname SWIFT_WARN_UNUSED_RESULT;
 - (id _Nonnull)getQuery:(uint32_t)target :(uint32_t)pname SWIFT_WARN_UNUSED_RESULT;
 - (id _Nonnull)getQueryParameter:(uint32_t)query :(uint32_t)pname SWIFT_WARN_UNUSED_RESULT;
 - (id _Nonnull)getSamplerParameter:(uint32_t)sampler :(uint32_t)pname SWIFT_WARN_UNUSED_RESULT;
@@ -1836,11 +1837,14 @@ SWIFT_CLASS_NAMED("TNS_WEBGL_lose_context")
 + (nonnull instancetype)new SWIFT_UNAVAILABLE_MSG("-init is unavailable");
 @end
 
+@class AVPlayer;
+@class AVPlayerItemVideoOutput;
 
 SWIFT_CLASS_NAMED("Utils")
 @interface Utils : NSObject
 + (CVOpenGLESTextureCacheRef _Nullable)createTextureCache:(TNSWebGLRenderingContext * _Nonnull)context SWIFT_WARN_UNUSED_RESULT;
 + (CVOpenGLESTextureRef _Nullable)createImage:(CVOpenGLESTextureCacheRef _Nonnull)texturecache :(CVImageBufferRef _Nonnull)buffer :(CFDictionaryRef _Nullable)textureAttributes :(GLenum)target :(GLint)internalFormat :(GLsizei)width :(GLsizei)height :(GLenum)format :(GLenum)type :(NSInteger)planeIndex SWIFT_WARN_UNUSED_RESULT;
++ (void)drawFrame:(AVPlayer * _Nonnull)player :(AVPlayerItemVideoOutput * _Nonnull)output :(CGSize)videoSize;
 - (nonnull instancetype)init OBJC_DESIGNATED_INITIALIZER;
 @end
 

packages/canvas/src-native/canvas-ios/CanvasNative/Dist/CanvasNative.xcframework/Info.plist

@@ -92,6 +92,14 @@ typedef enum {
 
 typedef struct Context Context;
 
+typedef struct {
+  uint8_t _unused[0];
+} ANativeWindow;
+
+typedef struct {
+  uint8_t _unused[0];
+} ASurfaceTexture;
+
 typedef struct {
   long long value;
   PaintStyleValueType value_type;
@@ -137,6 +145,120 @@ typedef struct {
   uintptr_t data_len;
 } U32Array;
 
+/**
+ * Returns a reference to an ANativeWindow (i.e. the Producer) for this SurfaceTexture.
+ * This is equivalent to Java's: Surface sur = new Surface(surfaceTexture);
+ *
+ * Available since API level 28.
+ *
+ * \param st A ASurfaceTexture reference acquired with ASurfaceTexture_fromSurfaceTexture()
+ * @return A reference to an ANativeWindow. This reference MUST BE released when no longer needed
+ * using ANativeWindow_release(). Failing to do so will result in leaked resources. nullptr is
+ * returned if \p st is null or if it's not an instance of android.graphics.SurfaceTexture
+ */
+extern ANativeWindow *ASurfaceTexture_acquireANativeWindow(ASurfaceTexture *st);
+
+/**
+ * Attach the SurfaceTexture to the OpenGL ES context that is current on the calling thread.  A
+ * new OpenGL ES texture object is created and populated with the SurfaceTexture image frame
+ * that was current at the time of the last call to {@link #detachFromGLContext}.  This new
+ * texture is bound to the GL_TEXTURE_EXTERNAL_OES texture target.
+ *
+ * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES
+ * contexts.  Note, however, that the image contents are only accessible from one OpenGL ES
+ * context at a time.
+ *
+ * Available since API level 28.
+ *
+ * \param st A ASurfaceTexture reference acquired with ASurfaceTexture_fromSurfaceTexture()
+ * \param texName The name of the OpenGL ES texture that will be created.  This texture name
+ * must be unusued in the OpenGL ES context that is current on the calling thread.
+ * \return 0 on success, negative posix error code otherwise (see <errno.h>)
+ */
+extern int ASurfaceTexture_attachToGLContext(ASurfaceTexture *st, uint32_t texName);
+
+/**
+ * Detach the SurfaceTexture from the OpenGL ES context that owns the OpenGL ES texture object.
+ * This call must be made with the OpenGL ES context current on the calling thread.  The OpenGL
+ * ES texture object will be deleted as a result of this call.  After calling this method all
+ * calls to {@link #updateTexImage} will fail until a successful call to {@link #attachToGLContext}
+ * is made.
+ *
+ * This can be used to access the SurfaceTexture image contents from multiple OpenGL ES
+ * contexts.  Note, however, that the image contents are only accessible from one OpenGL ES
+ * context at a time.
+ *
+ * Available since API level 28.
+ *
+ * \param st A ASurfaceTexture reference acquired with ASurfaceTexture_fromSurfaceTexture()
+ * \return 0 on success, negative posix error code otherwise (see <errno.h>)
+ */
+extern int ASurfaceTexture_detachFromGLContext(ASurfaceTexture *st);
+
+/**
+ * Retrieve the timestamp associated with the texture image set by the most recent call to
+ * updateTexImage.
+ *
+ * This timestamp is in nanoseconds, and is normally monotonically increasing. The timestamp
+ * should be unaffected by time-of-day adjustments, and for a camera should be strictly
+ * monotonic but for a MediaPlayer may be reset when the position is set.  The
+ * specific meaning and zero point of the timestamp depends on the source providing images to
+ * the SurfaceTexture. Unless otherwise specified by the image source, timestamps cannot
+ * generally be compared across SurfaceTexture instances, or across multiple program
+ * invocations. It is mostly useful for determining time offsets between subsequent frames.
+ *
+ * For EGL/Vulkan producers, this timestamp is the desired present time set with the
+ * EGL_ANDROID_presentation_time or VK_GOOGLE_display_timing extensions
+ *
+ * Available since API level 28.
+ *
+ * \param st A ASurfaceTexture reference acquired with ASurfaceTexture_fromSurfaceTexture()
+ */
+extern int64_t ASurfaceTexture_getTimestamp(ASurfaceTexture *st);
+
+/**
+ * Retrieve the 4x4 texture coordinate transform matrix associated with the texture image set by
+ * the most recent call to updateTexImage.
+ *
+ * This transform matrix maps 2D homogeneous texture coordinates of the form (s, t, 0, 1) with s
+ * and t in the inclusive range [0, 1] to the texture coordinate that should be used to sample
+ * that location from the texture.  Sampling the texture outside of the range of this transform
+ * is undefined.
+ *
+ * The matrix is stored in column-major order so that it may be passed directly to OpenGL ES via
+ * the glLoadMatrixf or glUniformMatrix4fv functions.
+ *
+ * Available since API level 28.
+ *
+ * \param st A ASurfaceTexture reference acquired with ASurfaceTexture_fromSurfaceTexture()
+ * \param mtx the array into which the 4x4 matrix will be stored.  The array must have exactly
+ *     16 elements.
+ */
+extern void ASurfaceTexture_getTransformMatrix(ASurfaceTexture *st, float *mtx);
+
+/**
+ * Release the reference to the native ASurfaceTexture acquired with
+ * ASurfaceTexture_fromSurfaceTexture().
+ * Failing to do so will result in leaked memory and graphic resources.
+ *
+ * Available since API level 28.
+ *
+ * \param st A ASurfaceTexture reference acquired with ASurfaceTexture_fromSurfaceTexture()
+ */
+extern void ASurfaceTexture_release(ASurfaceTexture *st);
+
+/**
+ * Update the texture image to the most recent frame from the image stream.  This may only be
+ * called while the OpenGL ES context that owns the texture is current on the calling thread.
+ * It will implicitly bind its texture to the GL_TEXTURE_EXTERNAL_OES texture target.
+ *
+ * Available since API level 28.
+ *
+ * \param st A ASurfaceTexture reference acquired with ASurfaceTexture_fromSurfaceTexture()
+ * \return 0 on success, negative posix error code otherwise (see <errno.h>)
+ */
+extern int ASurfaceTexture_updateTexImage(ASurfaceTexture *st);
+
 void context_arc(long long context,
                  float x,
                  float y,

packages/canvas/src-native/canvas-ios/CanvasNative/Dist/CanvasNative.xcframework/ios-arm64/CanvasNative.framework/CanvasNative

@@ -1,6 +1,7 @@
 // swift-interface-format-version: 1.0
 // swift-compiler-version: Apple Swift version 5.3.2 (swiftlang-1200.0.45 clang-1200.0.32.28)
 // swift-module-flags: -target arm64-apple-ios10.0 -enable-objc-interop -enable-library-evolution -swift-version 5 -enforce-exclusivity=checked -O -module-name CanvasNative
+import AVFoundation
 @_exported import CanvasNative
 import CoreVideo
 import Foundation
@@ -391,6 +392,7 @@ extension RangeReplaceableCollection {
   @objc public func getFragDataLocation(_ program: Swift.UInt32, _ name: Swift.String) -> Swift.Int32
   @objc public func getIndexedParameter(_ target: Swift.UInt32, _ index: Swift.UInt32) -> Any
   @objc public func getInternalformatParameter(_ target: Swift.UInt32, _ internalformat: Swift.UInt32, _ pname: Swift.UInt32) -> Any
+  @objc override public func getParameter(_ pname: Swift.UInt32) -> Any?
   @objc public func getQuery(_ target: Swift.UInt32, _ pname: Swift.UInt32) -> Any
   @objc public func getQueryParameter(_ query: Swift.UInt32, _ pname: Swift.UInt32) -> Any
   @objc public func getSamplerParameter(_ sampler: Swift.UInt32, _ pname: Swift.UInt32) -> Any
@@ -2454,6 +2456,7 @@ extension RangeReplaceableCollection {
 @_inheritsConvenienceInitializers @objc(Utils) public class Utils : ObjectiveC.NSObject {
   @objc public static func createTextureCache(_ context: CanvasNative.TNSWebGLRenderingContext) -> CoreVideo.CVOpenGLESTextureCache?
   @objc public static func createImage(_ texturecache: CoreVideo.CVOpenGLESTextureCache, _ buffer: CoreVideo.CVImageBuffer, _ textureAttributes: CoreFoundation.CFDictionary?, _ target: OpenGLES.GLenum, _ internalFormat: OpenGLES.GLint, _ width: OpenGLES.GLsizei, _ height: OpenGLES.GLsizei, _ format: OpenGLES.GLenum, _ type: OpenGLES.GLenum, _ planeIndex: Swift.Int) -> CoreVideo.CVOpenGLESTexture?
+  @objc public static func drawFrame(_ player: AVFoundation.AVPlayer, _ output: AVFoundation.AVPlayerItemVideoOutput, _ videoSize: CoreGraphics.CGSize)
   @objc deinit
   @objc override dynamic public init()
 }

packages/canvas/src-native/canvas-ios/CanvasNative/Dist/CanvasNative.xcframework/ios-arm64/CanvasNative.framework/Headers/CanvasNative-Swift.h

@@ -1,6 +1,7 @@
 // swift-interface-format-version: 1.0
 // swift-compiler-version: Apple Swift version 5.3.2 (swiftlang-1200.0.45 clang-1200.0.32.28)
 // swift-module-flags: -target arm64-apple-ios10.0 -enable-objc-interop -enable-library-evolution -swift-version 5 -enforce-exclusivity=checked -O -module-name CanvasNative
+import AVFoundation
 @_exported import CanvasNative
 import CoreVideo
 import Foundation
@@ -391,6 +392,7 @@ extension RangeReplaceableCollection {
   @objc public func getFragDataLocation(_ program: Swift.UInt32, _ name: Swift.String) -> Swift.Int32
   @objc public func getIndexedParameter(_ target: Swift.UInt32, _ index: Swift.UInt32) -> Any
   @objc public func getInternalformatParameter(_ target: Swift.UInt32, _ internalformat: Swift.UInt32, _ pname: Swift.UInt32) -> Any
+  @objc override public func getParameter(_ pname: Swift.UInt32) -> Any?
   @objc public func getQuery(_ target: Swift.UInt32, _ pname: Swift.UInt32) -> Any
   @objc public func getQueryParameter(_ query: Swift.UInt32, _ pname: Swift.UInt32) -> Any
   @objc public func getSamplerParameter(_ sampler: Swift.UInt32, _ pname: Swift.UInt32) -> Any
@@ -2454,6 +2456,7 @@ extension RangeReplaceableCollection {
 @_inheritsConvenienceInitializers @objc(Utils) public class Utils : ObjectiveC.NSObject {
   @objc public static func createTextureCache(_ context: CanvasNative.TNSWebGLRenderingContext) -> CoreVideo.CVOpenGLESTextureCache?
   @objc public static func createImage(_ texturecache: CoreVideo.CVOpenGLESTextureCache, _ buffer: CoreVideo.CVImageBuffer, _ textureAttributes: CoreFoundation.CFDictionary?, _ target: OpenGLES.GLenum, _ internalFormat: OpenGLES.GLint, _ width: OpenGLES.GLsizei, _ height: OpenGLES.GLsizei, _ format: OpenGLES.GLenum, _ type: OpenGLES.GLenum, _ planeIndex: Swift.Int) -> CoreVideo.CVOpenGLESTexture?
+  @objc public static func drawFrame(_ player: AVFoundation.AVPlayer, _ output: AVFoundation.AVPlayerItemVideoOutput, _ videoSize: CoreGraphics.CGSize)
   @objc deinit
   @objc override dynamic public init()
 }