feat(camera): update camera rtc

Author: JingyuanZhang

.eslintrc.json

@@ -120,7 +120,7 @@
         ],
         "max-nested-callbacks": "warn",
         "max-params": ["error", 6],
-        "max-statements": ["error", 50],
+        "max-statements": ["error", 70],
         "max-statements-per-line": "error",
         "multiline-comment-style": ["warn", "separate-lines"],
         "new-parens": "warn",

packages/paddlejs-backend-webgl/src/ops/shader/conv2d_depthwise.ts

@@ -2,66 +2,6 @@
  * @file conv2d_depthwise
  */
 
-function mainFunc(
-    { origin, filter },
-    { strides = [], paddings = [], dilations = [], fuse_relu }
-) {
-    const [stride_v = 1, stride_h = 1] = strides;
-    const [padTop = 0, padLeft = 0] = paddings;
-    const [dilation_v = 1, dilation_h = 1] = dilations;
-    return `
-    // start函数
-    void main(void) {
-        ivec4 oPos = getOutputTensorPos();
-        int x = oPos.a;
-        int c = oPos.g;
-        int y = oPos.b;
-        int b = oPos.r;
-        float res = 0.0;
-        int top = y * ${stride_v} - ${padTop};
-        int left = x * ${stride_h} - ${padLeft};
-        for (int fy = 0; fy < ${filter.height_shape}; fy++) {
-          int oy = top + fy * ${dilation_v};
-          if (oy >= ${origin.height_shape}) {
-              break;
-          }
-          if (oy < 0) {
-            continue;
-          }
-          for (int fx = 0; fx < ${filter.width_shape}; fx++) {
-            int ox = left + fx * ${dilation_h};
-            if (ox >= ${origin.width_shape}) {
-                break;
-            }
-            if (ox < 0) {
-                continue;
-            }
-            // b默认是0
-            float f = getValueFromTensorPos_filter(c, 0, fy, fx);
-            float o = getValueFromTensorPos_origin(b, c, oy, ox);
-            res += f * o;
-          }
-        }
-        float bi = getValueFromTensorPos_bias(0, 0, 0, c);
-        res += bi;
-        if (${fuse_relu}) {
-            res = max(0.0, res);
-        }
-        setOutput(res);
-    }
-`;
-}
-export default {
-    mainFunc,
-    params: [
-        'strides',
-        'paddings',
-        'dilations',
-        'fuse_relu'
-    ],
-    textureFuncConf: {
-        filter: ['getValueFromTensorPos'],
-        origin: ['getValueFromTensorPos'],
-        bias: ['getValueFromTensorPos']
-    }
-};
+import conv2d from './conv2d';
+
+export default conv2d;

packages/paddlejs-mediapipe/camera/src/index.ts

@@ -9,9 +9,9 @@ interface cameraOption {
     mirror?: boolean;
     targetCanvas?: HTMLCanvasElement;
     onSuccess?: () => void;
-    onError?: NavigatorUserMediaErrorCallback;
+    onError?: () => void;
     onNotSupported?: () => void;
-    onFrame?: (canvas: HTMLCanvasElement) => void;
+    onFrame?: (target: HTMLCanvasElement | HTMLVideoElement) => void;
     switchError?: () => void;
     videoLoaded?: () => void;
 }
@@ -42,8 +42,8 @@ export default class Camera {
     private canvas: HTMLCanvasElement;
     private context: CanvasRenderingContext2D;
     private requestAnimationId;
-    private stream: any;
-    private videoDevices: any;
+    private stream: MediaStream;
+    private videoDevices: string[];
     private videoDeviceId: string;
     private currentMode: string;
     private isIOS: boolean;
@@ -52,18 +52,19 @@ export default class Camera {
         const ua = navigator.userAgent;
         this.isIOS = !!ua.match(/\(i[^;]+;( U;)? CPU.+Mac OS X/);
         this.video.width = this.options.width || this.video.clientWidth;
-        if (this.options.height) {
-            this.video.height = this.options.height;
-        }
+        this.video.height = this.options.height || this.video.clientHeight;
         // 枚举设备摄像头
         this.enumerateDevices();
     }
 
     private handleStream() {
         const videoConstraints = {
             deviceId: { exact: this.videoDeviceId },
-            facingMode: this.currentMode
+            facingMode: this.currentMode,
+            width: this.video.width,
+            height: this.video.height
         };
+
         if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
             navigator.mediaDevices.getUserMedia({
                 video: videoConstraints
@@ -111,7 +112,7 @@ export default class Camera {
             })
             .catch(err => {
                 this.options.onNotSupported && this.options.onNotSupported();
-                console.log(err.name + ': ' + err.message);
+                console.error(err.name + ': ' + err.message);
             });
     }
 
@@ -122,7 +123,8 @@ export default class Camera {
         if ('srcObject' in this.video) {
             try {
                 this.video.srcObject = this.stream;
-            } catch (error) {
+            }
+            catch (error) {
                 this.video.src = URL.createObjectURL(this.stream) || this.stream;
             }
         }
@@ -134,6 +136,7 @@ export default class Camera {
             else {
                 this.video.height = this.video.clientHeight;
             }
+
             this.options.videoLoaded && this.options.videoLoaded();
             this.initCanvas();
         });
@@ -162,16 +165,18 @@ export default class Camera {
 
     private videoRequestAnimationFrame() {
         const drawImage = () => {
-            this.context.drawImage(this.video, 0, 0, this.video.width, this.video.height);
-            this.options.onFrame(this.canvas);
-            this.requestAnimationId = window.requestAnimationFrame(drawImage);
+            if (this.context) {
+                this.context.drawImage(this.video, 0, 0, this.video.width, this.video.height);
+            }
+            this.options.onFrame(this.video);
+            this.requestAnimationId = requestAnimationFrame(drawImage);
         };
         drawImage();
     }
 
     public start() {
         this.video && this.video.play();
-        if (!this.canvas || this.requestAnimationId) {
+        if (this.requestAnimationId) {
             return;
         }
         this.videoRequestAnimationFrame();