Small upgrades

build_cv_w_docker.sh

@@ -44,7 +44,7 @@ if [ $BUILD_SIMD ] ; then
     SIMD=" --simd "
 fi
 
-docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) -e "EMSCRIPTEN=/emsdk/upstream/emscripten"  emscripten/emsdk:3.1.26 emcmake python3 ./opencv/platforms/js/build_js.py opencv_js --config="./opencv.webarkit_config.py" $SIMD --build_wasm --cmake_option="-DBUILD_opencv_dnn=OFF"  --cmake_option="-DBUILD_opencv_objdetect=OFF" --cmake_option="-DBUILD_opencv_photo=OFF" --cmake_option="-DBUILD_opencv_imgcodecs=ON" --cmake_option="-DBUILD_opencv_xfeatures2d=ON"  --cmake_option="-DOPENCV_EXTRA_MODULES_PATH=../opencv_contrib/modules/" --build_flags="-Oz -s EXPORT_ES6=1 -s USE_ES6_IMPORT_META=0"
+docker run --rm -v $(pwd):/src -u $(id -u):$(id -g) -e "EMSCRIPTEN=/emsdk/upstream/emscripten"  emscripten/emsdk:3.1.26 emcmake python3 ./opencv/platforms/js/build_js.py opencv_js --config="./opencv.webarkit_config.py" $SIMD --build_wasm --cmake_option="-DBUILD_opencv_dnn=OFF"  --cmake_option="-DBUILD_opencv_objdetect=OFF" --cmake_option="-DBUILD_opencv_photo=OFF" --cmake_option="-DBUILD_opencv_imgcodecs=ON" --cmake_option="-DBUILD_opencv_xfeatures2d=ON"  --cmake_option="-DOPENCV_EXTRA_MODULES_PATH=../opencv_contrib/modules/" --build_flags=" -fwasm-exceptions -mbulk-memory -mnontrapping-fptoint -sWASM_BIGINT -sSUPPORT_LONGJMP=wasm "
 
 # copy the output to the build folder
 #cp -r opencv_js/bin/opencv_js.js build

emscripten/WebARKitJS.cpp

@@ -44,6 +44,15 @@ emscripten::val WebARKit::getPoseMatrix2() {
     return pose;
 }
 
+emscripten::val WebARKit::getTransformationMatrix() {
+    std::array<double, 16> transMatrix = manager.getTransformationMatrix();
+    emscripten::val tM = emscripten::val::array();
+    for(auto t:transMatrix) {
+        tM.call<void>("push", t);
+    }
+    return tM;
+}
+
 emscripten::val WebARKit::getGLViewMatrix() {
     cv::Mat glViewMatrix = manager.getGLViewMatrix();
     emscripten::val glView = emscripten::val::array();

emscripten/WebARKitJS.h

@@ -37,6 +37,7 @@ class WebARKit {
     emscripten::val getHomography();
     emscripten::val getPoseMatrix();
     emscripten::val getPoseMatrix2();
+    emscripten::val getTransformationMatrix();
     emscripten::val getGLViewMatrix();
     emscripten::val getCameraProjectionMatrix();
     emscripten::val getCorners();

emscripten/bindings.cpp

@@ -35,6 +35,7 @@ EMSCRIPTEN_BINDINGS(constant_bindings) {
       .function("getPoseMatrix", &WebARKit::getPoseMatrix)
       .function("getPoseMatrix2", &WebARKit::getPoseMatrix2)
       .function("getGLViewMatrix", &WebARKit::getGLViewMatrix)
+      .function("getTransformationMatrix", &WebARKit::getTransformationMatrix)
       .function("getCameraProjectionMatrix", &WebARKit::getCameraProjectionMatrix)
       .function("getCorners", &WebARKit::getCorners)
       .function("isValid", &WebARKit::isValid);

examples/threejs_worker_full_canvas_ES6.js

@@ -0,0 +1,224 @@
+function isMobile () {
+  return /Android|mobile|iPad|iPhone/i.test(navigator.userAgent);
+}
+
+const setMatrix = function (matrix, value) {
+  const array = [];
+  for (const key in value) {
+    array[key] = value[key];
+  }
+  if (typeof matrix.elements.set === "function") {
+    matrix.elements.set(array);
+  } else {
+    matrix.elements = [].slice.call(array);
+  }
+};
+
+function start(markerUrl, video, input_width, input_height, render_update, track_update) {
+  let vw, vh;
+  let sw, sh;
+  let pscale, sscale;
+  let w, h;
+  let pw, ph;
+  let ox, oy;
+  let worker;
+
+  let imageData;
+
+  //var canvas_process = document.getElementById('canvas_process');
+  const canvas_process = document.createElement('canvas');
+
+  const context_process = canvas_process.getContext('2d', {willReadFrequently: true});
+  const targetCanvas = document.querySelector("#canvas");
+
+  const renderer = new THREE.WebGLRenderer({canvas: targetCanvas, alpha: true, antialias: true});
+  renderer.setPixelRatio(window.devicePixelRatio);
+
+  const scene = new THREE.Scene();
+
+  let fov = (0.8 * 180) / Math.PI;
+  let ratio = input_width / input_height;
+
+  const cameraConfig = {
+    fov: fov,
+    aspect: ratio,
+    near: 0.01,
+    far: 1000,
+  };
+
+  const camera = new THREE.PerspectiveCamera(cameraConfig);
+  camera.matrixAutoUpdate = false;
+
+  scene.add(camera);
+
+  const sphere = new THREE.Mesh(
+      new THREE.SphereGeometry(0.5, 8, 8),
+      new THREE.MeshNormalMaterial()
+  );
+
+  const root = new THREE.Object3D();
+  scene.add(root);
+
+  sphere.material.flatShading;
+  sphere.scale.set(.5, .5, .5);
+
+  root.matrixAutoUpdate = false;
+  root.add(sphere);
+
+  const load = function () {
+    vw = input_width;
+    vh = input_height;
+
+    pscale = 320 / Math.max(vw, vh / 3 * 4);
+    sscale = isMobile() ? window.outerWidth / input_width : 1;
+
+    sw = vw * sscale;
+    sh = vh * sscale;
+
+    w = vw * pscale;
+    h = vh * pscale;
+    pw = Math.max(w, h / 3 * 4);
+    ph = Math.max(h, w / 4 * 3);
+    ox = (pw - w) / 2;
+    oy = (ph - h) / 2;
+    /*canvas_process.style.clientWidth = pw + "px";
+    canvas_process.style.clientHeight = ph + "px";
+    canvas_process.width = pw;
+    canvas_process.height = ph;*/
+
+    renderer.setSize(sw, sh);
+
+    worker = new Worker('./worker_threejs.js')
+
+    //const refIm = document.getElementById("refIm");
+
+    const type = setTrackerType();
+    const loadImage =  (URL) => {
+      fetch(URL)
+          .then(response => response.arrayBuffer())
+          .then(buff => {
+            let buffer = new Uint8Array(buff);
+            worker.postMessage({
+              type: "initTracker",
+              trackerType: type,
+              imageData: buffer,
+              //imgWidth: refIm.width,
+              imgWidth: 1637,
+              //imgHeight: refIm.height,
+              imgHeight: 2048,
+              //videoWidth: oWidth,
+              //videoHeight: oHeight,
+              videoWidth: vw,
+              videoHeight: vh,
+            });
+            return buffer;
+          })
+    }
+
+    loadImage(markerUrl)
+
+    worker.onmessage = function (ev) {
+      const msg = ev.data;
+      switch (msg.type) {
+        case "loadedTracker": {
+          const proj = JSON.parse(msg.cameraProjMat);
+          //console.log("proj: ", proj);
+          const ratioW = pw / w;
+          const ratioH = ph / h;
+          proj[0] *= ratioW;
+          proj[4] *= ratioW;
+          proj[8] *= ratioW;
+          proj[12] *= ratioW;
+          proj[1] *= ratioH;
+          proj[5] *= ratioH;
+          proj[9] *= ratioH;
+          proj[13] *= ratioH;
+          setMatrix(camera.projectionMatrix, proj);
+          process();
+          break;
+        }
+        case "endLoading": {
+          if (msg.end === true) {
+            // removing loader page if present
+            const loader = document.getElementById('loading');
+            if (loader) {
+              loader.querySelector('.loading-text').innerText = 'Start the tracking!';
+              setTimeout(function(){
+                loader.parentElement.removeChild(loader);
+              }, 2000);
+            }
+          }
+          break;
+        }
+        case 'found': {
+          found(msg);
+          process();
+          break;
+        }
+        case 'not found': {
+          found(null);
+          process();
+          break;
+        }
+      }
+      track_update();
+      process();
+    };
+  };
+
+  let world;
+
+  const found = function (msg) {
+    if (!msg) {
+      world = null;
+    } else {
+      world = JSON.parse(msg.pose);
+      //world = JSON.parse(msg.matrixGL_RH);
+      //world = JSON.parse(msg.viewMatrix_GL);
+
+    }
+  };
+
+  var lasttime = Date.now();
+  var time = 0;
+
+  const draw = function () {
+    render_update();
+    var now = Date.now();
+    var dt = now - lasttime;
+    time += dt;
+    lasttime = now;
+
+    if (!world) {
+      sphere.visible = false;
+    } else {
+      sphere.visible = true;
+      // set matrix of 'root' by detected 'world' matrix
+      //console.log("world: ", world);
+      setMatrix(root.matrix, world);
+    }
+    renderer.render(scene, camera);
+  };
+
+  const process = function () {
+    context_process.fillStyle = 'black';
+    //context_process.fillRect(0, 0, pw, ph);
+    context_process.fillRect(0, 0, vw, vh);
+    //context_process.drawImage(video, 0, 0, vw, vh, ox, oy, w, h);
+    context_process.drawImage(video, 0, 0, vw, vh);
+
+    const imageData = context_process.getImageData(0, 0, vw, vh);
+
+    //const imageData = context_process.getImageData(0, 0, pw, ph);
+    worker.postMessage({ type: 'process', imagedata: imageData }, [imageData.data.buffer]);
+  }
+
+  const tick = function () {
+    draw();
+    requestAnimationFrame(tick);
+  };
+
+  load();
+  tick();
+  process();
+}

examples/worker.js

@@ -44,6 +44,8 @@ function initTracker(msg) {
         corners: JSON.stringify(event.data.corners),
         matrix: JSON.stringify(event.data.matrix),
         matrixGL_RH: JSON.stringify(event.data.matrixGL_RH),
+        transMatrix: JSON.stringify(event.data.transMatrix),
+        viewMatrix_GL: JSON.stringify(event.data.viewMatrix_GL),
         pose: JSON.stringify(event.data.pose),
       };
     });

examples/worker_setup_advanced.js

@@ -60,6 +60,9 @@ var oHeight = window.innerHeight;
 var vw = oWidth;
 var vh = oHeight;
 
+var pscale = 320 / Math.max(vw, vh / 3 * 4);
+var sscale = isMobile() ? window.outerWidth / oWidth : 1;
+
 var sw = vw * sscale;
 var sh = vh * sscale;
 
@@ -71,11 +74,6 @@ var ph = Math.max(h, w / 4 * 3);
 var ox = (pw - w) / 2;
 var oy = (ph - h) / 2;
 
-
-var pscale = 320 / Math.max(vw, vh / 3 * 4);
-var sscale = isMobile() ? window.outerWidth / oWidth : 1;
-
-
 var videoEl;
 var arElem;
 var grayScaleVideo;
@@ -131,7 +129,7 @@ window.onload = async function () {
     switch (msg.type) {
       case "loadedTracker": {
         proj = JSON.parse(msg.cameraProjMat);
-        const ratioW = pw / w;
+        /*const ratioW = pw / w;
         const ratioH = ph / h;
         proj[0] *= ratioW;
         proj[4] *= ratioW;
@@ -140,7 +138,7 @@ window.onload = async function () {
         proj[1] *= ratioH;
         proj[5] *= ratioH;
         proj[9] *= ratioH;
-        proj[13] *= ratioH;
+        proj[13] *= ratioH;*/
         hideLoading();
         process();
         break;
@@ -180,8 +178,10 @@ window.onload = async function () {
     } else {
       //arElem.style.display = "block";
       arElem.style.display = "none";
-      console.log("matrixGL_RH matrix: ", JSON.parse(msg.matrixGL_RH));
-      const world = JSON.parse(msg.matrixGL_RH);
+      //console.log("matrixGL_RH matrix: ", JSON.parse(msg.matrixGL_RH));
+      console.log("viewMatrix_GL matrix: ", JSON.parse(msg.viewMatrix_GL));
+      console.log("transMatrix matrix: ", JSON.parse(msg.transMatrix));
+      const world = JSON.parse(msg.transMatrix);
       const mat = multiplyMatrices(proj, world);
 
       function glpointToCanvas(xyz) {
@@ -201,21 +201,29 @@ window.onload = async function () {
 
       const overlayCtx = overlayCanvas.getContext("2d");
       clearOverlayCtx();
-      drawCorners(JSON.parse(msg.corners));
+      //drawCorners(JSON.parse(msg.corners));
 
       //var width = marker.width;
       const mwidth = 1637;
       //var height = marker.height;
       const mheight = 2048;
-      const dpi = 150;
+      const dpi = 220;
 
       const w = mwidth / dpi * 2.54 * 10;
       const h = mheight / dpi * 2.54 * 10;
 
-      const p1 = drawpoint(0, 0, 0);
-      const p2 = drawpoint(w, 0, 0);
-      const p3 = drawpoint(w, h, 0);
-      const p4 = drawpoint(0, h, 0);
+      let p1 = drawpoint(0, 0, 0);
+      let p2 = drawpoint(w, 0, 0);
+      let p3 = drawpoint(w, h, 0);
+      let p4 = drawpoint(0, h, 0);
+
+      // this will draw a fixed red square, uncomment only for debugging...
+      /*p1 = {x:100, y: 100};
+      p2 = {x:200, y: 100};
+      p3 = {x:200, y: 200};
+      p4 = {x:100, y: 200};*/
+
+      console.log(p1, p2, p3, p4)
 
       overlayCtx.beginPath();
       overlayCtx.moveTo(p1.x, p1.y);