Main

extensions/src/poseBody/index.ts

@@ -1,4 +1,5 @@
 import { ArgumentType, BlockType, Extension, Block, DefineBlock, Environment, ExtensionMenuDisplayDetails, RuntimeEvent, ValueOf } from "$common";
+import "@tensorflow/tfjs-backend-webgl";
 import * as posenet from '@tensorflow-models/posenet';
 
 import { legacyFullSupport, info } from "./legacy";
@@ -36,6 +37,7 @@ type Details = {
  * Contains descriptions of the blocks of the Block Sensing extension
  */
 type Blocks = {
+  returnPart(coord: string, bodyPart: string): number;
   goToPart(bodyPart: string): void;
   // these video blocks are present in a few different extensions, perhaps making a file just for these?
   videoToggle(state: string): void;
@@ -76,12 +78,13 @@ export default class PoseBody extends Extension<Details, Blocks> {
    */
   bodyOptions = info.menus.PART.items;
 
+
   /**
    * Acts like class PoseBody's constructor (instead of a child class constructor)
    * @param env 
    */
   init(env: Environment) {
-
+    
     if (this.runtime.ioDevices) {
       this._loop();
     }
@@ -154,6 +157,7 @@ export default class PoseBody extends Extension<Details, Blocks> {
    * @returns 
    */
   async ensureBodyModelLoaded() {
+
     this.bodyModel ??= await posenet.load();
     return this.bodyModel;
   }
@@ -216,6 +220,32 @@ export default class PoseBody extends Extension<Details, Blocks> {
       }
     });
 
+    const returnPart = legacyDefinition.returnPart({
+      operation: (coord: string, bodyPart: string) => {
+
+        if (this.hasPose()) {
+          const { x, y } = this.tfCoordsToScratch(this.poseState.keypoints.find(point => point.part === bodyPart).position);
+          if (coord === 'x') {
+            return x;
+          } else {
+            return y;
+          }
+        }
+      },
+      argumentMethods: {
+        0: {
+          handler(coord: string) {
+            return ['x', 'y'].includes(coord) ? coord : 'x';
+          }
+        },
+        1: {
+          handler: (bodyPart: string) => {
+            return handlerOptions.includes(bodyPart) ? bodyPart : 'nose';
+          }
+        }
+      }
+    });
+
     const videoToggle = legacyDefinition.videoToggle({
       operation: (video_state) => {
         this.toggleVideo(video_state);
@@ -237,6 +267,7 @@ export default class PoseBody extends Extension<Details, Blocks> {
 
     return {
       goToPart,
+      returnPart,
       videoToggle,
       setVideoTransparency
     }

extensions/src/poseBody/legacy.ts

@@ -17,6 +17,24 @@ export const info = {
         }
       }
     },
+    {
+      "opcode": "returnPart",
+      "text": "get [COORD] of [PART]",
+      "blockType": "reporter",
+      "isTerminal": false,
+      "arguments": {
+        "COORD": {
+          "type": "string",
+          "defaultValue": "x",
+          "menu": "COORD"
+        },
+        "PART": {
+          "type": "string",
+          "defaultValue": "rightShoulder",
+          "menu": "PART"
+        }
+      }
+    },
     {
       "opcode": "videoToggle",
       "text": "turn video [VIDEO_STATE]",
@@ -115,6 +133,19 @@ export const info = {
         }
       ]
     },
+    "COORD": {
+      "acceptReporters": false,
+      "items": [
+        {
+          "text": "x",
+          "value": "x"
+        },
+        {
+          "text": "y",
+          "value": "y"
+        }
+      ]
+    },
     "ATTRIBUTE": {
       "acceptReporters": true,
       "items": [

extensions/src/poseBody/package.json

@@ -11,6 +11,8 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@tensorflow-models/posenet": "2.2.1"
+    "@tensorflow-models/posenet": "2.2.2",
+    "@tensorflow/tfjs-backend-webgl": "3.0.0-rc.1",
+    "@tensorflow/tfjs-core": "3.0.0-rc.1"
   }
 }

extensions/src/poseFace/index.ts

@@ -45,6 +45,7 @@ type Details = {
  */
 type Blocks = {
   affdexGoToPart(facePart: string): void;
+  affdexReturnPart(coord: string, facePart: string): number;
   affdexWhenExpression(expression: string): boolean;
   affdexExpressionAmount(expression: string): number;
   affdexIsExpression(expression: string): boolean;
@@ -199,6 +200,18 @@ export default class PoseFace extends Extension<Details, Blocks> {
     (util.target as any).setXY(x, y, false);
   }
 
+  returnPart(coord, part, util) {
+    if (!this.affdexState || !this.affdexState.featurePoints) return;
+
+    const featurePoint = this.affdexState.featurePoints[part];
+    const { x, y } = this.convertCoordsToScratch(featurePoint);
+    if (coord === 'x') {
+      return x;
+    } else {
+      return y;
+    }
+  }
+
   /**
    * If an expression is being expressed
    * @param expression 
@@ -302,6 +315,12 @@ export default class PoseFace extends Extension<Details, Blocks> {
       }
     });
 
+    const affdexReturnPart = legacyDefinition.affdexReturnPart({
+      operation: (coord: string, part: string, util: BlockUtility) => {
+        return this.returnPart(coord, part, util)
+      }
+    });
+
     const affdexWhenExpression = legacyDefinition.affdexWhenExpression({
       operation: (expression: string) => {
         return this.isExpression(expression);
@@ -403,6 +422,7 @@ export default class PoseFace extends Extension<Details, Blocks> {
 
     return {
       affdexGoToPart,
+      affdexReturnPart,
       affdexWhenExpression,
       affdexExpressionAmount,
       affdexIsExpression,

extensions/src/poseFace/legacy.ts

@@ -17,6 +17,24 @@ export const info = {
         }
       }
     },
+    {
+      "opcode": "affdexReturnPart",
+      "text": "get [COORD] from [AFFDEX_POINT]",
+      "blockType": "reporter",
+      "isTerminal": false,
+      "arguments": {
+        "COORD": {
+          "type": "string",
+          "defaultValue": "x",
+          "menu": "COORD"
+        },
+        "AFFDEX_POINT": {
+          "type": "string",
+          "defaultValue": "0",
+          "menu": "AFFDEX_POINT"
+        }
+      }
+    },
     {
       "opcode": "affdexWhenExpression",
       "text": "when [EXPRESSION] detected",
@@ -446,6 +464,19 @@ export const info = {
           "value": "on-flipped"
         }
       ]
+    },
+    "COORD": {
+      "acceptReporters": false,
+      "items": [
+        {
+          "text": "x",
+          "value": "x"
+        },
+        {
+          "text": "y",
+          "value": "y"
+        },
+      ]
     }
   }
 } as const;