diff --git a/extensions/src/poseBody/index.ts b/extensions/src/poseBody/index.ts index 7f9fb3395..23b6c6d3d 100644 --- a/extensions/src/poseBody/index.ts +++ b/extensions/src/poseBody/index.ts @@ -1,4 +1,5 @@ import { ArgumentType, BlockType, Extension, Block, DefineBlock, Environment, ExtensionMenuDisplayDetails, RuntimeEvent, ValueOf } from "$common"; +import "@tensorflow/tfjs-backend-webgl"; import * as posenet from '@tensorflow-models/posenet'; import { legacyFullSupport, info } from "./legacy"; @@ -36,6 +37,7 @@ type Details = { * Contains descriptions of the blocks of the Block Sensing extension */ type Blocks = { + returnPart(coord: string, bodyPart: string): number; goToPart(bodyPart: string): void; // these video blocks are present in a few different extensions, perhaps making a file just for these? videoToggle(state: string): void; @@ -76,12 +78,13 @@ export default class PoseBody extends Extension { */ bodyOptions = info.menus.PART.items; + /** * Acts like class PoseBody's constructor (instead of a child class constructor) * @param env */ init(env: Environment) { - + if (this.runtime.ioDevices) { this._loop(); } @@ -154,6 +157,7 @@ export default class PoseBody extends Extension { * @returns */ async ensureBodyModelLoaded() { + this.bodyModel ??= await posenet.load(); return this.bodyModel; } @@ -216,6 +220,32 @@ export default class PoseBody extends Extension { } }); + const returnPart = legacyDefinition.returnPart({ + operation: (coord: string, bodyPart: string) => { + + if (this.hasPose()) { + const { x, y } = this.tfCoordsToScratch(this.poseState.keypoints.find(point => point.part === bodyPart).position); + if (coord === 'x') { + return x; + } else { + return y; + } + } + }, + argumentMethods: { + 0: { + handler(coord: string) { + return ['x', 'y'].includes(coord) ? coord : 'x'; + } + }, + 1: { + handler: (bodyPart: string) => { + return handlerOptions.includes(bodyPart) ? bodyPart : 'nose'; + } + } + } + }); + const videoToggle = legacyDefinition.videoToggle({ operation: (video_state) => { this.toggleVideo(video_state); @@ -237,6 +267,7 @@ export default class PoseBody extends Extension { return { goToPart, + returnPart, videoToggle, setVideoTransparency } diff --git a/extensions/src/poseBody/legacy.ts b/extensions/src/poseBody/legacy.ts index fc5c13460..5e911fff7 100644 --- a/extensions/src/poseBody/legacy.ts +++ b/extensions/src/poseBody/legacy.ts @@ -17,6 +17,24 @@ export const info = { } } }, + { + "opcode": "returnPart", + "text": "get [COORD] of [PART]", + "blockType": "reporter", + "isTerminal": false, + "arguments": { + "COORD": { + "type": "string", + "defaultValue": "x", + "menu": "COORD" + }, + "PART": { + "type": "string", + "defaultValue": "rightShoulder", + "menu": "PART" + } + } + }, { "opcode": "videoToggle", "text": "turn video [VIDEO_STATE]", @@ -115,6 +133,19 @@ export const info = { } ] }, + "COORD": { + "acceptReporters": false, + "items": [ + { + "text": "x", + "value": "x" + }, + { + "text": "y", + "value": "y" + } + ] + }, "ATTRIBUTE": { "acceptReporters": true, "items": [ diff --git a/extensions/src/poseBody/package.json b/extensions/src/poseBody/package.json index 3c76f8705..9245e4b33 100644 --- a/extensions/src/poseBody/package.json +++ b/extensions/src/poseBody/package.json @@ -11,6 +11,8 @@ "author": "", "license": "ISC", "dependencies": { - "@tensorflow-models/posenet": "2.2.1" + "@tensorflow-models/posenet": "2.2.2", + "@tensorflow/tfjs-backend-webgl": "3.0.0-rc.1", + "@tensorflow/tfjs-core": "3.0.0-rc.1" } } \ No newline at end of file diff --git a/extensions/src/poseBody/pnpm-lock.yaml b/extensions/src/poseBody/pnpm-lock.yaml index ef5650a0d..940de8abd 100644 --- a/extensions/src/poseBody/pnpm-lock.yaml +++ b/extensions/src/poseBody/pnpm-lock.yaml @@ -9,43 +9,55 @@ importers: .: dependencies: '@tensorflow-models/posenet': - specifier: 2.2.1 - version: 2.2.1(@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.21.0))(@tensorflow/tfjs-core@3.21.0) + specifier: 2.2.2 + version: 2.2.2(@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1))(@tensorflow/tfjs-core@3.0.0-rc.1) + '@tensorflow/tfjs-backend-webgl': + specifier: 3.0.0-rc.1 + version: 3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1) + '@tensorflow/tfjs-core': + specifier: 3.0.0-rc.1 + version: 3.0.0-rc.1 packages: - '@tensorflow-models/posenet@2.2.1': - resolution: {integrity: sha512-n9/g6DfjAyrBTf/zt1haRCyWsgALxUCzg9/Ks3Y2mbYavRZVSCSTRPy/qlE5Hr4tLfyckGfDN14zmGTthNcg/g==} + '@tensorflow-models/posenet@2.2.2': + resolution: {integrity: sha512-0SXIksRet/IdX7WVH+JSD6W3upkGHix1hwtd3xykIoIMGR7zQ4SC5+wZcNt9ofASyxNYQoI+tUULUo4LNw0c3w==} peerDependencies: - '@tensorflow/tfjs-converter': ^1.3.0 - '@tensorflow/tfjs-core': ^1.3.0 + '@tensorflow/tfjs-converter': ^3.0.0-rc.1 + '@tensorflow/tfjs-core': ^3.0.0-rc.1 + + '@tensorflow/tfjs-backend-cpu@3.0.0-rc.1': + resolution: {integrity: sha512-9+13fs3vVpfOpDY+Aa/JXdda2SuBN/clxxoNYIX3pMY0GAGwpUaAkSECLuOOhnngHcl9pxuaJtgqrCcrq+A3/A==} + engines: {yarn: '>= 1.3.2'} + peerDependencies: + '@tensorflow/tfjs-core': 3.0.0-rc.1 + + '@tensorflow/tfjs-backend-webgl@3.0.0-rc.1': + resolution: {integrity: sha512-qEjtL/71HflBHJgWXhtj+8BVYB9Q+NHw282QGWi9QLA9SlWe3BRJhbGCaD28/Ddl+mSzBWFQnK+sKdZ04LL1Sg==} + engines: {yarn: '>= 1.3.2'} + peerDependencies: + '@tensorflow/tfjs-core': 3.0.0-rc.1 '@tensorflow/tfjs-converter@3.21.0': resolution: {integrity: sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==} peerDependencies: '@tensorflow/tfjs-core': 3.21.0 - '@tensorflow/tfjs-core@3.21.0': - resolution: {integrity: sha512-YSfsswOqWfd+M4bXIhT3hwtAb+IV8+ODwIxwdFR/7jTAPZP1wMVnSlpKnXHAN64HFOiP+Tm3HmKusEZ0+09A0w==} + '@tensorflow/tfjs-core@3.0.0-rc.1': + resolution: {integrity: sha512-h9a0TyWNJFqgmyfSklwI2q0SC93WFr8FBHwHorvsfm2mDK0ZrSs6bhWqa44tWRJjCrJNRq9J4GoTZQM292RTwg==} engines: {yarn: '>= 1.3.2'} - '@types/long@4.0.2': - resolution: {integrity: sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==} - '@types/offscreencanvas@2019.3.0': resolution: {integrity: sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==} - '@types/seedrandom@2.4.34': - resolution: {integrity: sha512-ytDiArvrn/3Xk6/vtylys5tlY6eo7Ane0hvcx++TKo6RxQXuVfW0AF/oeWqAj9dN29SyhtawuXstgmPlwNcv/A==} + '@types/seedrandom@2.4.27': + resolution: {integrity: sha512-YvMLqFak/7rt//lPBtEHv3M4sRNA+HGxrhFZ+DQs9K2IkYJbNwVIb8avtJfhDiuaUBX/AW0jnjv48FV8h3u9bQ==} '@types/webgl-ext@0.0.30': resolution: {integrity: sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==} - '@webgpu/types@0.1.16': - resolution: {integrity: sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A==} - - long@4.0.0: - resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==} + '@types/webgl2@0.0.5': + resolution: {integrity: sha512-oGaKsBbxQOY5+aJFV3KECDhGaXt+yZJt2y/OZsnQGLRkH6Fvr7rv4pCt3SRH1somIHfej/c4u7NSpCyd9x+1Ow==} node-fetch@2.6.13: resolution: {integrity: sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==} @@ -56,8 +68,8 @@ packages: encoding: optional: true - seedrandom@3.0.5: - resolution: {integrity: sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==} + seedrandom@2.4.3: + resolution: {integrity: sha512-2CkZ9Wn2dS4mMUWQaXLsOAfGD+irMlLEeSP3cMxpGbgyOOzJGFa+MWCOMTOCMyZinHRPxyOj/S/C57li/1to6Q==} tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} @@ -70,45 +82,54 @@ packages: snapshots: - '@tensorflow-models/posenet@2.2.1(@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.21.0))(@tensorflow/tfjs-core@3.21.0)': + '@tensorflow-models/posenet@2.2.2(@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1))(@tensorflow/tfjs-core@3.0.0-rc.1)': dependencies: - '@tensorflow/tfjs-converter': 3.21.0(@tensorflow/tfjs-core@3.21.0) - '@tensorflow/tfjs-core': 3.21.0 + '@tensorflow/tfjs-converter': 3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1) + '@tensorflow/tfjs-core': 3.0.0-rc.1 - '@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.21.0)': + '@tensorflow/tfjs-backend-cpu@3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1)': dependencies: - '@tensorflow/tfjs-core': 3.21.0 + '@tensorflow/tfjs-core': 3.0.0-rc.1 + '@types/seedrandom': 2.4.27 + seedrandom: 2.4.3 - '@tensorflow/tfjs-core@3.21.0': + '@tensorflow/tfjs-backend-webgl@3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1)': dependencies: - '@types/long': 4.0.2 + '@tensorflow/tfjs-backend-cpu': 3.0.0-rc.1(@tensorflow/tfjs-core@3.0.0-rc.1) + '@tensorflow/tfjs-core': 3.0.0-rc.1 '@types/offscreencanvas': 2019.3.0 - '@types/seedrandom': 2.4.34 + '@types/seedrandom': 2.4.27 + '@types/webgl-ext': 0.0.30 + '@types/webgl2': 0.0.5 + seedrandom: 2.4.3 + + '@tensorflow/tfjs-converter@3.21.0(@tensorflow/tfjs-core@3.0.0-rc.1)': + dependencies: + '@tensorflow/tfjs-core': 3.0.0-rc.1 + + '@tensorflow/tfjs-core@3.0.0-rc.1': + dependencies: + '@types/offscreencanvas': 2019.3.0 + '@types/seedrandom': 2.4.27 '@types/webgl-ext': 0.0.30 - '@webgpu/types': 0.1.16 - long: 4.0.0 node-fetch: 2.6.13 - seedrandom: 3.0.5 + seedrandom: 2.4.3 transitivePeerDependencies: - encoding - '@types/long@4.0.2': {} - '@types/offscreencanvas@2019.3.0': {} - '@types/seedrandom@2.4.34': {} + '@types/seedrandom@2.4.27': {} '@types/webgl-ext@0.0.30': {} - '@webgpu/types@0.1.16': {} - - long@4.0.0: {} + '@types/webgl2@0.0.5': {} node-fetch@2.6.13: dependencies: whatwg-url: 5.0.0 - seedrandom@3.0.5: {} + seedrandom@2.4.3: {} tr46@0.0.3: {} diff --git a/extensions/src/poseFace/index.ts b/extensions/src/poseFace/index.ts index ab49c8839..609f521d5 100644 --- a/extensions/src/poseFace/index.ts +++ b/extensions/src/poseFace/index.ts @@ -45,6 +45,7 @@ type Details = { */ type Blocks = { affdexGoToPart(facePart: string): void; + affdexReturnPart(coord: string, facePart: string): number; affdexWhenExpression(expression: string): boolean; affdexExpressionAmount(expression: string): number; affdexIsExpression(expression: string): boolean; @@ -199,6 +200,18 @@ export default class PoseFace extends Extension { (util.target as any).setXY(x, y, false); } + returnPart(coord, part, util) { + if (!this.affdexState || !this.affdexState.featurePoints) return; + + const featurePoint = this.affdexState.featurePoints[part]; + const { x, y } = this.convertCoordsToScratch(featurePoint); + if (coord === 'x') { + return x; + } else { + return y; + } + } + /** * If an expression is being expressed * @param expression @@ -302,6 +315,12 @@ export default class PoseFace extends Extension { } }); + const affdexReturnPart = legacyDefinition.affdexReturnPart({ + operation: (coord: string, part: string, util: BlockUtility) => { + return this.returnPart(coord, part, util) + } + }); + const affdexWhenExpression = legacyDefinition.affdexWhenExpression({ operation: (expression: string) => { return this.isExpression(expression); @@ -403,6 +422,7 @@ export default class PoseFace extends Extension { return { affdexGoToPart, + affdexReturnPart, affdexWhenExpression, affdexExpressionAmount, affdexIsExpression, diff --git a/extensions/src/poseFace/legacy.ts b/extensions/src/poseFace/legacy.ts index 8c6b0f4ba..887cd6366 100644 --- a/extensions/src/poseFace/legacy.ts +++ b/extensions/src/poseFace/legacy.ts @@ -17,6 +17,24 @@ export const info = { } } }, + { + "opcode": "affdexReturnPart", + "text": "get [COORD] from [AFFDEX_POINT]", + "blockType": "reporter", + "isTerminal": false, + "arguments": { + "COORD": { + "type": "string", + "defaultValue": "x", + "menu": "COORD" + }, + "AFFDEX_POINT": { + "type": "string", + "defaultValue": "0", + "menu": "AFFDEX_POINT" + } + } + }, { "opcode": "affdexWhenExpression", "text": "when [EXPRESSION] detected", @@ -446,6 +464,19 @@ export const info = { "value": "on-flipped" } ] + }, + "COORD": { + "acceptReporters": false, + "items": [ + { + "text": "x", + "value": "x" + }, + { + "text": "y", + "value": "y" + }, + ] } } } as const; diff --git a/extensions/src/poseHand/index.ts b/extensions/src/poseHand/index.ts index 5e566d404..08cb80106 100644 --- a/extensions/src/poseHand/index.ts +++ b/extensions/src/poseHand/index.ts @@ -1,7 +1,6 @@ import { ArgumentType, BlockType, Extension, Block, DefineBlock, Environment, ExtensionMenuDisplayDetails, RuntimeEvent } from "$common"; import { legacyFullSupport, info } from "./legacy"; - -import * as handpose from '@tensorflow-models/handpose'; +import { HandLandmarker, FilesetResolver } from '@mediapipe/tasks-vision'; const { legacyExtension, legacyDefinition } = legacyFullSupport.for(); // TODO: Add extension's health check (peripheral) @@ -37,6 +36,7 @@ type Details = { */ type Blocks = { goToHandPart(handPart: string, fingerPart: number): void; + returnHandPart(coord: string, handPart: string, fingerPart: number): number; // these video blocks are present in a few different extensions, perhaps making a file just for these? videoToggle(state: string): void; setVideoTransparency(transparency: number): void; @@ -82,12 +82,23 @@ export default class PoseHand extends Extension { * @param env */ init(env: Environment) { - + this.loadMediaPipeModel(); if (this.runtime.ioDevices) { this._loop(); } } + /** + * Converts the coordinates from the MediaPipe hand estimate to Scratch coordinates + * @param x + * @param y + * @param z + * @returns enum + */ + mediapipeCoordsToScratch(x, y, z) { + return this.tfCoordsToScratch({ x: (this.DIMENSIONS[0] * x), y: this.DIMENSIONS[1] * y, z }); + } + /** * Converts the coordinates from the hand pose estimate to Scratch coordinates * @param x @@ -113,8 +124,7 @@ export default class PoseHand extends Extension { * @returns {boolean} true if connected, false if not connected */ isConnected() { - console.log('connected'); - return !!this.handPoseState && this.handPoseState.length > 0; + return !!this.handPoseState && this.handPoseState.landmarks.length > 0; } /** @@ -123,41 +133,43 @@ export default class PoseHand extends Extension { * so as to prevent the entire program from slowing down. */ async _loop() { - while (true) { + const frame = this.runtime.ioDevices.video.getFrame({ - format: 'image-data', + format: 'canvas', dimensions: this.DIMENSIONS }); - - const time = +new Date(); - if (frame) { - this.handPoseState = await this.estimateHandPoseOnImage(frame); + + if (!this.handModel || !frame) { + requestAnimationFrame(this._loop.bind(this)); + return; } - const estimateThrottleTimeout = (+new Date() - time) / 4; - await new Promise(r => setTimeout(r, estimateThrottleTimeout)); - } - } - /** - * Estimates where the hand is on the video frame. - * @param imageElement - * @returns {Promise} - */ - async estimateHandPoseOnImage(imageElement) { - const handModel = await this.getLoadedHandModel(); - return await handModel.estimateHands(imageElement, { - flipHorizontal: false - }); - } + + if (this.handModel && frame) { + this.handPoseState = this.handModel.detect(frame); + } - /** - * Gets the hand model from handpose - * @returns hand model - */ - async getLoadedHandModel() { - this.handModel ??= await handpose.load(); - return this.handModel; + requestAnimationFrame(this._loop.bind(this)); + } + + + + async loadMediaPipeModel() { + const vision = await FilesetResolver.forVisionTasks( + // path/to/wasm/root + "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm" + ); + this.handModel = await HandLandmarker.createFromOptions( + vision, + { + baseOptions: { + modelAssetPath: "https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/latest/hand_landmarker.task", + delegate: "GPU" + }, + numHands: 2 + }); + } /** * Turns the video camera off/on/on and flipped. This is called in the operation of videoToggleBlock @@ -196,12 +208,46 @@ export default class PoseHand extends Extension { const handlerFingerOptions: Array = this.fingerOptions.map(finger => finger.value); + const handOptions = { + "thumb": { + 3: 4, + 1: 2, + 0: 1, + 2: 3 + }, + "indexFinger": { + 3: 8, + 1: 6, + 0: 5, + 2: 7 + }, + "middleFinger": { + 3: 12, + 1: 10, + 0: 9, + 2: 11 + }, + "ringFinger": { + 3: 16, + 1: 14, + 0: 13, + 2: 15 + }, + "pinky": { + 3: 20, + 1: 18, + 0: 17, + 2: 19 + }, + } + const goToHandPart = legacyDefinition.goToHandPart({ + operation: (handPart: string, fingerPart: number, util) => { + if (this.isConnected()) { - console.log('connected 2'); - const [x, y, z] = this.handPoseState[0].annotations[handPart][fingerPart]; - const { x: scratchX, y: scratchY } = this.tfCoordsToScratch({ x, y, z }); + const { x, y, z } = this.handPoseState.landmarks[0][handOptions[handPart][fingerPart]]; + const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z); (util.target as any).setXY(scratchX, scratchY, false); } }, @@ -219,6 +265,39 @@ export default class PoseHand extends Extension { } }); + const returnHandPart = legacyDefinition.returnHandPart({ + operation: (coord: string, handPart: string, fingerPart: number) => { + if (this.isConnected()) { + const { x, y, z } = this.handPoseState.landmarks[0][handOptions[handPart][fingerPart]]; + const { x: scratchX, y: scratchY } = this.mediapipeCoordsToScratch(x, y, z); + if (coord === 'x') { + return scratchX; + } else { + return scratchY; + } + } else { + return 0; + } + }, + argumentMethods: { + 0: { + handler: (coord: string) => { + return ['x', 'y'].includes(coord) ? coord : "x"; + } + }, + 1: { + handler: (finger: string) => { + return handlerFingerOptions.includes(finger) ? finger : "thumb"; + } + }, + 2: { + handler: (part: number) => { + return Math.max(Math.min(part, 3), 0) + } + } + } + }); + const videoToggle = legacyDefinition.videoToggle({ operation: (video_state) => { this.toggleVideo(video_state); @@ -241,6 +320,7 @@ export default class PoseHand extends Extension { return { goToHandPart, + returnHandPart, videoToggle, setVideoTransparency } diff --git a/extensions/src/poseHand/legacy.ts b/extensions/src/poseHand/legacy.ts index c739e853a..bbeee4dc5 100644 --- a/extensions/src/poseHand/legacy.ts +++ b/extensions/src/poseHand/legacy.ts @@ -22,6 +22,29 @@ export const info = { } } }, + { + "opcode": "returnHandPart", + "text": "get [COORD] from [HAND_PART] [HAND_SUB_PART]", + "blockType": "reporter", + "isTerminal": false, + "arguments": { + "COORD": { + "type": "string", + "defaultValue": "x", + "menu": "COORD" + }, + "HAND_PART": { + "type": "string", + "defaultValue": "thumb", + "menu": "HAND_PART" + }, + "HAND_SUB_PART": { + "type": "number", + "defaultValue": 3, + "menu": "HAND_SUB_PART" + } + } + }, { "opcode": "videoToggle", "text": "turn video [VIDEO_STATE]", @@ -119,6 +142,19 @@ export const info = { } ] }, + "COORD": { + "acceptReporters": false, + "items": [ + { + "text": "x", + "value": "x" + }, + { + "text": "y", + "value": "y" + } + ] + }, "VIDEO_STATE": { "acceptReporters": true, "items": [ diff --git a/extensions/src/poseHand/package.json b/extensions/src/poseHand/package.json index 32b84239f..ff504c1fb 100644 --- a/extensions/src/poseHand/package.json +++ b/extensions/src/poseHand/package.json @@ -11,6 +11,6 @@ "author": "", "license": "ISC", "dependencies": { - "@tensorflow-models/handpose": "^0.0.3" + "@mediapipe/tasks-vision": "^0.10.0" } } \ No newline at end of file diff --git a/extensions/src/poseHand/pnpm-lock.yaml b/extensions/src/poseHand/pnpm-lock.yaml index d1084d72f..08cc65caa 100644 --- a/extensions/src/poseHand/pnpm-lock.yaml +++ b/extensions/src/poseHand/pnpm-lock.yaml @@ -8,74 +8,15 @@ importers: .: dependencies: - '@tensorflow-models/handpose': - specifier: ^0.0.3 - version: 0.0.3(@tensorflow/tfjs-converter@1.7.4(@tensorflow/tfjs-core@1.7.4))(@tensorflow/tfjs-core@1.7.4) + '@mediapipe/tasks-vision': + specifier: ^0.10.0 + version: 0.10.21 packages: - '@tensorflow-models/handpose@0.0.3': - resolution: {integrity: sha512-U5SBwxeQUXVawACDn+e0r4XJEDEah/J1HlWAqApXcm8DXjCtGKxQm/8BmFsg6ebbtAQ/R1bripohaQ655fv29w==} - peerDependencies: - '@tensorflow/tfjs-converter': ^1.6.1 - '@tensorflow/tfjs-core': ^1.6.1 - - '@tensorflow/tfjs-converter@1.7.4': - resolution: {integrity: sha512-B/Ux9I3osI0CXoESGR0Xe5C6BsEfC04+g2xn5zVaW9KEuVEnGEgnuBQxgijRFzkqTwoyLv4ptAmjyIghVARX0Q==} - peerDependencies: - '@tensorflow/tfjs-core': 1.7.4 - - '@tensorflow/tfjs-core@1.7.4': - resolution: {integrity: sha512-3G4VKJ6nPs7iCt6gs3bjRj8chihKrYWenf63R0pm7D9MhlrVoX/tpN4LYVMGgBL7jHPxMLKdOkoAZJrn/J88HQ==} - engines: {yarn: '>= 1.3.2'} - - '@types/offscreencanvas@2019.3.0': - resolution: {integrity: sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==} - - '@types/seedrandom@2.4.27': - resolution: {integrity: sha512-YvMLqFak/7rt//lPBtEHv3M4sRNA+HGxrhFZ+DQs9K2IkYJbNwVIb8avtJfhDiuaUBX/AW0jnjv48FV8h3u9bQ==} - - '@types/webgl-ext@0.0.30': - resolution: {integrity: sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==} - - '@types/webgl2@0.0.4': - resolution: {integrity: sha512-PACt1xdErJbMUOUweSrbVM7gSIYm1vTncW2hF6Os/EeWi6TXYAYMPp+8v6rzHmypE5gHrxaxZNXgMkJVIdZpHw==} - - node-fetch@2.1.2: - resolution: {integrity: sha512-IHLHYskTc2arMYsHZH82PVX8CSKT5lzb7AXeyO06QnjGDKtkv+pv3mEki6S7reB/x1QPo+YPxQRNEVgR5V/w3Q==} - engines: {node: 4.x || >=6.0.0} - - seedrandom@2.4.3: - resolution: {integrity: sha512-2CkZ9Wn2dS4mMUWQaXLsOAfGD+irMlLEeSP3cMxpGbgyOOzJGFa+MWCOMTOCMyZinHRPxyOj/S/C57li/1to6Q==} + '@mediapipe/tasks-vision@0.10.21': + resolution: {integrity: sha512-TuhKH+credq4zLksGbYrnvJ1aLIWMc5r0UHwzxzql4BHECJwIAoBR61ZrqwGOW6ZmSBIzU1t4VtKj8hbxFaKeA==} snapshots: - '@tensorflow-models/handpose@0.0.3(@tensorflow/tfjs-converter@1.7.4(@tensorflow/tfjs-core@1.7.4))(@tensorflow/tfjs-core@1.7.4)': - dependencies: - '@tensorflow/tfjs-converter': 1.7.4(@tensorflow/tfjs-core@1.7.4) - '@tensorflow/tfjs-core': 1.7.4 - - '@tensorflow/tfjs-converter@1.7.4(@tensorflow/tfjs-core@1.7.4)': - dependencies: - '@tensorflow/tfjs-core': 1.7.4 - - '@tensorflow/tfjs-core@1.7.4': - dependencies: - '@types/offscreencanvas': 2019.3.0 - '@types/seedrandom': 2.4.27 - '@types/webgl-ext': 0.0.30 - '@types/webgl2': 0.0.4 - node-fetch: 2.1.2 - seedrandom: 2.4.3 - - '@types/offscreencanvas@2019.3.0': {} - - '@types/seedrandom@2.4.27': {} - - '@types/webgl-ext@0.0.30': {} - - '@types/webgl2@0.0.4': {} - - node-fetch@2.1.2: {} - - seedrandom@2.4.3: {} + '@mediapipe/tasks-vision@0.10.21': {} diff --git a/extensions/src/tables/View.svelte b/extensions/src/tables/View.svelte index e86435f87..555917978 100644 --- a/extensions/src/tables/View.svelte +++ b/extensions/src/tables/View.svelte @@ -71,15 +71,39 @@ - {#each [...Array(extension.tables[selected][0].length)] as _, i} - {i + 1} + {#each extension.columnNames[selected] as columnName, i} + + e.currentTarget.contentEditable = "true"} + on:blur={(e) => { + e.currentTarget.contentEditable = "false"; + invoke("renameColumn", { name: selected, column: i, value: e.currentTarget.innerText }); + }} + > + {columnName} + + {/each} {#each extension.tables[selected] as row, i} - {i + 1} + + e.currentTarget.contentEditable = "true"} + on:blur={(e) => { + e.currentTarget.contentEditable = "false"; + invoke("renameRow", { name: selected, row: i, value: e.currentTarget.innerText }); + }} + > + {extension.rowNames[selected][i]} + + {#each row as value, j} update(e, i, j)} data-testid="tableCell"> diff --git a/extensions/src/tables/index.ts b/extensions/src/tables/index.ts index 7e1ce764f..ae5b134e8 100644 --- a/extensions/src/tables/index.ts +++ b/extensions/src/tables/index.ts @@ -32,6 +32,8 @@ type Blocks = { @validGenericExtension() export default class Tables extends Extension { tables: Record; + columnNames: Record; + rowNames: Record; tableNamesArg: any; defaultNumberArg: any; @@ -46,6 +48,10 @@ export default class Tables extends Extension { if (!this.tables) { this.tables = {}; this.tables.myTable = []; + this.columnNames = {}; + this.rowNames = {}; + this.columnNames.myTable = ["Col 1"]; + this.rowNames.myTable = ["Row 1"]; this.tables.myTable.push([0]); } @@ -80,6 +86,24 @@ export default class Tables extends Extension { ); } + renameColumn(info: { name: string, column: number, value: string }) { + const { name, column, value } = info; + if (this.columnNames[name] && this.columnNames[name][column] !== undefined) { + this.columnNames[name][column] = value; + } else { + alert(`That table or column does not exist.`); + } + } + + renameRow(info: { name: string, row: number, value: string }) { + const { name, row, value } = info; + if (this.rowNames[name] && this.rowNames[name][row] !== undefined) { + this.rowNames[name][row] = value; + } else { + alert(`That table or row does not exist.`); + } + } + newTable(info: { name: string, rows: number, columns: number }) { const { name, rows, columns } = info; @@ -89,11 +113,15 @@ export default class Tables extends Extension { for (let j = 0; j < columns; j++) newRow.push(0); this.tables[name].push(newRow); } + + // Default row & column names + this.rowNames[name] = Array.from({ length: rows }, (_, i) => `Row ${i + 1}`); + this.columnNames[name] = Array.from({ length: columns }, (_, i) => `Col ${i + 1}`); } changeTableValue(info: { name: string, row: number, column: number, value: number }) { const { name, row, column, value } = info; - this.tables[name][row][column] = value; + this.tables[name][row][column] = value ? value : 0; } defineBlocks(): Tables["BlockDefinitions"] { @@ -150,6 +178,8 @@ export default class Tables extends Extension { for (let i = 0; i < this.tables[table].length; i++) { this.tables[table][i].push(0); } + const colCount = this.columnNames[table].length; + this.columnNames[table].push(`Col ${colCount + 1}`); } }), // add a row to the given table @@ -167,6 +197,8 @@ export default class Tables extends Extension { newRow.push(0); } this.tables[table].push(newRow); + const rowCount = this.rowNames[table].length; + this.rowNames[table].push(`Row ${rowCount + 1}`); } }), // change the value in a given table cell diff --git a/extensions/src/textClassification/index.ts b/extensions/src/textClassification/index.ts index 3be3d7505..716645b69 100644 --- a/extensions/src/textClassification/index.ts +++ b/extensions/src/textClassification/index.ts @@ -8,7 +8,7 @@ import timer from "./timer"; import voices, { Voice } from "./voices"; import Sentiment from "sentiment"; import { ToxicityClassifier, load as loadToxicity } from "@tensorflow-models/toxicity"; -import { getTranslationToEnglish } from "./services/translation"; +// import { getTranslationToEnglish } from "./services/translation"; import { Predictor, build, failure, success } from "./model"; const { legacyBlock, } = legacyFullSupport.for(); @@ -309,13 +309,15 @@ export default class TextClassification extends extension(details, "legacySuppor } private async getConfidence(text: string) { - const translation = await getTranslationToEnglish(text); + // const translation = await getTranslationToEnglish(text); + const translation = text; const { score } = await this.customPredictor(translation); return score; } private async getEmbeddings(text) { - const newText = await getTranslationToEnglish(text); //translates text from any language to english + // const newText = await getTranslationToEnglish(text); //translates text from any language to english + const newText = text; if (this.labels.length === 0 || !this.labels[0] || !this.customPredictor) return; const { label } = await this.customPredictor(newText); return label; diff --git a/scratch-packages/scratch-vm b/scratch-packages/scratch-vm index 8aca21fd7..cc723570e 160000 --- a/scratch-packages/scratch-vm +++ b/scratch-packages/scratch-vm @@ -1 +1 @@ -Subproject commit 8aca21fd7b82334d39c804f89d02096121242551 +Subproject commit cc723570e845e1c04dcc7a71e925905b61a7b431