diff --git a/extensions/src/poseFace/index.ts b/extensions/src/poseFace/index.ts index cd9138df6..76d5d8de7 100644 --- a/extensions/src/poseFace/index.ts +++ b/extensions/src/poseFace/index.ts @@ -1,6 +1,8 @@ import { Extension, Environment, untilExternalGlobalVariableLoaded, validGenericExtension, RuntimeEvent } from "$common"; import BlockUtility from "$root/packages/scratch-vm/src/engine/block-utility"; import { legacyFullSupport, info } from "./legacy"; +import { getLandmarkModel } from "./landmarkHelper"; +import { type Results, type FaceMesh } from "@mediapipe/face_mesh"; const { legacyExtension, legacyDefinition } = legacyFullSupport.for(); @@ -104,17 +106,26 @@ export default class PoseFace extends Extension { emotions = info.menus.EMOTION.items all_emotions = info.menus.EMOTION_ALL.items + landmarkDetector: FaceMesh; + landmarkResults: Results; + + private processResults(results: Results) { + this.landmarkResults = results; + } /** * Acts like class PoseHand's constructor (instead of a child class constructor) * @param env */ - init(env: Environment) { + async init(env: Environment) { + this.landmarkDetector = await getLandmarkModel((results) => this.processResults(results)); if (this.runtime.ioDevices) { this.runtime.on(RuntimeEvent.ProjectStart, this.projectStarted.bind(this)); this._loop(); } } + + projectStarted() { this.setTransparency(this.globalVideoTransparency); this.toggleVideo(this.globalVideoState); @@ -130,6 +141,16 @@ export default class PoseFace extends Extension { return { x: x - (this.DIMENSIONS[0] / 2), y: (this.DIMENSIONS[1] / 2) - y }; } + /** + * Converts the coordinates from the MediaPipe face estimate to Scratch coordinates + * @param x + * @param y + * @returns enum + */ + convertMediaPipeCoordsToScratch(x, y) { + return this.convertCoordsToScratch({ x: this.DIMENSIONS[0] * x, y: this.DIMENSIONS[1] * y }); + } + async _loop() { while (true) { const frame = this.runtime.ioDevices.video.getFrame({ @@ -137,9 +158,14 @@ export default class PoseFace extends Extension { dimensions: this.DIMENSIONS }); + const canvas = this.runtime.ioDevices.video.getFrame({ + format: 'canvas' + }); + const time = +new Date(); if (frame) { this.affdexState = await this.estimateAffdexOnImage(frame); + await this.landmarkDetector.send({ image: canvas }); // TODO: Once indicators are implemented, indicate the state of the extension based on this.affdexState } const estimateThrottleTimeout = (+new Date() - time) / 4; @@ -192,11 +218,17 @@ export default class PoseFace extends Extension { * @returns None */ goToPart(part, util) { - if (!this.affdexState || !this.affdexState.featurePoints) return; + if (part < 34) { + if (!this.affdexState || !this.affdexState.featurePoints) return; + const featurePoint = this.affdexState.featurePoints[part]; + const { x, y } = this.convertCoordsToScratch(featurePoint); + (util.target as any).setXY(x, y, false); + } else { + if (!this.landmarkResults) return; + const { x, y } = this.convertMediaPipeCoordsToScratch(this.landmarkResults.multiFaceLandmarks[0][10].x, this.landmarkResults.multiFaceLandmarks[0][10].y); + (util.target as any).setXY(x, y, false); + } - const featurePoint = this.affdexState.featurePoints[part]; - const { x, y } = this.convertCoordsToScratch(featurePoint); - (util.target as any).setXY(x, y, false); } /** diff --git a/extensions/src/poseFace/landmarkHelper.ts b/extensions/src/poseFace/landmarkHelper.ts new file mode 100644 index 000000000..e30ca6dac --- /dev/null +++ b/extensions/src/poseFace/landmarkHelper.ts @@ -0,0 +1,31 @@ +import { ResultsListener, type FaceMesh } from '@mediapipe/face_mesh'; +import '@tensorflow/tfjs-core'; +// Register WebGL backend. +import { untilExternalGlobalVariableLoaded } from "$common"; +import '@tensorflow/tfjs-backend-webgl'; + +export const getLandmarkModel = async (onFrame: ResultsListener) => { + + const packageURL = "https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh"; + const packageClassName = "FaceMesh"; + + const Class = await untilExternalGlobalVariableLoaded(packageURL, packageClassName); + + const faceMesh = new Class({ + locateFile: (file) => { + return `https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh/${file}`; + }, + }); + + // Initialize the mediaPipe model according to the documentation + faceMesh.setOptions({ + maxNumFaces: 1, + refineLandmarks: true, + minDetectionConfidence: 0.5, + minTrackingConfidence: 0.5 + }); + + faceMesh.onResults(onFrame); + await faceMesh.initialize(); + return faceMesh; +} \ No newline at end of file diff --git a/extensions/src/poseFace/legacy.ts b/extensions/src/poseFace/legacy.ts index 8c6b0f4ba..823d707e1 100644 --- a/extensions/src/poseFace/legacy.ts +++ b/extensions/src/poseFace/legacy.ts @@ -257,6 +257,10 @@ export const info = { { "text": "right lower eyelid", "value": "33" + }, + { + "text": "top of head", + "value": "34" } ], "acceptReporters": false @@ -450,4 +454,4 @@ export const info = { } } as const; export const legacyFullSupport = legacy(info); -export const legacyIncrementalSupport = legacy(info, {"incrementalDevelopment":true}); \ No newline at end of file +export const legacyIncrementalSupport = legacy(info, { "incrementalDevelopment": true }); \ No newline at end of file