scratchfoundation · MiroslavDionisiev · Dec 4, 2018 · Dec 5, 2018 · Dec 5, 2018 · Dec 6, 2018
@@ -57,7 +57,10 @@
     "watch": "webpack --watch"
   },
   "dependencies": {
+    "@mediapipe/face_detection": "0.4.1646425229",
     "@microbit/microbit-universal-hex": "0.2.2",
+    "@tensorflow-models/face-detection": "^1.0.3",
+    "@tensorflow/tfjs": "^4.22.0",
     "@scratch/scratch-render": "11.6.0-gui-standalone",
     "@scratch/scratch-svg-renderer": "11.6.0-gui-standalone",
     "@scratch/scratch-vm": "11.6.0-gui-standalone",

@@ -144,6 +144,12 @@ class Blocks extends React.Component {
         if (this.props.isVisible) {
             this.setLocale();
         }
+
+        window.addEventListener('load-extension', () => {
+            this.props.vm.extensionManager.loadExtensionURL('faceSensing').then(() => {
+                this.handleCategorySelected('faceSensing');
+            });
+        });
     }
     shouldComponentUpdate (nextProps, nextState) {
         return (

@@ -340,16 +340,29 @@ class Stage extends React.Component {
     }
     onStartDrag (x, y) {
         if (this.state.dragId) return;
-        const drawableId = this.renderer.pick(x, y);
+
+        // Targets with no attached drawable cannot be dragged.
+        let draggableTargets = this.props.vm.runtime.targets.filter(
+            target => Number.isFinite(target.drawableID)
+        );
+
+        // Because pick queries can be expensive, only perform them for drawables that are currently draggable.
+        // If we're in the editor, we can drag all targets. Otherwise, filter.
+        if (!this.props.useEditorDragStyle) {
+            draggableTargets = draggableTargets.filter(
+                target => target.draggable
+            );
+        }
+        if (draggableTargets.length === 0) return;
+
+        const draggableIDs = draggableTargets.map(target => target.drawableID);
+        const drawableId = this.renderer.pick(x, y, 1, 1, draggableIDs);
         if (drawableId === null) return;
         const targetId = this.props.vm.getTargetIdForDrawableId(drawableId);
         if (targetId === null) return;
 
         const target = this.props.vm.runtime.getTargetById(targetId);
 
-        // Do not start drag unless in editor drag mode or target is draggable
-        if (!(this.props.useEditorDragStyle || target.draggable)) return;
-
         // Dragging always brings the target to the front
         target.goToFront();
 

@@ -212,6 +212,20 @@ const alerts = [
         ),
         iconSpinner: true,
         level: AlertLevels.SUCCESS
+    },
+    {
+        alertId: 'loadingExtensionData',
+        alertType: AlertTypes.STANDARD,
+        clearList: [],
+        content: (
+            <FormattedMessage
+                defaultMessage="Loading extension..."
+                description="Message indicating that extension is in process of loading"
+                id="gui.alerts.loadingExtensionData"
+            />
+        ),
+        iconSpinner: true,
+        level: AlertLevels.SUCCESS
     }
 ];
 

@@ -101,7 +101,11 @@ const cloudManagerHOC = function (WrappedComponent) {
         handleExtensionAdded (categoryInfo) {
             // Note that props.vm.extensionManager.isExtensionLoaded('videoSensing') is still false
             // at the point of this callback, so it is difficult to reuse the canModifyCloudData logic.
-            if (categoryInfo.id === 'videoSensing' && this.isConnected()) {
+            if (
+                (categoryInfo.id === 'videoSensing' ||
+                    categoryInfo.id === 'faceSensing') &&
+                this.isConnected()
+            ) {
                 this.disconnectFromCloud();
             }
         }
@@ -157,9 +161,18 @@ const cloudManagerHOC = function (WrappedComponent) {
             isShowingWithId: getIsShowingWithId(loadingState),
             projectId: state.scratchGui.projectState.projectId,
             // if you're editing someone else's project, you can't modify cloud data
-            canModifyCloudData: (!state.scratchGui.mode.hasEverEnteredEditor || ownProps.canSave) &&
+            canModifyCloudData:
+                (!state.scratchGui.mode.hasEverEnteredEditor ||
+                    ownProps.canSave) &&
                 // possible security concern if the program attempts to encode webcam data over cloud variables
-                !ownProps.vm.extensionManager.isExtensionLoaded('videoSensing')
+                !(
+                    ownProps.vm.extensionManager.isExtensionLoaded(
+                        'videoSensing'
+                    ) ||
+                    ownProps.vm.extensionManager.isExtensionLoaded(
+                        'faceSensing'
+                    )
+                )
         };
     };
 

@@ -46,6 +46,9 @@ import gdxforInsetIconURL from './gdxfor/gdxfor-small.svg';
 import gdxforConnectionIconURL from './gdxfor/gdxfor-illustration.svg';
 import gdxforConnectionSmallIconURL from './gdxfor/gdxfor-small.svg';
 
+import faceSensingIconURL from './faceSensing/faceSensing.png';
+import faceSensingInsetIconURL from './faceSensing/faceSensing-small.svg';
+
 export default [
     {
         name: (
@@ -107,6 +110,26 @@ export default [
         ),
         featured: true
     },
+    {
+        name: (
+            <FormattedMessage
+                defaultMessage="Face Sensing"
+                description="Name for the 'Face Sensing' extension"
+                id="gui.extension.faceSensing.name"
+            />
+        ),
+        extensionId: 'faceSensing',
+        iconURL: faceSensingIconURL,
+        insetIconURL: faceSensingInsetIconURL,
+        description: (
+            <FormattedMessage
+                defaultMessage="Sense faces with the camera."
+                description="Description for the 'Face Sensing' extension"
+                id="gui.extension.faceSensing.description"
+            />
+        ),
+        featured: true
+    },
     {
         name: (
             <FormattedMessage

@@ -3,6 +3,7 @@ import penIcon from './extensions/penIcon.svg';
 import text2speechIcon from './extensions/text2speechIcon.svg';
 import translateIcon from './extensions/translateIcon.svg';
 import videoSensingIcon from './extensions/videoSensingIcon.svg';
+import faceSensingIcon from './extensions/faceSensingIcon.svg';
 
 const blockColors = {
     motion: {
@@ -101,6 +102,9 @@ const extensions = {
     },
     videoSensing: {
         blockIconURI: videoSensingIcon
+    },
+    faceSensing: {
+        blockIconURI: faceSensingIcon
     }
 };
 

@@ -10,7 +10,7 @@ import {updateBlockDrag} from '../reducers/block-drag';
 import {updateMonitors} from '../reducers/monitors';
 import {setProjectChanged, setProjectUnchanged} from '../reducers/project-changed';
 import {setRunningState, setTurboState, setStartedState} from '../reducers/vm-status';
-import {showExtensionAlert} from '../reducers/alerts';
+import {showExtensionAlert, showStandardAlert, closeAlertWithId} from '../reducers/alerts';
 import {updateMicIndicator} from '../reducers/mic-indicator';
 
 /*
@@ -46,6 +46,7 @@ const vmListenerHOC = function (WrappedComponent) {
             this.props.vm.on('PROJECT_START', this.props.onGreenFlag);
             this.props.vm.on('PERIPHERAL_CONNECTION_LOST_ERROR', this.props.onShowExtensionAlert);
             this.props.vm.on('MIC_LISTENING', this.props.onMicListeningUpdate);
+            this.props.vm.on('EXTENSION_DATA_LOADING', this.props.onExtensionDataLoading);
 
         }
         componentDidMount () {
@@ -125,6 +126,7 @@ const vmListenerHOC = function (WrappedComponent) {
                 onKeyDown,
                 onKeyUp,
                 onMicListeningUpdate,
+                onExtensionDataLoading,
                 onMonitorsUpdate,
                 onTargetsUpdate,
                 onProjectChanged,
@@ -144,6 +146,7 @@ const vmListenerHOC = function (WrappedComponent) {
     VMListener.propTypes = {
         attachKeyboardEvents: PropTypes.bool,
         onBlockDragUpdate: PropTypes.func.isRequired,
+        onExtensionDataLoading: PropTypes.func.isRequired,
         onGreenFlag: PropTypes.func,
         onKeyDown: PropTypes.func,
         onKeyUp: PropTypes.func,
@@ -205,6 +208,13 @@ const vmListenerHOC = function (WrappedComponent) {
         },
         onMicListeningUpdate: listening => {
             dispatch(updateMicIndicator(listening));
+        },
+        onExtensionDataLoading: loading => {
+            if (loading) {
+                dispatch(showStandardAlert('loadingExtensionData'));
+            } else {
+                dispatch(closeAlertWithId('loadingExtensionData'));
+            }
         }
     });
     return connect(

@@ -162,6 +162,25 @@ describe('CloudManagerHOC', () => {
         expect(CloudProvider).not.toHaveBeenCalled();
     });
 
+    test('when faceSensing extension is active, the cloud provider is not set on the vm', () => {
+        const Component = () => <div />;
+        const WrappedComponent = cloudManagerHOC(Component);
+        vm.extensionManager.isExtensionLoaded = jest.fn(extension => extension === 'faceSensing');
+
+        mount(
+            <WrappedComponent
+                hasCloudPermission
+                cloudHost="nonEmpty"
+                store={store}
+                username="user"
+                vm={vm}
+            />
+        );
+
+        expect(vm.setCloudProvider.mock.calls.length).toBe(0);
+        expect(CloudProvider).not.toHaveBeenCalled();
+    });
+
     test('if the isShowingWithId prop becomes true, it sets the cloud provider on the vm', () => {
         const Component = () => <div />;
         const WrappedComponent = cloudManagerHOC(Component);

@@ -84,6 +84,10 @@ const baseConfig = new ScratchWebpackConfigBuilder(
                 context: '../../node_modules/scratch-storage/dist/web',
                 from: 'chunks/fetch-worker.*.{js,js.map}',
                 noErrorOnMissing: true
+            },
+            {
+                from: '../../node_modules/@mediapipe/face_detection',
+                to: 'chunks/mediapipe/face_detection'
             }
         ]
     }));

@@ -1069,6 +1069,36 @@ class RenderWebGL extends EventEmitter {
         return false;
     }
 
+    drawableTouchingScratchRect (drawableID, left, top, right, bottom) {
+        const drawable = this._allDrawables[drawableID];
+        if (!drawable) {
+            return false;
+        }
+        const bounds = new Rectangle();
+        bounds.initFromBounds(left, right, bottom, top);
+        const worldPos = twgl.v3.create();
+
+        drawable.updateCPURenderAttributes();
+
+        for (worldPos[1] = bounds.bottom; worldPos[1] <= bounds.top; worldPos[1]++) {
+            for (worldPos[0] = bounds.left; worldPos[0] <= bounds.right; worldPos[0]++) {
+                if (drawable.isTouching(worldPos)) {
+                    return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    drawableTouchingScratchPoint (drawableID, x, y) {
+        const drawable = this._allDrawables[drawableID];
+        if (!drawable) {
+            return false;
+        }
+        drawable.updateCPURenderAttributes();
+        return drawable.isTouching([x, y]);
+    }
+
     /**
      * Detect which sprite, if any, is at the given location.
      * This function will pick all drawables that are visible, unless specific

@@ -677,6 +677,10 @@
         return 'MIC_LISTENING';
     }
 
+    static get EXTENSION_DATA_LOADING () {
+        return 'EXTENSION_DATA_LOADING';
+    }
+
     /**
      * Event name for reporting that blocksInfo was updated.
      * @const {string}
@@ -830,7 +834,7 @@
    _registerExtensionPrimitives (extensionInfo) {
        const categoryInfo = {
            id: extensionInfo.id,
            name: maybeFormatMessage(extensionInfo.name),
            showStatusButton: extensionInfo.showStatusButton,
            blockIconURI: extensionInfo.blockIconURI,
            menuIconURI: extensionInfo.menuIconURI
@@ -873,7 +877,7 @@
    _refreshExtensionPrimitives (extensionInfo) {
        const categoryInfo = this._blockInfo.find(info => info.id === extensionInfo.id);
        if (categoryInfo) {
            categoryInfo.name = maybeFormatMessage(extensionInfo.name);
            this._fillExtensionCategory(categoryInfo, extensionInfo);

            this.emit(Runtime.BLOCKSINFO_UPDATE, categoryInfo);
@@ -948,12 +952,12 @@
        if (typeof menuItems !== 'function') {
            const extensionMessageContext = this.makeMessageContextForTarget();
            return menuItems.map(item => {
                const formattedItem = maybeFormatMessage(item, extensionMessageContext);
                switch (typeof formattedItem) {
                case 'string':
                    return [formattedItem, formattedItem];
                case 'object':
                    return [maybeFormatMessage(item.text, extensionMessageContext), item.value];
                default:
                    throw new Error(`Can't interpret menu item: ${JSON.stringify(item)}`);
                }
@@ -1167,7 +1171,7 @@
        while (inTextNum < blockText.length || inBranchNum < blockInfo.branchCount) {
            if (inTextNum < blockText.length) {
                context.outLineNum = outLineNum;
                const lineText = maybeFormatMessage(blockText[inTextNum], extensionMessageContext);
                const convertedText = lineText.replace(/\[(.+?)]/g, convertPlaceholders);
                if (blockJSON[`message${outLineNum}`]) {
                    blockJSON[`message${outLineNum}`] += convertedText;
@@ -1248,7 +1252,7 @@
        }

        const extensionMessageContext = this.makeMessageContextForTarget();
        const buttonText = maybeFormatMessage(buttonInfo.text, extensionMessageContext);
        return {
            info: buttonInfo,
            xml: `<button text="${buttonText}" callbackKey="${buttonInfo.func}"></button>`
@@ -1319,7 +1323,7 @@

            const defaultValue =
                typeof argInfo.defaultValue === 'undefined' ? '' :
                    xmlEscape(maybeFormatMessage(argInfo.defaultValue, this.makeMessageContextForTarget()).toString());

            if (argTypeInfo.check) {
                // Right now the only type of 'check' we have specifies that the
@@ -1569,6 +1573,10 @@
         this.emit(Runtime.MIC_LISTENING, listening);
     }
 
+    emitExtensionLoading (loading) {
+        this.emit(Runtime.EXTENSION_DATA_LOADING, loading);
+    }
+
     /**
      * Retrieve the function associated with the given opcode.
      * @param {!string} opcode The opcode to look up.

@@ -23,7 +23,8 @@
     ev3: () => require('../extensions/scratch3_ev3'),
     makeymakey: () => require('../extensions/scratch3_makeymakey'),
     boost: () => require('../extensions/scratch3_boost'),
-    gdxfor: () => require('../extensions/scratch3_gdx_for')
+    gdxfor: () => require('../extensions/scratch3_gdx_for'),
+    faceSensing: () => require('../extensions/scratch3_face_sensing')
 };
 
 /**
@@ -350,11 +351,11 @@
        const menuFunc = extensionObject[menuItemFunctionName];
        const menuItems = menuFunc.call(extensionObject, editingTargetID).map(
            item => {
                item = maybeFormatMessage(item, extensionMessageContext);
                switch (typeof item) {
                case 'object':
                    return [
                        maybeFormatMessage(item.text, extensionMessageContext),
                        item.value
                    ];
                case 'string':