implemented withFaceExpressions

src/faceExpressionNet/FaceExpressionNet.ts

@@ -4,7 +4,7 @@ import { NetInput, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
 import { FaceFeatureExtractor } from '../faceFeatureExtractor/FaceFeatureExtractor';
 import { FaceFeatureExtractorParams } from '../faceFeatureExtractor/types';
 import { FaceProcessor } from '../faceProcessor/FaceProcessor';
-import { faceExpressionLabels } from './types';
+import { FaceExpression, faceExpressionLabels, FaceExpressionPrediction } from './types';
 
 export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams> {
 
@@ -18,12 +18,12 @@ export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams>
     return label
   }
 
-  public static decodeProbabilites(probabilities: number[] | Float32Array) {
+  public static decodeProbabilites(probabilities: number[] | Float32Array): FaceExpressionPrediction[] {
     if (probabilities.length !== 7) {
       throw new Error(`decodeProbabilites - expected probabilities.length to be 7, have: ${probabilities.length}`)
     }
 
-    return Object.keys(faceExpressionLabels)
+    return (Object.keys(faceExpressionLabels) as FaceExpression[])
       .map(expression => ({ expression, probability: probabilities[faceExpressionLabels[expression]] }))
   }
 

src/faceExpressionNet/types.ts

@@ -6,4 +6,11 @@ export const faceExpressionLabels = {
   fearful: 4,
   disgusted: 5,
   surprised:6
-}
\ No newline at end of file
+}
+
+export type FaceExpression = 'neutral' | 'happy' | 'sad' | 'angry' | 'fearful' | 'disgusted' | 'surprised'
+
+export type FaceExpressionPrediction = {
+  expression: FaceExpression,
+  probability: number
+}

src/faceFeatureExtractor/FaceFeatureExtractor.ts

 import * as tf from '@tensorflow/tfjs-core';
 import { NetInput, NeuralNetwork, normalize, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
-import { ConvParams, SeparableConvParams } from 'tfjs-tiny-yolov2';
 
-import { depthwiseSeparableConv } from './depthwiseSeparableConv';
+import { denseBlock4 } from './denseBlock';
 import { extractParams } from './extractParams';
 import { extractParamsFromWeigthMap } from './extractParamsFromWeigthMap';
-import { DenseBlock4Params, FaceFeatureExtractorParams, IFaceFeatureExtractor } from './types';
-
-function denseBlock(
-  x: tf.Tensor4D,
-  denseBlockParams: DenseBlock4Params,
-  isFirstLayer: boolean = false
-): tf.Tensor4D {
-  return tf.tidy(() => {
-    const out1 = tf.relu(
-      isFirstLayer
-        ? tf.add(
-          tf.conv2d(x, (denseBlockParams.conv0 as ConvParams).filters, [2, 2], 'same'),
-          denseBlockParams.conv0.bias
-        )
-        : depthwiseSeparableConv(x, denseBlockParams.conv0 as SeparableConvParams, [2, 2])
-    ) as tf.Tensor4D
-    const out2 = depthwiseSeparableConv(out1, denseBlockParams.conv1, [1, 1])
-
-    const in3 = tf.relu(tf.add(out1, out2)) as tf.Tensor4D
-    const out3 = depthwiseSeparableConv(in3, denseBlockParams.conv2, [1, 1])
-
-    const in4 = tf.relu(tf.add(out1, tf.add(out2, out3))) as tf.Tensor4D
-    const out4 = depthwiseSeparableConv(in4, denseBlockParams.conv3, [1, 1])
-
-    return tf.relu(tf.add(out1, tf.add(out2, tf.add(out3, out4)))) as tf.Tensor4D
-  })
-}
+import { FaceFeatureExtractorParams, IFaceFeatureExtractor } from './types';
 
 export class FaceFeatureExtractor extends NeuralNetwork<FaceFeatureExtractorParams> implements IFaceFeatureExtractor<FaceFeatureExtractorParams> {
 
@@ -52,10 +25,10 @@ export class FaceFeatureExtractor extends NeuralNetwork<FaceFeatureExtractorPara
       const meanRgb = [122.782, 117.001, 104.298]
       const normalized = normalize(batchTensor, meanRgb).div(tf.scalar(255)) as tf.Tensor4D
 
-      let out = denseBlock(normalized, params.dense0, true)
-      out = denseBlock(out, params.dense1)
-      out = denseBlock(out, params.dense2)
-      out = denseBlock(out, params.dense3)
+      let out = denseBlock4(normalized, params.dense0, true)
+      out = denseBlock4(out, params.dense1)
+      out = denseBlock4(out, params.dense2)
+      out = denseBlock4(out, params.dense3)
       out = tf.avgPool(out, [7, 7], [2, 2], 'valid')
 
       return out

src/faceFeatureExtractor/TinyFaceFeatureExtractor.ts

 import * as tf from '@tensorflow/tfjs-core';
 import { NetInput, NeuralNetwork, normalize, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
-import { ConvParams, SeparableConvParams } from 'tfjs-tiny-yolov2';
 
-import { depthwiseSeparableConv } from './depthwiseSeparableConv';
+import { denseBlock3 } from './denseBlock';
 import { extractParamsFromWeigthMapTiny } from './extractParamsFromWeigthMapTiny';
 import { extractParamsTiny } from './extractParamsTiny';
-import { DenseBlock3Params, IFaceFeatureExtractor, TinyFaceFeatureExtractorParams } from './types';
-
-function denseBlock(
-  x: tf.Tensor4D,
-  denseBlockParams: DenseBlock3Params,
-  isFirstLayer: boolean = false
-): tf.Tensor4D {
-  return tf.tidy(() => {
-    const out1 = tf.relu(
-      isFirstLayer
-        ? tf.add(
-          tf.conv2d(x, (denseBlockParams.conv0 as ConvParams).filters, [2, 2], 'same'),
-          denseBlockParams.conv0.bias
-        )
-        : depthwiseSeparableConv(x, denseBlockParams.conv0 as SeparableConvParams, [2, 2])
-    ) as tf.Tensor4D
-    const out2 = depthwiseSeparableConv(out1, denseBlockParams.conv1, [1, 1])
-
-    const in3 = tf.relu(tf.add(out1, out2)) as tf.Tensor4D
-    const out3 = depthwiseSeparableConv(in3, denseBlockParams.conv2, [1, 1])
-
-    return tf.relu(tf.add(out1, tf.add(out2, out3))) as tf.Tensor4D
-  })
-}
+import { IFaceFeatureExtractor, TinyFaceFeatureExtractorParams } from './types';
 
 export class TinyFaceFeatureExtractor extends NeuralNetwork<TinyFaceFeatureExtractorParams> implements IFaceFeatureExtractor<TinyFaceFeatureExtractorParams> {
 
@@ -49,9 +25,9 @@ export class TinyFaceFeatureExtractor extends NeuralNetwork<TinyFaceFeatureExtra
       const meanRgb = [122.782, 117.001, 104.298]
       const normalized = normalize(batchTensor, meanRgb).div(tf.scalar(255)) as tf.Tensor4D
 
-      let out = denseBlock(normalized, params.dense0, true)
-      out = denseBlock(out, params.dense1)
-      out = denseBlock(out, params.dense2)
+      let out = denseBlock3(normalized, params.dense0, true)
+      out = denseBlock3(out, params.dense1)
+      out = denseBlock3(out, params.dense2)
       out = tf.avgPool(out, [14, 14], [2, 2], 'valid')
 
       return out
@@ -63,7 +39,7 @@ export class TinyFaceFeatureExtractor extends NeuralNetwork<TinyFaceFeatureExtra
   }
 
   protected getDefaultModelName(): string {
-    return 'face_landmark_68_tiny_model'
+    return 'face_feature_extractor_tiny_model'
   }
 
   protected extractParamsFromWeigthMap(weightMap: tf.NamedTensorMap) {

src/faceFeatureExtractor/denseBlock.ts

+import * as tf from '@tensorflow/tfjs-core';
+import { ConvParams, SeparableConvParams } from 'tfjs-tiny-yolov2';
+
+import { depthwiseSeparableConv } from './depthwiseSeparableConv';
+import { DenseBlock3Params, DenseBlock4Params } from './types';
+
+export function denseBlock3(
+  x: tf.Tensor4D,
+  denseBlockParams: DenseBlock3Params,
+  isFirstLayer: boolean = false
+): tf.Tensor4D {
+  return tf.tidy(() => {
+    const out1 = tf.relu(
+      isFirstLayer
+        ? tf.add(
+          tf.conv2d(x, (denseBlockParams.conv0 as ConvParams).filters, [2, 2], 'same'),
+          denseBlockParams.conv0.bias
+        )
+        : depthwiseSeparableConv(x, denseBlockParams.conv0 as SeparableConvParams, [2, 2])
+    ) as tf.Tensor4D
+    const out2 = depthwiseSeparableConv(out1, denseBlockParams.conv1, [1, 1])
+
+    const in3 = tf.relu(tf.add(out1, out2)) as tf.Tensor4D
+    const out3 = depthwiseSeparableConv(in3, denseBlockParams.conv2, [1, 1])
+
+    return tf.relu(tf.add(out1, tf.add(out2, out3))) as tf.Tensor4D
+  })
+}
+
+export function denseBlock4(
+  x: tf.Tensor4D,
+  denseBlockParams: DenseBlock4Params,
+  isFirstLayer: boolean = false,
+  isScaleDown: boolean = true
+): tf.Tensor4D {
+  return tf.tidy(() => {
+    const out1 = tf.relu(
+      isFirstLayer
+        ? tf.add(
+          tf.conv2d(x, (denseBlockParams.conv0 as ConvParams).filters, isScaleDown ? [2, 2] : [1, 1], 'same'),
+          denseBlockParams.conv0.bias
+        )
+        : depthwiseSeparableConv(x, denseBlockParams.conv0 as SeparableConvParams, isScaleDown ? [2, 2] : [1, 1])
+    ) as tf.Tensor4D
+    const out2 = depthwiseSeparableConv(out1, denseBlockParams.conv1, [1, 1])
+
+    const in3 = tf.relu(tf.add(out1, out2)) as tf.Tensor4D
+    const out3 = depthwiseSeparableConv(in3, denseBlockParams.conv2, [1, 1])
+
+    const in4 = tf.relu(tf.add(out1, tf.add(out2, out3))) as tf.Tensor4D
+    const out4 = depthwiseSeparableConv(in4, denseBlockParams.conv3, [1, 1])
+
+    return tf.relu(tf.add(out1, tf.add(out2, tf.add(out3, out4)))) as tf.Tensor4D
+  })
+}

src/factories/WithFaceExpressions.ts

-// TODO
-export type FaceExpressions = number[]
+import { FaceExpressionPrediction } from '../faceExpressionNet/types';
 
 export type WithFaceExpressions<TSource> = TSource & {
-  expressions: FaceExpressions
+  expressions: FaceExpressionPrediction[]
 }
 
 export function extendWithFaceExpressions<
   TSource
 > (
   sourceObj: TSource,
-  expressions: FaceExpressions
+  expressions: FaceExpressionPrediction[]
 ): WithFaceExpressions<TSource> {
 
   const extension = { expressions }

src/globalApi/DetectFaceLandmarksTasks.ts

@@ -77,7 +77,6 @@ export class DetectSingleFaceLandmarksTask<
       ? await extractFaceTensors(this.input, [detection])
       : await extractFaces(this.input, [detection])
 
-
     const landmarks = await this.landmarkNet.detectLandmarks(faces[0]) as FaceLandmarks68
 
     faces.forEach(f => f instanceof tf.Tensor && f.dispose())

src/globalApi/PredictFaceExpressionsTask.ts

 import { TNetInput } from 'tfjs-image-recognition-base';
+import { tf } from 'tfjs-tiny-yolov2';
 
+import { extractFaces, extractFaceTensors } from '../dom';
+import { FaceExpressionPrediction } from '../faceExpressionNet/types';
 import { WithFaceDetection } from '../factories/WithFaceDetection';
 import { extendWithFaceExpressions, WithFaceExpressions } from '../factories/WithFaceExpressions';
 import { ComposableTask } from './ComposableTask';
 import { DetectAllFaceLandmarksTask, DetectSingleFaceLandmarksTask } from './DetectFaceLandmarksTasks';
+import { nets } from './nets';
 
 export class PredictFaceExpressionsTaskBase<TReturn, TParentReturn> extends ComposableTask<TReturn> {
   constructor(
@@ -22,9 +26,20 @@ export class PredictAllFaceExpressionsTask<
 
     const parentResults = await this.parentTask
 
-    // TODO: implement me
+    const detections = parentResults.map(parentResult => parentResult.detection)
+    const faces: Array<HTMLCanvasElement | tf.Tensor3D> = this.input instanceof tf.Tensor
+      ? await extractFaceTensors(this.input, detections)
+      : await extractFaces(this.input, detections)
 
-    return parentResults.map(parentResult => extendWithFaceExpressions<TSource>(parentResult, []))
+    const faceExpressionsByFace = await Promise.all(faces.map(
+      face => nets.faceExpressionNet.predictExpressions(face)
+    )) as FaceExpressionPrediction[][]
+
+    faces.forEach(f => f instanceof tf.Tensor && f.dispose())
+
+    return parentResults.map(
+      (parentResult, i) => extendWithFaceExpressions<TSource>(parentResult, faceExpressionsByFace[i])
+    )
   }
 
   withFaceLandmarks(): DetectAllFaceLandmarksTask<WithFaceExpressions<TSource>> {
@@ -43,9 +58,16 @@ export class PredictSingleFaceExpressionTask<
       return
     }
 
-    // TODO: implement me
+    const { detection } = parentResult
+    const faces: Array<HTMLCanvasElement | tf.Tensor3D> = this.input instanceof tf.Tensor
+      ? await extractFaceTensors(this.input, [detection])
+      : await extractFaces(this.input, [detection])
+
+    const faceExpressions = await nets.faceExpressionNet.predictExpressions(faces[0]) as FaceExpressionPrediction[]
+
+    faces.forEach(f => f instanceof tf.Tensor && f.dispose())
 
-    return extendWithFaceExpressions(parentResult, [])
+    return extendWithFaceExpressions(parentResult, faceExpressions)
   }
 
   withFaceLandmarks(): DetectSingleFaceLandmarksTask<WithFaceExpressions<TSource>> {

src/globalApi/nets.ts

@@ -4,7 +4,7 @@ import { ITinyYolov2Options } from 'tfjs-tiny-yolov2';
 import { FaceDetection } from '../classes/FaceDetection';
 import { FaceLandmarks5 } from '../classes/FaceLandmarks5';
 import { FaceLandmarks68 } from '../classes/FaceLandmarks68';
-import { FaceFeatureExtractor, TinyFaceFeatureExtractor } from '../faceFeatureExtractor';
+import { FaceExpressionNet } from '../faceExpressionNet/FaceExpressionNet';
 import { FaceLandmark68Net } from '../faceLandmarkNet/FaceLandmark68Net';
 import { FaceLandmark68TinyNet } from '../faceLandmarkNet/FaceLandmark68TinyNet';
 import { FaceRecognitionNet } from '../faceRecognitionNet/FaceRecognitionNet';
@@ -25,7 +25,8 @@ export const nets = {
   mtcnn: new Mtcnn(),
   faceLandmark68Net: new FaceLandmark68Net(),
   faceLandmark68TinyNet: new FaceLandmark68TinyNet(),
-  faceRecognitionNet: new FaceRecognitionNet()
+  faceRecognitionNet: new FaceRecognitionNet(),
+  faceExpressionNet: new FaceExpressionNet()
 }
 
 /**