refactored some stuff + train and test script for face expressions net with…

src/faceExpressionNet/FaceExpressionNet.ts

@@ -4,41 +4,35 @@ import { NetInput, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
 import { FaceFeatureExtractor } from '../faceFeatureExtractor/FaceFeatureExtractor';
 import { FaceFeatureExtractorParams } from '../faceFeatureExtractor/types';
 import { FaceProcessor } from '../faceProcessor/FaceProcessor';
-import { emotionLabels } from './types';
+import { faceExpressionLabels } from './types';
 
 export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams> {
 
-  public static getEmotionLabel(emotion: string) {
-    const label = emotionLabels[emotion]
+  public static getFaceExpressionLabel(faceExpression: string) {
+    const label = faceExpressionLabels[faceExpression]
 
     if (typeof label !== 'number') {
-      throw new Error(`getEmotionLabel - no label for emotion: ${emotion}`)
+      throw new Error(`getFaceExpressionLabel - no label for faceExpression: ${faceExpression}`)
     }
 
     return label
   }
 
-  public static decodeEmotions(probabilities: number[] | Float32Array) {
+  public static decodeProbabilites(probabilities: number[] | Float32Array) {
     if (probabilities.length !== 7) {
-      throw new Error(`decodeEmotions - expected probabilities.length to be 7, have: ${probabilities.length}`)
+      throw new Error(`decodeProbabilites - expected probabilities.length to be 7, have: ${probabilities.length}`)
     }
 
-    return Object.keys(emotionLabels).map(label => ({ label, probability: probabilities[emotionLabels[label]] }))
+    return Object.keys(faceExpressionLabels)
+      .map(expression => ({ expression, probability: probabilities[faceExpressionLabels[expression]] }))
   }
 
   constructor(faceFeatureExtractor: FaceFeatureExtractor = new FaceFeatureExtractor()) {
     super('FaceExpressionNet', faceFeatureExtractor)
   }
 
-  public runNet(input: NetInput | tf.Tensor4D): tf.Tensor2D {
-    return tf.tidy(() => {
-      const out = super.runNet(input)
-      return tf.softmax(out)
-    })
-  }
-
   public forwardInput(input: NetInput | tf.Tensor4D): tf.Tensor2D {
-    return tf.tidy(() => this.runNet(input))
+    return tf.tidy(() => tf.softmax(this.runNet(input)))
   }
 
   public async forward(input: TNetInput): Promise<tf.Tensor2D> {
@@ -52,14 +46,7 @@ export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams>
     out.dispose()
 
     const predictionsByBatch = probabilitesByBatch
-      .map(propablities => {
-        const predictions = {}
-        FaceExpressionNet.decodeEmotions(propablities as Float32Array)
-          .forEach(({ label, probability }) => {
-            predictions[label] = probability
-          })
-        return predictions
-      })
+      .map(propablities => FaceExpressionNet.decodeProbabilites(propablities as Float32Array))
 
     return netInput.isBatchInput
       ? predictionsByBatch

src/faceExpressionNet/types.ts

-export const emotionLabels = {
+export const faceExpressionLabels = {
   neutral: 0,
   happy: 1,
   sad: 2,

src/faceProcessor/FaceProcessor.ts

@@ -76,7 +76,7 @@ export abstract class FaceProcessor<
 
     const cIn = this.getClassifierChannelsIn()
     const cOut = this.getClassifierChannelsOut()
-    const classifierWeightSize = (cOut * cIn )+ cOut
+    const classifierWeightSize = (cOut * cIn ) + cOut
 
     const featureExtractorWeights = weights.slice(0, weights.length - classifierWeightSize)
     const classifierWeights = weights.slice(weights.length - classifierWeightSize)

tools/train/faceExpressions/public/js/commons.js

-function getImageUrl({ db, label, img }) {
-  if (db === 'kaggle') {
-    return `kaggle-face-expressions-db/${label}/${img}`
-  }
-
-  const id = parseInt(img.replace('.png'))
-  const dirNr = Math.floor(id / 5000)
-  return `cropped-faces/jpgs${dirNr + 1}/${img}`
-}
-

tools/train/faceExpressions/public/js/faceExpressionsCommons.js

+function getImageUrl({ db, label, img }) {
+  if (db === 'kaggle') {
+    return `kaggle-face-expressions-db/${label}/${img}`
+  }
+
+  const id = parseInt(img.replace('.png'))
+  const dirNr = Math.floor(id / 5000)
+  return `cropped-faces/jpgs${dirNr + 1}/${img}`
+}
+
+function prepareDataForEpoch(data) {
+  return faceapi.shuffleArray(
+    Object.keys(data).map(label => {
+      let dataForLabel = data[label].map(data => ({ ...data, label }))
+      // since train data for "disgusted" have less than 2000 samples
+      // use some data twice to ensure an even distribution
+      dataForLabel = label === 'disgusted'
+        ? faceapi.shuffleArray(dataForLabel.concat(dataForLabel).concat(dataForLabel)).slice(0, 2000)
+        : dataForLabel
+      return dataForLabel
+    }).reduce((flat, arr) => arr.concat(flat))
+  )
+}
+
+function getLabelOneHotVector(faceExpression) {
+  const label = faceapi.FaceExpressionNet.getFaceExpressionLabel(faceExpression)
+  return Array(7).fill(0).map((_, i) => i === label ? 1 : 0)
+}
+
+async function onEpochDone(epoch, params) {
+  saveWeights(params || window.net, `face_expression_model_${epoch}.weights`)
+
+  const loss = window.lossValues[epoch]
+  saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / (2000 * 7) })]), `face_expression_model_${epoch}.json`)
+
+}
+

tools/train/faceExpressions/public/js/test.js

+async function test(db) {
+  const faceExpressions = Object.keys(window.testData)
+  let errors = {}
+  let preds = {}
+  let thresh03 = {}
+  let thresh05 = {}
+  let thresh08 = {}
+  let sizes = {}
+
+  for (let faceExpression of faceExpressions) {
+
+    const container = document.getElementById('container')
+    const span = document.createElement('div')
+    container.appendChild(span)
+
+    console.log(faceExpression)
+
+    const dataForLabel = window.testData[faceExpression]
+      .filter(data => data.db === db)
+      .slice(0, window.numDataPerClass)
+
+    errors[faceExpression] = 0
+    preds[faceExpression] = 0
+    thresh03[faceExpression] = 0
+    thresh05[faceExpression] = 0
+    thresh08[faceExpression] = 0
+    sizes[faceExpression] = dataForLabel.length
+
+
+    for (let [idx, data] of dataForLabel.entries()) {
+      span.innerHTML =  faceExpression + ': ' + faceapi.round(idx / dataForLabel.length) * 100 + '%'
+
+      const img = await faceapi.fetchImage(getImageUrl({ ...data, label: faceExpression }))
+      const pred = await window.net.predictExpressions(img)
+      const bestPred = pred
+        .reduce((best, curr) => curr.probability < best.probability ? best : curr)
+
+      const { probability } = pred.find(p => p.expression === faceExpression)
+      thresh03[faceExpression] += (probability > 0.3 ? 1 : 0)
+      thresh05[faceExpression] += (probability > 0.5 ? 1 : 0)
+      thresh08[faceExpression] += (probability > 0.8 ? 1 : 0)
+      errors[faceExpression] += 1 - probability
+      preds[faceExpression] += (bestPred.expression === faceExpression ? 1 : 0)
+    }
+
+    span.innerHTML = faceExpression + ': 100%'
+
+  }
+
+  const totalError = faceExpressions.reduce((err, faceExpression) => err + errors[faceExpression], 0)
+
+  const relative = (obj) => {
+    res = {}
+    Object.keys(sizes).forEach((faceExpression) => {
+      res[faceExpression] = faceapi.round(
+        obj[faceExpression] / sizes[faceExpression]
+      )
+    })
+    return res
+  }
+
+  console.log('done...')
+  console.log('test set size:', sizes)
+  console.log('preds:', relative(preds))
+  console.log('thresh03:', relative(thresh03))
+  console.log('thresh05:', relative(thresh05))
+  console.log('thresh08:', relative(thresh08))
+  console.log('errors:', errors)
+  console.log('total error:', totalError)
+}
\ No newline at end of file

tools/train/faceExpressions/public/js/train.js

+async function train() {
+  await load()
+
+  for (let epoch = startEpoch; epoch < Infinity; epoch++) {
+
+    if (epoch !== startEpoch) {
+      // ugly hack to wait for loss datas for that epoch to be resolved
+      setTimeout(
+        () => onEpochDone(
+          epoch - 1,
+          new Float32Array(Array.from(window.net.faceFeatureExtractor.serializeParams())
+            .concat(Array.from(window.net.serializeParams()))
+          )
+        ),
+        10000
+      )
+    }
+    window.lossValues[epoch] = 0
+
+    const shuffledInputs = prepareDataForEpoch(window.trainData)
+      console.log(shuffledInputs)
+
+    for (let dataIdx = 0; dataIdx < shuffledInputs.length; dataIdx += window.batchSize) {
+      const tsIter = Date.now()
+
+      const batchData = shuffledInputs.slice(dataIdx, dataIdx + window.batchSize)
+      const bImages = await Promise.all(
+        batchData
+          .map(data => getImageUrl(data))
+          .map(imgUrl => faceapi.fetchImage(imgUrl))
+      )
+      const bOneHotVectors = batchData
+        .map(data => getLabelOneHotVector(data.label))
+
+      let tsBackward = Date.now()
+      let tsForward = Date.now()
+      const netInput = await faceapi.toNetInput(bImages)
+      tsForward = Date.now() - tsForward
+
+      const loss = optimizer.minimize(() => {
+        tsBackward = Date.now()
+        const labels = tf.tensor2d(bOneHotVectors)
+        const out = window.net.runNet(netInput)
+
+        const loss = tf.losses.softmaxCrossEntropy(
+          labels,
+          out,
+          tf.Reduction.MEAN
+        )
+
+        return loss
+      }, true)
+      tsBackward = Date.now() - tsBackward
+
+      // start next iteration without waiting for loss data
+
+      loss.data().then(data => {
+        const lossValue = data[0]
+        window.lossValues[epoch] += lossValue
+        window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - loss: ${lossValue}, ( ${window.lossValues[epoch]})`)
+        loss.dispose()
+      })
+
+      window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - forward: ${tsForward} ms, backprop: ${tsBackward} ms, iter: ${Date.now() - tsIter} ms`)
+      if (window.logsilly)  {
+        log(`fetch: ${tsFetch} ms, pts: ${tsFetchPts} ms, jpgs: ${tsFetchJpgs} ms, bufferToImage: ${tsBufferToImage} ms`)
+      }
+      if (window.iterDelay) {
+        await delay(window.iterDelay)
+      } else {
+        await tf.nextFrame()
+      }
+    }
+  }
+}
\ No newline at end of file

tools/train/faceExpressions/public/test.html

+<!DOCTYPE html>
+<html>
+<head>
+  <script src="face-api.js"></script>
+  <script src="FileSaver.js"></script>
+  <script src="commons.js"></script>
+  <script src="js/faceExpressionsCommons.js"></script>
+  <script src="js/test.js"></script>
+</head>
+<body>
+  <div id="container"></div>
+
+  <script>
+    tf = faceapi.tf
+
+    window.numDataPerClass = Infinity
+
+    // load the FaceLandmark68Net and use it's feature extractor since we only
+    // train the output layer of the FaceExpressionNet
+    window.net = new faceapi.FaceExpressionNet()
+
+    // uri to weights file of last checkpoint
+    const modelCheckpoint = 'tmp/full/face_expression_model_120.weights'
+
+    async function load() {
+      window.testData = await faceapi.fetchJson('testData.json')
+
+      // fetch the actual output layer weights
+      const weights = await faceapi.fetchNetWeights(modelCheckpoint)
+      await window.net.load(weights)
+
+      console.log('loaded')
+    }
+
+    load()
+
+  </script>
+
+</body>
+</html>
+

tools/train/faceExpressions/public/testClassifier.html

@@ -3,7 +3,8 @@
 <head>
   <script src="face-api.js"></script>
   <script src="FileSaver.js"></script>
-  <script src="js/commons.js"></script>
+  <script src="commons.js"></script>
+  <script src="js/faceExpressionsCommons.js"></script>
 </head>
 <body>
   <div id="container"></div>
@@ -11,13 +12,15 @@
   <script>
     tf = faceapi.tf
 
+    window.numDataPerClass = 10
+
     // load the FaceLandmark68Net and use it's feature extractor since we only
     // train the output layer of the FaceExpressionNet
     const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
     window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
 
     // uri to weights file of last checkpoint
-    const modelCheckpoint = 'tmp/face_expression_model_165.weights'
+    const modelCheckpoint = 'tmp/classifier/face_expression_model_118.weights'
 
     async function load() {
       window.testData = await faceapi.fetchJson('testData.json')
@@ -32,53 +35,6 @@
 
     load()
 
-    async function test() {
-      const emotions = Object.keys(window.testData)
-      let errors = {}
-      let preds = {}
-      let sizes = {}
-
-      for (let emotion of emotions) {
-
-        const container = document.getElementById('container')
-        const span = document.createElement('div')
-        container.appendChild(span)
-
-        console.log(emotion)
-
-        const dataForLabel = window.testData[emotion]
-
-        errors[emotion] = 0
-        preds[emotion] = 0
-        sizes[emotion] = dataForLabel.length
-
-
-        for (let [idx, data] of dataForLabel.entries()) {
-          span.innerHTML =  emotion + ': ' + faceapi.round(idx / dataForLabel.length) * 100 + '%'
-
-          const img = await faceapi.fetchImage(getImageUrl({ ...data, label: emotion }))
-          const pred = await window.net.predictExpressions(img)
-          const bestPred = Object.keys(pred)
-            .map(label => ({ label, probability: pred[label] }))
-            .reduce((best, curr) => curr.probability < best.probability ? curr : best)
-          errors[emotion] += (1 - pred[emotion])
-          pred[emotion] += (bestPred.label === emotion ? 1 : 0)
-
-        }
-
-        span.innerHTML = emotion + ': 100%'
-
-      }
-
-      const totalError = emotions.reduce((err, emotion) => err + errors[emotion], 0)
-
-      console.log('done...')
-      console.log('test set size:', sizes)
-      console.log('preds:', preds)
-      console.log('errors:', errors)
-      console.log('total error:', totalError)
-    }
-
   </script>
 </body>
 </html>
\ No newline at end of file

tools/train/faceExpressions/public/train.html

+<!DOCTYPE html>
+<html>
+<head>
+  <script src="face-api.js"></script>
+  <script src="FileSaver.js"></script>
+  <script src="commons.js"></script>
+  <script src="js/faceExpressionsCommons.js"></script>
+  <script src="js/train.js"></script>
+</head>
+<body>
+  <div id="container"></div>
+
+  <script>
+    tf = faceapi.tf
+
+    // load the FaceLandmark68Net and use it's feature extractor since we only
+    // train the output layer of the FaceExpressionNet
+    const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
+    window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
+
+    // uri to weights file of last checkpoint
+    const modelCheckpoint = 'tmp/full/face_expression_model_120.weights'
+    const startEpoch = 121
+
+    const learningRate = 0.00001
+    window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
+
+    window.saveEveryNthSample = Infinity
+
+    window.batchSize = 8
+    //window.batchSize = 32
+
+    window.lossValues = []
+
+    window.iterDelay = 0
+    window.withLogging = true
+
+    async function load() {
+      window.trainData = await faceapi.fetchJson('trainData.json')
+
+      // fetch the actual output layer weights
+      const weights = await faceapi.fetchNetWeights(modelCheckpoint)
+      await window.net.load(weights)
+      window.net.faceFeatureExtractor.variable()
+      window.net.variable()
+    }
+
+  </script>
+</body>
+</html>
\ No newline at end of file

tools/train/faceExpressions/public/trainClassifier.html

@@ -3,7 +3,8 @@
 <head>
   <script src="face-api.js"></script>
   <script src="FileSaver.js"></script>
-  <script src="js/commons.js"></script>
+  <script src="commons.js"></script>
+  <script src="js/faceExpressionsCommons.js"></script>
 </head>
 <body>
   <div id="container"></div>
@@ -17,10 +18,10 @@
     window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
 
     // uri to weights file of last checkpoint
-    const modelCheckpoint = 'tmp/face_expression_model_148.weights'
-    const startEpoch = 149
+    const modelCheckpoint = 'tmp/classifier/face_expression_model_121.weights'
+    const startEpoch = 0
 
-    const learningRate = 0.001
+    const learningRate = 0.0001
     window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
 
     window.saveEveryNthSample = Infinity
@@ -33,16 +34,6 @@
     window.iterDelay = 0
     window.withLogging = true
 
-    const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
-
-    function saveWeights(net, filename = 'train_tmp') {
-      saveAs(new Blob([net.serializeParams()]), filename)
-    }
-
-    async function delay(ms) {
-      return new Promise(res => setTimeout(res, ms))
-    }
-
     async function load() {
       window.trainData = await faceapi.fetchJson('trainData.json')
       await dummyLandmarkNet.load('/')
@@ -53,33 +44,6 @@
       window.net.variable()
     }
 
-    async function onEpochDone(epoch) {
-      saveWeights(window.net, `face_expression_model_${epoch}.weights`)
-
-      const loss = window.lossValues[epoch]
-      saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / (2000 * 7) })]), `face_expression_model_${epoch}.json`)
-
-    }
-
-    function prepareDataForEpoch() {
-      return faceapi.shuffleArray(
-        Object.keys(window.trainData).map(label => {
-          let dataForLabel = window.trainData[label].map(data => ({ ...data, label }))
-          // since train data for "disgusted" have less than 2000 samples
-          // use some data twice to ensure an even distribution
-          dataForLabel = label === 'disgusted'
-            ? faceapi.shuffleArray(dataForLabel.concat(dataForLabel).concat(dataForLabel)).slice(0, 2000)
-            : dataForLabel
-          return dataForLabel
-        }).reduce((flat, arr) => arr.concat(flat))
-      )
-    }
-
-    function getLabelOneHotVector(emotion) {
-      const label = faceapi.FaceExpressionNet.getEmotionLabel(emotion)
-      return Array(7).fill(0).map((_, i) => i === label ? 1 : 0)
-    }
-
     async function train() {
       await load()
 
@@ -91,8 +55,7 @@
         }
         window.lossValues[epoch] = 0
 
-        const shuffledInputs = prepareDataForEpoch()
-          console.log(shuffledInputs)
+        const shuffledInputs = prepareDataForEpoch(window.trainData)
 
         for (let dataIdx = 0; dataIdx < shuffledInputs.length; dataIdx += window.batchSize) {
           const tsIter = Date.now()
@@ -114,7 +77,7 @@
           const loss = optimizer.minimize(() => {
             tsBackward = Date.now()
             const labels = tf.tensor2d(bOneHotVectors)
-            const out = window.net.forwardInput(bottleneckFeatures)
+            const out = window.net.runNet(bottleneckFeatures)
 
             const loss = tf.losses.softmaxCrossEntropy(
               labels,

tools/train/faceExpressions/public/trainClassifier_debug.html

@@ -3,12 +3,13 @@
 <head>
   <script src="face-api.js"></script>
   <script src="FileSaver.js"></script>
-  <script src="js/commons.js"></script>
+  <script src="commons.js"></script>
+  <script src="js/faceExpressionsCommons.js"></script>
 </head>
 <body>
   <div id="container"></div>
   <div id="template" style="display: inline-flex; flex-direction: column;">
-    <span class="emotion-text"></span>
+    <span class="faceExpression-text"></span>
     <span class="predicted-text"></span>
   </div>
 
@@ -21,7 +22,7 @@
     window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
 
     // uri to weights file of last checkpoint
-    const modelCheckpoint = 'tmp/initial_classifier.weights'
+    const modelCheckpoint = 'tmp/classifier/initial_classifier.weights'
     const startEpoch = 0
 
     const learningRate = 0.1 // 0.001
@@ -32,8 +33,6 @@
     window.iterDelay = 0
     window.withLogging = true
 
-    const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
-
     async function load() {
       window.trainData = await faceapi.fetchJson('trainData.json')
       await dummyLandmarkNet.load('/')
@@ -44,29 +43,10 @@
       window.net.variable()
     }
 
-    function prepareDataForEpoch() {
-      return faceapi.shuffleArray(
-        Object.keys(window.trainData).map(label => {
-          let dataForLabel = window.trainData[label].map(data => ({ ...data, label }))
-          // since train data for "disgusted" have less than 2000 samples
-          // use some data twice to ensure an even distribution
-          dataForLabel = label === 'disgusted'
-            ? faceapi.shuffleArray(dataForLabel.concat(dataForLabel).concat(dataForLabel)).slice(0, 2000)
-            : dataForLabel
-          return dataForLabel
-        }).reduce((flat, arr) => arr.concat(flat))
-      )
-    }
-
-    function getLabelOneHotVector(emotion) {
-      const label = faceapi.FaceExpressionNet.getEmotionLabel(emotion)
-      return Array(7).fill(0).map((_, i) => i === label ? 1 : 0)
-    }
-
     async function train() {
       await load()
 
-      const shuffledInputs = prepareDataForEpoch().slice(0, window.batchSize)
+      const shuffledInputs = prepareDataForEpoch(window.trainData).slice(0, window.batchSize)
       const batchData = shuffledInputs
       const bImages = await Promise.all(
         batchData
@@ -83,21 +63,21 @@
         console.log(i, batchData[i].label, batchData[i].img)
 
         const squaredImg = faceapi.imageToSquare(img, 112, true)
-        const emotions = faceapi.FaceExpressionNet
-          .decodeEmotions(bOneHotVectors[i])
+        const faceExpressions = faceapi.FaceExpressionNet
+          .decodeProbabilites(bOneHotVectors[i])
           .filter(e => e.probability > 0)
 
         const clone = template.cloneNode(true)
         clone.id = i
         const span = clone.firstElementChild
-        span.innerHTML = i + ':' + emotions[0].label
+        span.innerHTML = i + ':' + faceExpressions[0].faceExpression
         clone.insertBefore(squaredImg, span)
         container.appendChild(clone)
       })
 
       for (let epoch = startEpoch; epoch < Infinity; epoch++) {
 
-        const bottleneckFeatures = await window.net.faceFeatureExtractor.forward(bImages)
+        const bottleneckFeatures = await window.net.faceFeatureExtractor.runNet(bImages)
 
         const loss = optimizer.minimize(() => {
           const labels = tf.tensor2d(bOneHotVectors)
@@ -109,18 +89,18 @@
             tf.Reduction.MEAN
           )
 
-          const predictedByBatch = tf.unstack(out)
+          const predictedByBatch = tf.unstack(tf.softmax(out))
           predictedByBatch.forEach((p, i) => {
             const probabilities = Array.from(p.dataSync())
-            const emotions = faceapi.FaceExpressionNet.decodeEmotions(probabilities)
+            const faceExpressions = faceapi.FaceExpressionNet.decodeProbabilites(probabilities)
             const container = document.getElementById(i)
 
-            const pred = emotions.reduce((best, curr) => curr.probability > best.probability ? curr : best)
+            const pred = faceExpressions.reduce((best, curr) => curr.probability > best.probability ? curr : best)
 
             const predNode = container.children[container.children.length - 1]
 
             predNode.innerHTML =
-              pred.label + ' (' +  faceapi.round(pred.probability) + ')'
+              pred.expression + ' (' +  faceapi.round(pred.probability) + ')'
           })
 
           return loss

tools/train/faceExpressions/public/train_debug.html

+<!DOCTYPE html>
+<html>
+<head>
+  <script src="face-api.js"></script>
+  <script src="FileSaver.js"></script>
+  <script src="commons.js"></script>
+  <script src="js/faceExpressionsCommons.js"></script>
+</head>
+<body>
+  <div id="container"></div>
+  <div id="template" style="display: inline-flex; flex-direction: column;">
+    <span class="faceExpression-text"></span>
+    <span class="predicted-text"></span>
+  </div>
+
+  <script>
+    tf = faceapi.tf
+
+    // load the FaceLandmark68Net and use it's feature extractor since we only
+    // train the output layer of the FaceExpressionNet
+    const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
+    window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
+
+    // uri to weights file of last checkpoint
+    const modelCheckpoint = 'tmp/full/initial_glorot.weights'
+    const startEpoch = 0
+
+    const learningRate = 0.001
+    window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
+
+    window.batchSize = 8
+
+    window.iterDelay = 0
+    window.withLogging = true
+
+    async function load() {
+      window.trainData = await faceapi.fetchJson('trainData.json')
+
+      // fetch the actual output layer weights
+      const weights = await faceapi.fetchNetWeights(modelCheckpoint)
+      await window.net.load(weights)
+      window.net.faceFeatureExtractor.variable()
+      window.net.variable()
+    }
+
+    async function train() {
+      await load()
+
+      const shuffledInputs = prepareDataForEpoch(window.trainData).slice(0, window.batchSize)
+      const batchData = shuffledInputs
+      const bImages = await Promise.all(
+        batchData
+          .map(data => getImageUrl(data))
+          .map(imgUrl => faceapi.fetchImage(imgUrl))
+      )
+      const bOneHotVectors = batchData
+        .map(data => getLabelOneHotVector(data.label))
+
+      const container = document.getElementById('container')
+      const template = document.getElementById('template')
+
+      bImages.forEach((img, i) => {
+        console.log(i, batchData[i].label, batchData[i].img)
+
+        const squaredImg = faceapi.imageToSquare(img, 112, true)
+        const faceExpressions = faceapi.FaceExpressionNet
+          .decodeProbabilites(bOneHotVectors[i])
+          .filter(e => e.probability > 0)
+
+        const clone = template.cloneNode(true)
+        clone.id = i
+        const span = clone.firstElementChild
+        span.innerHTML = i + ':' + faceExpressions[0].expression
+        clone.insertBefore(squaredImg, span)
+        container.appendChild(clone)
+      })
+
+      for (let epoch = startEpoch; epoch < Infinity; epoch++) {
+
+        const netInput = await faceapi.toNetInput(bImages)
+
+        const loss = optimizer.minimize(() => {
+          const labels = tf.tensor2d(bOneHotVectors)
+          const out = window.net.runNet(netInput)
+
+          const loss = tf.losses.softmaxCrossEntropy(
+            labels,
+            out,
+            tf.Reduction.MEAN
+          )
+
+          const predictedByBatch = tf.unstack(tf.softmax(out))
+          predictedByBatch.forEach((p, i) => {
+            const probabilities = Array.from(p.dataSync())
+            const faceExpressions = faceapi.FaceExpressionNet.decodeProbabilites(probabilities)
+            const container = document.getElementById(i)
+
+            const pred = faceExpressions.reduce((best, curr) => curr.probability > best.probability ? curr : best)
+
+            const predNode = container.children[container.children.length - 1]
+
+            predNode.innerHTML =
+              pred.expression + ' (' +  faceapi.round(pred.probability) + ')'
+          })
+
+          return loss
+        }, true)
+
+        // start next iteration without waiting for loss data
+
+        loss.data().then(data => {
+          const lossValue = data[0]
+          log(`epoch ${epoch}, loss: ${lossValue}`)
+          loss.dispose()
+        })
+
+        if (window.iterDelay) {
+          await delay(window.iterDelay)
+        } else {
+          await tf.nextFrame()
+        }
+      }
+    }
+
+  </script>
+</body>
+</html>
\ No newline at end of file

tools/train/faceExpressions/server.js

@@ -8,14 +8,16 @@ const app = express()
 
 const publicDir = path.join(__dirname, './public')
 app.use(express.static(publicDir))
-app.use(express.static(path.join(__dirname, '../shared')))
+app.use(express.static(path.join(__dirname, '../js')))
 app.use(express.static(path.join(__dirname, '../node_modules/file-saver')))
 app.use(express.static(path.join(__dirname, '../../../weights')))
 app.use(express.static(path.join(__dirname, '../../../dist')))
 app.use(express.static(path.resolve(process.env.DATA_PATH)))
 
 app.get('/', (req, res) => res.redirect('/train'))
-app.get('/train', (req, res) => res.sendFile(path.join(publicDir, 'trainClassifier.html')))
-app.get('/test', (req, res) => res.sendFile(path.join(publicDir, 'testClassifier.html')))
+app.get('/train', (req, res) => res.sendFile(path.join(publicDir, 'train.html')))
+app.get('/test', (req, res) => res.sendFile(path.join(publicDir, 'test.html')))
+app.get('/trainClassifier', (req, res) => res.sendFile(path.join(publicDir, 'trainClassifier.html')))
+app.get('/testClassifier', (req, res) => res.sendFile(path.join(publicDir, 'testClassifier.html')))
 
 app.listen(8000, () => console.log('Listening on port 8000!'))
\ No newline at end of file

tools/train/faceLandmarks/server.js

@@ -9,7 +9,7 @@ const app = express()
 const publicDir = path.join(__dirname, './public')
 app.use(express.static(publicDir))
 
-app.use(express.static(path.join(__dirname, '../shared')))
+app.use(express.static(path.join(__dirname, '../js')))
 app.use(express.static(path.join(__dirname, '../node_modules/file-saver')))
 app.use(express.static(path.join(__dirname, '../../../weights')))
 app.use(express.static(path.join(__dirname, '../../../dist')))

tools/train/js/commons.js

+const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
+
+function saveWeights(netOrParams, filename = 'train_tmp') {
+  saveAs(new Blob([netOrParams instanceof Float32Array ? netOrParams : netOrParams.serializeParams()]), filename)
+}
+
+async function delay(ms) {
+  return new Promise(res => setTimeout(res, ms))
+}
\ No newline at end of file