Skip to content

F
face
Commit ebaf7f22 by vincent
Options

fixed train and testdata splitting + init test script

parent 883f20c8
Showing with 287 additions and 24 deletions
src/faceExpressionNet/FaceExpressionNet.ts
...@@ -4,12 +4,12 @@ import { NetInput, TNetInput, toNetInput } from 'tfjs-image-recognition-base'; ...@@ -4,12 +4,12 @@ import { NetInput, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
import { FaceFeatureExtractor } from '../faceFeatureExtractor/FaceFeatureExtractor'; import { FaceFeatureExtractor } from '../faceFeatureExtractor/FaceFeatureExtractor';
import { FaceFeatureExtractorParams } from '../faceFeatureExtractor/types'; import { FaceFeatureExtractorParams } from '../faceFeatureExtractor/types';
import { FaceProcessor } from '../faceProcessor/FaceProcessor'; import { FaceProcessor } from '../faceProcessor/FaceProcessor';
import { EmotionLabels } from './types'; import { emotionLabels } from './types';
export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams> { export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams> {
public static getEmotionLabel(emotion: string) { public static getEmotionLabel(emotion: string) {
const label = EmotionLabels[emotion.toUpperCase()] const label = emotionLabels[emotion]
if (typeof label !== 'number') { if (typeof label !== 'number') {
throw new Error(`getEmotionLabel - no label for emotion: ${emotion}`) throw new Error(`getEmotionLabel - no label for emotion: ${emotion}`)
...@@ -22,7 +22,8 @@ export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams> ...@@ -22,7 +22,8 @@ export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams>
if (probabilities.length !== 7) { if (probabilities.length !== 7) {
throw new Error(`decodeEmotions - expected probabilities.length to be 7, have: ${probabilities.length}`) throw new Error(`decodeEmotions - expected probabilities.length to be 7, have: ${probabilities.length}`)
} }
return Object.keys(EmotionLabels).map(label => ({ label, probability: probabilities[EmotionLabels[label]] }))
return Object.keys(emotionLabels).map(label => ({ label, probability: probabilities[emotionLabels[label]] }))
} }
constructor(faceFeatureExtractor: FaceFeatureExtractor = new FaceFeatureExtractor()) { constructor(faceFeatureExtractor: FaceFeatureExtractor = new FaceFeatureExtractor()) {
...@@ -45,11 +46,24 @@ export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams> ...@@ -45,11 +46,24 @@ export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams>
} }
public async predictExpressions(input: TNetInput) { public async predictExpressions(input: TNetInput) {
const out = await this.forward(input) const netInput = await toNetInput(input)
const out = await this.forwardInput(netInput)
const probabilitesByBatch = await Promise.all(tf.unstack(out).map(t => t.data())) const probabilitesByBatch = await Promise.all(tf.unstack(out).map(t => t.data()))
out.dispose() out.dispose()
return probabilitesByBatch.map(propablities => FaceExpressionNet.decodeEmotions(propablities as Float32Array)) const predictionsByBatch = probabilitesByBatch
.map(propablities => {
const predictions = {}
FaceExpressionNet.decodeEmotions(propablities as Float32Array)
.forEach(({ label, probability }) => {
predictions[label] = probability
})
return predictions
})
return netInput.isBatchInput
? predictionsByBatch
: predictionsByBatch[0]
} }
public dispose(throwOnRedispose: boolean = true) { public dispose(throwOnRedispose: boolean = true) {
......
src/faceExpressionNet/types.ts
export enum EmotionLabels { export const emotionLabels = {
NEUTRAL = 0, neutral: 0,
HAPPY = 1, happy: 1,
SAD = 2, sad: 2,
ANGRY = 3, angry: 3,
FEARFUL = 4, fearful: 4,
DISGUSTED = 5, disgusted: 5,
SURPRISED = 6 surprised:6
} }
\ No newline at end of file
tools/train/faceExpressions/public/testClassifier.html 0 → 100644
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="js/commons.js"></script>
</head>
<body>
<div id="container"></div>
<script>
tf = faceapi.tf
// load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet
const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/face_expression_model_165.weights'
async function load() {
window.testData = await faceapi.fetchJson('testData.json')
await dummyLandmarkNet.load('/')
// fetch the actual output layer weights
const classifierWeights = await faceapi.fetchNetWeights(modelCheckpoint)
await window.net.loadClassifierParams(classifierWeights)
console.log('loaded')
}
load()
async function test() {
const emotions = Object.keys(window.testData)
let errors = {}
let preds = {}
let sizes = {}
for (let emotion of emotions) {
const container = document.getElementById('container')
const span = document.createElement('div')
container.appendChild(span)
console.log(emotion)
const dataForLabel = window.testData[emotion]
errors[emotion] = 0
preds[emotion] = 0
sizes[emotion] = dataForLabel.length
for (let [idx, data] of dataForLabel.entries()) {
span.innerHTML = emotion + ': ' + faceapi.round(idx / dataForLabel.length) * 100 + '%'
const img = await faceapi.fetchImage(getImageUrl({ ...data, label: emotion }))
const pred = await window.net.predictExpressions(img)
const bestPred = Object.keys(pred)
.map(label => ({ label, probability: pred[label] }))
.reduce((best, curr) => curr.probability < best.probability ? curr : best)
errors[emotion] += (1 - pred[emotion])
pred[emotion] += (bestPred.label === emotion ? 1 : 0)
}
span.innerHTML = emotion + ': 100%'
}
const totalError = emotions.reduce((err, emotion) => err + errors[emotion], 0)
console.log('done...')
console.log('test set size:', sizes)
console.log('preds:', preds)
console.log('errors:', errors)
console.log('total error:', totalError)
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceExpressions/public/testData.json
This source diff could not be displayed because it is too large. You can view the blob instead.
tools/train/faceExpressions/public/trainClassifier.html
...@@ -17,15 +17,15 @@ ...@@ -17,15 +17,15 @@
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor) window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint // uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/initial_classifier.weights' const modelCheckpoint = 'tmp/face_expression_model_148.weights'
const startEpoch = 0 const startEpoch = 149
const learningRate = 0.001 // 0.001 const learningRate = 0.001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8) window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.saveEveryNthSample = Infinity window.saveEveryNthSample = Infinity
window.batchSize = 16 window.batchSize = 32
//window.batchSize = 32 //window.batchSize = 32
window.lossValues = [] window.lossValues = []
...@@ -57,7 +57,7 @@ ...@@ -57,7 +57,7 @@
saveWeights(window.net, `face_expression_model_${epoch}.weights`) saveWeights(window.net, `face_expression_model_${epoch}.weights`)
const loss = window.lossValues[epoch] const loss = window.lossValues[epoch]
saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / window.trainIds.length })]), `face_expression_model_${epoch}.json`) saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / (2000 * 7) })]), `face_expression_model_${epoch}.json`)
} }
......
tools/train/faceExpressions/public/trainClassifier_debug.html 0 → 100644
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="js/commons.js"></script>
</head>
<body>
<div id="container"></div>
<div id="template" style="display: inline-flex; flex-direction: column;">
<span class="emotion-text"></span>
<span class="predicted-text"></span>
</div>
<script>
tf = faceapi.tf
// load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet
const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/initial_classifier.weights'
const startEpoch = 0
const learningRate = 0.1 // 0.001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.batchSize = 32
window.iterDelay = 0
window.withLogging = true
const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
async function load() {
window.trainData = await faceapi.fetchJson('trainData.json')
await dummyLandmarkNet.load('/')
// fetch the actual output layer weights
const classifierWeights = await faceapi.fetchNetWeights(modelCheckpoint)
await window.net.loadClassifierParams(classifierWeights)
window.net.variable()
}
function prepareDataForEpoch() {
return faceapi.shuffleArray(
Object.keys(window.trainData).map(label => {
let dataForLabel = window.trainData[label].map(data => ({ ...data, label }))
// since train data for "disgusted" have less than 2000 samples
// use some data twice to ensure an even distribution
dataForLabel = label === 'disgusted'
? faceapi.shuffleArray(dataForLabel.concat(dataForLabel).concat(dataForLabel)).slice(0, 2000)
: dataForLabel
return dataForLabel
}).reduce((flat, arr) => arr.concat(flat))
)
}
function getLabelOneHotVector(emotion) {
const label = faceapi.FaceExpressionNet.getEmotionLabel(emotion)
return Array(7).fill(0).map((_, i) => i === label ? 1 : 0)
}
async function train() {
await load()
const shuffledInputs = prepareDataForEpoch().slice(0, window.batchSize)
const batchData = shuffledInputs
const bImages = await Promise.all(
batchData
.map(data => getImageUrl(data))
.map(imgUrl => faceapi.fetchImage(imgUrl))
)
const bOneHotVectors = batchData
.map(data => getLabelOneHotVector(data.label))
const container = document.getElementById('container')
const template = document.getElementById('template')
bImages.forEach((img, i) => {
console.log(i, batchData[i].label, batchData[i].img)
const squaredImg = faceapi.imageToSquare(img, 112, true)
const emotions = faceapi.FaceExpressionNet
.decodeEmotions(bOneHotVectors[i])
.filter(e => e.probability > 0)
const clone = template.cloneNode(true)
clone.id = i
const span = clone.firstElementChild
span.innerHTML = i + ':' + emotions[0].label
clone.insertBefore(squaredImg, span)
container.appendChild(clone)
})
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
const bottleneckFeatures = await window.net.faceFeatureExtractor.forward(bImages)
const loss = optimizer.minimize(() => {
const labels = tf.tensor2d(bOneHotVectors)
const out = window.net.forwardInput(bottleneckFeatures)
const loss = tf.losses.softmaxCrossEntropy(
labels,
out,
tf.Reduction.MEAN
)
const predictedByBatch = tf.unstack(out)
predictedByBatch.forEach((p, i) => {
const probabilities = Array.from(p.dataSync())
const emotions = faceapi.FaceExpressionNet.decodeEmotions(probabilities)
const container = document.getElementById(i)
const pred = emotions.reduce((best, curr) => curr.probability > best.probability ? curr : best)
const predNode = container.children[container.children.length - 1]
predNode.innerHTML =
pred.label + ' (' + faceapi.round(pred.probability) + ')'
})
return loss
}, true)
bottleneckFeatures.dispose()
// start next iteration without waiting for loss data
loss.data().then(data => {
const lossValue = data[0]
log(`epoch ${epoch}, loss: ${lossValue}`)
loss.dispose()
})
if (window.iterDelay) {
await delay(window.iterDelay)
} else {
await tf.nextFrame()
}
}
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceExpressions/public/trainData.json
This source diff could not be displayed because it is too large. You can view the blob instead.
tools/train/faceExpressions/server.js
...@@ -16,5 +16,6 @@ app.use(express.static(path.resolve(process.env.DATA_PATH))) ...@@ -16,5 +16,6 @@ app.use(express.static(path.resolve(process.env.DATA_PATH)))
app.get('/', (req, res) => res.redirect('/train')) app.get('/', (req, res) => res.redirect('/train'))
app.get('/train', (req, res) => res.sendFile(path.join(publicDir, 'trainClassifier.html'))) app.get('/train', (req, res) => res.sendFile(path.join(publicDir, 'trainClassifier.html')))
app.get('/test', (req, res) => res.sendFile(path.join(publicDir, 'testClassifier.html')))
app.listen(8000, () => console.log('Listening on port 8000!')) app.listen(8000, () => console.log('Listening on port 8000!'))
\ No newline at end of file
tools/train/faceExpressions/splitDataSet.js
...@@ -34,11 +34,14 @@ const MAX_TRAIN_SAMPLES_PER_CLASS = 2000 ...@@ -34,11 +34,14 @@ const MAX_TRAIN_SAMPLES_PER_CLASS = 2000
require('./.env') require('./.env')
const { shuffleArray } = require('../../../') const { shuffleArray } = require('../../../')
const fs = require('fs') const fs = require('fs')
const path = require('path')
const createImageNameArray = (db, num, ext) => const dbEmotionMapping = JSON.parse(fs.readFileSync(
Array(num).fill(0) path.resolve(
.map((_, i) => `${i}${ext}`) process.env.DATA_PATH,
.map(img => ({ db, img })) 'face-expressions/emotionMapping.json'
)
).toString())
const splitArray = (arr, idx) => [arr.slice(0, idx), arr.slice(idx)] const splitArray = (arr, idx) => [arr.slice(0, idx), arr.slice(idx)]
...@@ -53,8 +56,16 @@ Object.keys(dataDistribution) ...@@ -53,8 +56,16 @@ Object.keys(dataDistribution)
const numDb = Math.floor(Math.min(0.7 * MAX_TRAIN_SAMPLES_PER_CLASS, 0.7 * db)) const numDb = Math.floor(Math.min(0.7 * MAX_TRAIN_SAMPLES_PER_CLASS, 0.7 * db))
const numKaggle = Math.floor(Math.min(MAX_TRAIN_SAMPLES_PER_CLASS - numDb, 0.7 * kaggle)) const numKaggle = Math.floor(Math.min(MAX_TRAIN_SAMPLES_PER_CLASS - numDb, 0.7 * kaggle))
const dbImages = shuffleArray(createImageNameArray('db', db, '.jpg')) const dbImages = shuffleArray(
const kaggleImages = shuffleArray(createImageNameArray('kaggle', kaggle, '.png')) dbEmotionMapping[label]
.map(img => ({ db: 'db', img }))
)
const kaggleImages = shuffleArray(
Array(kaggle).fill(0).map((_, i) => `${i}.png`)
.map(img => ({ db: 'kaggle', img }))
)
const [dbTrain, dbTest] = splitArray(dbImages, numDb) const [dbTrain, dbTest] = splitArray(dbImages, numDb)
const [kaggleTrain, kaggleTest] = splitArray(kaggleImages, numKaggle) const [kaggleTrain, kaggleTest] = splitArray(kaggleImages, numKaggle)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment