Commit da8b102c by vincent

refactored some stuff + train and test script for face expressions net with…

refactored some stuff + train and test script for face expressions net with trained feature extractor
parent ebaf7f22
...@@ -4,41 +4,35 @@ import { NetInput, TNetInput, toNetInput } from 'tfjs-image-recognition-base'; ...@@ -4,41 +4,35 @@ import { NetInput, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
import { FaceFeatureExtractor } from '../faceFeatureExtractor/FaceFeatureExtractor'; import { FaceFeatureExtractor } from '../faceFeatureExtractor/FaceFeatureExtractor';
import { FaceFeatureExtractorParams } from '../faceFeatureExtractor/types'; import { FaceFeatureExtractorParams } from '../faceFeatureExtractor/types';
import { FaceProcessor } from '../faceProcessor/FaceProcessor'; import { FaceProcessor } from '../faceProcessor/FaceProcessor';
import { emotionLabels } from './types'; import { faceExpressionLabels } from './types';
export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams> { export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams> {
public static getEmotionLabel(emotion: string) { public static getFaceExpressionLabel(faceExpression: string) {
const label = emotionLabels[emotion] const label = faceExpressionLabels[faceExpression]
if (typeof label !== 'number') { if (typeof label !== 'number') {
throw new Error(`getEmotionLabel - no label for emotion: ${emotion}`) throw new Error(`getFaceExpressionLabel - no label for faceExpression: ${faceExpression}`)
} }
return label return label
} }
public static decodeEmotions(probabilities: number[] | Float32Array) { public static decodeProbabilites(probabilities: number[] | Float32Array) {
if (probabilities.length !== 7) { if (probabilities.length !== 7) {
throw new Error(`decodeEmotions - expected probabilities.length to be 7, have: ${probabilities.length}`) throw new Error(`decodeProbabilites - expected probabilities.length to be 7, have: ${probabilities.length}`)
} }
return Object.keys(emotionLabels).map(label => ({ label, probability: probabilities[emotionLabels[label]] })) return Object.keys(faceExpressionLabels)
.map(expression => ({ expression, probability: probabilities[faceExpressionLabels[expression]] }))
} }
constructor(faceFeatureExtractor: FaceFeatureExtractor = new FaceFeatureExtractor()) { constructor(faceFeatureExtractor: FaceFeatureExtractor = new FaceFeatureExtractor()) {
super('FaceExpressionNet', faceFeatureExtractor) super('FaceExpressionNet', faceFeatureExtractor)
} }
public runNet(input: NetInput | tf.Tensor4D): tf.Tensor2D {
return tf.tidy(() => {
const out = super.runNet(input)
return tf.softmax(out)
})
}
public forwardInput(input: NetInput | tf.Tensor4D): tf.Tensor2D { public forwardInput(input: NetInput | tf.Tensor4D): tf.Tensor2D {
return tf.tidy(() => this.runNet(input)) return tf.tidy(() => tf.softmax(this.runNet(input)))
} }
public async forward(input: TNetInput): Promise<tf.Tensor2D> { public async forward(input: TNetInput): Promise<tf.Tensor2D> {
...@@ -52,14 +46,7 @@ export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams> ...@@ -52,14 +46,7 @@ export class FaceExpressionNet extends FaceProcessor<FaceFeatureExtractorParams>
out.dispose() out.dispose()
const predictionsByBatch = probabilitesByBatch const predictionsByBatch = probabilitesByBatch
.map(propablities => { .map(propablities => FaceExpressionNet.decodeProbabilites(propablities as Float32Array))
const predictions = {}
FaceExpressionNet.decodeEmotions(propablities as Float32Array)
.forEach(({ label, probability }) => {
predictions[label] = probability
})
return predictions
})
return netInput.isBatchInput return netInput.isBatchInput
? predictionsByBatch ? predictionsByBatch
......
export const emotionLabels = { export const faceExpressionLabels = {
neutral: 0, neutral: 0,
happy: 1, happy: 1,
sad: 2, sad: 2,
......
...@@ -76,7 +76,7 @@ export abstract class FaceProcessor< ...@@ -76,7 +76,7 @@ export abstract class FaceProcessor<
const cIn = this.getClassifierChannelsIn() const cIn = this.getClassifierChannelsIn()
const cOut = this.getClassifierChannelsOut() const cOut = this.getClassifierChannelsOut()
const classifierWeightSize = (cOut * cIn )+ cOut const classifierWeightSize = (cOut * cIn ) + cOut
const featureExtractorWeights = weights.slice(0, weights.length - classifierWeightSize) const featureExtractorWeights = weights.slice(0, weights.length - classifierWeightSize)
const classifierWeights = weights.slice(weights.length - classifierWeightSize) const classifierWeights = weights.slice(weights.length - classifierWeightSize)
......
function getImageUrl({ db, label, img }) {
if (db === 'kaggle') {
return `kaggle-face-expressions-db/${label}/${img}`
}
const id = parseInt(img.replace('.png'))
const dirNr = Math.floor(id / 5000)
return `cropped-faces/jpgs${dirNr + 1}/${img}`
}
function getImageUrl({ db, label, img }) {
if (db === 'kaggle') {
return `kaggle-face-expressions-db/${label}/${img}`
}
const id = parseInt(img.replace('.png'))
const dirNr = Math.floor(id / 5000)
return `cropped-faces/jpgs${dirNr + 1}/${img}`
}
function prepareDataForEpoch(data) {
return faceapi.shuffleArray(
Object.keys(data).map(label => {
let dataForLabel = data[label].map(data => ({ ...data, label }))
// since train data for "disgusted" have less than 2000 samples
// use some data twice to ensure an even distribution
dataForLabel = label === 'disgusted'
? faceapi.shuffleArray(dataForLabel.concat(dataForLabel).concat(dataForLabel)).slice(0, 2000)
: dataForLabel
return dataForLabel
}).reduce((flat, arr) => arr.concat(flat))
)
}
function getLabelOneHotVector(faceExpression) {
const label = faceapi.FaceExpressionNet.getFaceExpressionLabel(faceExpression)
return Array(7).fill(0).map((_, i) => i === label ? 1 : 0)
}
async function onEpochDone(epoch, params) {
saveWeights(params || window.net, `face_expression_model_${epoch}.weights`)
const loss = window.lossValues[epoch]
saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / (2000 * 7) })]), `face_expression_model_${epoch}.json`)
}
async function test(db) {
const faceExpressions = Object.keys(window.testData)
let errors = {}
let preds = {}
let thresh03 = {}
let thresh05 = {}
let thresh08 = {}
let sizes = {}
for (let faceExpression of faceExpressions) {
const container = document.getElementById('container')
const span = document.createElement('div')
container.appendChild(span)
console.log(faceExpression)
const dataForLabel = window.testData[faceExpression]
.filter(data => data.db === db)
.slice(0, window.numDataPerClass)
errors[faceExpression] = 0
preds[faceExpression] = 0
thresh03[faceExpression] = 0
thresh05[faceExpression] = 0
thresh08[faceExpression] = 0
sizes[faceExpression] = dataForLabel.length
for (let [idx, data] of dataForLabel.entries()) {
span.innerHTML = faceExpression + ': ' + faceapi.round(idx / dataForLabel.length) * 100 + '%'
const img = await faceapi.fetchImage(getImageUrl({ ...data, label: faceExpression }))
const pred = await window.net.predictExpressions(img)
const bestPred = pred
.reduce((best, curr) => curr.probability < best.probability ? best : curr)
const { probability } = pred.find(p => p.expression === faceExpression)
thresh03[faceExpression] += (probability > 0.3 ? 1 : 0)
thresh05[faceExpression] += (probability > 0.5 ? 1 : 0)
thresh08[faceExpression] += (probability > 0.8 ? 1 : 0)
errors[faceExpression] += 1 - probability
preds[faceExpression] += (bestPred.expression === faceExpression ? 1 : 0)
}
span.innerHTML = faceExpression + ': 100%'
}
const totalError = faceExpressions.reduce((err, faceExpression) => err + errors[faceExpression], 0)
const relative = (obj) => {
res = {}
Object.keys(sizes).forEach((faceExpression) => {
res[faceExpression] = faceapi.round(
obj[faceExpression] / sizes[faceExpression]
)
})
return res
}
console.log('done...')
console.log('test set size:', sizes)
console.log('preds:', relative(preds))
console.log('thresh03:', relative(thresh03))
console.log('thresh05:', relative(thresh05))
console.log('thresh08:', relative(thresh08))
console.log('errors:', errors)
console.log('total error:', totalError)
}
\ No newline at end of file
async function train() {
await load()
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
if (epoch !== startEpoch) {
// ugly hack to wait for loss datas for that epoch to be resolved
setTimeout(
() => onEpochDone(
epoch - 1,
new Float32Array(Array.from(window.net.faceFeatureExtractor.serializeParams())
.concat(Array.from(window.net.serializeParams()))
)
),
10000
)
}
window.lossValues[epoch] = 0
const shuffledInputs = prepareDataForEpoch(window.trainData)
console.log(shuffledInputs)
for (let dataIdx = 0; dataIdx < shuffledInputs.length; dataIdx += window.batchSize) {
const tsIter = Date.now()
const batchData = shuffledInputs.slice(dataIdx, dataIdx + window.batchSize)
const bImages = await Promise.all(
batchData
.map(data => getImageUrl(data))
.map(imgUrl => faceapi.fetchImage(imgUrl))
)
const bOneHotVectors = batchData
.map(data => getLabelOneHotVector(data.label))
let tsBackward = Date.now()
let tsForward = Date.now()
const netInput = await faceapi.toNetInput(bImages)
tsForward = Date.now() - tsForward
const loss = optimizer.minimize(() => {
tsBackward = Date.now()
const labels = tf.tensor2d(bOneHotVectors)
const out = window.net.runNet(netInput)
const loss = tf.losses.softmaxCrossEntropy(
labels,
out,
tf.Reduction.MEAN
)
return loss
}, true)
tsBackward = Date.now() - tsBackward
// start next iteration without waiting for loss data
loss.data().then(data => {
const lossValue = data[0]
window.lossValues[epoch] += lossValue
window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - loss: ${lossValue}, ( ${window.lossValues[epoch]})`)
loss.dispose()
})
window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - forward: ${tsForward} ms, backprop: ${tsBackward} ms, iter: ${Date.now() - tsIter} ms`)
if (window.logsilly) {
log(`fetch: ${tsFetch} ms, pts: ${tsFetchPts} ms, jpgs: ${tsFetchJpgs} ms, bufferToImage: ${tsBufferToImage} ms`)
}
if (window.iterDelay) {
await delay(window.iterDelay)
} else {
await tf.nextFrame()
}
}
}
}
\ No newline at end of file
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
<script src="js/test.js"></script>
</head>
<body>
<div id="container"></div>
<script>
tf = faceapi.tf
window.numDataPerClass = Infinity
// load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet
window.net = new faceapi.FaceExpressionNet()
// uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/full/face_expression_model_120.weights'
async function load() {
window.testData = await faceapi.fetchJson('testData.json')
// fetch the actual output layer weights
const weights = await faceapi.fetchNetWeights(modelCheckpoint)
await window.net.load(weights)
console.log('loaded')
}
load()
</script>
</body>
</html>
...@@ -3,7 +3,8 @@ ...@@ -3,7 +3,8 @@
<head> <head>
<script src="face-api.js"></script> <script src="face-api.js"></script>
<script src="FileSaver.js"></script> <script src="FileSaver.js"></script>
<script src="js/commons.js"></script> <script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
</head> </head>
<body> <body>
<div id="container"></div> <div id="container"></div>
...@@ -11,13 +12,15 @@ ...@@ -11,13 +12,15 @@
<script> <script>
tf = faceapi.tf tf = faceapi.tf
window.numDataPerClass = 10
// load the FaceLandmark68Net and use it's feature extractor since we only // load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet // train the output layer of the FaceExpressionNet
const dummyLandmarkNet = new faceapi.FaceLandmark68Net() const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor) window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint // uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/face_expression_model_165.weights' const modelCheckpoint = 'tmp/classifier/face_expression_model_118.weights'
async function load() { async function load() {
window.testData = await faceapi.fetchJson('testData.json') window.testData = await faceapi.fetchJson('testData.json')
...@@ -32,53 +35,6 @@ ...@@ -32,53 +35,6 @@
load() load()
async function test() {
const emotions = Object.keys(window.testData)
let errors = {}
let preds = {}
let sizes = {}
for (let emotion of emotions) {
const container = document.getElementById('container')
const span = document.createElement('div')
container.appendChild(span)
console.log(emotion)
const dataForLabel = window.testData[emotion]
errors[emotion] = 0
preds[emotion] = 0
sizes[emotion] = dataForLabel.length
for (let [idx, data] of dataForLabel.entries()) {
span.innerHTML = emotion + ': ' + faceapi.round(idx / dataForLabel.length) * 100 + '%'
const img = await faceapi.fetchImage(getImageUrl({ ...data, label: emotion }))
const pred = await window.net.predictExpressions(img)
const bestPred = Object.keys(pred)
.map(label => ({ label, probability: pred[label] }))
.reduce((best, curr) => curr.probability < best.probability ? curr : best)
errors[emotion] += (1 - pred[emotion])
pred[emotion] += (bestPred.label === emotion ? 1 : 0)
}
span.innerHTML = emotion + ': 100%'
}
const totalError = emotions.reduce((err, emotion) => err + errors[emotion], 0)
console.log('done...')
console.log('test set size:', sizes)
console.log('preds:', preds)
console.log('errors:', errors)
console.log('total error:', totalError)
}
</script> </script>
</body> </body>
</html> </html>
\ No newline at end of file
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
<script src="js/train.js"></script>
</head>
<body>
<div id="container"></div>
<script>
tf = faceapi.tf
// load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet
const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/full/face_expression_model_120.weights'
const startEpoch = 121
const learningRate = 0.00001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.saveEveryNthSample = Infinity
window.batchSize = 8
//window.batchSize = 32
window.lossValues = []
window.iterDelay = 0
window.withLogging = true
async function load() {
window.trainData = await faceapi.fetchJson('trainData.json')
// fetch the actual output layer weights
const weights = await faceapi.fetchNetWeights(modelCheckpoint)
await window.net.load(weights)
window.net.faceFeatureExtractor.variable()
window.net.variable()
}
</script>
</body>
</html>
\ No newline at end of file
...@@ -3,7 +3,8 @@ ...@@ -3,7 +3,8 @@
<head> <head>
<script src="face-api.js"></script> <script src="face-api.js"></script>
<script src="FileSaver.js"></script> <script src="FileSaver.js"></script>
<script src="js/commons.js"></script> <script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
</head> </head>
<body> <body>
<div id="container"></div> <div id="container"></div>
...@@ -17,10 +18,10 @@ ...@@ -17,10 +18,10 @@
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor) window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint // uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/face_expression_model_148.weights' const modelCheckpoint = 'tmp/classifier/face_expression_model_121.weights'
const startEpoch = 149 const startEpoch = 0
const learningRate = 0.001 const learningRate = 0.0001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8) window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.saveEveryNthSample = Infinity window.saveEveryNthSample = Infinity
...@@ -33,16 +34,6 @@ ...@@ -33,16 +34,6 @@
window.iterDelay = 0 window.iterDelay = 0
window.withLogging = true window.withLogging = true
const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
function saveWeights(net, filename = 'train_tmp') {
saveAs(new Blob([net.serializeParams()]), filename)
}
async function delay(ms) {
return new Promise(res => setTimeout(res, ms))
}
async function load() { async function load() {
window.trainData = await faceapi.fetchJson('trainData.json') window.trainData = await faceapi.fetchJson('trainData.json')
await dummyLandmarkNet.load('/') await dummyLandmarkNet.load('/')
...@@ -53,33 +44,6 @@ ...@@ -53,33 +44,6 @@
window.net.variable() window.net.variable()
} }
async function onEpochDone(epoch) {
saveWeights(window.net, `face_expression_model_${epoch}.weights`)
const loss = window.lossValues[epoch]
saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / (2000 * 7) })]), `face_expression_model_${epoch}.json`)
}
function prepareDataForEpoch() {
return faceapi.shuffleArray(
Object.keys(window.trainData).map(label => {
let dataForLabel = window.trainData[label].map(data => ({ ...data, label }))
// since train data for "disgusted" have less than 2000 samples
// use some data twice to ensure an even distribution
dataForLabel = label === 'disgusted'
? faceapi.shuffleArray(dataForLabel.concat(dataForLabel).concat(dataForLabel)).slice(0, 2000)
: dataForLabel
return dataForLabel
}).reduce((flat, arr) => arr.concat(flat))
)
}
function getLabelOneHotVector(emotion) {
const label = faceapi.FaceExpressionNet.getEmotionLabel(emotion)
return Array(7).fill(0).map((_, i) => i === label ? 1 : 0)
}
async function train() { async function train() {
await load() await load()
...@@ -91,8 +55,7 @@ ...@@ -91,8 +55,7 @@
} }
window.lossValues[epoch] = 0 window.lossValues[epoch] = 0
const shuffledInputs = prepareDataForEpoch() const shuffledInputs = prepareDataForEpoch(window.trainData)
console.log(shuffledInputs)
for (let dataIdx = 0; dataIdx < shuffledInputs.length; dataIdx += window.batchSize) { for (let dataIdx = 0; dataIdx < shuffledInputs.length; dataIdx += window.batchSize) {
const tsIter = Date.now() const tsIter = Date.now()
...@@ -114,7 +77,7 @@ ...@@ -114,7 +77,7 @@
const loss = optimizer.minimize(() => { const loss = optimizer.minimize(() => {
tsBackward = Date.now() tsBackward = Date.now()
const labels = tf.tensor2d(bOneHotVectors) const labels = tf.tensor2d(bOneHotVectors)
const out = window.net.forwardInput(bottleneckFeatures) const out = window.net.runNet(bottleneckFeatures)
const loss = tf.losses.softmaxCrossEntropy( const loss = tf.losses.softmaxCrossEntropy(
labels, labels,
......
...@@ -3,12 +3,13 @@ ...@@ -3,12 +3,13 @@
<head> <head>
<script src="face-api.js"></script> <script src="face-api.js"></script>
<script src="FileSaver.js"></script> <script src="FileSaver.js"></script>
<script src="js/commons.js"></script> <script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
</head> </head>
<body> <body>
<div id="container"></div> <div id="container"></div>
<div id="template" style="display: inline-flex; flex-direction: column;"> <div id="template" style="display: inline-flex; flex-direction: column;">
<span class="emotion-text"></span> <span class="faceExpression-text"></span>
<span class="predicted-text"></span> <span class="predicted-text"></span>
</div> </div>
...@@ -21,7 +22,7 @@ ...@@ -21,7 +22,7 @@
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor) window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint // uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/initial_classifier.weights' const modelCheckpoint = 'tmp/classifier/initial_classifier.weights'
const startEpoch = 0 const startEpoch = 0
const learningRate = 0.1 // 0.001 const learningRate = 0.1 // 0.001
...@@ -32,8 +33,6 @@ ...@@ -32,8 +33,6 @@
window.iterDelay = 0 window.iterDelay = 0
window.withLogging = true window.withLogging = true
const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
async function load() { async function load() {
window.trainData = await faceapi.fetchJson('trainData.json') window.trainData = await faceapi.fetchJson('trainData.json')
await dummyLandmarkNet.load('/') await dummyLandmarkNet.load('/')
...@@ -44,29 +43,10 @@ ...@@ -44,29 +43,10 @@
window.net.variable() window.net.variable()
} }
function prepareDataForEpoch() {
return faceapi.shuffleArray(
Object.keys(window.trainData).map(label => {
let dataForLabel = window.trainData[label].map(data => ({ ...data, label }))
// since train data for "disgusted" have less than 2000 samples
// use some data twice to ensure an even distribution
dataForLabel = label === 'disgusted'
? faceapi.shuffleArray(dataForLabel.concat(dataForLabel).concat(dataForLabel)).slice(0, 2000)
: dataForLabel
return dataForLabel
}).reduce((flat, arr) => arr.concat(flat))
)
}
function getLabelOneHotVector(emotion) {
const label = faceapi.FaceExpressionNet.getEmotionLabel(emotion)
return Array(7).fill(0).map((_, i) => i === label ? 1 : 0)
}
async function train() { async function train() {
await load() await load()
const shuffledInputs = prepareDataForEpoch().slice(0, window.batchSize) const shuffledInputs = prepareDataForEpoch(window.trainData).slice(0, window.batchSize)
const batchData = shuffledInputs const batchData = shuffledInputs
const bImages = await Promise.all( const bImages = await Promise.all(
batchData batchData
...@@ -83,21 +63,21 @@ ...@@ -83,21 +63,21 @@
console.log(i, batchData[i].label, batchData[i].img) console.log(i, batchData[i].label, batchData[i].img)
const squaredImg = faceapi.imageToSquare(img, 112, true) const squaredImg = faceapi.imageToSquare(img, 112, true)
const emotions = faceapi.FaceExpressionNet const faceExpressions = faceapi.FaceExpressionNet
.decodeEmotions(bOneHotVectors[i]) .decodeProbabilites(bOneHotVectors[i])
.filter(e => e.probability > 0) .filter(e => e.probability > 0)
const clone = template.cloneNode(true) const clone = template.cloneNode(true)
clone.id = i clone.id = i
const span = clone.firstElementChild const span = clone.firstElementChild
span.innerHTML = i + ':' + emotions[0].label span.innerHTML = i + ':' + faceExpressions[0].faceExpression
clone.insertBefore(squaredImg, span) clone.insertBefore(squaredImg, span)
container.appendChild(clone) container.appendChild(clone)
}) })
for (let epoch = startEpoch; epoch < Infinity; epoch++) { for (let epoch = startEpoch; epoch < Infinity; epoch++) {
const bottleneckFeatures = await window.net.faceFeatureExtractor.forward(bImages) const bottleneckFeatures = await window.net.faceFeatureExtractor.runNet(bImages)
const loss = optimizer.minimize(() => { const loss = optimizer.minimize(() => {
const labels = tf.tensor2d(bOneHotVectors) const labels = tf.tensor2d(bOneHotVectors)
...@@ -109,18 +89,18 @@ ...@@ -109,18 +89,18 @@
tf.Reduction.MEAN tf.Reduction.MEAN
) )
const predictedByBatch = tf.unstack(out) const predictedByBatch = tf.unstack(tf.softmax(out))
predictedByBatch.forEach((p, i) => { predictedByBatch.forEach((p, i) => {
const probabilities = Array.from(p.dataSync()) const probabilities = Array.from(p.dataSync())
const emotions = faceapi.FaceExpressionNet.decodeEmotions(probabilities) const faceExpressions = faceapi.FaceExpressionNet.decodeProbabilites(probabilities)
const container = document.getElementById(i) const container = document.getElementById(i)
const pred = emotions.reduce((best, curr) => curr.probability > best.probability ? curr : best) const pred = faceExpressions.reduce((best, curr) => curr.probability > best.probability ? curr : best)
const predNode = container.children[container.children.length - 1] const predNode = container.children[container.children.length - 1]
predNode.innerHTML = predNode.innerHTML =
pred.label + ' (' + faceapi.round(pred.probability) + ')' pred.expression + ' (' + faceapi.round(pred.probability) + ')'
}) })
return loss return loss
......
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
</head>
<body>
<div id="container"></div>
<div id="template" style="display: inline-flex; flex-direction: column;">
<span class="faceExpression-text"></span>
<span class="predicted-text"></span>
</div>
<script>
tf = faceapi.tf
// load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet
const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/full/initial_glorot.weights'
const startEpoch = 0
const learningRate = 0.001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.batchSize = 8
window.iterDelay = 0
window.withLogging = true
async function load() {
window.trainData = await faceapi.fetchJson('trainData.json')
// fetch the actual output layer weights
const weights = await faceapi.fetchNetWeights(modelCheckpoint)
await window.net.load(weights)
window.net.faceFeatureExtractor.variable()
window.net.variable()
}
async function train() {
await load()
const shuffledInputs = prepareDataForEpoch(window.trainData).slice(0, window.batchSize)
const batchData = shuffledInputs
const bImages = await Promise.all(
batchData
.map(data => getImageUrl(data))
.map(imgUrl => faceapi.fetchImage(imgUrl))
)
const bOneHotVectors = batchData
.map(data => getLabelOneHotVector(data.label))
const container = document.getElementById('container')
const template = document.getElementById('template')
bImages.forEach((img, i) => {
console.log(i, batchData[i].label, batchData[i].img)
const squaredImg = faceapi.imageToSquare(img, 112, true)
const faceExpressions = faceapi.FaceExpressionNet
.decodeProbabilites(bOneHotVectors[i])
.filter(e => e.probability > 0)
const clone = template.cloneNode(true)
clone.id = i
const span = clone.firstElementChild
span.innerHTML = i + ':' + faceExpressions[0].expression
clone.insertBefore(squaredImg, span)
container.appendChild(clone)
})
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
const netInput = await faceapi.toNetInput(bImages)
const loss = optimizer.minimize(() => {
const labels = tf.tensor2d(bOneHotVectors)
const out = window.net.runNet(netInput)
const loss = tf.losses.softmaxCrossEntropy(
labels,
out,
tf.Reduction.MEAN
)
const predictedByBatch = tf.unstack(tf.softmax(out))
predictedByBatch.forEach((p, i) => {
const probabilities = Array.from(p.dataSync())
const faceExpressions = faceapi.FaceExpressionNet.decodeProbabilites(probabilities)
const container = document.getElementById(i)
const pred = faceExpressions.reduce((best, curr) => curr.probability > best.probability ? curr : best)
const predNode = container.children[container.children.length - 1]
predNode.innerHTML =
pred.expression + ' (' + faceapi.round(pred.probability) + ')'
})
return loss
}, true)
// start next iteration without waiting for loss data
loss.data().then(data => {
const lossValue = data[0]
log(`epoch ${epoch}, loss: ${lossValue}`)
loss.dispose()
})
if (window.iterDelay) {
await delay(window.iterDelay)
} else {
await tf.nextFrame()
}
}
}
</script>
</body>
</html>
\ No newline at end of file
...@@ -8,14 +8,16 @@ const app = express() ...@@ -8,14 +8,16 @@ const app = express()
const publicDir = path.join(__dirname, './public') const publicDir = path.join(__dirname, './public')
app.use(express.static(publicDir)) app.use(express.static(publicDir))
app.use(express.static(path.join(__dirname, '../shared'))) app.use(express.static(path.join(__dirname, '../js')))
app.use(express.static(path.join(__dirname, '../node_modules/file-saver'))) app.use(express.static(path.join(__dirname, '../node_modules/file-saver')))
app.use(express.static(path.join(__dirname, '../../../weights'))) app.use(express.static(path.join(__dirname, '../../../weights')))
app.use(express.static(path.join(__dirname, '../../../dist'))) app.use(express.static(path.join(__dirname, '../../../dist')))
app.use(express.static(path.resolve(process.env.DATA_PATH))) app.use(express.static(path.resolve(process.env.DATA_PATH)))
app.get('/', (req, res) => res.redirect('/train')) app.get('/', (req, res) => res.redirect('/train'))
app.get('/train', (req, res) => res.sendFile(path.join(publicDir, 'trainClassifier.html'))) app.get('/train', (req, res) => res.sendFile(path.join(publicDir, 'train.html')))
app.get('/test', (req, res) => res.sendFile(path.join(publicDir, 'testClassifier.html'))) app.get('/test', (req, res) => res.sendFile(path.join(publicDir, 'test.html')))
app.get('/trainClassifier', (req, res) => res.sendFile(path.join(publicDir, 'trainClassifier.html')))
app.get('/testClassifier', (req, res) => res.sendFile(path.join(publicDir, 'testClassifier.html')))
app.listen(8000, () => console.log('Listening on port 8000!')) app.listen(8000, () => console.log('Listening on port 8000!'))
\ No newline at end of file
...@@ -9,7 +9,7 @@ const app = express() ...@@ -9,7 +9,7 @@ const app = express()
const publicDir = path.join(__dirname, './public') const publicDir = path.join(__dirname, './public')
app.use(express.static(publicDir)) app.use(express.static(publicDir))
app.use(express.static(path.join(__dirname, '../shared'))) app.use(express.static(path.join(__dirname, '../js')))
app.use(express.static(path.join(__dirname, '../node_modules/file-saver'))) app.use(express.static(path.join(__dirname, '../node_modules/file-saver')))
app.use(express.static(path.join(__dirname, '../../../weights'))) app.use(express.static(path.join(__dirname, '../../../weights')))
app.use(express.static(path.join(__dirname, '../../../dist'))) app.use(express.static(path.join(__dirname, '../../../dist')))
......
const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
function saveWeights(netOrParams, filename = 'train_tmp') {
saveAs(new Blob([netOrParams instanceof Float32Array ? netOrParams : netOrParams.serializeParams()]), filename)
}
async function delay(ms) {
return new Promise(res => setTimeout(res, ms))
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment