Skip to content

F
face
Commit e63ed57a by vincent
Options

moved training and tool scripts to external repo

parent 5061313f
Showing with 0 additions and 2198 deletions
tools/quantize/package.json deleted 100644 → 0
{
"scripts": {
"start": "node server.js"
},
"author": "justadudewhohacks",
"license": "MIT",
"dependencies": {
"express": "^4.16.3",
"file-saver": "^1.3.8"
}
}
tools/quantize/public/quantization.js deleted 100644 → 0
function getQuantizationRange(min, max, qdtype) {
if (qdtype !== 0 && qdtype !== 1) {
throw new Error('qdtype !== 0 && qdtype !== 1: ' + qdtype)
}
const quantMax = qdtype === 0 ? 255 : 65535
const scale = (max - min) / quantMax
let result = { scale, min, max }
if (min <= 0 && 0 <= max) {
const quantizedZeroPoint = (0 - min) / scale
const nudgedZeroPoint = Math.round(quantizedZeroPoint)
result.min = (-nudgedZeroPoint) * scale
result.max = quantMax * scale + result.min
}
return result
}
function quantizeWeights(tensor, qdtype = 0) {
const min = tensor.min().dataSync()[0]
const max = tensor.max().dataSync()[0]
if (min === max) {
return {
scale: 1.0,
min,
qdata: qdtype === 0 ? new Uint8Array(tensor.size) : new Uint16Array(tensor.size)
}
}
const q = getQuantizationRange(min, max, qdtype)
const qdata = tf.round(tf.clipByValue(tensor, q.min, q.max).sub(tf.scalar(q.min)).div(tf.scalar(q.scale))).dataSync()
return {
scale: q.scale,
min: q.min,
qdata: qdtype === 0 ? new Uint8Array(qdata) : new Uint16Array(qdata)
}
}
function dequantizeWeights(qdata, scale, min) {
if (qdata.qdata && qdata.scale && qdata.min) {
return Float32Array.from(qdata.qdata, v => (v * qdata.scale) + qdata.min)
}
return Float32Array.from(qdata, v => (v * scale) + min)
}
tools/quantize/server.js deleted 100644 → 0
const express = require('express')
const path = require('path')
const app = express()
const viewsDir = path.join(__dirname, 'views')
app.use(express.static(viewsDir))
app.use(express.static(path.join(__dirname, './public')))
app.use(express.static(path.join(__dirname, './node_modules/file-saver')))
app.use(express.static(path.join(__dirname, '../../examples/public')))
app.use(express.static(path.join(__dirname, '../../weights')))
app.use(express.static(path.join(__dirname, '../../weights_uncompressed')))
app.use(express.static(path.join(__dirname, '../../dist')))
app.get('/', (req, res) => res.redirect('/quantize_model'))
app.get('/quantize_model', (req, res) => res.sendFile(path.join(viewsDir, 'quantizeModel.html')))
app.listen(3000, () => console.log('Listening on port 3000!'))
\ No newline at end of file
tools/quantize/views/quantizeModel.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="quantization.js"></script>
</head>
<body>
<script>
tf = faceapi.tf
const uncompressedWeightsUri = `tiny_face_detector_model.weights`
const net = new faceapi.TinyFaceDetector()
async function load() {
await net.load(new Float32Array(await (await fetch(uncompressedWeightsUri)).arrayBuffer()))
console.log('net loaded')
}
function getNamedTensors() {
return net.getParamList().map(({ path, tensor }) => ({ name: path, tensor }))
}
const modelName = 'tiny_face_detector'
function makeShards(weightArray) {
const maxLength = 4096 * 1024
const shards = []
let shardIdx = 0
for (let i = 0; i < weightArray.length; i++) {
if (!shards[shardIdx]) {
shards[shardIdx] = []
}
shards[shardIdx].push(weightArray[i])
if (shards[shardIdx].length >= maxLength) {
shardIdx += 1
}
}
return shards.map(shardArray => new Uint8Array(shardArray))
}
async function quantizeAndSave() {
const quantizedTensorArrays = []
const weightEntries = []
getNamedTensors().forEach(({ name, tensor, isSkipQuantization }) => {
const weightEntry = {
name,
shape: tensor.shape,
dtype: tensor.dtype
}
if (isSkipQuantization) {
quantizedTensorArrays.push(new Uint8Array(tensor.dataSync().buffer))
weightEntries.push(weightEntry)
return
}
const { scale, min, qdata } = quantizeWeights(tensor)
console.log(name, { scale, min })
const quantizedWeightEntry = {
...weightEntry,
quantization: { dtype: 'uint8', scale, min }
}
quantizedTensorArrays.push(qdata)
weightEntries.push(quantizedWeightEntry)
})
const quantizedWeights = quantizedTensorArrays
.map(typedArray => Array.from(typedArray))
.reduce((flat, arr) => flat.concat(arr))
const shards = makeShards(quantizedWeights)
console.log('num shards: ', shards.length)
const paths = []
shards.forEach((shardData, i) => {
const shardName = `${modelName}_model-shard${i + 1}`
paths.push(shardName)
saveAs(new Blob([shardData]), shardName)
})
const weightManifest = [{
weights: weightEntries,
paths
}]
saveAs(new Blob([JSON.stringify(weightManifest)]), `${modelName}_model-weights_manifest.json`)
}
load()
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceExpressions/public/initWeights.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.12.0"> </script>
<script src="../../node_modules/file-saver/FileSaver.js"></script>
</head>
<body>
<div class="row side-by-side">
<button
class="waves-effect waves-light btn"
onclick="save()"
>
Save
</button>
</div>
<script>
function toDataArray(tensor) {
return Array.from(tensor.dataSync())
}
function flatten(arrs) {
return arrs.reduce((flat, arr) => flat.concat(arr))
}
function initWeights(initializer) {
const numOutputFilters = 256
const outputSize = 7
const weights = toDataArray(initializer.apply([1, 1, numOutputFilters, 7]))
const bias = toDataArray(tf.zeros([outputSize]))
return new Float32Array(weights.concat(bias))
}
function save() {
const initialWeights = initWeights(
tf.initializers.glorotNormal()
)
saveAs(new Blob([initialWeights]), `initial_glorot.weights`)
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceExpressions/public/js/faceExpressionsCommons.js deleted 100644 → 0
function getImageUrl({ db, label, img }) {
if (db === 'kaggle') {
return `kaggle-face-expressions-db/${label}/${img}`
}
const id = parseInt(img.replace('.png'))
const dirNr = Math.floor(id / 5000)
return `cropped-faces/jpgs${dirNr + 1}/${img}`
}
function prepareDataForEpoch(data) {
return faceapi.shuffleArray(
Object.keys(data).map(label => {
let dataForLabel = data[label].map(data => ({ ...data, label }))
// since train data for "disgusted" have less than 2000 samples
// use some data twice to ensure an even distribution
dataForLabel = label === 'disgusted'
? faceapi.shuffleArray(dataForLabel.concat(dataForLabel).concat(dataForLabel)).slice(0, 2000)
: dataForLabel
return dataForLabel
}).reduce((flat, arr) => arr.concat(flat))
)
}
function getLabelOneHotVector(faceExpression) {
const label = faceapi.FaceExpressionNet.getFaceExpressionLabel(faceExpression)
return Array(7).fill(0).map((_, i) => i === label ? 1 : 0)
}
async function onEpochDone(epoch, params) {
saveWeights(params || window.net, `face_expression_model_${epoch}.weights`)
const loss = window.lossValues[epoch]
saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / (2000 * 7) })]), `face_expression_model_${epoch}.json`)
}
tools/train/faceExpressions/public/js/test.js deleted 100644 → 0
async function test(db) {
const faceExpressions = Object.keys(window.testData)
let errors = {}
let preds = {}
let thresh03 = {}
let thresh05 = {}
let thresh08 = {}
let sizes = {}
for (let faceExpression of faceExpressions) {
const container = document.getElementById('container')
const span = document.createElement('div')
container.appendChild(span)
console.log(faceExpression)
const dataForLabel = window.testData[faceExpression]
.filter(data => data.db === db)
.slice(0, window.numDataPerClass)
errors[faceExpression] = 0
preds[faceExpression] = 0
thresh03[faceExpression] = 0
thresh05[faceExpression] = 0
thresh08[faceExpression] = 0
sizes[faceExpression] = dataForLabel.length
for (let [idx, data] of dataForLabel.entries()) {
span.innerHTML = faceExpression + ': ' + faceapi.round(idx / dataForLabel.length) * 100 + '%'
const img = await faceapi.fetchImage(getImageUrl({ ...data, label: faceExpression }))
const pred = await window.net.predictExpressions(img)
const bestPred = pred
.reduce((best, curr) => curr.probability < best.probability ? best : curr)
const { probability } = pred.find(p => p.expression === faceExpression)
thresh03[faceExpression] += (probability > 0.3 ? 1 : 0)
thresh05[faceExpression] += (probability > 0.5 ? 1 : 0)
thresh08[faceExpression] += (probability > 0.8 ? 1 : 0)
errors[faceExpression] += 1 - probability
preds[faceExpression] += (bestPred.expression === faceExpression ? 1 : 0)
}
span.innerHTML = faceExpression + ': 100%'
}
const totalError = faceExpressions.reduce((err, faceExpression) => err + errors[faceExpression], 0)
const relative = (obj) => {
res = {}
Object.keys(sizes).forEach((faceExpression) => {
res[faceExpression] = faceapi.round(
obj[faceExpression] / sizes[faceExpression]
)
})
return res
}
console.log('done...')
console.log('test set size:', sizes)
console.log('preds:', relative(preds))
console.log('thresh03:', relative(thresh03))
console.log('thresh05:', relative(thresh05))
console.log('thresh08:', relative(thresh08))
console.log('errors:', errors)
console.log('total error:', totalError)
}
\ No newline at end of file
tools/train/faceExpressions/public/js/train.js deleted 100644 → 0
async function train() {
await load()
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
if (epoch !== startEpoch) {
// ugly hack to wait for loss datas for that epoch to be resolved
setTimeout(
() => onEpochDone(
epoch - 1,
new Float32Array(Array.from(window.net.faceFeatureExtractor.serializeParams())
.concat(Array.from(window.net.serializeParams()))
)
),
10000
)
}
window.lossValues[epoch] = 0
const shuffledInputs = prepareDataForEpoch(window.trainData)
console.log(shuffledInputs)
for (let dataIdx = 0; dataIdx < shuffledInputs.length; dataIdx += window.batchSize) {
const tsIter = Date.now()
const batchData = shuffledInputs.slice(dataIdx, dataIdx + window.batchSize)
const bImages = await Promise.all(
batchData
.map(data => getImageUrl(data))
.map(imgUrl => faceapi.fetchImage(imgUrl))
)
const bOneHotVectors = batchData
.map(data => getLabelOneHotVector(data.label))
let tsBackward = Date.now()
let tsForward = Date.now()
const netInput = await faceapi.toNetInput(bImages)
tsForward = Date.now() - tsForward
const loss = optimizer.minimize(() => {
tsBackward = Date.now()
const labels = tf.tensor2d(bOneHotVectors)
const out = window.net.runNet(netInput)
const loss = tf.losses.softmaxCrossEntropy(
labels,
out,
tf.Reduction.MEAN
)
return loss
}, true)
tsBackward = Date.now() - tsBackward
// start next iteration without waiting for loss data
loss.data().then(data => {
const lossValue = data[0]
window.lossValues[epoch] += lossValue
window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - loss: ${lossValue}, ( ${window.lossValues[epoch]})`)
loss.dispose()
})
window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - forward: ${tsForward} ms, backprop: ${tsBackward} ms, iter: ${Date.now() - tsIter} ms`)
if (window.logsilly) {
log(`fetch: ${tsFetch} ms, pts: ${tsFetchPts} ms, jpgs: ${tsFetchJpgs} ms, bufferToImage: ${tsBufferToImage} ms`)
}
if (window.iterDelay) {
await delay(window.iterDelay)
} else {
await tf.nextFrame()
}
}
}
}
\ No newline at end of file
tools/train/faceExpressions/public/test.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
<script src="js/test.js"></script>
</head>
<body>
<div id="container"></div>
<script>
tf = faceapi.tf
window.numDataPerClass = Infinity
// load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet
window.net = new faceapi.FaceExpressionNet()
// uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/full/face_expression_model_120.weights'
async function load() {
window.testData = await faceapi.fetchJson('testData.json')
// fetch the actual output layer weights
const weights = await faceapi.fetchNetWeights(modelCheckpoint)
await window.net.load(weights)
console.log('loaded')
}
load()
</script>
</body>
</html>
tools/train/faceExpressions/public/testClassifier.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
</head>
<body>
<div id="container"></div>
<script>
tf = faceapi.tf
window.numDataPerClass = 10
// load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet
const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/classifier/face_expression_model_118.weights'
async function load() {
window.testData = await faceapi.fetchJson('testData.json')
await dummyLandmarkNet.load('/')
// fetch the actual output layer weights
const classifierWeights = await faceapi.fetchNetWeights(modelCheckpoint)
await window.net.loadClassifierParams(classifierWeights)
console.log('loaded')
}
load()
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceExpressions/public/testData.json deleted 100644 → 0
This source diff could not be displayed because it is too large. You can view the blob instead.
tools/train/faceExpressions/public/train.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
<script src="js/train.js"></script>
</head>
<body>
<div id="container"></div>
<script>
tf = faceapi.tf
// load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet
const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/full/face_expression_model_120.weights'
const startEpoch = 121
const learningRate = 0.00001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.saveEveryNthSample = Infinity
window.batchSize = 8
//window.batchSize = 32
window.lossValues = []
window.iterDelay = 0
window.withLogging = true
async function load() {
window.trainData = await faceapi.fetchJson('trainData.json')
// fetch the actual output layer weights
const weights = await faceapi.fetchNetWeights(modelCheckpoint)
await window.net.load(weights)
window.net.faceFeatureExtractor.variable()
window.net.variable()
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceExpressions/public/trainClassifier.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
</head>
<body>
<div id="container"></div>
<script>
tf = faceapi.tf
// load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet
const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/classifier/face_expression_model_121.weights'
const startEpoch = 0
const learningRate = 0.0001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.saveEveryNthSample = Infinity
window.batchSize = 32
//window.batchSize = 32
window.lossValues = []
window.iterDelay = 0
window.withLogging = true
async function load() {
window.trainData = await faceapi.fetchJson('trainData.json')
await dummyLandmarkNet.load('/')
// fetch the actual output layer weights
const classifierWeights = await faceapi.fetchNetWeights(modelCheckpoint)
await window.net.loadClassifierParams(classifierWeights)
window.net.variable()
}
async function train() {
await load()
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
if (epoch !== startEpoch) {
// ugly hack to wait for loss datas for that epoch to be resolved
setTimeout(() => onEpochDone(epoch - 1), 10000)
}
window.lossValues[epoch] = 0
const shuffledInputs = prepareDataForEpoch(window.trainData)
for (let dataIdx = 0; dataIdx < shuffledInputs.length; dataIdx += window.batchSize) {
const tsIter = Date.now()
const batchData = shuffledInputs.slice(dataIdx, dataIdx + window.batchSize)
const bImages = await Promise.all(
batchData
.map(data => getImageUrl(data))
.map(imgUrl => faceapi.fetchImage(imgUrl))
)
const bOneHotVectors = batchData
.map(data => getLabelOneHotVector(data.label))
let tsBackward = Date.now()
let tsForward = Date.now()
const bottleneckFeatures = await window.net.faceFeatureExtractor.forward(bImages)
tsForward = Date.now() - tsForward
const loss = optimizer.minimize(() => {
tsBackward = Date.now()
const labels = tf.tensor2d(bOneHotVectors)
const out = window.net.runNet(bottleneckFeatures)
const loss = tf.losses.softmaxCrossEntropy(
labels,
out,
tf.Reduction.MEAN
)
return loss
}, true)
tsBackward = Date.now() - tsBackward
bottleneckFeatures.dispose()
// start next iteration without waiting for loss data
loss.data().then(data => {
const lossValue = data[0]
window.lossValues[epoch] += lossValue
window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - loss: ${lossValue}, ( ${window.lossValues[epoch]})`)
loss.dispose()
})
window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - forward: ${tsForward} ms, backprop: ${tsBackward} ms, iter: ${Date.now() - tsIter} ms`)
if (window.logsilly) {
log(`fetch: ${tsFetch} ms, pts: ${tsFetchPts} ms, jpgs: ${tsFetchJpgs} ms, bufferToImage: ${tsBufferToImage} ms`)
}
if (window.iterDelay) {
await delay(window.iterDelay)
} else {
await tf.nextFrame()
}
}
}
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceExpressions/public/trainClassifier_debug.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
</head>
<body>
<div id="container"></div>
<div id="template" style="display: inline-flex; flex-direction: column;">
<span class="faceExpression-text"></span>
<span class="predicted-text"></span>
</div>
<script>
tf = faceapi.tf
// load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet
const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/classifier/initial_classifier.weights'
const startEpoch = 0
const learningRate = 0.1 // 0.001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.batchSize = 32
window.iterDelay = 0
window.withLogging = true
async function load() {
window.trainData = await faceapi.fetchJson('trainData.json')
await dummyLandmarkNet.load('/')
// fetch the actual output layer weights
const classifierWeights = await faceapi.fetchNetWeights(modelCheckpoint)
await window.net.loadClassifierParams(classifierWeights)
window.net.variable()
}
async function train() {
await load()
const shuffledInputs = prepareDataForEpoch(window.trainData).slice(0, window.batchSize)
const batchData = shuffledInputs
const bImages = await Promise.all(
batchData
.map(data => getImageUrl(data))
.map(imgUrl => faceapi.fetchImage(imgUrl))
)
const bOneHotVectors = batchData
.map(data => getLabelOneHotVector(data.label))
const container = document.getElementById('container')
const template = document.getElementById('template')
bImages.forEach((img, i) => {
console.log(i, batchData[i].label, batchData[i].img)
const squaredImg = faceapi.imageToSquare(img, 112, true)
const faceExpressions = faceapi.FaceExpressionNet
.decodeProbabilites(bOneHotVectors[i])
.filter(e => e.probability > 0)
const clone = template.cloneNode(true)
clone.id = i
const span = clone.firstElementChild
span.innerHTML = i + ':' + faceExpressions[0].faceExpression
clone.insertBefore(squaredImg, span)
container.appendChild(clone)
})
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
const bottleneckFeatures = await window.net.faceFeatureExtractor.runNet(bImages)
const loss = optimizer.minimize(() => {
const labels = tf.tensor2d(bOneHotVectors)
const out = window.net.forwardInput(bottleneckFeatures)
const loss = tf.losses.softmaxCrossEntropy(
labels,
out,
tf.Reduction.MEAN
)
const predictedByBatch = tf.unstack(tf.softmax(out))
predictedByBatch.forEach((p, i) => {
const probabilities = Array.from(p.dataSync())
const faceExpressions = faceapi.FaceExpressionNet.decodeProbabilites(probabilities)
const container = document.getElementById(i)
const pred = faceExpressions.reduce((best, curr) => curr.probability > best.probability ? curr : best)
const predNode = container.children[container.children.length - 1]
predNode.innerHTML =
pred.expression + ' (' + faceapi.round(pred.probability) + ')'
})
return loss
}, true)
bottleneckFeatures.dispose()
// start next iteration without waiting for loss data
loss.data().then(data => {
const lossValue = data[0]
log(`epoch ${epoch}, loss: ${lossValue}`)
loss.dispose()
})
if (window.iterDelay) {
await delay(window.iterDelay)
} else {
await tf.nextFrame()
}
}
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceExpressions/public/trainData.json deleted 100644 → 0
This source diff could not be displayed because it is too large. You can view the blob instead.
tools/train/faceExpressions/public/train_debug.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="FileSaver.js"></script>
<script src="commons.js"></script>
<script src="js/faceExpressionsCommons.js"></script>
</head>
<body>
<div id="container"></div>
<div id="template" style="display: inline-flex; flex-direction: column;">
<span class="faceExpression-text"></span>
<span class="predicted-text"></span>
</div>
<script>
tf = faceapi.tf
// load the FaceLandmark68Net and use it's feature extractor since we only
// train the output layer of the FaceExpressionNet
const dummyLandmarkNet = new faceapi.FaceLandmark68Net()
window.net = new faceapi.FaceExpressionNet(dummyLandmarkNet.faceFeatureExtractor)
// uri to weights file of last checkpoint
const modelCheckpoint = 'tmp/full/initial_glorot.weights'
const startEpoch = 0
const learningRate = 0.001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.batchSize = 8
window.iterDelay = 0
window.withLogging = true
async function load() {
window.trainData = await faceapi.fetchJson('trainData.json')
// fetch the actual output layer weights
const weights = await faceapi.fetchNetWeights(modelCheckpoint)
await window.net.load(weights)
window.net.faceFeatureExtractor.variable()
window.net.variable()
}
async function train() {
await load()
const shuffledInputs = prepareDataForEpoch(window.trainData).slice(0, window.batchSize)
const batchData = shuffledInputs
const bImages = await Promise.all(
batchData
.map(data => getImageUrl(data))
.map(imgUrl => faceapi.fetchImage(imgUrl))
)
const bOneHotVectors = batchData
.map(data => getLabelOneHotVector(data.label))
const container = document.getElementById('container')
const template = document.getElementById('template')
bImages.forEach((img, i) => {
console.log(i, batchData[i].label, batchData[i].img)
const squaredImg = faceapi.imageToSquare(img, 112, true)
const faceExpressions = faceapi.FaceExpressionNet
.decodeProbabilites(bOneHotVectors[i])
.filter(e => e.probability > 0)
const clone = template.cloneNode(true)
clone.id = i
const span = clone.firstElementChild
span.innerHTML = i + ':' + faceExpressions[0].expression
clone.insertBefore(squaredImg, span)
container.appendChild(clone)
})
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
const netInput = await faceapi.toNetInput(bImages)
const loss = optimizer.minimize(() => {
const labels = tf.tensor2d(bOneHotVectors)
const out = window.net.runNet(netInput)
const loss = tf.losses.softmaxCrossEntropy(
labels,
out,
tf.Reduction.MEAN
)
const predictedByBatch = tf.unstack(tf.softmax(out))
predictedByBatch.forEach((p, i) => {
const probabilities = Array.from(p.dataSync())
const faceExpressions = faceapi.FaceExpressionNet.decodeProbabilites(probabilities)
const container = document.getElementById(i)
const pred = faceExpressions.reduce((best, curr) => curr.probability > best.probability ? curr : best)
const predNode = container.children[container.children.length - 1]
predNode.innerHTML =
pred.expression + ' (' + faceapi.round(pred.probability) + ')'
})
return loss
}, true)
// start next iteration without waiting for loss data
loss.data().then(data => {
const lossValue = data[0]
log(`epoch ${epoch}, loss: ${lossValue}`)
loss.dispose()
})
if (window.iterDelay) {
await delay(window.iterDelay)
} else {
await tf.nextFrame()
}
}
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceExpressions/server.js deleted 100644 → 0
require('./.env')
const express = require('express')
const path = require('path')
const fs = require('fs')
const app = express()
const publicDir = path.join(__dirname, './public')
app.use(express.static(publicDir))
app.use(express.static(path.join(__dirname, '../js')))
app.use(express.static(path.join(__dirname, '../node_modules/file-saver')))
app.use(express.static(path.join(__dirname, '../../../weights')))
app.use(express.static(path.join(__dirname, '../../../dist')))
app.use(express.static(path.resolve(process.env.DATA_PATH)))
app.get('/', (req, res) => res.redirect('/train'))
app.get('/train', (req, res) => res.sendFile(path.join(publicDir, 'train.html')))
app.get('/test', (req, res) => res.sendFile(path.join(publicDir, 'test.html')))
app.get('/trainClassifier', (req, res) => res.sendFile(path.join(publicDir, 'trainClassifier.html')))
app.get('/testClassifier', (req, res) => res.sendFile(path.join(publicDir, 'testClassifier.html')))
app.listen(8000, () => console.log('Listening on port 8000!'))
\ No newline at end of file
tools/train/faceExpressions/splitDataSet.js deleted 100644 → 0
const dataDistribution = {
angry: {
db: 1147,
kaggle: 4953
},
disgusted: {
db: 690,
kaggle: 547
},
fearful: {
db: 844,
kaggle: 5121
},
happy: {
db: 8634,
kaggle: 8989
},
neutral: {
db: 1262,
kaggle: 6198
},
sad: {
db: 929,
kaggle: 6077
},
surprised: {
db: 1264,
kaggle: 4002
}
}
const MAX_TRAIN_SAMPLES_PER_CLASS = 2000
require('./.env')
const { shuffleArray } = require('../../../')
const fs = require('fs')
const path = require('path')
const dbEmotionMapping = JSON.parse(fs.readFileSync(
path.resolve(
process.env.DATA_PATH,
'face-expressions/emotionMapping.json'
)
).toString())
const splitArray = (arr, idx) => [arr.slice(0, idx), arr.slice(idx)]
const trainData = {}
const testData = {}
Object.keys(dataDistribution)
.forEach(label => {
const { db, kaggle } = dataDistribution[label]
// take max 0.7 percent of db, take rest from kaggle db
const numDb = Math.floor(Math.min(0.7 * MAX_TRAIN_SAMPLES_PER_CLASS, 0.7 * db))
const numKaggle = Math.floor(Math.min(MAX_TRAIN_SAMPLES_PER_CLASS - numDb, 0.7 * kaggle))
const dbImages = shuffleArray(
dbEmotionMapping[label]
.map(img => ({ db: 'db', img }))
)
const kaggleImages = shuffleArray(
Array(kaggle).fill(0).map((_, i) => `${i}.png`)
.map(img => ({ db: 'kaggle', img }))
)
const [dbTrain, dbTest] = splitArray(dbImages, numDb)
const [kaggleTrain, kaggleTest] = splitArray(kaggleImages, numKaggle)
console.log()
console.log('%s:', label)
console.log('train data - db: %s, kaggle: %s', dbTrain.length, kaggleTrain.length)
console.log('test data - db: %s, kaggle: %s', dbTest.length, kaggleTest.length)
trainData[label] = dbTrain.concat(kaggleTrain)
testData[label] = dbTest.concat(kaggleTest)
})
fs.writeFileSync('trainData.json', JSON.stringify(trainData))
fs.writeFileSync('testData.json', JSON.stringify(testData))
tools/train/faceLandmarks/public/initWeightsDensenet.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.12.0"> </script>
<script src="../../node_modules/file-saver/FileSaver.js"></script>
</head>
<body>
<div class="row side-by-side">
<button
class="waves-effect waves-light btn"
onclick="save()"
>
Save
</button>
<button
class="waves-effect waves-light btn"
onclick="save(true)"
>
Save Tiny
</button>
</div>
<script>
function toDataArray(tensor) {
return Array.from(tensor.dataSync())
}
function flatten(arrs) {
return arrs.reduce((flat, arr) => flat.concat(arr))
}
function initWeights(initializer, isTiny) {
function initConvWeights(inChannels, outChannels) {
return flatten(
[
// filter
initializer.apply([3, 3, inChannels, outChannels]),
// bias
tf.zeros([outChannels])
]
.map(toDataArray)
)
}
function initSeparableConvWeights(inChannels, outChannels) {
return flatten(
[
// depthwise filter
initializer.apply([3, 3, inChannels, 1]),
// pointwise filter
initializer.apply([1, 1, inChannels, outChannels]),
// bias
tf.zeros([outChannels])
]
.map(toDataArray)
)
}
const separableConvWeights = isTiny
? flatten([
initConvWeights(3, 32),
initSeparableConvWeights(32, 32),
initSeparableConvWeights(32, 32),
initSeparableConvWeights(32, 64),
initSeparableConvWeights(64, 64),
initSeparableConvWeights(64, 64),
initSeparableConvWeights(64, 128),
initSeparableConvWeights(128, 128),
initSeparableConvWeights(128, 128)
])
: flatten([
initConvWeights(3, 32),
initSeparableConvWeights(32, 32),
initSeparableConvWeights(32, 32),
initSeparableConvWeights(32, 32),
initSeparableConvWeights(32, 64),
initSeparableConvWeights(64, 64),
initSeparableConvWeights(64, 64),
initSeparableConvWeights(64, 64),
initSeparableConvWeights(64, 128),
initSeparableConvWeights(128, 128),
initSeparableConvWeights(128, 128),
initSeparableConvWeights(128, 128),
initSeparableConvWeights(128, 256),
initSeparableConvWeights(256, 256),
initSeparableConvWeights(256, 256),
initSeparableConvWeights(256, 256)
])
const fc = flatten(
[
initializer.apply([1, 1, isTiny ? 128 : 256, 136]),
// bias
tf.zeros([136])
]
.map(toDataArray)
)
return new Float32Array(separableConvWeights.concat(fc))
}
function save(isTiny = false) {
const initialWeights = initWeights(
tf.initializers.glorotNormal(),
isTiny
)
saveAs(new Blob([initialWeights]), `initial_glorot.weights`)
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceLandmarks/public/js/indexedDb.js deleted 100644 → 0
const indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB || window.shimIndexedDB
let jsonsDb = null
let jpgsDb = null
const openJsons = indexedDB.open('jsons', 1)
const openJpgs = indexedDB.open('jpgs', 1)
openJsons.onupgradeneeded = function() {
jsonsDb = openJsons.result
jsonsDb.createObjectStore('jsons', { keyPath: 'id' })
}
openJpgs.onupgradeneeded = function() {
jpgsDb = openJpgs.result
jpgsDb.createObjectStore('jpgs', { keyPath: 'id' })
}
openJsons.onsuccess = function() {
console.log('connected to jsons')
jsonsDb = openJsons.result
}
openJpgs.onsuccess = function() {
console.log('connected to jpgs')
jpgsDb = openJpgs.result
}
function putReq(store, obj) {
return new Promise((res, rej) => {
const req = store.put(obj)
req.onsuccess = res
req.onerror = rej
})
}
function getReq(store, id, throwIfNoResult = true) {
return new Promise((res, rej) => {
const req = store.get(id)
req.onsuccess = () => {
if (!req.result && throwIfNoResult) {
return rej(`no result for id: ${id}`)
}
res(req.result)
}
req.onerror = rej
})
}
function existsReq(store, id) {
return getReq(store, id, false)
}
async function getNotFound(store, ids) {
return (await Promise.all(ids.map(async id => ({ id, exists: await existsReq(store, id) }))))
.filter(({ exists }) => !exists)
.map(({ id }) => id)
}
async function getNotFoundPts(ids) {
const store = jsonsDb.transaction('jsons', 'readonly').objectStore('jsons')
return getNotFound(store, ids)
}
async function getNotFoundJpegs(ids) {
const store = jpgsDb.transaction('jpgs', 'readonly').objectStore('jpgs')
return getNotFound(store, ids)
}
async function persistPts(ptsById, overwrite = false) {
const store = jsonsDb.transaction('jsons', 'readwrite').objectStore('jsons')
for (let i = 0; i < ptsById.length; i++) {
const { id, pts } = ptsById[i]
if (!await existsReq(store, id)) {
console.log('persistPts - inserting %s', id)
await putReq(store, { id, pts })
}
}
}
function getPts(ids) {
const store = jsonsDb.transaction('jsons', 'readonly').objectStore('jsons')
return Promise.all(ids.map(id => getReq(store, id)))
}
async function persistJpgs(jpgsById, overwrite = false) {
const store = jpgsDb.transaction('jpgs', 'readwrite').objectStore('jpgs')
for (let i = 0; i < jpgsById.length; i++) {
const { id, blob } = jpgsById[i]
if (!await existsReq(store, id)) {
console.log('persistJpgs - inserting %s', id)
await putReq(store, { id, blob })
}
}
}
function getJpgs(ids) {
const store = jpgsDb.transaction('jpgs', 'readonly').objectStore('jpgs')
return Promise.all(ids.map(id => getReq(store, id)))
}
tools/train/faceLandmarks/public/js/printIndexDbSizes.js deleted 100644 → 0
// https://gist.github.com/tralves/9e5de2bd9f582007a52708d7d4209865
var getTableSize = function(db, dbName){
return new Promise((resolve,reject) => {
if (db == null) {
return reject();
}
var size = 0;
db = event.target.result;
var transaction = db.transaction([dbName])
.objectStore(dbName)
.openCursor();
transaction.onsuccess = function(event){
var cursor = event.target.result;
if(cursor){
var storedObject = cursor.value;
var json = JSON.stringify(storedObject);
size += json.length;
cursor.continue();
}
else{
resolve(size);
}
}.bind(this);
transaction.onerror = function(err){
reject("error in " + dbName + ": " + err);
}
});
};
var getDatabaseSize = function (dbName) {
var request = indexedDB.open(dbName);
var db;
var dbSize = 0;
request.onerror = function(event) {
alert("Why didn't you allow my web app to use IndexedDB?!");
};
request.onsuccess = function(event) {
db = event.target.result;
var tableNames = [ ...db.objectStoreNames ];
(function(tableNames, db) {
var tableSizeGetters = tableNames
.reduce( (acc, tableName) => {
acc.push( getTableSize(db, tableName) );
return acc;
}, []);
Promise.all(tableSizeGetters)
.then(sizes => {
console.log('--------- ' + db.name + ' -------------');
tableNames.forEach( (tableName,i) => {
console.log(" - " + tableName + "\t: " + humanReadableSize(sizes[i]));
});
var total = sizes.reduce(function(acc, val) {
return acc + val;
}, 0);
console.log("TOTAL: " + humanReadableSize(total))
});
})(tableNames, db);
};
};
var humanReadableSize = function (bytes) {
var thresh = 1024;
if(Math.abs(bytes) < thresh) {
return bytes + ' B';
}
var units = ['KB','MB','GB','TB','PB','EB','ZB','YB'];
var u = -1;
do {
bytes /= thresh;
++u;
} while(Math.abs(bytes) >= thresh && u < units.length - 1);
return bytes.toFixed(1)+' '+units[u];
}
getDatabaseSize('jsons')
getDatabaseSize('jpgs')
\ No newline at end of file
tools/train/faceLandmarks/public/js/randomCrop.js deleted 100644 → 0
function randomCrop(img, pts) {
const xCoords = pts.map(pt => pt.x)
const yCoords = pts.map(pt => pt.y)
const minX = xCoords.reduce((x, min) => x < min ? x : min, img.width)
const minY = yCoords.reduce((y, min) => y < min ? y : min, img.height)
const maxX = xCoords.reduce((x, max) => x < max ? max : x, 0)
const maxY = yCoords.reduce((y, max) => y < max ? max : y, 0)
const x0 = Math.random() * minX
const y0 = Math.random() * minY
const x1 = (Math.random() * Math.abs(img.width - maxX)) + maxX
const y1 = (Math.random() * Math.abs(img.height - maxY)) + maxY
const targetCanvas = faceapi.createCanvas({ width: x1 - x0, height: y1 - y0 })
const inputCanvas = img instanceof HTMLCanvasElement ? img : faceapi.createCanvasFromMedia(img)
const region = faceapi.getContext2dOrThrow(inputCanvas)
.getImageData(x0, y0, targetCanvas.width, targetCanvas.height)
faceapi.getContext2dOrThrow(targetCanvas).putImageData(region, 0, 0)
return {
croppedImage: targetCanvas,
shiftedPts: pts.map(pt => new faceapi.Point(pt.x - x0, pt.y - y0))
}
}
\ No newline at end of file
tools/train/faceLandmarks/public/js/synchronizeDb.js deleted 100644 → 0
async function fillPtsDb(ids) {
const numFetchPerIteration = 4000
for (let i = 0; i < Math.floor(ids.length / numFetchPerIteration) + 1; i++) {
const ptsById = await Promise.all(
ids.slice(i * numFetchPerIteration, (i + 1) * numFetchPerIteration)
.map(async id => ({
id,
pts: await (await fetch(`${id}.json`)).json()
}))
)
console.log('persistPts')
console.time('persistPts')
await persistPts(ptsById)
console.timeEnd('persistPts')
}
}
async function fillJpgsDb(ids) {
const numFetchPerIteration = 4000
for (let i = 0; i < Math.floor(ids.length / numFetchPerIteration) + 1; i++) {
const jpgsById = await Promise.all(
ids.slice(i * numFetchPerIteration, (i + 1) * numFetchPerIteration)
.map(async id => ({
id,
blob: await fetchImage(`${id}.jpg`)
}))
)
console.log('persistJpgs')
console.time('persistJpgs')
await persistJpgs(jpgsById)
console.timeEnd('persistJpgs')
}
}
\ No newline at end of file
tools/train/faceLandmarks/public/test.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<script src="js/randomCrop.js"></script>
<script src="js/indexedDb.js"></script>
<script src="js/synchronizeDb.js"></script>
<link rel="stylesheet" href="styles.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.css">
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/js/materialize.min.js"></script>
</head>
<body>
<div id="result" style="position: relative" class="margin">
<img id="inputImg" src="" style="max-width: 400px;" />
<canvas id="overlay" />
</div>
<script>
const net = new faceapi.FaceLandmark68Net()
const modelCheckpoint = 'tmp/densenet4/checkpoints/landmarks_epoch46_lr00001_12_lr000001_18.weights'
const crops = 4
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}
async function load() {
await net.load('./')
//const weights = await loadNetWeights(modelCheckpoint)
//await net.load(weights)
console.log('model loaded')
}
load()
function pointsToFlatArray(pts) {
return pts.map(pt => [pt.x, pt.y]).reduce((flat, arr) => flat.concat(arr))
}
async function showResults(landmarks) {
const inputImgEl = $('#inputImg').get(0)
const { width, height } = inputImgEl
const canvas = $('#overlay').get(0)
canvas.width = width
canvas.height = height
faceapi.drawLandmarks('overlay', landmarks, { drawLines: true })
}
async function testAll() {
window.testIds = (await fetch('/face_landmarks_test_ids').then(res => res.json()))
test()
}
async function testSidewaysX(useTrainSet = false) {
window.testIds = (await fetch(useTrainSet ? '/sideways_x_train' : '/sideways_x_test').then(res => res.json()))
test()
}
async function testStrongSidewaysX(useTrainSet = false) {
window.testIds = (await fetch(useTrainSet ? '/strong_sideways_x_train' : '/strong_sideways_x_test').then(res => res.json()))
test()
}
async function testStrongSidewaysY(useTrainSet = false) {
window.testIds = (await fetch(useTrainSet ? '/strong_sideways_y_train' : '/strong_sideways_y_test').then(res => res.json()))
test()
}
async function test() {
let totalError = 0
for (let i = 0; i < window.testIds.length; i++) {
const dataId = window.testIds[i]
let originalImage = (await getJpgs([dataId]))[0]
originalImage = await faceapi.bufferToImage(originalImage.blob)
const groundTruthPts = (await getPts([dataId])).map(res => res.pts)[0]
for (let c = 0; c < (crops || 1); c++) {
console.log(i, c, i / window.testIds.length)
const { croppedImage, shiftedPts } = crops > 0
? randomCrop(originalImage, groundTruthPts)
: { croppedImage: originalImage, shiftedPts: groundTruthPts }
const squaredImg = faceapi.imageToSquare(croppedImage, 112, true)
const landmarks = (await net.detectLandmarks(squaredImg)).forSize(croppedImage.width, croppedImage.height)
const dist = faceapi.euclideanDistance(
pointsToFlatArray(landmarks.getPositions()),
pointsToFlatArray(shiftedPts)
)
const distByCoordinate = dist / 136
totalError += distByCoordinate
if (window.debug) {
const inputImgEl = $('#inputImg').get(0)
inputImgEl.src = croppedImage instanceof HTMLImageElement ? croppedImage.src : croppedImage.toDataURL()
inputImgEl.onload = () => showResults(landmarks)
for (let i = 0; i < 30; i++)
await faceapi.tf.nextFrame()
}
}
}
console.log('done...')
console.log('test set size:', window.testIds.length)
console.log('total error:', totalError / (crops || 1))
console.log('average error:', totalError / ((crops || 1) * window.testIds.length))
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceLandmarks/public/train.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<script src="js/randomCrop.js"></script>
<script src="js/indexedDb.js"></script>
<script src="js/synchronizeDb.js"></script>
<script src="FileSaver.js"></script>
</head>
<body>
<div id="container"></div>
<script>
tf = faceapi.tf
// uri to weights file of last checkpoint
window.net = new faceapi.FaceLandmark68TinyNet()
const modelCheckpoint = 'tmp/face_landmark_68_tiny_model_lr00001_19.weights'
const startEpoch = 20
const learningRate = 0.0001 // 0.001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.saveEveryNthSample = Infinity
window.withRandomCrop = true
window.batchSize = 12
//window.batchSize = 32
window.lossValues = []
window.iterDelay = 0
window.withLogging = true
const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
function saveWeights(net, filename = 'train_tmp') {
saveAs(new Blob([net.serializeParams()]), filename)
}
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}
async function delay(ms) {
return new Promise(res => setTimeout(res, ms))
}
async function fetchTrainDataIds() {
return fetch('/face_landmarks_train_ids').then(res => res.json())
}
async function load() {
window.trainIds = (await fetchTrainDataIds())
const weights = await loadNetWeights(modelCheckpoint)
await window.net.load(weights)
window.net.variable()
}
async function onEpochDone(epoch) {
saveWeights(window.net, `face_landmark_68_tiny_model_lr00001_${epoch}.weights`)
const loss = window.lossValues[epoch]
saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / window.trainIds.length })]), `face_landmark_68_tiny_model_lr00001_${epoch}.json`)
}
async function train() {
await load()
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
if (epoch !== startEpoch) {
// ugly hack to wait for loss datas for that epoch to be resolved
setTimeout(() => onEpochDone(epoch - 1), 10000)
}
window.lossValues[epoch] = 0
const shuffledInputs = faceapi.shuffleArray(window.trainIds)
for (let dataIdx = 0; dataIdx < shuffledInputs.length; dataIdx += window.batchSize) {
const tsIter = Date.now()
const ids = shuffledInputs.slice(dataIdx, dataIdx + window.batchSize)
// fetch image and ground truth bounding boxes
let tsFetch = Date.now()
let tsFetchPts = Date.now()
const allPts = (await getPts(ids)).map(res => res.pts)
tsFetchPts = Date.now() - tsFetchPts
let tsFetchJpgs = Date.now()
const allJpgBlobs = (await getJpgs(ids)).map(res => res.blob)
tsFetchJpgs = Date.now() - tsFetchJpgs
let tsBufferToImage = Date.now()
const allJpgs = await Promise.all(
allJpgBlobs.map(blob => faceapi.bufferToImage(blob))
)
tsBufferToImage = Date.now() - tsBufferToImage
tsFetch = Date.now() - tsFetch
const bImages = []
const bPts = []
for (let batchIdx = 0; batchIdx < ids.length; batchIdx += 1) {
const pts = allPts[batchIdx]
const img = allJpgs[batchIdx]
const { croppedImage, shiftedPts } = window.withRandomCrop
? randomCrop(img, pts)
: { croppedImage: img, shiftedPts: pts }
const squaredImg = faceapi.imageToSquare(croppedImage, 112, true)
const reshapedDimensions = faceapi.computeReshapedDimensions(croppedImage, 112)
const pad = Math.abs(reshapedDimensions.width - reshapedDimensions.height) / (2 * 112)
const padX = reshapedDimensions.width < reshapedDimensions.height ? pad : 0
const padY = reshapedDimensions.height < reshapedDimensions.width ? pad : 0
const groundTruthLandmarks = shiftedPts.map(pt => new faceapi.Point(
padX + (pt.x / croppedImage.width) * (reshapedDimensions.width / 112),
padY + (pt.y / croppedImage.height)* (reshapedDimensions.height / 112)
))
bImages.push(squaredImg)
bPts.push(groundTruthLandmarks)
}
const netInput = await faceapi.toNetInput(bImages)
let tsBackward = Date.now()
let tsForward = Date.now()
const loss = optimizer.minimize(() => {
const out = window.net.runNet(netInput)
tsForward = Date.now() - tsForward
tsBackward = Date.now()
const landmarksTensor = tf.tensor2d(
bPts
.reduce((flat, arr) => flat.concat(arr))
.map(pt => [pt.x, pt.y])
.reduce((flat, arr) => flat.concat(arr)),
[ids.length, 136]
)
const loss = tf.losses.meanSquaredError(
landmarksTensor,
out,
tf.Reduction.MEAN
)
return loss
}, true)
tsBackward = Date.now() - tsBackward
// start next iteration without waiting for loss data
loss.data().then(data => {
const lossValue = data[0]
window.lossValues[epoch] += lossValue
window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - loss: ${lossValue}, ( ${window.lossValues[epoch]})`)
loss.dispose()
})
window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - forward: ${tsForward} ms, backprop: ${tsBackward} ms, iter: ${Date.now() - tsIter} ms`)
if (window.logsilly) {
log(`fetch: ${tsFetch} ms, pts: ${tsFetchPts} ms, jpgs: ${tsFetchJpgs} ms, bufferToImage: ${tsBufferToImage} ms`)
}
if (window.iterDelay) {
await delay(window.iterDelay)
} else {
await tf.nextFrame()
}
}
}
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceLandmarks/public/trainData.json deleted 100644 → 0
This source diff could not be displayed because it is too large. You can view the blob instead.
tools/train/faceLandmarks/public/train_debug.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<script src="js/randomCrop.js"></script>
<script src="js/indexedDb.js"></script>
<script src="FileSaver.js"></script>
</head>
<body>
<div id="container"></div>
<script>
tf = faceapi.tf
// uri to weights file of last checkpoint
window.net = new faceapi.FaceLandmark68LargeNet()
const modelCheckpoint = '/tmp/initial_glorot.weights'
const startEpoch = 0
const learningRate = 0.001 // 0.001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.saveEveryNthSample = 50000000
window.withRandomCrop = false
window.batchSize = 12
window.lossValue = 0
window.drawResults = true
const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
function saveWeights(net, filename = 'train_tmp') {
saveAs(new Blob([net.serializeParams()]), filename)
}
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}
async function fetchTrainDataFilenames() {
return fetch('/face_landmarks_train_ids').then(res => res.json())
}
async function load() {
window.ptsFiles = await fetchTrainDataFilenames()
window.ptsFiles = window.ptsFiles.slice(1000, 1000 + window.batchSize)
//window.ptsFiles = window.ptsFiles.slice(7, 8)
const weights = await loadNetWeights(modelCheckpoint)
await window.net.load(weights)
window.net.variable()
}
function randomCrop(img, pts) {
const xCoords = pts.map(pt => pt.x)
const yCoords = pts.map(pt => pt.y)
const minX = xCoords.reduce((x, min) => x < min ? x : min, img.width)
const minY = yCoords.reduce((y, min) => y < min ? y : min, img.height)
const maxX = xCoords.reduce((x, max) => x < max ? max : x, 0)
const maxY = yCoords.reduce((y, max) => y < max ? max : y, 0)
const x0 = Math.random() * minX
const y0 = Math.random() * minY
const x1 = (Math.random() * Math.abs(img.width - maxX)) + maxX
const y1 = (Math.random() * Math.abs(img.height - maxY)) + maxY
const targetCanvas = faceapi.createCanvas({ width: x1 - x0, height: y1 - y0 })
const inputCanvas = img instanceof HTMLCanvasElement ? img : faceapi.createCanvasFromMedia(img)
const region = faceapi.getContext2dOrThrow(inputCanvas)
.getImageData(x0, y0, targetCanvas.width, targetCanvas.height)
faceapi.getContext2dOrThrow(targetCanvas).putImageData(region, 0, 0)
return {
croppedImage: targetCanvas,
shiftedPts: pts.map(pt => new faceapi.Point(pt.x - x0, pt.y - y0))
}
}
async function train() {
await load()
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
if (epoch !== startEpoch) {
//saveWeights(window.net, `landmarks_epoch${epoch - 1}.weights`)
//saveAs(new Blob([JSON.stringify({ loss: window.lossValue, avgLoss: window.lossValue / 20000 })]), `landmarks_epoch${epoch - 1}.json`)
window.lossValue = 0
}
const shuffledInputs = window.drawResults ? window.ptsFiles : faceapi.shuffleArray(window.ptsFiles)
for (let dataIdx = 0; dataIdx < shuffledInputs.length; dataIdx += window.batchSize) {
const ids = shuffledInputs.slice(dataIdx, dataIdx + window.batchSize)
// fetch image and ground truth bounding boxes
console.time('getPts')
const allPts = (await getPts(ids)).map(res => res.pts)
console.timeEnd('getPts')
console.time('getJpgs')
const allJpgs = await Promise.all(
(await getJpgs(ids)).map(res => res.blob).map(blob => faceapi.bufferToImage(blob))
)
console.timeEnd('getJpgs')
const bSquaredImages = []
const bCroppedImages = []
const bImages = []
const bPts = []
for (let batchIdx = 0; batchIdx < ids.length; batchIdx += 1) {
const pts = allPts[batchIdx]
const img = allJpgs[batchIdx]
const { croppedImage, shiftedPts } = window.withRandomCrop
? randomCrop(img, pts)
: { croppedImage: img, shiftedPts: pts }
const squaredImg = faceapi.imageToSquare(croppedImage, 112, true)
const reshapedDimensions = faceapi.computeReshapedDimensions(croppedImage, 112)
const pad = Math.abs(reshapedDimensions.width - reshapedDimensions.height) / (2 * 112)
const padX = reshapedDimensions.width < reshapedDimensions.height ? pad : 0
const padY = reshapedDimensions.height < reshapedDimensions.width ? pad : 0
const groundTruthLandmarks = shiftedPts.map(pt => new faceapi.Point(
padX + (pt.x / croppedImage.width) * (reshapedDimensions.width / 112),
padY + (pt.y / croppedImage.height)* (reshapedDimensions.height / 112)
))
bSquaredImages.push(squaredImg)
bCroppedImages.push(croppedImage)
bImages.push(squaredImg)
bPts.push(groundTruthLandmarks)
}
const netInput = await faceapi.toNetInput(bImages)
const ts = Date.now()
const loss = optimizer.minimize(() => {
console.time('runNet')
const out = window.net.runNet(netInput)
console.timeEnd('runNet')
if (window.drawResults) {
// debug
const landmarkTensors = tf.unstack(out)
landmarkTensors.forEach((t, i) => {
const croppedImage = bSquaredImages[i]
const landmarksArray = Array.from(t.dataSync())
const xCoords = landmarksArray.filter((_, i) => faceapi.isEven(i))
const yCoords = landmarksArray.filter((_, i) => !faceapi.isEven(i))
/*
const landmarksArray = bPts[i]
const xCoords = landmarksArray.map(pt => pt.x)
const yCoords = landmarksArray.map(pt => pt.y)
*/
const marks = new faceapi.FaceLandmarks68(
Array(68).fill(0).map((_, i) => new faceapi.Point(xCoords[i], yCoords[i])),
croppedImage
)
const canvas = croppedImage instanceof HTMLCanvasElement ? croppedImage : faceapi.createCanvasFromMedia(croppedImage)
faceapi.drawLandmarks(canvas, marks, { drawLines: true })
const container = document.getElementById('container')
if (container.children[i]) {
const oldChild = container.children[i]
container.insertBefore(canvas, oldChild)
container.removeChild(oldChild)
} else {
container.appendChild(canvas)
}
})
// debug
}
const landmarksTensor = tf.tensor2d(
bPts
.reduce((flat, arr) => flat.concat(arr))
.map(pt => [pt.x, pt.y])
.reduce((flat, arr) => flat.concat(arr)),
[window.batchSize, 136]
)
const loss = tf.losses.meanSquaredError(
landmarksTensor,
out,
tf.Reduction.MEAN
)
//return tf.sum(tf.abs(tf.sub(landmarksTensor, out)))
return loss
}, true)
// start next iteration without waiting for loss data
loss.data().then(data => {
const lossValue = data[0]
window.lossValue += lossValue
log(`epoch ${epoch}, dataIdx ${dataIdx} - loss: ${lossValue}`)
loss.dispose()
})
if (window.iterDelay) {
await delay(window.iterDelay)
} else {
await tf.nextFrame()
}
}
}
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceLandmarks/public/verify.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<script src="js/randomCrop.js"></script>
<link rel="stylesheet" href="styles.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.css">
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/js/materialize.min.js"></script>
</head>
<body>
<div id="navbar"></div>
<div class="center-content page-container">
<div class="progress" id="loader">
<div class="indeterminate"></div>
</div>
<div id="results" class="side-by-side">
<div id="result0" style="position: relative" class="margin">
<img id="inputImg0" src="" style="max-width: 400px;" />
<canvas class="overlay" id="overlay0" />
</div>
<div id="result1" style="position: relative" class="margin">
<img id="inputImg1" src="" style="max-width: 400px;" />
<canvas class="overlay" id="overlay1" />
</div>
<div id="result2" style="position: relative" class="margin">
<img id="inputImg2" src="" style="max-width: 400px;" />
<canvas class="overlay" id="overlay2" />
</div>
</div>
<div class="row">
<label for="imgByNr">Enter image NR: </label>
<input id="imgByNr" type="text" class="bold" value="0"/>
</div>
</div>
<script>
const modelCheckpoints = [
'tmp/face_landmark_68_tiny_model_lr00001_0.weights',
'tmp/face_landmark_68_tiny_model_lr00001_13.weights',
'tmp/face_landmark_68_tiny_model_lr00001_24.weights'
]
let originalImage = null
function cloneImage(img) {
var image = document.createElement("img")
image.src = img.src
return image
}
function getInputValue(input) {
input = input || $('#imgByNr').get(0)
return input.value || 0
}
function onKeyDown(e) {
e.target.value = (
parseInt(e.target.value) + (e.keyCode === 38 ? 1 : (e.keyCode === 40 ? -1 : 0))
) || getInputValue(e.target)
const imgUri = `${window.testIds[e.target.value]}.jpg`
console.log(imgUri)
onSelectionChanged(imgUri)
}
async function updateResults(_landmarks) {
const inputImgEl = $('#inputImg0').get(0)
const { width, height } = inputImgEl
for (let i = 0; i < window.nets.length; i++) {
const net = window.nets[i]
const canvas = $(`#overlay${i}`).get(0)
canvas.width = width
canvas.height = height
console.time('detectLandmarks')
const landmarks = await net.detectLandmarks(inputImgEl)
console.timeEnd('detectLandmarks')
faceapi.drawLandmarks(`overlay${i}`, landmarks.forSize(width, height), { drawLines: true })
}
}
async function onSelectionChanged(imgUri) {
const img = await faceapi.bufferToImage(await fetchImage(imgUri))
window.nets.forEach(async (_, i) => {
$(`#inputImg${i}`).get(0).src = img.src
})
originalImage = cloneImage(img)
updateResults()
}
async function applyRandomCrop() {
const dataId = window.testIds[getInputValue()]
const pts = await (await fetch(`${dataId}.json`)).json()
const img = cloneImage(originalImage)
const { croppedImage, shiftedPts } = randomCrop(img, pts)
img.src = croppedImage.toDataURL()
$(`#result${0}`).get(0).removeChild($(`#inputImg${0}`).get(0))
$(`#result${0}`).get(0).appendChild(img)
window.nets.slice(1).forEach((_, i) => {
const im = cloneImage(img)
im.id = `#inputImg${i + 1}`
$(`#result${i + 1}`).get(0).removeChild($(`#inputImg${i + 1}`).get(0))
$(`#result${i + 1}`).get(0).appendChild(im)
})
const { width, height } = croppedImage
img.onload = () => updateResults(
new faceapi.FaceLandmarks68(
shiftedPts.map(pt =>
new faceapi.Point(
pt.x / width,
pt.y / height
)
),
{ width, height }
)
)
}
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}
async function run() {
$('#imgByNr').keydown(onKeyDown)
//faceapi.loadFaceLandmarkModel('./')
//window.trainIds = (await fetch('/face_landmarks_train_ids').then(res => res.json()))
//window.testIds = (await fetch('/face_landmarks_test_ids').then(res => res.json()))
window.testIds = (await fetch('/strong_sideways_test').then(res => res.json()))
window.nets = []
//window.nets.push(faceapi)
//const net = new faceapi.FaceLandmark68TinyNet()
//await net.load('./')
//window.nets.push(net)
modelCheckpoints.forEach(async checkpoint => {
const net = new faceapi.FaceLandmark68TinyNet()
const weights = await loadNetWeights(checkpoint)
await net.load(weights)
window.nets.push(net)
})
$('#loader').hide()
}
$(document).ready(() => run())
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceLandmarks/server.js deleted 100644 → 0
require('./.env')
const express = require('express')
const path = require('path')
const fs = require('fs')
const app = express()
const publicDir = path.join(__dirname, './public')
app.use(express.static(publicDir))
app.use(express.static(path.join(__dirname, '../js')))
app.use(express.static(path.join(__dirname, '../node_modules/file-saver')))
app.use(express.static(path.join(__dirname, '../../../weights')))
app.use(express.static(path.join(__dirname, '../../../dist')))
const trainDataPath = path.resolve(process.env.TRAIN_DATA_PATH)
const pngPath = path.join(trainDataPath, 'png')
const jpgPath = path.join(trainDataPath, 'jpg')
const groundTruthPath = path.join(trainDataPath, 'pts')
app.use(express.static(jpgPath))
app.use(express.static(groundTruthPath))
const trainIds = JSON.parse(fs.readFileSync(path.join(publicDir, './trainData.json')))
const trainIdsSet = new Set(trainIds)
const testFileIds = fs.readdirSync(groundTruthPath)
.map(file => file.replace('.json', ''))
.filter(file => !trainIdsSet.has(file))
app.get('/face_landmarks_train_ids', (req, res) => res.status(202).send(trainIds))
app.get('/face_landmarks_test_ids', (req, res) => res.status(202).send(testFileIds))
const poseAnchors = JSON.parse(fs.readFileSync(path.join(trainDataPath, './pose-anchors.json')))
const sidewaysXTrain = []
const strongSidewaysXTrain = []
const strongSidewaysYTrain = []
const strongSidewaysTrain = []
const sidewaysXTest = []
const strongSidewaysXTest = []
const strongSidewaysYTest = []
const strongSidewaysTest = []
poseAnchors.forEach((pt, idx) => {
const id = `${idx}`
if (Math.abs(0.5 - pt.x) > 0.15) {
(trainIdsSet.has(id) ? sidewaysXTrain : sidewaysXTest).push(id)
}
if (Math.abs(0.5 - pt.x) > 0.2) {
(trainIdsSet.has(id) ? strongSidewaysXTrain : strongSidewaysXTest).push(id)
}
if (Math.abs(0.44 - pt.y) > 0.1) {
(trainIdsSet.has(id) ? strongSidewaysYTrain : strongSidewaysYTest).push(id)
}
if (Math.abs(0.5 - pt.x) > 0.2 || Math.abs(0.44 - pt.y) > 0.1) {
(trainIdsSet.has(id) ? strongSidewaysTrain : strongSidewaysTest).push(id)
}
})
console.log(sidewaysXTrain.length)
console.log(strongSidewaysXTrain.length)
console.log(strongSidewaysYTrain.length)
console.log(strongSidewaysTrain.length)
console.log(sidewaysXTest.length)
console.log(strongSidewaysXTest.length)
console.log(strongSidewaysYTest.length)
console.log(strongSidewaysTrain.length)
app.get('/sideways_x_train', (req, res) => res.status(202).send(sidewaysXTrain))
app.get('/strong_sideways_x_train', (req, res) => res.status(202).send(strongSidewaysXTrain))
app.get('/strong_sideways_y_train', (req, res) => res.status(202).send(strongSidewaysYTrain))
app.get('/strong_sideways_train', (req, res) => res.status(202).send(strongSidewaysTrain))
app.get('/sideways_x_test', (req, res) => res.status(202).send(sidewaysXTest))
app.get('/strong_sideways_x_test', (req, res) => res.status(202).send(strongSidewaysXTest))
app.get('/strong_sideways_y_test', (req, res) => res.status(202).send(strongSidewaysYTest))
app.get('/strong_sideways_test', (req, res) => res.status(202).send(strongSidewaysTest))
app.get('/', (req, res) => res.redirect('/train'))
app.get('/train', (req, res) => res.sendFile(path.join(publicDir, 'train.html')))
app.get('/verify', (req, res) => res.sendFile(path.join(publicDir, 'verify.html')))
app.get('/test', (req, res) => res.sendFile(path.join(publicDir, 'test.html')))
app.listen(8000, () => console.log('Listening on port 8000!'))
\ No newline at end of file
tools/train/js/commons.js deleted 100644 → 0
const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
function saveWeights(netOrParams, filename = 'train_tmp') {
saveAs(new Blob([netOrParams instanceof Float32Array ? netOrParams : netOrParams.serializeParams()]), filename)
}
async function delay(ms) {
return new Promise(res => setTimeout(res, ms))
}
\ No newline at end of file
tools/train/package.json deleted 100644 → 0
{
"author": "justadudewhohacks",
"license": "MIT",
"dependencies": {
"express": "^4.16.3",
"file-saver": "^1.3.8"
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment