Skip to content

F
face
Commit 0bfaa037 by vincent
Options

check in retrained tiny 68 point landmark model

parent 7126f5c0
Showing with 1418 additions and 1411 deletions
src/faceLandmarkNet/FaceLandmark68TinyNet.ts 0 → 100644
import * as tf from '@tensorflow/tfjs-core';
import { NetInput, normalize } from 'tfjs-image-recognition-base';
import { ConvParams } from 'tfjs-tiny-yolov2';
import { SeparableConvParams } from 'tfjs-tiny-yolov2/build/tinyYolov2/types';
import { depthwiseSeparableConv } from './depthwiseSeparableConv';
import { extractParamsTiny } from './extractParamsTiny';
import { FaceLandmark68NetBase } from './FaceLandmark68NetBase';
import { fullyConnectedLayer } from './fullyConnectedLayer';
import { loadQuantizedParamsTiny } from './loadQuantizedParamsTiny';
import { DenseBlockParams, TinyNetParams } from './types';
function denseBlock(
x: tf.Tensor4D,
denseBlockParams: DenseBlockParams,
isFirstLayer: boolean = false
): tf.Tensor4D {
return tf.tidy(() => {
const out1 = tf.relu(
isFirstLayer
? tf.add(
tf.conv2d(x, (denseBlockParams.conv0 as ConvParams).filters, [2, 2], 'same'),
denseBlockParams.conv0.bias
)
: depthwiseSeparableConv(x, denseBlockParams.conv0 as SeparableConvParams, [2, 2])
) as tf.Tensor4D
const out2 = depthwiseSeparableConv(out1, denseBlockParams.conv1, [1, 1])
const in3 = tf.relu(tf.add(out1, out2)) as tf.Tensor4D
const out3 = depthwiseSeparableConv(in3, denseBlockParams.conv2, [1, 1])
return tf.relu(tf.add(out1, tf.add(out2, out3))) as tf.Tensor4D
})
}
export class FaceLandmark68TinyNet extends FaceLandmark68NetBase<TinyNetParams> {
constructor() {
super('FaceLandmark68TinyNet')
}
public runNet(input: NetInput): tf.Tensor2D {
const { params } = this
if (!params) {
throw new Error('FaceLandmark68TinyNet - load model before inference')
}
return tf.tidy(() => {
const batchTensor = input.toBatchTensor(112, true)
const meanRgb = [122.782, 117.001, 104.298]
const normalized = normalize(batchTensor, meanRgb).div(tf.scalar(255)) as tf.Tensor4D
let out = denseBlock(normalized, params.dense0, true)
out = denseBlock(out, params.dense1)
out = denseBlock(out, params.dense2)
out = tf.avgPool(out, [14, 14], [2, 2], 'valid')
return fullyConnectedLayer(out.as2D(out.shape[0], -1), params.fc)
})
}
protected loadQuantizedParams(uri: string | undefined) {
return loadQuantizedParamsTiny(uri)
}
protected extractParams(weights: Float32Array) {
return extractParamsTiny(weights)
}
}
\ No newline at end of file
src/faceLandmarkNet/depthwiseSeparableConv.ts 0 → 100644
import * as tf from '@tensorflow/tfjs-core';
import { SeparableConvParams } from 'tfjs-tiny-yolov2/build/tinyYolov2/types';
export function depthwiseSeparableConv(
x: tf.Tensor4D,
params: SeparableConvParams,
stride: [number, number]
): tf.Tensor4D {
return tf.tidy(() => {
let out = tf.separableConv2d(x, params.depthwise_filter, params.pointwise_filter, stride, 'same')
out = tf.add(out, params.bias)
return out
})
}
\ No newline at end of file
src/faceLandmarkNet/extractParamsTiny.ts 0 → 100644
import * as tf from '@tensorflow/tfjs-core';
import { extractWeightsFactory, ExtractWeightsFunction, ParamMapping } from 'tfjs-image-recognition-base';
import { extractConvParamsFactory, FCParams } from 'tfjs-tiny-yolov2';
import { SeparableConvParams } from 'tfjs-tiny-yolov2/build/tinyYolov2/types';
import { DenseBlockParams, TinyNetParams } from './types';
function extractorsFactory(extractWeights: ExtractWeightsFunction, paramMappings: ParamMapping[]) {
function extractSeparableConvParams(channelsIn: number, channelsOut: number, mappedPrefix: string): SeparableConvParams {
const depthwise_filter = tf.tensor4d(extractWeights(3 * 3 * channelsIn), [3, 3, channelsIn, 1])
const pointwise_filter = tf.tensor4d(extractWeights(channelsIn * channelsOut), [1, 1, channelsIn, channelsOut])
const bias = tf.tensor1d(extractWeights(channelsOut))
paramMappings.push(
{ paramPath: `${mappedPrefix}/depthwise_filter` },
{ paramPath: `${mappedPrefix}/pointwise_filter` },
{ paramPath: `${mappedPrefix}/bias` }
)
return new SeparableConvParams(
depthwise_filter,
pointwise_filter,
bias
)
}
function extractFCParams(channelsIn: number, channelsOut: number, mappedPrefix: string): FCParams {
const weights = tf.tensor2d(extractWeights(channelsIn * channelsOut), [channelsIn, channelsOut])
const bias = tf.tensor1d(extractWeights(channelsOut))
paramMappings.push(
{ paramPath: `${mappedPrefix}/weights` },
{ paramPath: `${mappedPrefix}/bias` }
)
return {
weights,
bias
}
}
return {
extractSeparableConvParams,
extractFCParams
}
}
export function extractParamsTiny(weights: Float32Array): { params: TinyNetParams, paramMappings: ParamMapping[] } {
const paramMappings: ParamMapping[] = []
const {
extractWeights,
getRemainingWeights
} = extractWeightsFactory(weights)
const {
extractSeparableConvParams,
extractFCParams
} = extractorsFactory(extractWeights, paramMappings)
const extractConvParams = extractConvParamsFactory(extractWeights, paramMappings)
function extractDenseBlockParams(channelsIn: number, channelsOut: number, mappedPrefix: string, isFirstLayer: boolean = false): DenseBlockParams {
const conv0 = isFirstLayer
? extractConvParams(channelsIn, channelsOut, 3, `${mappedPrefix}/conv0`)
: extractSeparableConvParams(channelsIn, channelsOut, `${mappedPrefix}/conv0`)
const conv1 = extractSeparableConvParams(channelsOut, channelsOut, `${mappedPrefix}/conv1`)
const conv2 = extractSeparableConvParams(channelsOut, channelsOut, `${mappedPrefix}/conv2`)
return { conv0, conv1, conv2 }
}
const dense0 = extractDenseBlockParams(3, 32, 'dense0', true)
const dense1 = extractDenseBlockParams(32, 64, 'dense1')
const dense2 = extractDenseBlockParams(64, 128, 'dense2')
const fc = extractFCParams(128, 136, 'fc')
if (getRemainingWeights().length !== 0) {
throw new Error(`weights remaing after extract: ${getRemainingWeights().length}`)
}
return {
paramMappings,
params: { dense0, dense1, dense2, fc }
}
}
\ No newline at end of file
src/faceLandmarkNet/index.ts
import { FaceLandmark68Net } from './FaceLandmark68Net';
export * from './FaceLandmark68Net';
export * from './FaceLandmark68TinyNet';
export class FaceLandmarkNet extends FaceLandmark68Net {}
......
src/faceLandmarkNet/loadQuantizedParamsTiny.ts 0 → 100644
import * as tf from '@tensorflow/tfjs-core';
import {
disposeUnusedWeightTensors,
extractWeightEntryFactory,
loadWeightMap,
ParamMapping,
} from 'tfjs-image-recognition-base';
import { ConvParams, FCParams } from 'tfjs-tiny-yolov2';
import { SeparableConvParams } from 'tfjs-tiny-yolov2/build/tinyYolov2/types';
import { DenseBlockParams, TinyNetParams } from './types';
const DEFAULT_MODEL_NAME = 'face_landmark_68_tiny_model'
function extractorsFactory(weightMap: any, paramMappings: ParamMapping[]) {
const extractWeightEntry = extractWeightEntryFactory(weightMap, paramMappings)
function extractConvParams(prefix: string): ConvParams {
const filters = extractWeightEntry<tf.Tensor4D>(`${prefix}/filters`, 4)
const bias = extractWeightEntry<tf.Tensor1D>(`${prefix}/bias`, 1)
return { filters, bias }
}
function extractSeparableConvParams(prefix: string): SeparableConvParams {
const depthwise_filter = extractWeightEntry<tf.Tensor4D>(`${prefix}/depthwise_filter`, 4)
const pointwise_filter = extractWeightEntry<tf.Tensor4D>(`${prefix}/pointwise_filter`, 4)
const bias = extractWeightEntry<tf.Tensor1D>(`${prefix}/bias`, 1)
return new SeparableConvParams(
depthwise_filter,
pointwise_filter,
bias
)
}
function extractDenseBlockParams(prefix: string, isFirstLayer: boolean = false): DenseBlockParams {
const conv0 = isFirstLayer
? extractConvParams(`${prefix}/conv0`)
: extractSeparableConvParams(`${prefix}/conv0`)
const conv1 = extractSeparableConvParams(`${prefix}/conv1`)
const conv2 = extractSeparableConvParams(`${prefix}/conv2`)
return { conv0, conv1, conv2 }
}
function extractFcParams(prefix: string): FCParams {
const weights = extractWeightEntry<tf.Tensor2D>(`${prefix}/weights`, 2)
const bias = extractWeightEntry<tf.Tensor1D>(`${prefix}/bias`, 1)
return { weights, bias }
}
return {
extractDenseBlockParams,
extractFcParams
}
}
export async function loadQuantizedParamsTiny(
uri: string | undefined
): Promise<{ params: TinyNetParams, paramMappings: ParamMapping[] }> {
const weightMap = await loadWeightMap(uri, DEFAULT_MODEL_NAME)
const paramMappings: ParamMapping[] = []
const {
extractDenseBlockParams,
extractFcParams
} = extractorsFactory(weightMap, paramMappings)
const params = {
dense0: extractDenseBlockParams('dense0', true),
dense1: extractDenseBlockParams('dense1'),
dense2: extractDenseBlockParams('dense2'),
fc: extractFcParams('fc')
}
disposeUnusedWeightTensors(weightMap, paramMappings)
return { params, paramMappings }
}
\ No newline at end of file
src/faceLandmarkNet/types.ts
import * as tf from '@tensorflow/tfjs-core';
import { ConvParams, FCParams } from 'tfjs-tiny-yolov2';
import { SeparableConvParams } from 'tfjs-tiny-yolov2/build/tinyYolov2/types';
export type NetParams = {
conv0: ConvParams
......@@ -11,4 +13,49 @@ export type NetParams = {
conv7: ConvParams
fc0: FCParams
fc1: FCParams
}
\ No newline at end of file
}
export type ConvWithBatchNormParams = BatchNormParams & {
filter: tf.Tensor4D
}
export type BatchNormParams = {
mean: tf.Tensor1D
variance: tf.Tensor1D
scale: tf.Tensor1D
offset: tf.Tensor1D
}
export type SeparableConvWithBatchNormParams = {
depthwise: ConvWithBatchNormParams
pointwise: ConvWithBatchNormParams
}
export declare type FCWithBatchNormParams = BatchNormParams & {
weights: tf.Tensor2D
}
export type DenseBlockParams = {
conv0: SeparableConvParams | ConvParams
conv1: SeparableConvParams
conv2: SeparableConvParams
//conv3: SeparableConvParams
}
export type TinyNetParams = {
dense0: DenseBlockParams
dense1: DenseBlockParams
dense2: DenseBlockParams
fc: FCParams
}
export type MobileResnetParams = {
conv0: SeparableConvParams
conv1: SeparableConvParams
conv2: SeparableConvParams
conv3: SeparableConvParams
conv4: SeparableConvParams
conv5: SeparableConvParams
fc: FCParams
}
tools/quantize/views/quantizeModel.html
......@@ -10,9 +10,9 @@
<script>
tf = faceapi.tf
const modelName = 'tiny_yolov2_separable_conv'
const uncompressedWeightsUri = `tiny_yolov2_separable_conv_model_v1.weights`
const net = new faceapi.TinyYolov2(true)
const modelName = 'face_landmark_68_tiny_model'
const uncompressedWeightsUri = `face_landmark_68_tiny_model.weights`
const net = new faceapi.FaceLandmark68TinyNet()
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
......
tools/train/faceLandmarks/faceLandmarksTrain.js deleted 100644 → 0
async function trainStep(batchCreators) {
await promiseSequential(batchCreators.map((batchCreator, dataIdx) => async () => {
const { batchInput, landmarksBatchTensor } = await batchCreator()
let ts = Date.now()
const cost = optimizer.minimize(() => {
const out = window.trainNet.forwardInput(batchInput.managed())
const loss = lossFunction(
landmarksBatchTensor,
out
)
return tf.sum(out)
}, true)
ts = Date.now() - ts
console.log(`loss[${dataIdx}]: ${await cost.data()}, ${ts} ms (${ts / batchInput.batchSize} ms / batch element)`)
landmarksBatchTensor.dispose()
cost.dispose()
await tf.nextFrame()
console.log(tf.memory())
}))
}
function createBatchCreators(data, batchSize) {
if (batchSize < 1) {
throw new Error('invalid batch size: ' + batchSize)
}
const batches = []
const pushToBatch = (remaining) => {
if (remaining.length) {
batches.push(remaining.slice(0, batchSize))
pushToBatch(remaining.slice(batchSize))
}
return batches
}
pushToBatch(data)
const batchCreators = batches.map(dataForBatch => async () => {
const imgs = dataForBatch.map(d => d.img)
const allLandmarks = dataForBatch.map(d => landmarkPositionsToArray(d.landmarks))
const batchInput = await faceapi.toNetInput(imgs)
const landmarksBatchTensor = tf.tidy(() => {
const landmarkTensors = allLandmarks.map(arr => tf.tensor2d(arr, [1, 136]))
return tf.stack(landmarkTensors, 0).as2D(-1, 136)
})
return {
batchInput,
landmarksBatchTensor
}
})
return batchCreators
}
function landmarkPositionsToArray(landmarks) {
return landmarks.getRelativePositions().map(pt => [pt.x, pt.y])
.reduce((flat, arr) => flat.concat(arr))
}
function toFaceLandmarks(landmarks, { naturalWidth, naturalHeight }) {
return new faceapi.FaceLandmarks68(
landmarks.map(l => new faceapi.Point(l.x / naturalWidth, l.y / naturalHeight)),
{ width: naturalWidth, height: naturalHeight }
)
}
async function loadImagesInBatch(allLandmarks, offset = 0) {
return Promise.all(allLandmarks.map(async (landmarks, i) => {
const imgUri = `/train_images/${i + offset}.png`
const img = await faceapi.bufferToImage(await fetchImage(imgUri))
return {
imgUri,
img,
landmarks: toFaceLandmarks(landmarks, img)
}
}))
}
async function getTrainData() {
const landmarksJson = (await (await fetch('/train_landmarks.json')).json())
const numLandmarks = Object.keys(landmarksJson).length
const allLandmarks = Array.from(
new Array(numLandmarks),
(_, i) => landmarksJson[i]
)
return await loadImagesInBatch(allLandmarks.slice(0, 100))
/**
const batch1 = await loadImagesInBatch(allLandmarks.slice(0, 4000))
const batch2 = await loadImagesInBatch(allLandmarks.slice(4000), 4000)
return batch1.concat(batch2)
*/
}
\ No newline at end of file
tools/train/faceLandmarks/faceLandmarksUi.js deleted 100644 → 0
function onKeyDown(e) {
e.target.value = (
parseInt(e.target.value) + (e.keyCode === 38 ? 1 : (e.keyCode === 40 ? -1 : 0))
) || e.target.value || 0
const uri = '/train_images/' + e.target.value + '.png'
console.log(uri)
onSelectionChanged(uri)
}
function onChangeDrawLines(e) {
window.drawLines = $(e.target).prop('checked')
redraw()
}
async function onSelectionChanged(uri) {
const imgBuf = await fetchImage(uri)
window.currentImg = await faceapi.bufferToImage(imgBuf)
window.landmarks = await Promise.all(window.nets.map(
async net => net.detectLandmarks(window.currentImg)
))
redraw(uri)
}
function drawLandmarkCanvas(img, landmarks, drawOpts) {
const canvas = faceapi.createCanvasFromMedia(img)
$('#faceContainer').append(canvas)
faceapi.drawLandmarks(
canvas,
landmarks,
drawOpts
)
}
function redraw(uri) {
$('#faceContainer').empty()
const drawOpts = { lineWidth: window.drawLines ? 2 : 4, drawLines: window.drawLines }
window.landmarks.forEach(landmarks => {
drawLandmarkCanvas(window.currentImg, landmarks, drawOpts)
})
const trainImgAndLandmarks = window.trainData.find(t => t.imgUri === uri)
if (trainImgAndLandmarks) {
drawLandmarkCanvas(trainImgAndLandmarks.img, trainImgAndLandmarks.landmarks, drawOpts)
}
}
async function loadNet(file) {
const res = await fetch(file)
const weights = new Float32Array(await res.arrayBuffer())
return faceapi.faceLandmarkNet(weights)
}
async function init() {
//await faceapi.loadFaceLandmarkModel('/')
//window.nets.push(faceapi.landmarkNet)
//window.nets.push(await loadNet('retrained/landmarks_v0.weights'))
//window.nets.push(await loadNet('retrained/landmarks_v2.weights'))
window.trainNet = await loadNet('/tmp/retrained/landmarks_v9.weights')
window.nets.push(trainNet)
$('#loader').hide()
await run()
await onSelectionChanged($('#selectList select').val())
}
$(document).ready(function() {
$('#imgByNr').keydown(onKeyDown);
renderFaceImageSelectList(
'#selectList',
onSelectionChanged,
{ className: 'sheldon', imageIdx: 1 }
)
init()
})
\ No newline at end of file
tools/train/faceLandmarks/initWeightsDensenet.html 0 → 100644
<!DOCTYPE html>
<html>
<head>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.12.0"> </script>
<script src="../node_modules/file-saver/FileSaver.js"></script>
</head>
<body>
<div class="row side-by-side">
<button
class="waves-effect waves-light btn"
onclick="save()"
>
Save
</button>
<button
class="waves-effect waves-light btn"
onclick="save(true)"
>
Save Tiny
</button>
</div>
<script>
function toDataArray(tensor) {
return Array.from(tensor.dataSync())
}
function flatten(arrs) {
return arrs.reduce((flat, arr) => flat.concat(arr))
}
function initWeights(initializer, isTiny) {
function initConvWeights(inChannels, outChannels) {
return flatten(
[
// filter
initializer.apply([3, 3, inChannels, outChannels]),
// bias
tf.zeros([outChannels])
]
.map(toDataArray)
)
}
function initSeparableConvWeights(inChannels, outChannels) {
return flatten(
[
// depthwise filter
initializer.apply([3, 3, inChannels, 1]),
// pointwise filter
initializer.apply([1, 1, inChannels, outChannels]),
// bias
tf.zeros([outChannels])
]
.map(toDataArray)
)
}
const separableConvWeights = isTiny
? flatten([
initConvWeights(3, 32),
initSeparableConvWeights(32, 32),
initSeparableConvWeights(32, 32),
initSeparableConvWeights(32, 64),
initSeparableConvWeights(64, 64),
initSeparableConvWeights(64, 64),
initSeparableConvWeights(64, 128),
initSeparableConvWeights(128, 128),
initSeparableConvWeights(128, 128)
])
: flatten([
initConvWeights(3, 32),
initSeparableConvWeights(32, 32),
initSeparableConvWeights(32, 32),
initSeparableConvWeights(32, 32),
initSeparableConvWeights(32, 64),
initSeparableConvWeights(64, 64),
initSeparableConvWeights(64, 64),
initSeparableConvWeights(64, 64),
initSeparableConvWeights(64, 128),
initSeparableConvWeights(128, 128),
initSeparableConvWeights(128, 128),
initSeparableConvWeights(128, 128),
initSeparableConvWeights(256, 256),
initSeparableConvWeights(256, 256),
initSeparableConvWeights(256, 256),
initSeparableConvWeights(256, 256)
])
const fc = flatten(
[
initializer.apply([1, 1, isTiny ? 128 : 256, 136]),
// bias
tf.zeros([136])
]
.map(toDataArray)
)
return new Float32Array(separableConvWeights.concat(fc))
}
function save(isTiny = false) {
const initialWeights = initWeights(
tf.initializers.glorotNormal(),
isTiny
)
saveAs(new Blob([initialWeights]), `initial_glorot.weights`)
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceLandmarks/js/indexedDb.js 0 → 100644
const indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB || window.shimIndexedDB
let jsonsDb = null
let jpgsDb = null
const openJsons = indexedDB.open('jsons', 1)
const openJpgs = indexedDB.open('jpgs', 1)
openJsons.onupgradeneeded = function() {
jsonsDb = openJsons.result
jsonsDb.createObjectStore('jsons', { keyPath: 'id' })
}
openJpgs.onupgradeneeded = function() {
jpgsDb = openJpgs.result
jpgsDb.createObjectStore('jpgs', { keyPath: 'id' })
}
openJsons.onsuccess = function() {
console.log('connected to jsons')
jsonsDb = openJsons.result
}
openJpgs.onsuccess = function() {
console.log('connected to jpgs')
jpgsDb = openJpgs.result
}
function putReq(store, obj) {
return new Promise((res, rej) => {
const req = store.put(obj)
req.onsuccess = res
req.onerror = rej
})
}
function getReq(store, id, throwIfNoResult = true) {
return new Promise((res, rej) => {
const req = store.get(id)
req.onsuccess = () => {
if (!req.result && throwIfNoResult) {
return rej(`no result for id: ${id}`)
}
res(req.result)
}
req.onerror = rej
})
}
function existsReq(store, id) {
return getReq(store, id, false)
}
async function getNotFound(store, ids) {
return (await Promise.all(ids.map(async id => ({ id, exists: await existsReq(store, id) }))))
.filter(({ exists }) => !exists)
.map(({ id }) => id)
}
async function getNotFoundPts(ids) {
const store = jsonsDb.transaction('jsons', 'readonly').objectStore('jsons')
return getNotFound(store, ids)
}
async function getNotFoundJpegs(ids) {
const store = jpgsDb.transaction('jpgs', 'readonly').objectStore('jpgs')
return getNotFound(store, ids)
}
async function persistPts(ptsById, overwrite = false) {
const store = jsonsDb.transaction('jsons', 'readwrite').objectStore('jsons')
for (let i = 0; i < ptsById.length; i++) {
const { id, pts } = ptsById[i]
if (!await existsReq(store, id)) {
console.log('persistPts - inserting %s', id)
await putReq(store, { id, pts })
}
}
}
function getPts(ids) {
const store = jsonsDb.transaction('jsons', 'readonly').objectStore('jsons')
return Promise.all(ids.map(id => getReq(store, id)))
}
async function persistJpgs(jpgsById, overwrite = false) {
const store = jpgsDb.transaction('jpgs', 'readwrite').objectStore('jpgs')
for (let i = 0; i < jpgsById.length; i++) {
const { id, blob } = jpgsById[i]
if (!await existsReq(store, id)) {
console.log('persistJpgs - inserting %s', id)
await putReq(store, { id, blob })
}
}
}
function getJpgs(ids) {
const store = jpgsDb.transaction('jpgs', 'readonly').objectStore('jpgs')
return Promise.all(ids.map(id => getReq(store, id)))
}
tools/train/faceLandmarks/js/printIndexDbSizes.js 0 → 100644
// https://gist.github.com/tralves/9e5de2bd9f582007a52708d7d4209865
var getTableSize = function(db, dbName){
return new Promise((resolve,reject) => {
if (db == null) {
return reject();
}
var size = 0;
db = event.target.result;
var transaction = db.transaction([dbName])
.objectStore(dbName)
.openCursor();
transaction.onsuccess = function(event){
var cursor = event.target.result;
if(cursor){
var storedObject = cursor.value;
var json = JSON.stringify(storedObject);
size += json.length;
cursor.continue();
}
else{
resolve(size);
}
}.bind(this);
transaction.onerror = function(err){
reject("error in " + dbName + ": " + err);
}
});
};
var getDatabaseSize = function (dbName) {
var request = indexedDB.open(dbName);
var db;
var dbSize = 0;
request.onerror = function(event) {
alert("Why didn't you allow my web app to use IndexedDB?!");
};
request.onsuccess = function(event) {
db = event.target.result;
var tableNames = [ ...db.objectStoreNames ];
(function(tableNames, db) {
var tableSizeGetters = tableNames
.reduce( (acc, tableName) => {
acc.push( getTableSize(db, tableName) );
return acc;
}, []);
Promise.all(tableSizeGetters)
.then(sizes => {
console.log('--------- ' + db.name + ' -------------');
tableNames.forEach( (tableName,i) => {
console.log(" - " + tableName + "\t: " + humanReadableSize(sizes[i]));
});
var total = sizes.reduce(function(acc, val) {
return acc + val;
}, 0);
console.log("TOTAL: " + humanReadableSize(total))
});
})(tableNames, db);
};
};
var humanReadableSize = function (bytes) {
var thresh = 1024;
if(Math.abs(bytes) < thresh) {
return bytes + ' B';
}
var units = ['KB','MB','GB','TB','PB','EB','ZB','YB'];
var u = -1;
do {
bytes /= thresh;
++u;
} while(Math.abs(bytes) >= thresh && u < units.length - 1);
return bytes.toFixed(1)+' '+units[u];
}
getDatabaseSize('jsons')
getDatabaseSize('jpgs')
\ No newline at end of file
tools/train/faceLandmarks/js/randomCrop.js 0 → 100644
function randomCrop(img, pts) {
const xCoords = pts.map(pt => pt.x)
const yCoords = pts.map(pt => pt.y)
const minX = xCoords.reduce((x, min) => x < min ? x : min, img.width)
const minY = yCoords.reduce((y, min) => y < min ? y : min, img.height)
const maxX = xCoords.reduce((x, max) => x < max ? max : x, 0)
const maxY = yCoords.reduce((y, max) => y < max ? max : y, 0)
const x0 = Math.random() * minX
const y0 = Math.random() * minY
const x1 = (Math.random() * Math.abs(img.width - maxX)) + maxX
const y1 = (Math.random() * Math.abs(img.height - maxY)) + maxY
const targetCanvas = faceapi.createCanvas({ width: x1 - x0, height: y1 - y0 })
const inputCanvas = img instanceof HTMLCanvasElement ? img : faceapi.createCanvasFromMedia(img)
const region = faceapi.getContext2dOrThrow(inputCanvas)
.getImageData(x0, y0, targetCanvas.width, targetCanvas.height)
faceapi.getContext2dOrThrow(targetCanvas).putImageData(region, 0, 0)
return {
croppedImage: targetCanvas,
shiftedPts: pts.map(pt => new faceapi.Point(pt.x - x0, pt.y - y0))
}
}
\ No newline at end of file
tools/train/faceLandmarks/js/synchronizeDb.js 0 → 100644
async function fillPtsDb(ids) {
const numFetchPerIteration = 4000
for (let i = 0; i < Math.floor(ids.length / numFetchPerIteration) + 1; i++) {
const ptsById = await Promise.all(
ids.slice(i * numFetchPerIteration, (i + 1) * numFetchPerIteration)
.map(async id => ({
id,
pts: await (await fetch(`${id}.json`)).json()
}))
)
console.log('persistPts')
console.time('persistPts')
await persistPts(ptsById)
console.timeEnd('persistPts')
}
}
async function fillJpgsDb(ids) {
const numFetchPerIteration = 4000
for (let i = 0; i < Math.floor(ids.length / numFetchPerIteration) + 1; i++) {
const jpgsById = await Promise.all(
ids.slice(i * numFetchPerIteration, (i + 1) * numFetchPerIteration)
.map(async id => ({
id,
blob: await fetchImage(`${id}.jpg`)
}))
)
console.log('persistJpgs')
console.time('persistJpgs')
await persistJpgs(jpgsById)
console.timeEnd('persistJpgs')
}
}
\ No newline at end of file
tools/train/faceLandmarks/test.html 0 → 100644
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<script src="js/randomCrop.js"></script>
<script src="js/indexedDb.js"></script>
<script src="js/synchronizeDb.js"></script>
<link rel="stylesheet" href="styles.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.css">
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/js/materialize.min.js"></script>
</head>
<body>
<div id="result" style="position: relative" class="margin">
<img id="inputImg" src="" style="max-width: 400px;" />
<canvas id="overlay" />
</div>
<script>
const net = new faceapi.FaceLandmark68TinyNet()
const modelCheckpoint = 'tmp/densenet3_conv0/checkpoints/landmarks_epoch59.weights'
const crops = 4
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}
async function load() {
await net.load('./')
//const weights = await loadNetWeights(modelCheckpoint)
//await net.load(weights)
console.log('model loaded')
}
load()
function pointsToFlatArray(pts) {
return pts.map(pt => [pt.x, pt.y]).reduce((flat, arr) => flat.concat(arr))
}
async function showResults(landmarks) {
const inputImgEl = $('#inputImg').get(0)
const { width, height } = inputImgEl
const canvas = $('#overlay').get(0)
canvas.width = width
canvas.height = height
faceapi.drawLandmarks('overlay', landmarks, { drawLines: true })
}
async function testAll() {
window.testIds = (await fetch('/face_landmarks_test_ids').then(res => res.json()))
test()
}
async function testSidewaysX(useTrainSet = false) {
window.testIds = (await fetch(useTrainSet ? '/sideways_x_train' : '/sideways_x_test').then(res => res.json()))
test()
}
async function testStrongSidewaysX(useTrainSet = false) {
window.testIds = (await fetch(useTrainSet ? '/strong_sideways_x_train' : '/strong_sideways_x_test').then(res => res.json()))
test()
}
async function testStrongSidewaysY(useTrainSet = false) {
window.testIds = (await fetch(useTrainSet ? '/strong_sideways_y_train' : '/strong_sideways_y_test').then(res => res.json()))
test()
}
async function test() {
let totalError = 0
for (let i = 0; i < window.testIds.length; i++) {
const dataId = window.testIds[i]
let originalImage = (await getJpgs([dataId]))[0]
originalImage = await faceapi.bufferToImage(originalImage.blob)
const groundTruthPts = (await getPts([dataId])).map(res => res.pts)[0]
for (let c = 0; c < (crops || 1); c++) {
console.log(i, c, i / window.testIds.length)
const { croppedImage, shiftedPts } = crops > 0
? randomCrop(originalImage, groundTruthPts)
: { croppedImage: originalImage, shiftedPts: groundTruthPts }
const squaredImg = faceapi.imageToSquare(croppedImage, 112, true)
const landmarks = (await net.detectLandmarks(squaredImg)).forSize(croppedImage.width, croppedImage.height)
const dist = faceapi.euclideanDistance(
pointsToFlatArray(landmarks.getPositions()),
pointsToFlatArray(shiftedPts)
)
const distByCoordinate = dist / 136
totalError += distByCoordinate
if (window.debug) {
const inputImgEl = $('#inputImg').get(0)
inputImgEl.src = croppedImage instanceof HTMLImageElement ? croppedImage.src : croppedImage.toDataURL()
inputImgEl.onload = () => showResults(landmarks)
for (let i = 0; i < 30; i++)
await faceapi.tf.nextFrame()
}
}
}
console.log('done...')
console.log('test set size:', window.testIds.length)
console.log('total error:', totalError / (crops || 1))
console.log('average error:', totalError / ((crops || 1) * window.testIds.length))
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceLandmarks/train.html
<!DOCTYPE html>
<html>
<head>
<link rel="stylesheet" href="styles.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.css">
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/js/materialize.min.js"></script>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<script src="faceLandmarksUi.js"></script>
<script src="faceLandmarksTrain.js"></script>
<script src="js/randomCrop.js"></script>
<script src="js/indexedDb.js"></script>
<script src="js/synchronizeDb.js"></script>
<script src="FileSaver.js"></script>
<script src="trainUtils.js"></script>
</head>
<body>
<div id="navbar"></div>
<div class="center-content page-container">
<div>
<div class="progress" id="loader">
<div class="indeterminate"></div>
</div>
<div class="row side-by-side">
<div class="center-content">
<div id="faceContainer"></div>
<div id="selectList"></div>
</div>
</div>
<p>
<input type="checkbox" id="drawLinesCheckbox" checked="checked" onchange="onChangeDrawLines(event)" />
<label for="drawLinesCheckbox">Draw Lines</label>
</p>
<div class="row">
<label for="imgByNr">Enter image NR: </label>
<input id="imgByNr" type="text" class="bold">
</div>
</div>
</div>
<div id="container"></div>
<script>
tf = faceapi.tf
window.saveEveryNthIteration = 2
window.drawLines = true
window.nets = []
window.landmarks = []
window.trainSteps = 100
window.learningRate = 0.01
//window.optimizer = tf.train.sgd(learningRate)
window.optimizer = tf.train.adam(0.001, 0.9, 0.999, 1e-8)
function lossFunction(labels, out) {
return tf.losses.meanSquaredError(labels, out)
// uri to weights file of last checkpoint
window.net = new faceapi.FaceLandmark68TinyNet()
const modelCheckpoint = '/tmp/initial_glorot.weights.weights'
const startEpoch = 0
const learningRate = 0.001 // 0.001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.saveEveryNthSample = Infinity
window.withRandomCrop = true
window.batchSize = 12
//window.batchSize = 32
window.lossValues = []
window.iterDelay = 0
window.withLogging = true
const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
function saveWeights(net, filename = 'train_tmp') {
saveAs(new Blob([net.serializeParams()]), filename)
}
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}
async function delay(ms) {
return new Promise(res => setTimeout(res, ms))
}
async function run() {
window.trainData = await getTrainData()
window.trainNet.variable()
async function fetchTrainDataIds() {
return fetch('/face_landmarks_train_ids').then(res => res.json())
}
async function load() {
window.trainIds = (await fetchTrainDataIds())
const weights = await loadNetWeights(modelCheckpoint)
await window.net.load(weights)
window.net.variable()
}
async function onEpochDone(epoch) {
saveWeights(window.net, `landmarks_epoch59_sideways71_epoch8_sideways35_epoch${epoch}.weights`)
return
const loss = window.lossValues[epoch]
saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / window.trainIds.length })]), `landmarks_epoch59_sideways71_epoch8_sideways35_epoch${epoch}.json`)
await train()
}
async function train(batchSize = 1) {
for (let i = 0; i < trainSteps; i++) {
console.log('step', i)
const batchCreators = createBatchCreators(shuffle(window.trainData), batchSize)
let ts = Date.now()
await trainStep(batchCreators)
ts = Date.now() - ts
console.log('step %s done (%s ms)', i, ts)
if (((i + 1) % saveEveryNthIteration) === 0) {
//saveWeights(window.trainNet, 'landmark_trained_weights_' + idx + '.weights')
async function train() {
await load()
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
if (epoch !== startEpoch) {
// ugly hack to wait for loss datas for that epoch to be resolved
setTimeout(() => onEpochDone(epoch - 1), 10000)
}
window.lossValues[epoch] = 0
const shuffledInputs = faceapi.shuffleArray(window.trainIds)
for (let dataIdx = 0; dataIdx < shuffledInputs.length; dataIdx += window.batchSize) {
const tsIter = Date.now()
const ids = shuffledInputs.slice(dataIdx, dataIdx + window.batchSize)
// fetch image and ground truth bounding boxes
let tsFetch = Date.now()
let tsFetchPts = Date.now()
const allPts = (await getPts(ids)).map(res => res.pts)
tsFetchPts = Date.now() - tsFetchPts
let tsFetchJpgs = Date.now()
const allJpgBlobs = (await getJpgs(ids)).map(res => res.blob)
tsFetchJpgs = Date.now() - tsFetchJpgs
let tsBufferToImage = Date.now()
const allJpgs = await Promise.all(
allJpgBlobs.map(blob => faceapi.bufferToImage(blob))
)
tsBufferToImage = Date.now() - tsBufferToImage
tsFetch = Date.now() - tsFetch
const bImages = []
const bPts = []
for (let batchIdx = 0; batchIdx < ids.length; batchIdx += 1) {
const pts = allPts[batchIdx]
const img = allJpgs[batchIdx]
const { croppedImage, shiftedPts } = window.withRandomCrop
? randomCrop(img, pts)
: { croppedImage: img, shiftedPts: pts }
const squaredImg = faceapi.imageToSquare(croppedImage, 112, true)
const reshapedDimensions = faceapi.computeReshapedDimensions(croppedImage, 112)
const pad = Math.abs(reshapedDimensions.width - reshapedDimensions.height) / (2 * 112)
const padX = reshapedDimensions.width < reshapedDimensions.height ? pad : 0
const padY = reshapedDimensions.height < reshapedDimensions.width ? pad : 0
const groundTruthLandmarks = shiftedPts.map(pt => new faceapi.Point(
padX + (pt.x / croppedImage.width) * (reshapedDimensions.width / 112),
padY + (pt.y / croppedImage.height)* (reshapedDimensions.height / 112)
))
bImages.push(squaredImg)
bPts.push(groundTruthLandmarks)
}
const netInput = await faceapi.toNetInput(bImages)
let tsBackward = Date.now()
let tsForward = Date.now()
const loss = optimizer.minimize(() => {
const out = window.net.runNet(netInput)
tsForward = Date.now() - tsForward
tsBackward = Date.now()
const landmarksTensor = tf.tensor2d(
bPts
.reduce((flat, arr) => flat.concat(arr))
.map(pt => [pt.x, pt.y])
.reduce((flat, arr) => flat.concat(arr)),
[ids.length, 136]
)
const loss = tf.losses.meanSquaredError(
landmarksTensor,
out,
tf.Reduction.MEAN
)
return loss
}, true)
tsBackward = Date.now() - tsBackward
// start next iteration without waiting for loss data
loss.data().then(data => {
const lossValue = data[0]
window.lossValues[epoch] += lossValue
window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - loss: ${lossValue}, ( ${window.lossValues[epoch]})`)
loss.dispose()
})
window.withLogging && log(`epoch ${epoch}, dataIdx ${dataIdx} - forward: ${tsForward} ms, backprop: ${tsBackward} ms, iter: ${Date.now() - tsIter} ms`)
if (window.logsilly) {
log(`fetch: ${tsFetch} ms, pts: ${tsFetchPts} ms, jpgs: ${tsFetchJpgs} ms, bufferToImage: ${tsBufferToImage} ms`)
}
if (window.iterDelay) {
await delay(window.iterDelay)
} else {
await tf.nextFrame()
}
}
}
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceLandmarks/trainData.json 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
tools/train/faceLandmarks/train_debug.html 0 → 100644
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<script src="js/randomCrop.js"></script>
<script src="js/indexedDb.js"></script>
<script src="FileSaver.js"></script>
</head>
<body>
<div id="container"></div>
<script>
tf = faceapi.tf
// uri to weights file of last checkpoint
window.net = new faceapi.FaceLandmark68TinyNet()
const modelCheckpoint = '/tmp/initial_glorot.weights'
const startEpoch = 0
const learningRate = 0.001 // 0.001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.saveEveryNthSample = 50000000
window.withRandomCrop = false
window.batchSize = 18
window.lossValue = 0
window.drawResults = true
const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
function saveWeights(net, filename = 'train_tmp') {
saveAs(new Blob([net.serializeParams()]), filename)
}
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}
async function fetchTrainDataFilenames() {
return fetch('/face_landmarks_train_ids').then(res => res.json())
}
async function load() {
window.ptsFiles = await fetchTrainDataFilenames()
window.ptsFiles = window.ptsFiles.slice(1000, 1000 + window.batchSize)
//window.ptsFiles = window.ptsFiles.slice(7, 8)
const weights = await loadNetWeights(modelCheckpoint)
await window.net.load(weights)
window.net.variable()
}
function randomCrop(img, pts) {
const xCoords = pts.map(pt => pt.x)
const yCoords = pts.map(pt => pt.y)
const minX = xCoords.reduce((x, min) => x < min ? x : min, img.width)
const minY = yCoords.reduce((y, min) => y < min ? y : min, img.height)
const maxX = xCoords.reduce((x, max) => x < max ? max : x, 0)
const maxY = yCoords.reduce((y, max) => y < max ? max : y, 0)
const x0 = Math.random() * minX
const y0 = Math.random() * minY
const x1 = (Math.random() * Math.abs(img.width - maxX)) + maxX
const y1 = (Math.random() * Math.abs(img.height - maxY)) + maxY
const targetCanvas = faceapi.createCanvas({ width: x1 - x0, height: y1 - y0 })
const inputCanvas = img instanceof HTMLCanvasElement ? img : faceapi.createCanvasFromMedia(img)
const region = faceapi.getContext2dOrThrow(inputCanvas)
.getImageData(x0, y0, targetCanvas.width, targetCanvas.height)
faceapi.getContext2dOrThrow(targetCanvas).putImageData(region, 0, 0)
return {
croppedImage: targetCanvas,
shiftedPts: pts.map(pt => new faceapi.Point(pt.x - x0, pt.y - y0))
}
}
async function train() {
await load()
for (let epoch = startEpoch; epoch < Infinity; epoch++) {
if (epoch !== startEpoch) {
//saveWeights(window.net, `landmarks_epoch${epoch - 1}.weights`)
//saveAs(new Blob([JSON.stringify({ loss: window.lossValue, avgLoss: window.lossValue / 20000 })]), `landmarks_epoch${epoch - 1}.json`)
window.lossValue = 0
}
const shuffledInputs = window.drawResults ? window.ptsFiles : faceapi.shuffleArray(window.ptsFiles)
for (let dataIdx = 0; dataIdx < shuffledInputs.length; dataIdx += window.batchSize) {
const ids = shuffledInputs.slice(dataIdx, dataIdx + window.batchSize)
// fetch image and ground truth bounding boxes
console.time('getPts')
const allPts = (await getPts(ids)).map(res => res.pts)
console.timeEnd('getPts')
console.time('getJpgs')
const allJpgs = await Promise.all(
(await getJpgs(ids)).map(res => res.blob).map(blob => faceapi.bufferToImage(blob))
)
console.timeEnd('getJpgs')
const bSquaredImages = []
const bCroppedImages = []
const bImages = []
const bPts = []
for (let batchIdx = 0; batchIdx < ids.length; batchIdx += 1) {
const pts = allPts[batchIdx]
const img = allJpgs[batchIdx]
const { croppedImage, shiftedPts } = window.withRandomCrop
? randomCrop(img, pts)
: { croppedImage: img, shiftedPts: pts }
const squaredImg = faceapi.imageToSquare(croppedImage, 112, true)
const reshapedDimensions = faceapi.computeReshapedDimensions(croppedImage, 112)
const pad = Math.abs(reshapedDimensions.width - reshapedDimensions.height) / (2 * 112)
const padX = reshapedDimensions.width < reshapedDimensions.height ? pad : 0
const padY = reshapedDimensions.height < reshapedDimensions.width ? pad : 0
const groundTruthLandmarks = shiftedPts.map(pt => new faceapi.Point(
padX + (pt.x / croppedImage.width) * (reshapedDimensions.width / 112),
padY + (pt.y / croppedImage.height)* (reshapedDimensions.height / 112)
))
bSquaredImages.push(squaredImg)
bCroppedImages.push(croppedImage)
bImages.push(squaredImg)
bPts.push(groundTruthLandmarks)
}
const netInput = await faceapi.toNetInput(bImages)
const ts = Date.now()
const loss = optimizer.minimize(() => {
console.time('runNet')
const out = window.net.runNet(netInput)
console.timeEnd('runNet')
if (window.drawResults) {
// debug
const landmarkTensors = tf.unstack(out)
landmarkTensors.forEach((t, i) => {
const croppedImage = bSquaredImages[i]
const landmarksArray = Array.from(t.dataSync())
const xCoords = landmarksArray.filter((_, i) => faceapi.isEven(i))
const yCoords = landmarksArray.filter((_, i) => !faceapi.isEven(i))
/*
const landmarksArray = bPts[i]
const xCoords = landmarksArray.map(pt => pt.x)
const yCoords = landmarksArray.map(pt => pt.y)
*/
const marks = new faceapi.FaceLandmarks68(
Array(68).fill(0).map((_, i) => new faceapi.Point(xCoords[i], yCoords[i])),
croppedImage
)
const canvas = croppedImage instanceof HTMLCanvasElement ? croppedImage : faceapi.createCanvasFromMedia(croppedImage)
faceapi.drawLandmarks(canvas, marks, { drawLines: true })
const container = document.getElementById('container')
if (container.children[i]) {
const oldChild = container.children[i]
container.insertBefore(canvas, oldChild)
container.removeChild(oldChild)
} else {
container.appendChild(canvas)
}
})
// debug
}
const landmarksTensor = tf.tensor2d(
bPts
.reduce((flat, arr) => flat.concat(arr))
.map(pt => [pt.x, pt.y])
.reduce((flat, arr) => flat.concat(arr)),
[window.batchSize, 136]
)
const loss = tf.losses.meanSquaredError(
landmarksTensor,
out,
tf.Reduction.MEAN
)
//return tf.sum(tf.abs(tf.sub(landmarksTensor, out)))
return loss
}, true)
// start next iteration without waiting for loss data
loss.data().then(data => {
const lossValue = data[0]
window.lossValue += lossValue
log(`epoch ${epoch}, dataIdx ${dataIdx} - loss: ${lossValue}`)
loss.dispose()
})
if (window.iterDelay) {
await delay(window.iterDelay)
} else {
await tf.nextFrame()
}
}
}
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceLandmarks/verify.html 0 → 100644
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<script src="js/randomCrop.js"></script>
<link rel="stylesheet" href="styles.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.css">
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/js/materialize.min.js"></script>
</head>
<body>
<div id="navbar"></div>
<div class="center-content page-container">
<div class="progress" id="loader">
<div class="indeterminate"></div>
</div>
<div id="results" class="side-by-side">
<div id="result0" style="position: relative" class="margin">
<img id="inputImg0" src="" style="max-width: 400px;" />
<canvas class="overlay" id="overlay0" />
</div>
<div id="result1" style="position: relative" class="margin">
<img id="inputImg1" src="" style="max-width: 400px;" />
<canvas class="overlay" id="overlay1" />
</div>
<div id="result2" style="position: relative" class="margin">
<img id="inputImg2" src="" style="max-width: 400px;" />
<canvas class="overlay" id="overlay2" />
</div>
</div>
<div class="row">
<label for="imgByNr">Enter image NR: </label>
<input id="imgByNr" type="text" class="bold" value="0"/>
</div>
</div>
<script>
const modelCheckpoints = [
//'tmp/densenet3_conv0/checkpoints/landmarks_epoch59.weights'
]
let originalImage = null
function cloneImage(img) {
var image = document.createElement("img")
image.src = img.src
return image
}
function getInputValue(input) {
input = input || $('#imgByNr').get(0)
return input.value || 0
}
function onKeyDown(e) {
e.target.value = (
parseInt(e.target.value) + (e.keyCode === 38 ? 1 : (e.keyCode === 40 ? -1 : 0))
) || getInputValue(e.target)
const imgUri = `${window.testIds[e.target.value]}.jpg`
console.log(imgUri)
onSelectionChanged(imgUri)
}
async function updateResults(_landmarks) {
const inputImgEl = $('#inputImg0').get(0)
const { width, height } = inputImgEl
for (let i = 0; i < window.nets.length; i++) {
const net = window.nets[i]
const canvas = $(`#overlay${i}`).get(0)
canvas.width = width
canvas.height = height
console.time('detectLandmarks')
const landmarks = await net.detectLandmarks(inputImgEl)
console.timeEnd('detectLandmarks')
faceapi.drawLandmarks(`overlay${i}`, landmarks.forSize(width, height), { drawLines: true })
}
}
async function onSelectionChanged(imgUri) {
const img = await faceapi.bufferToImage(await fetchImage(imgUri))
window.nets.forEach(async (_, i) => {
$(`#inputImg${i}`).get(0).src = img.src
})
originalImage = cloneImage(img)
updateResults()
}
async function applyRandomCrop() {
const dataId = window.testIds[getInputValue()]
const pts = await (await fetch(`${dataId}.json`)).json()
const img = cloneImage(originalImage)
const { croppedImage, shiftedPts } = randomCrop(img, pts)
img.src = croppedImage.toDataURL()
$(`#result${0}`).get(0).removeChild($(`#inputImg${0}`).get(0))
$(`#result${0}`).get(0).appendChild(img)
window.nets.slice(1).forEach((_, i) => {
const im = cloneImage(img)
im.id = `#inputImg${i + 1}`
$(`#result${i + 1}`).get(0).removeChild($(`#inputImg${i + 1}`).get(0))
$(`#result${i + 1}`).get(0).appendChild(im)
})
const { width, height } = croppedImage
img.onload = () => updateResults(
new faceapi.FaceLandmarks68(
shiftedPts.map(pt =>
new faceapi.Point(
pt.x / width,
pt.y / height
)
),
{ width, height }
)
)
}
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}
async function run() {
$('#imgByNr').keydown(onKeyDown)
faceapi.loadFaceLandmarkModel('./')
//window.trainIds = (await fetch('/face_landmarks_train_ids').then(res => res.json()))
//window.testIds = (await fetch('/face_landmarks_test_ids').then(res => res.json()))
window.testIds = (await fetch('/strong_sideways_test').then(res => res.json()))
window.nets = []
window.nets.push(faceapi)
const net = new faceapi.FaceLandmark68TinyNet()
await net.load('./')
window.nets.push(net)
modelCheckpoints.forEach(async checkpoint => {
const net = new faceapi.FaceLandmark68TinyNet()
const weights = await loadNetWeights(checkpoint)
await net.load(weights)
window.nets.push(net)
})
$('#loader').hide()
}
$(document).ready(() => run())
</script>
</body>
</html>
\ No newline at end of file
tools/train/serveFaceLandmarks.js
......@@ -2,6 +2,7 @@ require('./faceLandmarks/.env')
const express = require('express')
const path = require('path')
const fs = require('fs')
const app = express()
......@@ -14,9 +15,72 @@ app.use(express.static(path.join(__dirname, '../../weights')))
app.use(express.static(path.join(__dirname, '../../dist')))
const trainDataPath = path.resolve(process.env.TRAIN_DATA_PATH)
app.use(express.static(trainDataPath))
app.get('/', (req, res) => res.redirect('/face_landmarks'))
app.get('/face_landmarks', (req, res) => res.sendFile(path.join(publicDir, 'train.html')))
const pngPath = path.join(trainDataPath, 'png')
const jpgPath = path.join(trainDataPath, 'jpg')
const groundTruthPath = path.join(trainDataPath, 'pts')
app.listen(3000, () => console.log('Listening on port 3000!'))
\ No newline at end of file
app.use(express.static(jpgPath))
app.use(express.static(groundTruthPath))
const trainIds = JSON.parse(fs.readFileSync(path.join(publicDir, './trainData.json')))
const trainIdsSet = new Set(trainIds)
const testFileIds = fs.readdirSync(groundTruthPath)
.map(file => file.replace('.json', ''))
.filter(file => !trainIdsSet.has(file))
app.get('/face_landmarks_train_ids', (req, res) => res.status(202).send(trainIds))
app.get('/face_landmarks_test_ids', (req, res) => res.status(202).send(testFileIds))
const poseAnchors = JSON.parse(fs.readFileSync(path.join(trainDataPath, './pose-anchors.json')))
const sidewaysXTrain = []
const strongSidewaysXTrain = []
const strongSidewaysYTrain = []
const strongSidewaysTrain = []
const sidewaysXTest = []
const strongSidewaysXTest = []
const strongSidewaysYTest = []
const strongSidewaysTest = []
poseAnchors.forEach((pt, idx) => {
const id = `${idx}`
if (Math.abs(0.5 - pt.x) > 0.15) {
(trainIdsSet.has(id) ? sidewaysXTrain : sidewaysXTest).push(id)
}
if (Math.abs(0.5 - pt.x) > 0.2) {
(trainIdsSet.has(id) ? strongSidewaysXTrain : strongSidewaysXTest).push(id)
}
if (Math.abs(0.44 - pt.y) > 0.1) {
(trainIdsSet.has(id) ? strongSidewaysYTrain : strongSidewaysYTest).push(id)
}
if (Math.abs(0.5 - pt.x) > 0.2 || Math.abs(0.44 - pt.y) > 0.1) {
(trainIdsSet.has(id) ? strongSidewaysTrain : strongSidewaysTest).push(id)
}
})
console.log(sidewaysXTrain.length)
console.log(strongSidewaysXTrain.length)
console.log(strongSidewaysYTrain.length)
console.log(strongSidewaysTrain.length)
console.log(sidewaysXTest.length)
console.log(strongSidewaysXTest.length)
console.log(strongSidewaysYTest.length)
console.log(strongSidewaysTrain.length)
app.get('/sideways_x_train', (req, res) => res.status(202).send(sidewaysXTrain))
app.get('/strong_sideways_x_train', (req, res) => res.status(202).send(strongSidewaysXTrain))
app.get('/strong_sideways_y_train', (req, res) => res.status(202).send(strongSidewaysYTrain))
app.get('/strong_sideways_train', (req, res) => res.status(202).send(strongSidewaysTrain))
app.get('/sideways_x_test', (req, res) => res.status(202).send(sidewaysXTest))
app.get('/strong_sideways_x_test', (req, res) => res.status(202).send(strongSidewaysXTest))
app.get('/strong_sideways_y_test', (req, res) => res.status(202).send(strongSidewaysYTest))
app.get('/strong_sideways_test', (req, res) => res.status(202).send(strongSidewaysTest))
app.get('/', (req, res) => res.redirect('/train'))
app.get('/train', (req, res) => res.sendFile(path.join(publicDir, 'train.html')))
app.get('/verify', (req, res) => res.sendFile(path.join(publicDir, 'verify.html')))
app.get('/test', (req, res) => res.sendFile(path.join(publicDir, 'test.html')))
app.listen(8000, () => console.log('Listening on port 8000!'))
\ No newline at end of file
tools/train/serveTinyYolov2.js deleted 100644 → 0
require('./tinyYolov2/.env')
const express = require('express')
const path = require('path')
const fs = require('fs')
const app = express()
const publicDir = path.join(__dirname, './tinyYolov2')
app.use(express.static(publicDir))
app.use(express.static(path.join(__dirname, './shared')))
app.use(express.static(path.join(__dirname, './node_modules/file-saver')))
app.use(express.static(path.join(__dirname, '../../examples/public')))
app.use(express.static(path.join(__dirname, '../../weights')))
app.use(express.static(path.join(__dirname, '../../dist')))
const trainDataPath = path.resolve(process.env.TRAIN_DATA_PATH)
const testDataPath = path.resolve(process.env.TEST_DATA_PATH)
const imagesPath = path.join(trainDataPath, './final_images')
const detectionsPath = path.join(trainDataPath, './final_detections')
app.use(express.static(imagesPath))
app.use(express.static(detectionsPath))
app.use(express.static(testDataPath))
const detectionFilenames = fs.readdirSync(detectionsPath)
const detectionFilenamesMultibox = JSON.parse(fs.readFileSync(path.join(__dirname, './tinyYolov2/multibox.json')))
app.use(express.static(trainDataPath))
app.get('/detection_filenames', (req, res) => res.status(202).send(detectionFilenames))
app.get('/detection_filenames_multibox', (req, res) => res.status(202).send(detectionFilenamesMultibox))
app.get('/', (req, res) => res.sendFile(path.join(publicDir, 'train.html')))
app.get('/verify', (req, res) => res.sendFile(path.join(publicDir, 'verify.html')))
app.get('/test', (req, res) => res.sendFile(path.join(publicDir, 'test.html')))
app.listen(3000, () => console.log('Listening on port 3000!'))
\ No newline at end of file
tools/train/shared/trainUtils.js deleted 100644 → 0
const log = (str, ...args) => console.log(`[${[(new Date()).toTimeString().substr(0, 8)]}] ${str || ''}`, ...args)
async function promiseSequential(promises) {
const curr = promises[0]
if (!curr) {
return
}
await curr()
return promiseSequential(promises.slice(1))
}
// https://stackoverflow.com/questions/6274339/how-can-i-shuffle-an-array
function shuffle(a) {
var j, x, i;
for (i = a.length - 1; i > 0; i--) {
j = Math.floor(Math.random() * (i + 1));
x = a[i];
a[i] = a[j];
a[j] = x;
}
return a;
}
function saveWeights(net, filename = 'train_tmp') {
const binaryWeights = new Float32Array(
net.getParamList()
.map(({ tensor }) => Array.from(tensor.dataSync()))
.reduce((flat, arr) => flat.concat(arr))
)
saveAs(new Blob([binaryWeights]), filename)
}
function imageToSquare(img) {
const scale = 608 / Math.max(img.height, img.width)
const width = scale * img.width
const height = scale * img.height
const canvas1 = faceapi.createCanvasFromMedia(img)
const targetCanvas = faceapi.createCanvas({ width: 608, height: 608 })
targetCanvas.getContext('2d').putImageData(canvas1.getContext('2d').getImageData(0, 0, width, height), 0, 0)
return targetCanvas
}
function getPaddingsAndReshapedSize(img, inputSize) {
const [h, w] = [img.height, img.width]
const maxDim = Math.max(h, w)
const f = inputSize / maxDim
const reshapedImgDims = {
height: Math.floor(h * f),
width: Math.floor(w * f)
}
const paddings = new faceapi.Point(
maxDim / img.width,
maxDim / img.height
)
return { paddings, reshapedImgDims }
}
\ No newline at end of file
tools/train/tinyYolov2/initWeights.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs@0.12.0"> </script>
<script src="FileSaver.js"></script>
</head>
<body>
<script>
const glorotNormal = tf.initializers.glorotNormal()
saveWeights()
function initSeparableConvWeights(inChannels, outChannels) {
return {
depthwiseFilter: glorotNormal.apply([3, 3, inChannels, 1]),
pointwiseFilter: glorotNormal.apply([1, 1, inChannels, outChannels]),
bias: tf.zeros([outChannels])
}
}
function initTinyYolov2SeparableWeights() {
const conv0 = initSeparableConvWeights(3, 16)
const conv1 = initSeparableConvWeights(16, 32)
const conv2 = initSeparableConvWeights(32, 64)
const conv3 = initSeparableConvWeights(64, 128)
const conv4 = initSeparableConvWeights(128, 256)
const conv5 = initSeparableConvWeights(256, 512)
const conv6 = initSeparableConvWeights(512, 1024)
const conv7 = initSeparableConvWeights(1024, 1024)
const conv8 = {
filters: glorotNormal.apply([1, 1, 1024, 25]),
bias: tf.zeros([25])
}
return {
conv0,
conv1,
conv2,
conv3,
conv4,
conv5,
conv6,
conv7,
conv8
}
}
function saveWeights() {
const w = initTinyYolov2SeparableWeights()
const binaryWeights = new Float32Array(
Array(8).fill(0)
.map((_, i) => w[`conv${i}`])
.map(ps => [ps.depthwiseFilter, ps.pointwiseFilter, ps.bias])
.reduce((flat, arr) => flat.concat(arr))
.concat([w.conv8.filters, w.conv8.bias])
.map(tensor => console.log(tensor) || Array.from(tensor.dataSync()))
.reduce((flat, arr) => flat.concat(arr))
)
saveAs(new Blob([binaryWeights]), 'foo.weights')
}
</script>
</body>
</html>
\ No newline at end of file
tools/train/tinyYolov2/loss.js deleted 100644 → 0
const CELL_SIZE = 32
const getNumCells = inputSize => inputSize / CELL_SIZE
const inverseSigmoid = x => Math.log(x / (1 - x))
function getAnchors() {
return window.net.anchors
}
function squaredSumOverMask(lossTensors, mask) {
return tf.tidy(() => tf.sum(tf.square(tf.mul(mask, lossTensors))))
}
function assignBoxesToAnchors(groundTruthBoxes, reshapedImgDims) {
const inputSize = Math.max(reshapedImgDims.width, reshapedImgDims.height)
const numCells = getNumCells(inputSize)
return groundTruthBoxes.map(box => {
const { left, top, width, height } = box.rescale(reshapedImgDims)
const ctX = left + (width / 2)
const ctY = top + (height / 2)
const col = Math.floor((ctX / inputSize) * numCells)
const row = Math.floor((ctY / inputSize) * numCells)
const anchorsByIou = getAnchors().map((anchor, idx) => ({
idx,
iou: faceapi.iou(
new faceapi.BoundingBox(0, 0, anchor.x * CELL_SIZE, anchor.y * CELL_SIZE),
new faceapi.BoundingBox(0, 0, width, height)
)
})).sort((a1, a2) => a2.iou - a1.iou)
const anchor = anchorsByIou[0].idx
return { row, col, anchor, box }
})
}
function getGroundTruthMask(groundTruthBoxes, inputSize) {
const numCells = getNumCells(inputSize)
const mask = tf.zeros([numCells, numCells, 25])
const buf = mask.buffer()
groundTruthBoxes.forEach(({ row, col, anchor }) => {
const anchorOffset = anchor * 5
for (let i = 0; i < 5; i++) {
buf.set(1, row, col, anchorOffset + i)
}
})
return mask
}
function getCoordAndScoreMasks(inputSize) {
const numCells = getNumCells(inputSize)
const coordMask = tf.zeros([numCells, numCells, 25])
const scoreMask = tf.zeros([numCells, numCells, 25])
const coordBuf = coordMask.buffer()
const scoreBuf = scoreMask.buffer()
for (let row = 0; row < numCells; row++) {
for (let col = 0; col < numCells; col++) {
for (let anchor = 0; anchor < 5; anchor++) {
const anchorOffset = 5 * anchor
for (let i = 0; i < 4; i++) {
coordBuf.set(1, row, col, anchorOffset + i)
}
scoreBuf.set(1, row, col, anchorOffset + 4)
}
}
}
return { coordMask, scoreMask }
}
function computeBoxAdjustments(groundTruthBoxes, reshapedImgDims) {
const inputSize = Math.max(reshapedImgDims.width, reshapedImgDims.height)
const numCells = getNumCells(inputSize)
const adjustments = tf.zeros([numCells, numCells, 25])
const buf = adjustments.buffer()
groundTruthBoxes.forEach(({ row, col, anchor, box }) => {
const { left, top, right, bottom, width, height } = box.rescale(reshapedImgDims)
const centerX = (left + right) / 2
const centerY = (top + bottom) / 2
//const dCenterX = centerX - (col * CELL_SIZE + (CELL_SIZE / 2))
//const dCenterY = centerY - (row * CELL_SIZE + (CELL_SIZE / 2))
const dCenterX = centerX - (col * CELL_SIZE)
const dCenterY = centerY - (row * CELL_SIZE)
const dx = inverseSigmoid(dCenterX / CELL_SIZE)
const dy = inverseSigmoid(dCenterY / CELL_SIZE)
//const dx = dCenterX / CELL_SIZE
//const dy = dCenterY / CELL_SIZE
const dw = Math.log((width / CELL_SIZE) / getAnchors()[anchor].x)
const dh = Math.log((height / CELL_SIZE) / getAnchors()[anchor].y)
const anchorOffset = anchor * 5
buf.set(dx, row, col, anchorOffset + 0)
buf.set(dy, row, col, anchorOffset + 1)
buf.set(dw, row, col, anchorOffset + 2)
buf.set(dh, row, col, anchorOffset + 3)
})
return adjustments
}
function computeIous(predBoxes, groundTruthBoxes, reshapedImgDims) {
const inputSize = Math.max(reshapedImgDims.width, reshapedImgDims.height)
const numCells = getNumCells(inputSize)
const isSameAnchor = p1 => p2 =>
p1.row === p2.row
&& p1.col === p2.col
&& p1.anchor === p2.anchor
const ious = tf.zeros([numCells, numCells, 25])
const buf = ious.buffer()
groundTruthBoxes.forEach(({ row, col, anchor, box }) => {
const predBox = predBoxes.find(isSameAnchor({ row, col, anchor }))
if (!predBox) {
console.log(groundTruthBoxes)
console.log(predBoxes)
throw new Error(`no output box found for: row ${row}, col ${col}, anchor ${anchor}`)
}
const iou = faceapi.iou(
box.rescale(reshapedImgDims),
predBox.box.rescale(reshapedImgDims)
)
if (window.debug) {
console.log('ground thruth box:', box.rescale(reshapedImgDims).toRect())
console.log('predicted box:', predBox.box.rescale(reshapedImgDims).toRect())
console.log('predicted score:', predBox.score)
console.log('iou:', iou)
}
const anchorOffset = anchor * 5
buf.set(iou, row, col, anchorOffset + 4)
})
return ious
}
window.computeNoObjectLoss = function(outTensor, mask) {
return tf.tidy(() => {
const lossTensor = tf.sigmoid(outTensor)
return squaredSumOverMask(lossTensor, mask)
})
}
function computeObjectLoss(outTensor, groundTruthBoxes, reshapedImgDims, paddings, mask) {
return tf.tidy(() => {
const predBoxes = window.net.postProcess(
outTensor,
{ paddings }
)
const ious = computeIous(
predBoxes,
groundTruthBoxes,
reshapedImgDims
)
const lossTensor = tf.sub(ious, tf.sigmoid(outTensor))
return squaredSumOverMask(lossTensor, mask)
})
}
function computeCoordLoss(groundTruthBoxes, outTensor, reshapedImgDims, mask, paddings) {
return tf.tidy(() => {
const boxAdjustments = computeBoxAdjustments(
groundTruthBoxes,
reshapedImgDims
)
if (window.debug) {
const indToPos = []
const numCells = outTensor.shape[1]
for (let row = 0; row < numCells; row++) {
for (let col = 0; col < numCells; col++) {
for (let anchor = 0; anchor < 25; anchor++) {
indToPos.push({ row, col, anchor: parseInt(anchor / 5) })
}
}
}
const indices = Array.from(mask.dataSync()).map((val, ind) => ({ val, ind })).filter(v => v.val !== 0).map(v => v.ind)
const gt = Array.from(boxAdjustments.dataSync())
const out = Array.from(outTensor.dataSync())
const comp = indices.map(i => (
{
pos: indToPos[i],
gt: gt[i],
out: out[i]
}
))
console.log(comp.map(c => `gt: ${c.gt}, out: ${c.out}`))
const getBbox = (which) => {
const { row, col, anchor } = comp[0].pos
const ctX = ((col + faceapi.sigmoid(comp[0][which])) / numCells) * paddings.x
const ctY = ((row + faceapi.sigmoid(comp[1][which])) / numCells) * paddings.y
const width = ((Math.exp(comp[2][which]) * getAnchors()[anchor].x) / numCells) * paddings.x
const height = ((Math.exp(comp[3][which]) * getAnchors()[anchor].y) / numCells) * paddings.y
const x = (ctX - (width / 2))
const y = (ctY - (height / 2))
return new faceapi.BoundingBox(x, y, x + width, y + height)
}
const outRect = getBbox('out').rescale(reshapedImgDims).toRect()
const gtRect = getBbox('gt').rescale(reshapedImgDims).toRect()
console.log('out', outRect)
console.log('gtRect', gtRect)
}
const lossTensor = tf.sub(boxAdjustments, outTensor)
return squaredSumOverMask(lossTensor, mask)
})
}
function computeLoss(outTensor, groundTruth, reshapedImgDims, paddings) {
const inputSize = Math.max(reshapedImgDims.width, reshapedImgDims.height)
if (!inputSize) {
throw new Error(`invalid inputSize: ${inputSize}`)
}
let groundTruthBoxes = assignBoxesToAnchors(
groundTruth
.map(({ x, y, width, height }) => new faceapi.Rect(x, y, width, height))
.map(rect => rect.toBoundingBox()),
reshapedImgDims
)
const groundTruthMask = getGroundTruthMask(groundTruthBoxes, inputSize)
const { coordMask, scoreMask } = getCoordAndScoreMasks(inputSize)
const noObjectLossMask = tf.tidy(() => tf.mul(scoreMask, tf.sub(tf.scalar(1), groundTruthMask)))
const objectLossMask = tf.tidy(() => tf.mul(scoreMask, groundTruthMask))
const coordLossMask = tf.tidy(() => tf.mul(coordMask, groundTruthMask))
const noObjectLoss = tf.tidy(() =>
tf.mul(
tf.scalar(noObjectScale),
computeNoObjectLoss(outTensor, noObjectLossMask)
)
)
const objectLoss = tf.tidy(() =>
tf.mul(
tf.scalar(objectScale),
computeObjectLoss(outTensor, groundTruthBoxes, reshapedImgDims, paddings, objectLossMask)
)
)
const coordLoss = tf.tidy(() =>
tf.mul(
tf.scalar(coordScale),
computeCoordLoss(groundTruthBoxes, outTensor, reshapedImgDims, coordLossMask, paddings)
)
)
const totalLoss = tf.tidy(() => noObjectLoss.add(objectLoss).add(coordLoss))
return {
noObjectLoss,
objectLoss,
coordLoss,
totalLoss
}
}
\ No newline at end of file
tools/train/tinyYolov2/loss.test.ts deleted 100644 → 0
import * as _tf from '@tensorflow/tfjs-core';
const faceapi = require('../../../dist/face-api.js')
const tf: typeof _tf = faceapi.tf
require('./loss')
window['faceapi'] = faceapi
window['tf'] = tf
const anchors = [
new faceapi.Point(1.603231, 2.094468),
new faceapi.Point(6.041143, 7.080126),
new faceapi.Point(2.882459, 3.518061),
new faceapi.Point(4.266906, 5.178857),
new faceapi.Point(9.041765, 10.66308)
]
window['net'] = {
getAnchors() {
return anchors
}
}
describe('loss', () => {
describe('computeNoObjectLoss', () => {
const computeNoObjectLoss = window['computeNoObjectLoss']
it('should only compute loss over scores, 1x1 grid', () => tf.tidy(() => {
const outTensor = tf.zeros([1, 1, 1, 25])
const loss = tf.sum(computeNoObjectLoss(outTensor)).dataSync()[0]
expect(loss).toEqual(0.5 * 0.5 * 5)
}))
it('should only compute loss over scores, 13x13 grid', () => tf.tidy(() => {
const outTensor = tf.zeros([1, 13, 13, 25])
const loss = tf.sum(computeNoObjectLoss(outTensor)).dataSync()[0]
expect(loss).toEqual(0.5 * 0.5 * 5 * 13 * 13)
}))
it('should only compute loss over scores, 13x13 grid, batchSize: 10', () => tf.tidy(() => {
const outTensor = tf.zeros([10, 13, 13, 25])
const loss = tf.sum(computeNoObjectLoss(outTensor)).dataSync()[0]
expect(loss).toEqual(0.5 * 0.5 * 5 * 13 * 13 * 10)
}))
})
})
\ No newline at end of file
tools/train/tinyYolov2/test.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<script src="trainUtils.js"></script>
<script src="loss.js"></script>
<script src="FileSaver.js"></script>
<link rel="stylesheet" href="styles.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.css">
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/js/materialize.min.js"></script>
</head>
<body>
<script>
window.tf = faceapi.tf
// hyper parameters
window.objectScale = 5
window.noObjectScale = 1
window.coordScale = 1
const weightsUrl = `/tmp/tmp__224_35060__320_41188__416_31050__608_16520.weights`
//const inputSizes = [160, 224, 320, 416]
const inputSizes = [512, 608]
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}
async function fetchFddbJson() {
return fetch('/fddb-detections.json').then(res => res.json())
}
async function run() {
window.fddbJson = await fetchFddbJson()
const weights = await loadNetWeights(weightsUrl)
window.net = new faceapi.TinyYolov2(true)
await window.net.load(weights)
await promiseSequential(inputSizes.map(inputSize => async () => {
await promiseSequential(window.fddbJson.map(({ filePath, rects }) => () => {
return test(filePath, rects, inputSize)
}))
const losses = Object.keys(window.lossMap[inputSize]).map(k => window.lossMap[inputSize][k])
const totalLoss = losses
.map(l => l.totalLoss)
.reduce((sum, l) => sum + l)
const avgLoss = totalLoss / losses.length
log(`totalLoss (${inputSize}): ${totalLoss}`)
log(`avgLoss (${inputSize}): ${avgLoss}`)
window.losses = window.losses || {}
window.losses[inputSize] = { totalLoss, avgLoss }
}))
console.log(window.losses)
}
async function test(fileUri, rects, inputSize) {
const img = await faceapi.bufferToImage(await fetchImage(fileUri))
const groundTruthBoxes = rects
.map(({ x, y, width, height }) => new faceapi.Rect(x, y, width, height))
.map(rect => rect.clipAtImageBorders(img.width, img.height))
.map(({ x, y, width, height }) => ({
x: x / img.width,
y: y / img.height,
width: width / img.width,
height: height / img.height,
}))
const { reshapedImgDims, paddings } = getPaddingsAndReshapedSize(img, inputSize)
const squareImg = imageToSquare(img)
const netInput = (await faceapi.toNetInput(squareImg)).managed()
const losses = tf.tidy(() => {
const outTensor = window.net.forwardInput(netInput, inputSize)
const {
noObjectLoss,
objectLoss,
coordLoss,
totalLoss
} = computeLoss(
outTensor,
groundTruthBoxes,
reshapedImgDims,
paddings
)
const losses = {
totalLoss: totalLoss.dataSync()[0],
noObjectLoss: noObjectLoss.dataSync()[0],
objectLoss: objectLoss.dataSync()[0],
coordLoss: coordLoss.dataSync()[0]
}
return losses
})
log(`${fileUri}:`)
log(`ground truth boxes: ${groundTruthBoxes.length}`)
log(`noObjectLoss: ${losses.noObjectLoss}`)
log(`objectLoss: ${losses.objectLoss}`)
log(`coordLoss: ${losses.coordLoss}`)
log(`totalLoss: ${losses.totalLoss}`)
if (Object.keys(losses).map(k => losses[k]).some(loss => isNaN(loss) || loss === Infinity)) {
console.log(groundTruthBoxes)
console.log(img)
console.log(losses)
throw new Error('corrupted loss value')
}
window.lossMap = window.lossMap || {}
window.lossMap[inputSize] = window.lossMap[inputSize] || {}
window.lossMap[inputSize][fileUri] = losses
}
$(document).ready(function() {
run()
})
</script>
</body>
</html>
\ No newline at end of file
tools/train/tinyYolov2/train.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<link rel="stylesheet" href="styles.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.css">
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/js/materialize.min.js"></script>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<script src="FileSaver.js"></script>
<script src="trainUtils.js"></script>
<script src="train.js"></script>
<script src="loss.js"></script>
</head>
<body>
<script>
tf = faceapi.tf
const startIdx160 = 26600
const startIdx224 = 61660
const startIdx320 = 67788
const startIdx416 = 57650
const startIdx608 = 16520
//const weightsUrl = `/tmp/tmp__160_${startIdx160}__224_${startIdx224}__320_${startIdx320}__416_${startIdx416}__608_${startIdx608}.weights`
const weightsUrl = `/tmp/tmp_multiscale_count_8700.weights`
const fromEpoch = 0
const trainOnlyMultibox = false
const trainSizes = [416, 512, 608]
//const trainSizes = [608]
window.debug = false
window.logTrainSteps = true
window.count = 0
// hyper parameters
window.objectScale = 5
window.noObjectScale = 1
window.coordScale = 1
const rescaleEveryNthBatch = 100
window.saveEveryNthDataIdx = trainSizes.length * rescaleEveryNthBatch
window.trainSteps = 4000
//window.optimizer = tf.train.sgd(0.001)
window.optimizer = tf.train.adam(0.001, 0.9, 0.999, 1e-8)
// all samples
//const dataStartIdx = 8000
const dataStartIdx = 0
const numTrainSamples = Infinity
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}
async function fetchDetectionFilenames() {
return fetch('/detection_filenames').then(res => res.json())
}
async function fetchDetectionFilenamesMultibox() {
return fetch('/detection_filenames_multibox').then(res => res.json())
}
async function run() {
const weights = await loadNetWeights(weightsUrl)
window.net = new faceapi.TinyYolov2(true)
window.net.load(weights)
window.net.variable()
const fetchDetectionsFn = trainOnlyMultibox
? fetchDetectionFilenamesMultibox
: fetchDetectionFilenames
window.detectionFilenames = (await fetchDetectionsFn()).slice(dataStartIdx, dataStartIdx + numTrainSamples)
window.lossMap = {}
console.log('ready')
}
function logLossChange(lossType) {
const { currentLoss, prevLoss, detectionFilenames } = window
log(`${lossType} : ${faceapi.round(currentLoss[lossType])} (avg: ${faceapi.round(currentLoss[lossType] / detectionFilenames.length)}) (delta: ${currentLoss[lossType] - prevLoss[lossType]})`)
}
function onBatchProcessed(dataIdx, inputSize) {
window.count++
const idx = (dataIdx + 1) + (window.epoch * window.detectionFilenames.length)
console.log('dataIdx', dataIdx)
if ((window.count % saveEveryNthDataIdx) === 0) {
saveWeights(window.net, `tmp_multiscale_count_${window.count}.weights`)
}
}
function _onBatchProcessed(dataIdx, inputSize) {
const idx = (dataIdx + 1) + (window.epoch * window.detectionFilenames.length)
console.log('idx', idx)
if ((idx % saveEveryNthDataIdx) === 0) {
saveWeights(window.net, `tmp__224_${startIdx224 + (inputSize === 224 ? idx : 0)}__320_${startIdx320 + (inputSize === 320 ? idx : 0)}__416_${startIdx416 + (inputSize === 416 ? idx : 0)}__608_${startIdx608 + (inputSize === 608 ? idx : 0)}.weights`)
}
}
async function train() {
const batchSize = 1
for (let i = fromEpoch; i < trainSteps; i++) {
window.epoch = i
log('step', i)
let ts2 = Date.now()
const batchCreators = createBatchCreators(shuffle(window.detectionFilenames), batchSize)
await trainStep(batchCreators, trainSizes, rescaleEveryNthBatch, onBatchProcessed)
ts2 = Date.now() - ts2
}
ts = Date.now() - ts
log()
log('--------------------')
log()
log('step %s done (%s ms)', i, ts)
window.prevLoss = window.currentLoss
window.currentLoss = Object.keys(lossMap)
.map(filename => lossMap[filename])
.reduce((accumulatedLosses, losses) =>
Object.keys(losses)
.map(key => ({
[key]: (accumulatedLosses[key] || 0) + losses[key]
}))
.reduce((map, curr) => ({ ...map, ...curr }), {}),
{}
)
if (window.prevLoss) {
logLossChange('noObjectLoss')
logLossChange('objectLoss')
logLossChange('coordLoss')
logLossChange('totalLoss')
}
log()
log('--------------------')
log()
}
run()
</script>
</body>
</html>
\ No newline at end of file
tools/train/tinyYolov2/train.js deleted 100644 → 0
const batchIdx = 0
function minimize(groundTruthBoxes, batchInput, inputSize, batch, { reshapedImgDims, paddings }) {
const filename = batch.filenames[batchIdx]
const { dataIdx } = batch
return optimizer.minimize(() => {
const outTensor = window.net.forwardInput(batchInput, inputSize)
const {
noObjectLoss,
objectLoss,
coordLoss,
totalLoss
} = computeLoss(
outTensor,
groundTruthBoxes,
reshapedImgDims,
paddings
)
const losses = {
totalLoss: totalLoss.dataSync()[0],
noObjectLoss: noObjectLoss.dataSync()[0],
objectLoss: objectLoss.dataSync()[0],
coordLoss: coordLoss.dataSync()[0]
}
const lossKey = `${filename}_${inputSize}`
if (window.logTrainSteps) {
log(`ground truth boxes: ${groundTruthBoxes.length}`)
log(`noObjectLoss[${dataIdx}]: ${losses.noObjectLoss}`)
log(`objectLoss[${dataIdx}]: ${losses.objectLoss}`)
log(`coordLoss[${dataIdx}]: ${losses.coordLoss}`)
log(`totalLoss[${dataIdx}]: ${losses.totalLoss}`)
if (window.lossMap[lossKey]) {
log(`loss change: ${losses.totalLoss - window.lossMap[lossKey].totalLoss}`)
}
}
window.lossMap[lossKey] = losses
return totalLoss
}, true)
}
async function trainStep(batchCreators, inputSizes, rescaleEveryNthBatch, onBatchProcessed = () => {}) {
async function step(currentBatchCreators) {
if (!currentBatchCreators.batchCreators.length) {
return
}
await promiseSequential(inputSizes.map(inputSize => async () => {
await promiseSequential(currentBatchCreators.batchCreators.map(batchCreator => async () => {
const batch = await batchCreator()
const { imgs, groundTruthBoxes, filenames, dataIdx } = batch
const img = imgs[0]
const { reshapedImgDims, paddings } = getPaddingsAndReshapedSize(img, inputSize)
const squareImg = imageToSquare(img)
const batchInput = await faceapi.toNetInput(squareImg)
const [imgHeight, imgWidth] = batchInput.inputs[batchIdx].shape
// skip groundTruthBoxes, which are too tiny
const scaleFactor = inputSize / Math.max(imgHeight, imgWidth)
const filteredGroundTruthBoxes = groundTruthBoxes[batchIdx].filter(({ x, y, width, height }) => {
const box = (new faceapi.Rect(x, y, width, height))
.toBoundingBox()
.rescale({ height: imgHeight, width: imgWidth })
.rescale(scaleFactor)
const isTooTiny = box.width < 40 || box.height < 40
if (isTooTiny && window.debug) {
log(`skipping box for input size ${inputSize}: (${Math.floor(box.width)} x ${Math.floor(box.height)})`)
}
return !isTooTiny
})
if (!filteredGroundTruthBoxes.length) {
if (window.debug) {
log(`no boxes for input size ${inputSize}, ${groundTruthBoxes[batchIdx].length} boxes were too small`)
}
batchInput.dispose()
onBatchProcessed(dataIdx, inputSize)
return
}
let ts = Date.now()
const loss = minimize(filteredGroundTruthBoxes, batchInput, inputSize, batch, { reshapedImgDims, paddings })
ts = Date.now() - ts
if (window.logTrainSteps) {
log(`trainStep time for dataIdx ${dataIdx} (${inputSize}): ${ts} ms`)
}
loss.dispose()
batchInput.dispose()
onBatchProcessed(dataIdx, inputSize)
await tf.nextFrame()
}))
}))
await step(currentBatchCreators.next(rescaleEveryNthBatch))
}
await step(batchCreators.next(rescaleEveryNthBatch))
}
async function fetchGroundTruthBoxesForFile(file) {
const boxes = await fetch(file).then(res => res.json())
return {
file,
boxes
}
}
function createBatchCreators(detectionFilenames, batchSize) {
if (batchSize < 1) {
throw new Error('invalid batch size: ' + batchSize)
}
const batches = []
const pushToBatch = (remaining) => {
if (remaining.length) {
batches.push(remaining.slice(0, batchSize))
pushToBatch(remaining.slice(batchSize))
}
return batches
}
pushToBatch(detectionFilenames)
const batchCreators = batches.map((filenamesForBatch, dataIdx) => async () => {
const groundTruthBoxes = (await Promise.all(filenamesForBatch.map(fetchGroundTruthBoxesForFile)))
.map(({ boxes }) => boxes)
const imgs = await Promise.all(filenamesForBatch.map(
async file => await faceapi.bufferToImage(await fetchImage(file.replace('.json', '')))
))
return {
imgs,
groundTruthBoxes,
filenames: filenamesForBatch,
dataIdx
}
})
let idx = 0
function next(n) {
const nextBatchCreators = batchCreators.slice(idx, idx + n)
idx += n
return {
batchCreators: nextBatchCreators,
next
}
}
return {
data: batchCreators,
next
}
}
\ No newline at end of file
tools/train/tinyYolov2/verify.html deleted 100644 → 0
<!DOCTYPE html>
<html>
<head>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<link rel="stylesheet" href="styles.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/css/materialize.css">
<script type="text/javascript" src="https://code.jquery.com/jquery-2.1.1.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/materialize/0.100.2/js/materialize.min.js"></script>
</head>
<body>
<div id="navbar"></div>
<div class="center-content page-container">
<div class="progress" id="loader">
<div class="indeterminate"></div>
</div>
<div style="position: relative" class="margin">
<img id="inputImg" src="" style="max-width: 800px;" />
<canvas id="overlay" />
</div>
<div class="row side-by-side">
<div id="selectList"></div>
<div class="row">
<label for="imgUrlInput">Get image from URL:</label>
<input id="imgUrlInput" type="text" class="bold">
</div>
<button
class="waves-effect waves-light btn"
onclick="loadImageFromUrl()"
>
Ok
</button>
</div>
<div class="row side-by-side">
<div class="row input-field" style="margin-right: 20px;">
<select id="sizeType">
<option value="128">128 x 128</option>
<option value="160">160 x 160</option>
<option value="224">224 x 224</option>
<option value="320">320 x 320</option>
<option value="416">416 x 416</option>
<option value="608">608 x 608</option>
</select>
<label>Input Size</label>
</div>
<div class="row">
<label for="scoreThreshold">Score Threshold:</label>
<input disabled value="0.5" id="scoreThreshold" type="text" class="bold">
</div>
<button
class="waves-effect waves-light btn"
onclick="onDecreaseThreshold()"
>
<i class="material-icons left">-</i>
</button>
<button
class="waves-effect waves-light btn"
onclick="onIncreaseThreshold()"
>
<i class="material-icons left">+</i>
</button>
</div>
<div class="row">
<label for="imgByNr">Enter image NR: </label>
<input id="imgByNr" type="text" class="bold">
</div>
</div>
<script>
let scoreThreshold = 0.5
let sizeType = 608
function onKeyDown(e) {
e.target.value = (
parseInt(e.target.value) + (e.keyCode === 38 ? 1 : (e.keyCode === 40 ? -1 : 0))
) || e.target.value || 0
const imgUri = window.imgs[e.target.value]
console.log(imgUri)
onSelectionChanged(imgUri)
}
function onIncreaseThreshold() {
scoreThreshold = Math.min(faceapi.round(scoreThreshold + 0.1), 1.0)
$('#scoreThreshold').val(scoreThreshold)
updateResults()
}
function onDecreaseThreshold() {
scoreThreshold = Math.max(faceapi.round(scoreThreshold - 0.1), 0.1)
$('#scoreThreshold').val(scoreThreshold)
updateResults()
}
function onSizeTypeChanged(e, c) {
sizeType = e.target.value
$('#sizeType').val(sizeType)
updateResults()
}
async function loadImageFromUrl(url) {
const img = await requestExternalImage($('#imgUrlInput').val())
$('#inputImg').get(0).src = img.src
updateResults()
}
async function updateResults() {
const inputImgEl = $('#inputImg').get(0)
const { width, height } = inputImgEl
const canvas = $('#overlay').get(0)
canvas.width = width
canvas.height = height
const forwardParams = {
inputSize: parseInt(sizeType),
scoreThreshold
}
const detections = await window.net.locateFaces(inputImgEl, forwardParams)
faceapi.drawDetection('overlay', detections.map(det => det.forSize(width, height)))
}
async function onSelectionChanged(uri) {
const imgBuf = await fetchImage(uri)
$(`#inputImg`).get(0).src = (await faceapi.bufferToImage(imgBuf)).src
updateResults()
}
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
}
async function fetchDetectionFilenames() {
return fetch('/detection_filenames').then(res => res.json())
}
async function run() {
$('#imgByNr').keydown(onKeyDown)
const startIdx224 = 35060
const startIdx320 = 41188
const startIdx416 = 31050
const startIdx608 = 16520
//const weightsUrl = `/tmp/tmp__224_${startIdx224}__320_${startIdx320}__416_${startIdx416}__608_${startIdx608}.weights`
const weightsUrl = `/tmp/tmp_multiscale_count_4200.weights`
const weights = await loadNetWeights(weightsUrl)
window.net = new faceapi.TinyYolov2(true)
await window.net.load(weights)
window.imgs = (await fetchDetectionFilenames()).map(f => f.replace('.json', ''))
$('#loader').hide()
onSelectionChanged($('#selectList select').val())
}
$(document).ready(function() {
renderNavBar('#navbar', 'tiny_yolov2_face_detection')
renderImageSelectList(
'#selectList',
async (uri) => {
await onSelectionChanged(uri)
},
'bbt1.jpg'
)
const sizeTypeSelect = $('#sizeType')
sizeTypeSelect.val(sizeType)
sizeTypeSelect.on('change', onSizeTypeChanged)
sizeTypeSelect.material_select()
run()
})
</script>
</body>
</html>
\ No newline at end of file
weights/face_landmark_68_tiny_model-weights_manifest.json 0 → 100644
[{"weights":[{"name":"dense0/conv0/filters","shape":[3,3,3,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.007822331026488659,"min":-0.8761010749667297}},{"name":"dense0/conv0/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0061304877786075365,"min":-0.8460073134478401}},{"name":"dense0/conv1/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006931193669637044,"min":-0.928779951731364}},{"name":"dense0/conv1/pointwise_filter","shape":[1,1,32,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.01667951135074391,"min":-2.201695498298196}},{"name":"dense0/conv1/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.00381035027550716,"min":-0.7658804053769392}},{"name":"dense0/conv2/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006376917455710617,"min":-0.9437837834451713}},{"name":"dense0/conv2/pointwise_filter","shape":[1,1,32,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.013428742745343376,"min":-1.2757305608076208}},{"name":"dense0/conv2/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.003760007666606529,"min":-0.6091212419902577}},{"name":"dense1/conv0/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.003171338637669881,"min":-0.38373197515805557}},{"name":"dense1/conv0/pointwise_filter","shape":[1,1,32,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.014757957645491059,"min":-1.8447447056863824}},{"name":"dense1/conv0/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0032720065584369734,"min":-0.369736741103378}},{"name":"dense1/conv1/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006221360318800983,"min":-0.6096933112424963}},{"name":"dense1/conv1/pointwise_filter","shape":[1,1,64,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.011515324723486807,"min":-1.6236607860116399}},{"name":"dense1/conv1/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002638959300284292,"min":-0.316675116034115}},{"name":"dense1/conv2/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.005790043110940971,"min":-0.7237553888676214}},{"name":"dense1/conv2/pointwise_filter","shape":[1,1,64,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.013147306442260743,"min":-1.6960025310516358}},{"name":"dense1/conv2/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0036364357845455994,"min":-0.5163738814054751}},{"name":"dense2/conv0/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0025823556909374164,"min":-0.3279591727490519}},{"name":"dense2/conv0/pointwise_filter","shape":[1,1,64,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.014951597475538066,"min":-1.8689496844422582}},{"name":"dense2/conv0/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0034451217043633556,"min":-0.561554837811227}},{"name":"dense2/conv1/depthwise_filter","shape":[3,3,128,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.00580287587408926,"min":-0.7775853671279609}},{"name":"dense2/conv1/pointwise_filter","shape":[1,1,128,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.020558691024780272,"min":-2.2203386306762694}},{"name":"dense2/conv1/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.003580525459027758,"min":-0.47262936059166405}},{"name":"dense2/conv2/depthwise_filter","shape":[3,3,128,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.007787894267661899,"min":-0.996850466260723}},{"name":"dense2/conv2/pointwise_filter","shape":[1,1,128,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0294371997608858,"min":-3.6207755705889535}},{"name":"dense2/conv2/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0037439118413364184,"min":-0.40434247886433317}},{"name":"fc/weights","shape":[128,136],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.013749843485215131,"min":-1.8424790270188276}},{"name":"fc/bias","shape":[136],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002984718394045736,"min":0.0885973796248436}}],"paths":["face_landmark_68_tiny_model-shard1"]}]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment