Skip to content

F
face
Commit 4d3706fd by vincent
Options

retrained landmark 68 and tiny landmark 68 point models

parent 0bfaa037
Showing with 304 additions and 284 deletions
examples/public/styles.css
......@@ -67,6 +67,12 @@
left: 0;
}
.overlay {
position: absolute;
top: 0;
left: 0;
}
#facesContainer canvas {
margin: 10px;
}
\ No newline at end of file
src/faceLandmarkNet/FaceLandmark68Net.ts
import * as tf from '@tensorflow/tfjs-core';
import { NetInput } from 'tfjs-image-recognition-base';
import { convLayer, ConvParams } from 'tfjs-tiny-yolov2';
import { NetInput, normalize } from 'tfjs-image-recognition-base';
import { ConvParams } from 'tfjs-tiny-yolov2';
import { SeparableConvParams } from 'tfjs-tiny-yolov2/build/tinyYolov2/types';
import { depthwiseSeparableConv } from './depthwiseSeparableConv';
import { extractParams } from './extractParams';
import { FaceLandmark68NetBase } from './FaceLandmark68NetBase';
import { fullyConnectedLayer } from './fullyConnectedLayer';
import { loadQuantizedParams } from './loadQuantizedParams';
import { NetParams } from './types';
import { DenseBlock4Params, NetParams } from './types';
function conv(x: tf.Tensor4D, params: ConvParams): tf.Tensor4D {
return convLayer(x, params, 'valid', true)
}
function denseBlock(
x: tf.Tensor4D,
denseBlockParams: DenseBlock4Params,
isFirstLayer: boolean = false
): tf.Tensor4D {
return tf.tidy(() => {
const out1 = tf.relu(
isFirstLayer
? tf.add(
tf.conv2d(x, (denseBlockParams.conv0 as ConvParams).filters, [2, 2], 'same'),
denseBlockParams.conv0.bias
)
: depthwiseSeparableConv(x, denseBlockParams.conv0 as SeparableConvParams, [2, 2])
) as tf.Tensor4D
const out2 = depthwiseSeparableConv(out1, denseBlockParams.conv1, [1, 1])
const in3 = tf.relu(tf.add(out1, out2)) as tf.Tensor4D
const out3 = depthwiseSeparableConv(in3, denseBlockParams.conv2, [1, 1])
const in4 = tf.relu(tf.add(out1, tf.add(out2, out3))) as tf.Tensor4D
const out4 = depthwiseSeparableConv(in4, denseBlockParams.conv3, [1, 1])
function maxPool(x: tf.Tensor4D, strides: [number, number] = [2, 2]): tf.Tensor4D {
return tf.maxPool(x, [2, 2], strides, 'valid')
return tf.relu(tf.add(out1, tf.add(out2, tf.add(out3, out4)))) as tf.Tensor4D
})
}
export class FaceLandmark68Net extends FaceLandmark68NetBase<NetParams> {
constructor() {
super('FaceLandmark68Net')
super('FaceLandmark68LargeNet')
}
public runNet(input: NetInput): tf.Tensor2D {
......@@ -27,27 +47,21 @@ export class FaceLandmark68Net extends FaceLandmark68NetBase<NetParams> {
const { params } = this
if (!params) {
throw new Error('FaceLandmark68Net - load model before inference')
throw new Error('FaceLandmark68LargeNet - load model before inference')
}
return tf.tidy(() => {
const batchTensor = input.toBatchTensor(128, true).toFloat() as tf.Tensor4D
let out = conv(batchTensor, params.conv0)
out = maxPool(out)
out = conv(out, params.conv1)
out = conv(out, params.conv2)
out = maxPool(out)
out = conv(out, params.conv3)
out = conv(out, params.conv4)
out = maxPool(out)
out = conv(out, params.conv5)
out = conv(out, params.conv6)
out = maxPool(out, [1, 1])
out = conv(out, params.conv7)
const fc0 = tf.relu(fullyConnectedLayer(out.as2D(out.shape[0], -1), params.fc0))
return fullyConnectedLayer(fc0, params.fc1)
const batchTensor = input.toBatchTensor(112, true)
const meanRgb = [122.782, 117.001, 104.298]
const normalized = normalize(batchTensor, meanRgb).div(tf.scalar(255)) as tf.Tensor4D
let out = denseBlock(normalized, params.dense0, true)
out = denseBlock(out, params.dense1)
out = denseBlock(out, params.dense2)
out = denseBlock(out, params.dense3)
out = tf.avgPool(out, [7, 7], [2, 2], 'valid')
return fullyConnectedLayer(out.as2D(out.shape[0], -1), params.fc)
})
}
......@@ -55,6 +69,7 @@ export class FaceLandmark68Net extends FaceLandmark68NetBase<NetParams> {
return loadQuantizedParams(uri)
}
protected extractParams(weights: Float32Array) {
return extractParams(weights)
}
......
src/faceLandmarkNet/FaceLandmark68TinyNet.ts
......@@ -8,11 +8,11 @@ import { extractParamsTiny } from './extractParamsTiny';
import { FaceLandmark68NetBase } from './FaceLandmark68NetBase';
import { fullyConnectedLayer } from './fullyConnectedLayer';
import { loadQuantizedParamsTiny } from './loadQuantizedParamsTiny';
import { DenseBlockParams, TinyNetParams } from './types';
import { DenseBlock3Params, TinyNetParams } from './types';
function denseBlock(
x: tf.Tensor4D,
denseBlockParams: DenseBlockParams,
denseBlockParams: DenseBlock3Params,
isFirstLayer: boolean = false
): tf.Tensor4D {
return tf.tidy(() => {
......
src/faceLandmarkNet/extractParams.ts
import { extractWeightsFactory, ParamMapping } from 'tfjs-image-recognition-base';
import { extractConvParamsFactory, extractFCParamsFactory } from 'tfjs-tiny-yolov2';
import { extractorsFactory } from './extractorsFactory';
import { NetParams } from './types';
export function extractParams(weights: Float32Array): { params: NetParams, paramMappings: ParamMapping[] } {
......@@ -12,19 +12,16 @@ export function extractParams(weights: Float32Array): { params: NetParams, param
getRemainingWeights
} = extractWeightsFactory(weights)
const extractConvParams = extractConvParamsFactory(extractWeights, paramMappings)
const extractFCParams = extractFCParamsFactory(extractWeights, paramMappings)
const {
extractDenseBlock4Params,
extractFCParams
} = extractorsFactory(extractWeights, paramMappings)
const conv0 = extractConvParams(3, 32, 3, 'conv0')
const conv1 = extractConvParams(32, 64, 3, 'conv1')
const conv2 = extractConvParams(64, 64, 3, 'conv2')
const conv3 = extractConvParams(64, 64, 3, 'conv3')
const conv4 = extractConvParams(64, 64, 3, 'conv4')
const conv5 = extractConvParams(64, 128, 3, 'conv5')
const conv6 = extractConvParams(128, 128, 3, 'conv6')
const conv7 = extractConvParams(128, 256, 3, 'conv7')
const fc0 = extractFCParams(6400, 1024, 'fc0')
const fc1 = extractFCParams(1024, 136, 'fc1')
const dense0 = extractDenseBlock4Params(3, 32, 'dense0', true)
const dense1 = extractDenseBlock4Params(32, 64, 'dense1')
const dense2 = extractDenseBlock4Params(64, 128, 'dense2')
const dense3 = extractDenseBlock4Params(128, 256, 'dense3')
const fc = extractFCParams(256, 136, 'fc')
if (getRemainingWeights().length !== 0) {
throw new Error(`weights remaing after extract: ${getRemainingWeights().length}`)
......@@ -32,17 +29,6 @@ export function extractParams(weights: Float32Array): { params: NetParams, param
return {
paramMappings,
params: {
conv0,
conv1,
conv2,
conv3,
conv4,
conv5,
conv6,
conv7,
fc0,
fc1
}
params: { dense0, dense1, dense2, dense3, fc }
}
}
\ No newline at end of file
src/faceLandmarkNet/extractParamsTiny.ts
import * as tf from '@tensorflow/tfjs-core';
import { extractWeightsFactory, ExtractWeightsFunction, ParamMapping } from 'tfjs-image-recognition-base';
import { extractConvParamsFactory, FCParams } from 'tfjs-tiny-yolov2';
import { SeparableConvParams } from 'tfjs-tiny-yolov2/build/tinyYolov2/types';
import { extractWeightsFactory, ParamMapping } from 'tfjs-image-recognition-base';
import { DenseBlockParams, TinyNetParams } from './types';
function extractorsFactory(extractWeights: ExtractWeightsFunction, paramMappings: ParamMapping[]) {
function extractSeparableConvParams(channelsIn: number, channelsOut: number, mappedPrefix: string): SeparableConvParams {
const depthwise_filter = tf.tensor4d(extractWeights(3 * 3 * channelsIn), [3, 3, channelsIn, 1])
const pointwise_filter = tf.tensor4d(extractWeights(channelsIn * channelsOut), [1, 1, channelsIn, channelsOut])
const bias = tf.tensor1d(extractWeights(channelsOut))
paramMappings.push(
{ paramPath: `${mappedPrefix}/depthwise_filter` },
{ paramPath: `${mappedPrefix}/pointwise_filter` },
{ paramPath: `${mappedPrefix}/bias` }
)
return new SeparableConvParams(
depthwise_filter,
pointwise_filter,
bias
)
}
function extractFCParams(channelsIn: number, channelsOut: number, mappedPrefix: string): FCParams {
const weights = tf.tensor2d(extractWeights(channelsIn * channelsOut), [channelsIn, channelsOut])
const bias = tf.tensor1d(extractWeights(channelsOut))
paramMappings.push(
{ paramPath: `${mappedPrefix}/weights` },
{ paramPath: `${mappedPrefix}/bias` }
)
return {
weights,
bias
}
}
return {
extractSeparableConvParams,
extractFCParams
}
}
import { extractorsFactory } from './extractorsFactory';
import { TinyNetParams } from './types';
export function extractParamsTiny(weights: Float32Array): { params: TinyNetParams, paramMappings: ParamMapping[] } {
......@@ -57,26 +13,13 @@ export function extractParamsTiny(weights: Float32Array): { params: TinyNetParam
} = extractWeightsFactory(weights)
const {
extractSeparableConvParams,
extractDenseBlock3Params,
extractFCParams
} = extractorsFactory(extractWeights, paramMappings)
const extractConvParams = extractConvParamsFactory(extractWeights, paramMappings)
function extractDenseBlockParams(channelsIn: number, channelsOut: number, mappedPrefix: string, isFirstLayer: boolean = false): DenseBlockParams {
const conv0 = isFirstLayer
? extractConvParams(channelsIn, channelsOut, 3, `${mappedPrefix}/conv0`)
: extractSeparableConvParams(channelsIn, channelsOut, `${mappedPrefix}/conv0`)
const conv1 = extractSeparableConvParams(channelsOut, channelsOut, `${mappedPrefix}/conv1`)
const conv2 = extractSeparableConvParams(channelsOut, channelsOut, `${mappedPrefix}/conv2`)
return { conv0, conv1, conv2 }
}
const dense0 = extractDenseBlockParams(3, 32, 'dense0', true)
const dense1 = extractDenseBlockParams(32, 64, 'dense1')
const dense2 = extractDenseBlockParams(64, 128, 'dense2')
const dense0 = extractDenseBlock3Params(3, 32, 'dense0', true)
const dense1 = extractDenseBlock3Params(32, 64, 'dense1')
const dense2 = extractDenseBlock3Params(64, 128, 'dense2')
const fc = extractFCParams(128, 136, 'fc')
if (getRemainingWeights().length !== 0) {
......
src/faceLandmarkNet/extractorsFactory.ts 0 → 100644
import * as tf from '@tensorflow/tfjs-core';
import { ExtractWeightsFunction, ParamMapping } from 'tfjs-image-recognition-base';
import { extractConvParamsFactory, FCParams } from 'tfjs-tiny-yolov2';
import { SeparableConvParams } from 'tfjs-tiny-yolov2/build/tinyYolov2/types';
import { DenseBlock3Params, DenseBlock4Params } from './types';
export function extractorsFactory(extractWeights: ExtractWeightsFunction, paramMappings: ParamMapping[]) {
function extractSeparableConvParams(channelsIn: number, channelsOut: number, mappedPrefix: string): SeparableConvParams {
const depthwise_filter = tf.tensor4d(extractWeights(3 * 3 * channelsIn), [3, 3, channelsIn, 1])
const pointwise_filter = tf.tensor4d(extractWeights(channelsIn * channelsOut), [1, 1, channelsIn, channelsOut])
const bias = tf.tensor1d(extractWeights(channelsOut))
paramMappings.push(
{ paramPath: `${mappedPrefix}/depthwise_filter` },
{ paramPath: `${mappedPrefix}/pointwise_filter` },
{ paramPath: `${mappedPrefix}/bias` }
)
return new SeparableConvParams(
depthwise_filter,
pointwise_filter,
bias
)
}
function extractFCParams(channelsIn: number, channelsOut: number, mappedPrefix: string): FCParams {
const weights = tf.tensor2d(extractWeights(channelsIn * channelsOut), [channelsIn, channelsOut])
const bias = tf.tensor1d(extractWeights(channelsOut))
paramMappings.push(
{ paramPath: `${mappedPrefix}/weights` },
{ paramPath: `${mappedPrefix}/bias` }
)
return {
weights,
bias
}
}
const extractConvParams = extractConvParamsFactory(extractWeights, paramMappings)
function extractDenseBlock3Params(channelsIn: number, channelsOut: number, mappedPrefix: string, isFirstLayer: boolean = false): DenseBlock3Params {
const conv0 = isFirstLayer
? extractConvParams(channelsIn, channelsOut, 3, `${mappedPrefix}/conv0`)
: extractSeparableConvParams(channelsIn, channelsOut, `${mappedPrefix}/conv0`)
const conv1 = extractSeparableConvParams(channelsOut, channelsOut, `${mappedPrefix}/conv1`)
const conv2 = extractSeparableConvParams(channelsOut, channelsOut, `${mappedPrefix}/conv2`)
return { conv0, conv1, conv2 }
}
function extractDenseBlock4Params(channelsIn: number, channelsOut: number, mappedPrefix: string, isFirstLayer: boolean = false): DenseBlock4Params {
const { conv0, conv1, conv2 } = extractDenseBlock3Params(channelsIn, channelsOut, mappedPrefix, isFirstLayer)
const conv3 = extractSeparableConvParams(channelsOut, channelsOut, `${mappedPrefix}/conv3`)
return { conv0, conv1, conv2, conv3 }
}
return {
extractDenseBlock3Params,
extractDenseBlock4Params,
extractFCParams
}
}
\ No newline at end of file
src/faceLandmarkNet/loadParamsFactory.ts 0 → 100644
import * as tf from '@tensorflow/tfjs-core';
import { extractWeightEntryFactory, ParamMapping } from 'tfjs-image-recognition-base';
import { ConvParams, FCParams } from 'tfjs-tiny-yolov2';
import { SeparableConvParams } from 'tfjs-tiny-yolov2/build/tinyYolov2/types';
import { DenseBlock3Params, DenseBlock4Params } from './types';
export function loadParamsFactory(weightMap: any, paramMappings: ParamMapping[]) {
const extractWeightEntry = extractWeightEntryFactory(weightMap, paramMappings)
function extractConvParams(prefix: string): ConvParams {
const filters = extractWeightEntry<tf.Tensor4D>(`${prefix}/filters`, 4)
const bias = extractWeightEntry<tf.Tensor1D>(`${prefix}/bias`, 1)
return { filters, bias }
}
function extractSeparableConvParams(prefix: string): SeparableConvParams {
const depthwise_filter = extractWeightEntry<tf.Tensor4D>(`${prefix}/depthwise_filter`, 4)
const pointwise_filter = extractWeightEntry<tf.Tensor4D>(`${prefix}/pointwise_filter`, 4)
const bias = extractWeightEntry<tf.Tensor1D>(`${prefix}/bias`, 1)
return new SeparableConvParams(
depthwise_filter,
pointwise_filter,
bias
)
}
function extractDenseBlock3Params(prefix: string, isFirstLayer: boolean = false): DenseBlock3Params {
const conv0 = isFirstLayer
? extractConvParams(`${prefix}/conv0`)
: extractSeparableConvParams(`${prefix}/conv0`)
const conv1 = extractSeparableConvParams(`${prefix}/conv1`)
const conv2 = extractSeparableConvParams(`${prefix}/conv2`)
return { conv0, conv1, conv2 }
}
function extractDenseBlock4Params(prefix: string, isFirstLayer: boolean = false): DenseBlock4Params {
const conv0 = isFirstLayer
? extractConvParams(`${prefix}/conv0`)
: extractSeparableConvParams(`${prefix}/conv0`)
const conv1 = extractSeparableConvParams(`${prefix}/conv1`)
const conv2 = extractSeparableConvParams(`${prefix}/conv2`)
const conv3 = extractSeparableConvParams(`${prefix}/conv3`)
return { conv0, conv1, conv2, conv3 }
}
function extractFcParams(prefix: string): FCParams {
const weights = extractWeightEntry<tf.Tensor2D>(`${prefix}/weights`, 2)
const bias = extractWeightEntry<tf.Tensor1D>(`${prefix}/bias`, 1)
return { weights, bias }
}
return {
extractDenseBlock3Params,
extractDenseBlock4Params,
extractFcParams
}
}
\ No newline at end of file
src/faceLandmarkNet/loadQuantizedParams.ts
import * as tf from '@tensorflow/tfjs-core';
import {
disposeUnusedWeightTensors,
extractWeightEntryFactory,
loadWeightMap,
ParamMapping,
} from 'tfjs-image-recognition-base';
import { ConvParams, FCParams } from 'tfjs-tiny-yolov2';
import { disposeUnusedWeightTensors, loadWeightMap, ParamMapping } from 'tfjs-image-recognition-base';
import { loadParamsFactory } from './loadParamsFactory';
import { NetParams } from './types';
const DEFAULT_MODEL_NAME = 'face_landmark_68_model'
function extractorsFactory(weightMap: any, paramMappings: ParamMapping[]) {
const extractWeightEntry = extractWeightEntryFactory(weightMap, paramMappings)
function extractConvParams(prefix: string, mappedPrefix: string): ConvParams {
const filters = extractWeightEntry<tf.Tensor4D>(`${prefix}/kernel`, 4, `${mappedPrefix}/filters`)
const bias = extractWeightEntry<tf.Tensor1D>(`${prefix}/bias`, 1, `${mappedPrefix}/bias`)
return { filters, bias }
}
function extractFcParams(prefix: string, mappedPrefix: string): FCParams {
const weights = extractWeightEntry<tf.Tensor2D>(`${prefix}/kernel`, 2, `${mappedPrefix}/weights`)
const bias = extractWeightEntry<tf.Tensor1D>(`${prefix}/bias`, 1, `${mappedPrefix}/bias`)
return { weights, bias }
}
return {
extractConvParams,
extractFcParams
}
}
export async function loadQuantizedParams(
uri: string | undefined
): Promise<{ params: NetParams, paramMappings: ParamMapping[] }> {
......@@ -43,21 +13,16 @@ export async function loadQuantizedParams(
const paramMappings: ParamMapping[] = []
const {
extractConvParams,
extractDenseBlock4Params,
extractFcParams
} = extractorsFactory(weightMap, paramMappings)
} = loadParamsFactory(weightMap, paramMappings)
const params = {
conv0: extractConvParams('conv2d_0', 'conv0'),
conv1: extractConvParams('conv2d_1', 'conv1'),
conv2: extractConvParams('conv2d_2', 'conv2'),
conv3: extractConvParams('conv2d_3', 'conv3'),
conv4: extractConvParams('conv2d_4', 'conv4'),
conv5: extractConvParams('conv2d_5', 'conv5'),
conv6: extractConvParams('conv2d_6', 'conv6'),
conv7: extractConvParams('conv2d_7', 'conv7'),
fc0: extractFcParams('dense', 'fc0'),
fc1: extractFcParams('logits', 'fc1')
dense0: extractDenseBlock4Params('dense0', true),
dense1: extractDenseBlock4Params('dense1'),
dense2: extractDenseBlock4Params('dense2'),
dense3: extractDenseBlock4Params('dense3'),
fc: extractFcParams('fc')
}
disposeUnusedWeightTensors(weightMap, paramMappings)
......
src/faceLandmarkNet/loadQuantizedParamsTiny.ts
import * as tf from '@tensorflow/tfjs-core';
import {
disposeUnusedWeightTensors,
extractWeightEntryFactory,
loadWeightMap,
ParamMapping,
} from 'tfjs-image-recognition-base';
import { ConvParams, FCParams } from 'tfjs-tiny-yolov2';
import { SeparableConvParams } from 'tfjs-tiny-yolov2/build/tinyYolov2/types';
import { disposeUnusedWeightTensors, loadWeightMap, ParamMapping } from 'tfjs-image-recognition-base';
import { DenseBlockParams, TinyNetParams } from './types';
import { loadParamsFactory } from './loadParamsFactory';
import { TinyNetParams } from './types';
const DEFAULT_MODEL_NAME = 'face_landmark_68_tiny_model'
function extractorsFactory(weightMap: any, paramMappings: ParamMapping[]) {
const extractWeightEntry = extractWeightEntryFactory(weightMap, paramMappings)
function extractConvParams(prefix: string): ConvParams {
const filters = extractWeightEntry<tf.Tensor4D>(`${prefix}/filters`, 4)
const bias = extractWeightEntry<tf.Tensor1D>(`${prefix}/bias`, 1)
return { filters, bias }
}
function extractSeparableConvParams(prefix: string): SeparableConvParams {
const depthwise_filter = extractWeightEntry<tf.Tensor4D>(`${prefix}/depthwise_filter`, 4)
const pointwise_filter = extractWeightEntry<tf.Tensor4D>(`${prefix}/pointwise_filter`, 4)
const bias = extractWeightEntry<tf.Tensor1D>(`${prefix}/bias`, 1)
return new SeparableConvParams(
depthwise_filter,
pointwise_filter,
bias
)
}
function extractDenseBlockParams(prefix: string, isFirstLayer: boolean = false): DenseBlockParams {
const conv0 = isFirstLayer
? extractConvParams(`${prefix}/conv0`)
: extractSeparableConvParams(`${prefix}/conv0`)
const conv1 = extractSeparableConvParams(`${prefix}/conv1`)
const conv2 = extractSeparableConvParams(`${prefix}/conv2`)
return { conv0, conv1, conv2 }
}
function extractFcParams(prefix: string): FCParams {
const weights = extractWeightEntry<tf.Tensor2D>(`${prefix}/weights`, 2)
const bias = extractWeightEntry<tf.Tensor1D>(`${prefix}/bias`, 1)
return { weights, bias }
}
return {
extractDenseBlockParams,
extractFcParams
}
}
export async function loadQuantizedParamsTiny(
uri: string | undefined
): Promise<{ params: TinyNetParams, paramMappings: ParamMapping[] }> {
......@@ -66,14 +13,14 @@ export async function loadQuantizedParamsTiny(
const paramMappings: ParamMapping[] = []
const {
extractDenseBlockParams,
extractDenseBlock3Params,
extractFcParams
} = extractorsFactory(weightMap, paramMappings)
} = loadParamsFactory(weightMap, paramMappings)
const params = {
dense0: extractDenseBlockParams('dense0', true),
dense1: extractDenseBlockParams('dense1'),
dense2: extractDenseBlockParams('dense2'),
dense0: extractDenseBlock3Params('dense0', true),
dense1: extractDenseBlock3Params('dense1'),
dense2: extractDenseBlock3Params('dense2'),
fc: extractFcParams('fc')
}
......
src/faceLandmarkNet/types.ts
......@@ -2,19 +2,6 @@ import * as tf from '@tensorflow/tfjs-core';
import { ConvParams, FCParams } from 'tfjs-tiny-yolov2';
import { SeparableConvParams } from 'tfjs-tiny-yolov2/build/tinyYolov2/types';
export type NetParams = {
conv0: ConvParams
conv1: ConvParams
conv2: ConvParams
conv3: ConvParams
conv4: ConvParams
conv5: ConvParams
conv6: ConvParams
conv7: ConvParams
fc0: FCParams
fc1: FCParams
}
export type ConvWithBatchNormParams = BatchNormParams & {
filter: tf.Tensor4D
}
......@@ -35,27 +22,28 @@ export declare type FCWithBatchNormParams = BatchNormParams & {
weights: tf.Tensor2D
}
export type DenseBlockParams = {
export type DenseBlock3Params = {
conv0: SeparableConvParams | ConvParams
conv1: SeparableConvParams
conv2: SeparableConvParams
//conv3: SeparableConvParams
}
export type DenseBlock4Params = DenseBlock3Params & {
conv3: SeparableConvParams
}
export type TinyNetParams = {
dense0: DenseBlockParams
dense1: DenseBlockParams
dense2: DenseBlockParams
dense0: DenseBlock3Params
dense1: DenseBlock3Params
dense2: DenseBlock3Params
fc: FCParams
}
export type MobileResnetParams = {
conv0: SeparableConvParams
conv1: SeparableConvParams
conv2: SeparableConvParams
conv3: SeparableConvParams
conv4: SeparableConvParams
conv5: SeparableConvParams
export type NetParams = {
dense0: DenseBlock4Params
dense1: DenseBlock4Params
dense2: DenseBlock4Params
dense3: DenseBlock4Params
fc: FCParams
}
src/globalApi.ts
......@@ -8,6 +8,7 @@ import { FaceLandmarks68 } from './classes/FaceLandmarks68';
import { FullFaceDescription } from './classes/FullFaceDescription';
import { FaceDetectionNet } from './faceDetectionNet/FaceDetectionNet';
import { FaceLandmark68Net } from './faceLandmarkNet/FaceLandmark68Net';
import { FaceLandmark68TinyNet } from './faceLandmarkNet/FaceLandmark68TinyNet';
import { FaceRecognitionNet } from './faceRecognitionNet/FaceRecognitionNet';
import { Mtcnn } from './mtcnn/Mtcnn';
import { MtcnnForwardParams, MtcnnResult } from './mtcnn/types';
......@@ -22,6 +23,7 @@ export const recognitionNet = new FaceRecognitionNet()
export const nets = {
ssdMobilenetv1: detectionNet,
faceLandmark68Net: landmarkNet,
faceLandmark68TinyNet: new FaceLandmark68TinyNet(),
faceRecognitionNet: recognitionNet,
mtcnn: new Mtcnn(),
tinyYolov2: new TinyYolov2()
......@@ -35,6 +37,10 @@ export function loadFaceLandmarkModel(url: string) {
return nets.faceLandmark68Net.load(url)
}
export function loadFaceLandmarkTinyModel(url: string) {
return nets.faceLandmark68TinyNet.load(url)
}
export function loadFaceRecognitionModel(url: string) {
return nets.faceRecognitionNet.load(url)
}
......@@ -52,6 +58,7 @@ export function loadFaceDetectionModel(url: string) {
}
export function loadModels(url: string) {
console.warn('loadModels will be deprecated in future')
return Promise.all([
loadSsdMobilenetv1Model(url),
loadFaceLandmarkModel(url),
......@@ -76,6 +83,11 @@ export function detectLandmarks(
): Promise<FaceLandmarks68 | FaceLandmarks68[]> {
return nets.faceLandmark68Net.detectLandmarks(input)
}
export function detectLandmarksTiny(
input: TNetInput
): Promise<FaceLandmarks68 | FaceLandmarks68[]> {
return nets.faceLandmark68TinyNet.detectLandmarks(input)
}
export function computeFaceDescriptor(
input: TNetInput
......
tools/quantize/views/quantizeModel.html
......@@ -2,7 +2,6 @@
<html>
<head>
<script src="face-api.js"></script>
<script src="commons.js"></script>
<script src="FileSaver.js"></script>
<script src="quantization.js"></script>
</head>
......@@ -10,14 +9,20 @@
<script>
tf = faceapi.tf
const modelName = 'face_landmark_68_tiny_model'
const uncompressedWeightsUri = `face_landmark_68_tiny_model.weights`
const net = new faceapi.FaceLandmark68TinyNet()
const uncompressedWeightsUri = `face_landmark_68_model.weights`
const net = new faceapi.FaceLandmark68LargeNet()
async function loadNetWeights(uri) {
return new Float32Array(await (await fetch(uri)).arrayBuffer())
async function load() {
await net.load(new Float32Array(await (await fetch(uncompressedWeightsUri)).arrayBuffer()))
console.log('net loaded')
}
function getNamedTensors() {
return net.getParamList().map(({ path, tensor }) => ({ name: path, tensor }))
}
const modelName = 'face_landmark_68'
function makeShards(weightArray) {
const maxLength = 4096 * 1024
......@@ -41,25 +46,34 @@
async function quantizeAndSave() {
await net.load(await loadNetWeights(uncompressedWeightsUri))
const quantizedTensorArrays = []
const weightEntries = []
net.getParamList().forEach(({ path, tensor }) => {
const { scale, min, qdata } = quantizeWeights(tensor)
getNamedTensors().forEach(({ name, tensor, isSkipQuantization }) => {
const weightEntry = {
name : path,
name,
shape: tensor.shape,
dtype: tensor.dtype,
quantization: { dtype: 'uint8', scale, min }
dtype: tensor.dtype
}
console.log({ scale, min })
if (isSkipQuantization) {
quantizedTensorArrays.push(tensor.dataSync())
weightEntries.push(weightEntry)
return
}
const { scale, min, qdata } = quantizeWeights(tensor)
console.log(name, { scale, min })
const quantizedWeightEntry = {
...weightEntry,
quantization: { dtype: 'uint8', scale, min }
}
quantizedTensorArrays.push(qdata)
weightEntries.push(weightEntry)
weightEntries.push(quantizedWeightEntry)
})
const quantizedWeights = quantizedTensorArrays
......@@ -83,6 +97,9 @@
saveAs(new Blob([JSON.stringify(weightManifest)]), `${modelName}_model-weights_manifest.json`)
}
load()
</script>
</body>
</html>
\ No newline at end of file
tools/train/faceLandmarks/initWeightsDensenet.html
......@@ -82,7 +82,7 @@
initSeparableConvWeights(128, 128),
initSeparableConvWeights(128, 128),
initSeparableConvWeights(128, 128),
initSeparableConvWeights(256, 256),
initSeparableConvWeights(128, 256),
initSeparableConvWeights(256, 256),
initSeparableConvWeights(256, 256),
initSeparableConvWeights(256, 256)
......
tools/train/faceLandmarks/test.html
......@@ -18,8 +18,8 @@
</div>
<script>
const net = new faceapi.FaceLandmark68TinyNet()
const modelCheckpoint = 'tmp/densenet3_conv0/checkpoints/landmarks_epoch59.weights'
const net = new faceapi.FaceLandmark68LargeNet()
const modelCheckpoint = 'tmp/densenet4/checkpoints/landmarks_epoch46_lr00001_12_lr000001_18.weights'
const crops = 4
async function loadNetWeights(uri) {
......@@ -27,9 +27,9 @@
}
async function load() {
await net.load('./')
//const weights = await loadNetWeights(modelCheckpoint)
//await net.load(weights)
//await net.load('./')
const weights = await loadNetWeights(modelCheckpoint)
await net.load(weights)
console.log('model loaded')
}
......
tools/train/faceLandmarks/train.html
......@@ -16,10 +16,10 @@
// uri to weights file of last checkpoint
window.net = new faceapi.FaceLandmark68TinyNet()
const modelCheckpoint = '/tmp/initial_glorot.weights.weights'
const startEpoch = 0
const modelCheckpoint = 'tmp/face_landmark_68_tiny_model_lr00001_19.weights'
const startEpoch = 20
const learningRate = 0.001 // 0.001
const learningRate = 0.0001 // 0.001
window.optimizer = tf.train.adam(learningRate, 0.9, 0.999, 1e-8)
window.saveEveryNthSample = Infinity
......@@ -60,10 +60,10 @@
}
async function onEpochDone(epoch) {
saveWeights(window.net, `landmarks_epoch59_sideways71_epoch8_sideways35_epoch${epoch}.weights`)
saveWeights(window.net, `face_landmark_68_tiny_model_lr00001_${epoch}.weights`)
const loss = window.lossValues[epoch]
saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / window.trainIds.length })]), `landmarks_epoch59_sideways71_epoch8_sideways35_epoch${epoch}.json`)
saveAs(new Blob([JSON.stringify({ loss, avgLoss: loss / window.trainIds.length })]), `face_landmark_68_tiny_model_lr00001_${epoch}.json`)
}
......
tools/train/faceLandmarks/train_debug.html
......@@ -14,7 +14,7 @@
tf = faceapi.tf
// uri to weights file of last checkpoint
window.net = new faceapi.FaceLandmark68TinyNet()
window.net = new faceapi.FaceLandmark68LargeNet()
const modelCheckpoint = '/tmp/initial_glorot.weights'
const startEpoch = 0
......@@ -25,7 +25,7 @@
window.withRandomCrop = false
window.batchSize = 18
window.batchSize = 12
window.lossValue = 0
......
tools/train/faceLandmarks/verify.html
......@@ -38,7 +38,9 @@
<script>
const modelCheckpoints = [
//'tmp/densenet3_conv0/checkpoints/landmarks_epoch59.weights'
'tmp/face_landmark_68_tiny_model_lr00001_0.weights',
'tmp/face_landmark_68_tiny_model_lr00001_13.weights',
'tmp/face_landmark_68_tiny_model_lr00001_24.weights'
]
let originalImage = null
......@@ -131,17 +133,17 @@
async function run() {
$('#imgByNr').keydown(onKeyDown)
faceapi.loadFaceLandmarkModel('./')
//faceapi.loadFaceLandmarkModel('./')
//window.trainIds = (await fetch('/face_landmarks_train_ids').then(res => res.json()))
//window.testIds = (await fetch('/face_landmarks_test_ids').then(res => res.json()))
window.testIds = (await fetch('/strong_sideways_test').then(res => res.json()))
window.nets = []
window.nets.push(faceapi)
const net = new faceapi.FaceLandmark68TinyNet()
await net.load('./')
window.nets.push(net)
//window.nets.push(faceapi)
//const net = new faceapi.FaceLandmark68TinyNet()
//await net.load('./')
//window.nets.push(net)
modelCheckpoints.forEach(async checkpoint => {
const net = new faceapi.FaceLandmark68TinyNet()
const weights = await loadNetWeights(checkpoint)
......
weights/face_landmark_68_model-shard2 deleted 100644 → 0
This source diff could not be displayed because it is too large. You can view the blob instead.
weights/face_landmark_68_tiny_model-weights_manifest.json
[{"weights":[{"name":"dense0/conv0/filters","shape":[3,3,3,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.007822331026488659,"min":-0.8761010749667297}},{"name":"dense0/conv0/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0061304877786075365,"min":-0.8460073134478401}},{"name":"dense0/conv1/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006931193669637044,"min":-0.928779951731364}},{"name":"dense0/conv1/pointwise_filter","shape":[1,1,32,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.01667951135074391,"min":-2.201695498298196}},{"name":"dense0/conv1/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.00381035027550716,"min":-0.7658804053769392}},{"name":"dense0/conv2/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006376917455710617,"min":-0.9437837834451713}},{"name":"dense0/conv2/pointwise_filter","shape":[1,1,32,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.013428742745343376,"min":-1.2757305608076208}},{"name":"dense0/conv2/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.003760007666606529,"min":-0.6091212419902577}},{"name":"dense1/conv0/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.003171338637669881,"min":-0.38373197515805557}},{"name":"dense1/conv0/pointwise_filter","shape":[1,1,32,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.014757957645491059,"min":-1.8447447056863824}},{"name":"dense1/conv0/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0032720065584369734,"min":-0.369736741103378}},{"name":"dense1/conv1/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006221360318800983,"min":-0.6096933112424963}},{"name":"dense1/conv1/pointwise_filter","shape":[1,1,64,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.011515324723486807,"min":-1.6236607860116399}},{"name":"dense1/conv1/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002638959300284292,"min":-0.316675116034115}},{"name":"dense1/conv2/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.005790043110940971,"min":-0.7237553888676214}},{"name":"dense1/conv2/pointwise_filter","shape":[1,1,64,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.013147306442260743,"min":-1.6960025310516358}},{"name":"dense1/conv2/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0036364357845455994,"min":-0.5163738814054751}},{"name":"dense2/conv0/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0025823556909374164,"min":-0.3279591727490519}},{"name":"dense2/conv0/pointwise_filter","shape":[1,1,64,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.014951597475538066,"min":-1.8689496844422582}},{"name":"dense2/conv0/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0034451217043633556,"min":-0.561554837811227}},{"name":"dense2/conv1/depthwise_filter","shape":[3,3,128,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.00580287587408926,"min":-0.7775853671279609}},{"name":"dense2/conv1/pointwise_filter","shape":[1,1,128,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.020558691024780272,"min":-2.2203386306762694}},{"name":"dense2/conv1/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.003580525459027758,"min":-0.47262936059166405}},{"name":"dense2/conv2/depthwise_filter","shape":[3,3,128,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.007787894267661899,"min":-0.996850466260723}},{"name":"dense2/conv2/pointwise_filter","shape":[1,1,128,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0294371997608858,"min":-3.6207755705889535}},{"name":"dense2/conv2/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0037439118413364184,"min":-0.40434247886433317}},{"name":"fc/weights","shape":[128,136],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.013749843485215131,"min":-1.8424790270188276}},{"name":"fc/bias","shape":[136],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002984718394045736,"min":0.0885973796248436}}],"paths":["face_landmark_68_tiny_model-shard1"]}]
\ No newline at end of file
[{"weights":[{"name":"dense0/conv0/filters","shape":[3,3,3,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.008194216092427571,"min":-0.9423348506291708}},{"name":"dense0/conv0/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006839508168837603,"min":-0.8412595047670252}},{"name":"dense0/conv1/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.009194007106855804,"min":-1.2779669878529567}},{"name":"dense0/conv1/pointwise_filter","shape":[1,1,32,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0036026100317637128,"min":-0.3170296827952067}},{"name":"dense0/conv1/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.000740380117706224,"min":-0.06367269012273527}},{"name":"dense0/conv2/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":1,"min":0}},{"name":"dense0/conv2/pointwise_filter","shape":[1,1,32,32],"dtype":"float32","quantization":{"dtype":"uint8","scale":1,"min":0}},{"name":"dense0/conv2/bias","shape":[32],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0037702228508743585,"min":-0.6220867703942692}},{"name":"dense1/conv0/depthwise_filter","shape":[3,3,32,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0033707996209462483,"min":-0.421349952618281}},{"name":"dense1/conv0/pointwise_filter","shape":[1,1,32,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.014611541991140328,"min":-1.8556658328748217}},{"name":"dense1/conv0/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002832523046755323,"min":-0.30307996600281956}},{"name":"dense1/conv1/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.006593170586754294,"min":-0.6329443763284123}},{"name":"dense1/conv1/pointwise_filter","shape":[1,1,64,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.012215249211180444,"min":-1.6001976466646382}},{"name":"dense1/conv1/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002384825547536214,"min":-0.3028728445370992}},{"name":"dense1/conv2/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.005859645441466687,"min":-0.7617539073906693}},{"name":"dense1/conv2/pointwise_filter","shape":[1,1,64,64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.013121426806730382,"min":-1.7845140457153321}},{"name":"dense1/conv2/bias","shape":[64],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0032247188044529336,"min":-0.46435950784122243}},{"name":"dense2/conv0/depthwise_filter","shape":[3,3,64,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.002659512618008782,"min":-0.32977956463308894}},{"name":"dense2/conv0/pointwise_filter","shape":[1,1,64,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.015499923743453681,"min":-1.9839902391620712}},{"name":"dense2/conv0/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0032450980999890497,"min":-0.522460794098237}},{"name":"dense2/conv1/depthwise_filter","shape":[3,3,128,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.005911862382701799,"min":-0.792189559282041}},{"name":"dense2/conv1/pointwise_filter","shape":[1,1,128,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.021025861478319356,"min":-2.2077154552235325}},{"name":"dense2/conv1/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.00349616945958605,"min":-0.46149436866535865}},{"name":"dense2/conv2/depthwise_filter","shape":[3,3,128,1],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.008104994250278847,"min":-1.013124281284856}},{"name":"dense2/conv2/pointwise_filter","shape":[1,1,128,128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.029337059282789044,"min":-3.5791212325002633}},{"name":"dense2/conv2/bias","shape":[128],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0038808938334969913,"min":-0.4230174278511721}},{"name":"fc/weights","shape":[128,136],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.014016061670639936,"min":-1.8921683255363912}},{"name":"fc/bias","shape":[136],"dtype":"float32","quantization":{"dtype":"uint8","scale":0.0029505149698724935,"min":0.088760145008564}}],"paths":["face_landmark_68_tiny_model-shard1"]}]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment