Skip to content

F
face
Unverified Commit 1c56acb1 by justadudewhohacks Committed by GitHub
Options

Merge pull request #93 from justadudewhohacks/fixes-and-performance-improvements

fixed landmark postprocessing + resize input canvases instead of tensors to net input size, which is much more performant
parents f512f8b3 8b1d5771
Showing with 672 additions and 492 deletions
examples/views/detectAndDrawFaces.html
......@@ -84,12 +84,9 @@
const detections = await faceapi.locateFaces(input, minConfidence)
faceapi.drawDetection('overlay', detections.map(det => det.forSize(width, height)))
const faceImages = await faceapi.extractFaces(input.inputs[0], detections)
const faceImages = await faceapi.extractFaces(inputImgEl, detections)
$('#facesContainer').empty()
faceImages.forEach(canvas => $('#facesContainer').append(canvas))
// free memory for input tensors
input.dispose()
}
async function onSelectionChanged(uri) {
......
examples/views/detectAndDrawLandmarks.html
......@@ -89,11 +89,8 @@
const input = await faceapi.toNetInput(inputImgEl)
const locations = await faceapi.locateFaces(input, minConfidence)
const faceTensors = (await faceapi.extractFaceTensors(input, locations))
let landmarksByFace = await Promise.all(faceTensors.map(t => faceapi.detectLandmarks(t)))
// free memory for face image tensors after we computed their descriptors
faceTensors.forEach(t => t.dispose())
const faces = await faceapi.extractFaces(input, locations)
let landmarksByFace = await Promise.all(faces.map(face => faceapi.detectLandmarks(face)))
// shift and scale the face landmarks to the face image position in the canvas
landmarksByFace = landmarksByFace.map((landmarks, i) => {
......@@ -103,9 +100,6 @@
faceapi.drawLandmarks(canvas, landmarksByFace, { lineWidth: drawLines ? 2 : 4, drawLines, color: 'red' })
faceapi.drawDetection('overlay', locations.map(det => det.forSize(width, height)))
// free memory for input tensors
input.dispose()
}
async function run() {
......
examples/views/faceAlignment.html
......@@ -86,26 +86,17 @@
}
async function locateAndAlignFacesWithMtcnn(inputImgEl) {
const input = await faceapi.toNetInput(
inputImgEl,
// dispose input manually
false,
// keep canvases (required for mtcnn)
true
)
const input = await faceapi.toNetInput(inputImgEl)
const results = await faceapi.mtcnn(input, { minFaceSize: 100 })
const unalignedFaceImages = await faceapi.extractFaces(input.inputs[0], results.map(res => res.faceDetection))
const unalignedFaceImages = await faceapi.extractFaces(input.getInput(0), results.map(res => res.faceDetection))
const alignedFaceBoxes = results
.filter(res => res.faceDetection.score > minConfidence)
.map(res => res.faceLandmarks.align())
const alignedFaceImages = await faceapi.extractFaces(input.inputs[0], alignedFaceBoxes)
// free memory for input tensors
input.dispose()
const alignedFaceImages = await faceapi.extractFaces(input.getInput(0), alignedFaceBoxes)
return {
unalignedFaceImages,
......@@ -118,7 +109,7 @@
const locations = await faceapi.locateFaces(input, minConfidence)
const unalignedFaceImages = await faceapi.extractFaces(input.inputs[0], locations)
const unalignedFaceImages = await faceapi.extractFaces(input.getInput(0), locations)
// detect landmarks and get the aligned face image bounding boxes
const alignedFaceBoxes = await Promise.all(unalignedFaceImages.map(
......@@ -127,10 +118,7 @@
return faceLandmarks.align(locations[i])
}
))
const alignedFaceImages = await faceapi.extractFaces(input.inputs[0], alignedFaceBoxes)
// free memory for input tensors
input.dispose()
const alignedFaceImages = await faceapi.extractFaces(input.getInput(0), alignedFaceBoxes)
return {
unalignedFaceImages,
......
package-lock.json
......@@ -5838,18 +5838,20 @@
"dev": true
},
"tfjs-image-recognition-base": {
"version": "git+https://github.com/justadudewhohacks/tfjs-image-recognition-base.git#2f2072f883dd098bc539e2e89a61878720e400a1",
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/tfjs-image-recognition-base/-/tfjs-image-recognition-base-0.1.0.tgz",
"integrity": "sha512-rgbDz+96qwDaH3dUbyMAfCc+ptlTH9jV0M8ucuCfSmxTKwvQ7alAOI5EblubSaKq0y+ioNRoK2eBkkfIenPOSQ==",
"requires": {
"@tensorflow/tfjs-core": "0.12.14"
}
},
"tfjs-tiny-yolov2": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/tfjs-tiny-yolov2/-/tfjs-tiny-yolov2-0.0.2.tgz",
"integrity": "sha512-NtiPErN2tIP9EkZA7rGjW5wF7iN4JAjI0LwoC1HvMfd4oPpmqwFSbXxwDvFbZY3a5mPUyW6E4/AqoXjSg4w3yA==",
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/tfjs-tiny-yolov2/-/tfjs-tiny-yolov2-0.1.0.tgz",
"integrity": "sha512-UZyvukF61XExoGAGvvCTYKALe6mz0ZNUczgfW4P+qE6mVbPzIFHiY1yweSwc9u4LZm6FstQf5EgiBFgXoFoxgw==",
"requires": {
"@tensorflow/tfjs-core": "0.12.14",
"tfjs-image-recognition-base": "git+https://github.com/justadudewhohacks/tfjs-image-recognition-base.git#2f2072f883dd098bc539e2e89a61878720e400a1"
"tfjs-image-recognition-base": "0.1.0"
}
},
"through2": {
......@@ -5983,8 +5985,7 @@
"tslib": {
"version": "1.9.3",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz",
"integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==",
"dev": true
"integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ=="
},
"tty-browserify": {
"version": "0.0.0",
......
package.json
......@@ -22,8 +22,8 @@
"license": "MIT",
"dependencies": {
"@tensorflow/tfjs-core": "^0.12.14",
"tfjs-image-recognition-base": "^0.0.0",
"tfjs-tiny-yolov2": "0.0.2",
"tfjs-image-recognition-base": "0.1.0",
"tfjs-tiny-yolov2": "0.1.0",
"tslib": "^1.9.3"
},
"devDependencies": {
......
src/allFacesFactory.ts
......@@ -5,9 +5,9 @@ import { TinyYolov2 } from '.';
import { FaceDetection } from './classes/FaceDetection';
import { FaceLandmarks68 } from './classes/FaceLandmarks68';
import { FullFaceDescription } from './classes/FullFaceDescription';
import { extractFaceTensors } from './dom';
import { extractFaces } from './dom';
import { FaceDetectionNet } from './faceDetectionNet/FaceDetectionNet';
import { FaceLandmarkNet } from './faceLandmarkNet/FaceLandmarkNet';
import { FaceLandmark68Net } from './faceLandmarkNet/FaceLandmark68Net';
import { FaceRecognitionNet } from './faceRecognitionNet/FaceRecognitionNet';
import { Mtcnn } from './mtcnn/Mtcnn';
import { MtcnnForwardParams } from './mtcnn/types';
......@@ -16,23 +16,21 @@ function computeDescriptorsFactory(
recognitionNet: FaceRecognitionNet
) {
return async function(input: TNetInput, alignedFaceBoxes: Rect[], useBatchProcessing: boolean) {
const alignedFaceTensors = await extractFaceTensors(input, alignedFaceBoxes)
const alignedFaceCanvases = await extractFaces(input, alignedFaceBoxes)
const descriptors = useBatchProcessing
? await recognitionNet.computeFaceDescriptor(alignedFaceTensors) as Float32Array[]
: await Promise.all(alignedFaceTensors.map(
faceTensor => recognitionNet.computeFaceDescriptor(faceTensor)
? await recognitionNet.computeFaceDescriptor(alignedFaceCanvases) as Float32Array[]
: await Promise.all(alignedFaceCanvases.map(
canvas => recognitionNet.computeFaceDescriptor(canvas)
)) as Float32Array[]
alignedFaceTensors.forEach(t => t.dispose())
return descriptors
}
}
function allFacesFactory(
detectFaces: (input: TNetInput) => Promise<FaceDetection[]>,
landmarkNet: FaceLandmarkNet,
landmarkNet: FaceLandmark68Net,
recognitionNet: FaceRecognitionNet
) {
const computeDescriptors = computeDescriptorsFactory(recognitionNet)
......@@ -43,15 +41,14 @@ function allFacesFactory(
): Promise<FullFaceDescription[]> {
const detections = await detectFaces(input)
const faceTensors = await extractFaceTensors(input, detections)
const faceCanvases = await extractFaces(input, detections)
const faceLandmarksByFace = useBatchProcessing
? await landmarkNet.detectLandmarks(faceTensors) as FaceLandmarks68[]
: await Promise.all(faceTensors.map(
faceTensor => landmarkNet.detectLandmarks(faceTensor)
? await landmarkNet.detectLandmarks(faceCanvases) as FaceLandmarks68[]
: await Promise.all(faceCanvases.map(
canvas => landmarkNet.detectLandmarks(canvas)
)) as FaceLandmarks68[]
faceTensors.forEach(t => t.dispose())
const alignedFaceBoxes = faceLandmarksByFace.map(
(landmarks, i) => landmarks.align(detections[i].getBox())
......@@ -74,7 +71,7 @@ function allFacesFactory(
export function allFacesSsdMobilenetv1Factory(
ssdMobilenetv1: FaceDetectionNet,
landmarkNet: FaceLandmarkNet,
landmarkNet: FaceLandmark68Net,
recognitionNet: FaceRecognitionNet
) {
return async function(
......@@ -90,7 +87,7 @@ export function allFacesSsdMobilenetv1Factory(
export function allFacesTinyYolov2Factory(
tinyYolov2: TinyYolov2,
landmarkNet: FaceLandmarkNet,
landmarkNet: FaceLandmark68Net,
recognitionNet: FaceRecognitionNet
) {
return async function(
......
src/dom/extractFaceTensors.ts
import * as tf from '@tensorflow/tfjs-core';
import { Rect, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
import { isTensor4D, Rect } from 'tfjs-image-recognition-base';
import { FaceDetection } from '../classes/FaceDetection';
......@@ -9,28 +9,21 @@ import { FaceDetection } from '../classes/FaceDetection';
* Using this method is faster then extracting a canvas for each face and
* converting them to tensors individually.
*
* @param input The image that face detection has been performed on.
* @param imageTensor The image tensor that face detection has been performed on.
* @param detections The face detection results or face bounding boxes for that image.
* @returns Tensors of the corresponding image region for each detected face.
*/
export async function extractFaceTensors(
input: TNetInput,
imageTensor: tf.Tensor3D | tf.Tensor4D,
detections: Array<FaceDetection | Rect>
): Promise<tf.Tensor4D[]> {
): Promise<tf.Tensor3D[]> {
const netInput = await toNetInput(input, true)
if (netInput.batchSize > 1) {
if (netInput.isManaged) {
netInput.dispose()
}
if (isTensor4D(imageTensor) && imageTensor.shape[0] > 1) {
throw new Error('extractFaceTensors - batchSize > 1 not supported')
}
return tf.tidy(() => {
const imgTensor = netInput.inputs[0].expandDims().toFloat() as tf.Tensor4D
const [imgHeight, imgWidth, numChannels] = imgTensor.shape.slice(1)
const [imgHeight, imgWidth, numChannels] = imageTensor.shape.slice(isTensor4D(imageTensor) ? 1 : 0)
const boxes = detections.map(
det => det instanceof FaceDetection
......@@ -40,12 +33,9 @@ export async function extractFaceTensors(
.map(box => box.clipAtImageBorders(imgWidth, imgHeight))
const faceTensors = boxes.map(({ x, y, width, height }) =>
tf.slice4d(imgTensor, [0, y, x, 0], [1, height, width, numChannels])
tf.slice3d(imageTensor.as3D(imgHeight, imgWidth, numChannels), [y, x, 0], [height, width, numChannels])
)
if (netInput.isManaged) {
netInput.dispose()
}
return faceTensors
})
}
\ No newline at end of file
src/dom/extractFaces.ts
......@@ -24,24 +24,19 @@ export async function extractFaces(
let canvas = input as HTMLCanvasElement
if (!(input instanceof HTMLCanvasElement)) {
const netInput = await toNetInput(input, true)
const netInput = await toNetInput(input)
if (netInput.batchSize > 1) {
if (netInput.isManaged) {
netInput.dispose()
}
throw new Error('extractFaces - batchSize > 1 not supported')
}
canvas = await imageTensorToCanvas(netInput.inputs[0])
if (netInput.isManaged) {
netInput.dispose()
}
const tensorOrCanvas = netInput.getInput(0)
canvas = tensorOrCanvas instanceof HTMLCanvasElement
? tensorOrCanvas
: await imageTensorToCanvas(tensorOrCanvas)
}
const ctx = getContext2dOrThrow(canvas)
const boxes = detections.map(
det => det instanceof FaceDetection
? det.forSize(canvas.width, canvas.height).getBox().floor()
......
src/faceDetectionNet/FaceDetectionNet.ts
......@@ -25,7 +25,7 @@ export class FaceDetectionNet extends NeuralNetwork<NetParams> {
}
return tf.tidy(() => {
const batchTensor = input.toBatchTensor(512, false)
const batchTensor = input.toBatchTensor(512, false).toFloat()
const x = tf.sub(tf.mul(batchTensor, tf.scalar(0.007843137718737125)), tf.scalar(1)) as tf.Tensor4D
const features = mobileNetV1(x, params.mobilenetv1)
......@@ -40,7 +40,7 @@ export class FaceDetectionNet extends NeuralNetwork<NetParams> {
}
public async forward(input: TNetInput) {
return this.forwardInput(await toNetInput(input, true))
return this.forwardInput(await toNetInput(input))
}
public async locateFaces(
......@@ -49,7 +49,7 @@ export class FaceDetectionNet extends NeuralNetwork<NetParams> {
maxResults: number = 100
): Promise<FaceDetection[]> {
const netInput = await toNetInput(input, true)
const netInput = await toNetInput(input)
const {
boxes: _boxes,
......@@ -77,18 +77,21 @@ export class FaceDetectionNet extends NeuralNetwork<NetParams> {
minConfidence
)
const paddings = netInput.getRelativePaddings(0)
const reshapedDims = netInput.getReshapedInputDimensions(0)
const inputSize = netInput.inputSize as number
const padX = inputSize / reshapedDims.width
const padY = inputSize / reshapedDims.height
const results = indices
.map(idx => {
const [top, bottom] = [
Math.max(0, boxes.get(idx, 0)),
Math.min(1.0, boxes.get(idx, 2))
].map(val => val * paddings.y)
].map(val => val * padY)
const [left, right] = [
Math.max(0, boxes.get(idx, 1)),
Math.min(1.0, boxes.get(idx, 3))
].map(val => val * paddings.x)
].map(val => val * padX)
return new FaceDetection(
scoresData[idx],
new Rect(
......
src/faceLandmarkNet/FaceLandmarkNet.ts → src/faceLandmarkNet/FaceLandmark68Net.ts
import * as tf from '@tensorflow/tfjs-core';
import { isEven, NetInput, NeuralNetwork, Point, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
import { NetInput } from 'tfjs-image-recognition-base';
import { convLayer, ConvParams } from 'tfjs-tiny-yolov2';
import { FaceLandmarks68 } from '../classes/FaceLandmarks68';
import { extractParams } from './extractParams';
import { FaceLandmark68NetBase } from './FaceLandmark68NetBase';
import { fullyConnectedLayer } from './fullyConnectedLayer';
import { loadQuantizedParams } from './loadQuantizedParams';
import { NetParams } from './types';
......@@ -16,22 +16,22 @@ function maxPool(x: tf.Tensor4D, strides: [number, number] = [2, 2]): tf.Tensor4
return tf.maxPool(x, [2, 2], strides, 'valid')
}
export class FaceLandmarkNet extends NeuralNetwork<NetParams> {
export class FaceLandmark68Net extends FaceLandmark68NetBase<NetParams> {
constructor() {
super('FaceLandmarkNet')
super('FaceLandmark68Net')
}
public forwardInput(input: NetInput): tf.Tensor2D {
public runNet(input: NetInput): tf.Tensor2D {
const { params } = this
if (!params) {
throw new Error('FaceLandmarkNet - load model before inference')
throw new Error('FaceLandmark68Net - load model before inference')
}
return tf.tidy(() => {
const batchTensor = input.toBatchTensor(128, true)
const batchTensor = input.toBatchTensor(128, true).toFloat() as tf.Tensor4D
let out = conv(batchTensor, params.conv0)
out = maxPool(out)
......@@ -46,77 +46,11 @@ export class FaceLandmarkNet extends NeuralNetwork<NetParams> {
out = maxPool(out, [1, 1])
out = conv(out, params.conv7)
const fc0 = tf.relu(fullyConnectedLayer(out.as2D(out.shape[0], -1), params.fc0))
const fc1 = fullyConnectedLayer(fc0, params.fc1)
const createInterleavedTensor = (fillX: number, fillY: number) =>
tf.stack([
tf.fill([68], fillX),
tf.fill([68], fillY)
], 1).as2D(1, 136).as1D()
/* shift coordinates back, to undo centered padding
x = ((x * widthAfterPadding) - shiftX) / width
y = ((y * heightAfterPadding) - shiftY) / height
*/
const landmarkTensors = fc1
.mul(tf.stack(Array.from(Array(input.batchSize), (_, batchIdx) =>
createInterleavedTensor(
input.getPaddings(batchIdx).x + input.getInputWidth(batchIdx),
input.getPaddings(batchIdx).y + input.getInputHeight(batchIdx)
)
)))
.sub(tf.stack(Array.from(Array(input.batchSize), (_, batchIdx) =>
createInterleavedTensor(
Math.floor(input.getPaddings(batchIdx).x / 2),
Math.floor(input.getPaddings(batchIdx).y / 2)
)
)))
.div(tf.stack(Array.from(Array(input.batchSize), (_, batchIdx) =>
createInterleavedTensor(
input.getInputWidth(batchIdx),
input.getInputHeight(batchIdx)
)
)))
return landmarkTensors as tf.Tensor2D
return fullyConnectedLayer(fc0, params.fc1)
})
}
public async forward(input: TNetInput): Promise<tf.Tensor2D> {
return this.forwardInput(await toNetInput(input, true))
}
public async detectLandmarks(input: TNetInput): Promise<FaceLandmarks68 | FaceLandmarks68[]> {
const netInput = await toNetInput(input, true)
const landmarkTensors = tf.tidy(
() => tf.unstack(this.forwardInput(netInput))
)
const landmarksForBatch = await Promise.all(landmarkTensors.map(
async (landmarkTensor, batchIdx) => {
const landmarksArray = Array.from(await landmarkTensor.data())
const xCoords = landmarksArray.filter((_, i) => isEven(i))
const yCoords = landmarksArray.filter((_, i) => !isEven(i))
return new FaceLandmarks68(
Array(68).fill(0).map((_, i) => new Point(xCoords[i], yCoords[i])),
{
height: netInput.getInputHeight(batchIdx),
width : netInput.getInputWidth(batchIdx),
}
)
}
))
landmarkTensors.forEach(t => t.dispose())
return netInput.isBatchInput
? landmarksForBatch
: landmarksForBatch[0]
}
protected loadQuantizedParams(uri: string | undefined) {
return loadQuantizedParams(uri)
}
......
src/faceLandmarkNet/FaceLandmark68NetBase.ts 0 → 100644
import * as tf from '@tensorflow/tfjs-core';
import { isEven, NetInput, NeuralNetwork, Point, TNetInput, toNetInput, Dimensions } from 'tfjs-image-recognition-base';
import { FaceLandmarks68 } from '../classes/FaceLandmarks68';
export class FaceLandmark68NetBase<NetParams> extends NeuralNetwork<NetParams> {
// TODO: make super.name protected
private __name: string
constructor(_name: string) {
super(_name)
this.__name = _name
}
public runNet(_: NetInput): tf.Tensor2D {
throw new Error(`${this.__name} - runNet not implemented`)
}
public postProcess(output: tf.Tensor2D, inputSize: number, originalDimensions: Dimensions[]): tf.Tensor2D {
const inputDimensions = originalDimensions.map(({ width, height }) => {
const scale = inputSize / Math.max(height, width)
return {
width: width * scale,
height: height * scale
}
})
const batchSize = inputDimensions.length
return tf.tidy(() => {
const createInterleavedTensor = (fillX: number, fillY: number) =>
tf.stack([
tf.fill([68], fillX),
tf.fill([68], fillY)
], 1).as2D(1, 136).as1D()
const getPadding = (batchIdx: number, cond: (w: number, h: number) => boolean): number => {
const { width, height } = inputDimensions[batchIdx]
return cond(width, height) ? Math.abs(width - height) / 2 : 0
}
const getPaddingX = (batchIdx: number) => getPadding(batchIdx, (w, h) => w < h)
const getPaddingY = (batchIdx: number) => getPadding(batchIdx, (w, h) => h < w)
const landmarkTensors = output
.mul(tf.fill([batchSize, 136], inputSize))
.sub(tf.stack(Array.from(Array(batchSize), (_, batchIdx) =>
createInterleavedTensor(
getPaddingX(batchIdx),
getPaddingY(batchIdx)
)
)))
.div(tf.stack(Array.from(Array(batchSize), (_, batchIdx) =>
createInterleavedTensor(
inputDimensions[batchIdx].width,
inputDimensions[batchIdx].height
)
)))
return landmarkTensors as tf.Tensor2D
})
}
public forwardInput(input: NetInput): tf.Tensor2D {
return tf.tidy(() => {
const out = this.runNet(input)
return this.postProcess(
out,
input.inputSize as number,
input.inputDimensions.map(([height, width]) => ({ height, width }))
)
})
}
public async forward(input: TNetInput): Promise<tf.Tensor2D> {
return this.forwardInput(await toNetInput(input))
}
public async detectLandmarks(input: TNetInput): Promise<FaceLandmarks68 | FaceLandmarks68[]> {
const netInput = await toNetInput(input)
const landmarkTensors = tf.tidy(
() => tf.unstack(this.forwardInput(netInput))
)
const landmarksForBatch = await Promise.all(landmarkTensors.map(
async (landmarkTensor, batchIdx) => {
const landmarksArray = Array.from(await landmarkTensor.data())
const xCoords = landmarksArray.filter((_, i) => isEven(i))
const yCoords = landmarksArray.filter((_, i) => !isEven(i))
return new FaceLandmarks68(
Array(68).fill(0).map((_, i) => new Point(xCoords[i], yCoords[i])),
{
height: netInput.getInputHeight(batchIdx),
width : netInput.getInputWidth(batchIdx),
}
)
}
))
landmarkTensors.forEach(t => t.dispose())
return netInput.isBatchInput
? landmarksForBatch
: landmarksForBatch[0]
}
}
\ No newline at end of file
src/faceLandmarkNet/index.ts
import { FaceLandmarkNet } from './FaceLandmarkNet';
import { FaceLandmark68Net } from './FaceLandmark68Net';
export * from './FaceLandmarkNet';
export * from './FaceLandmark68Net';
export class FaceLandmarkNet extends FaceLandmark68Net {}
export function createFaceLandmarkNet(weights: Float32Array) {
const net = new FaceLandmarkNet()
......
src/faceRecognitionNet/FaceRecognitionNet.ts
......@@ -23,7 +23,7 @@ export class FaceRecognitionNet extends NeuralNetwork<NetParams> {
}
return tf.tidy(() => {
const batchTensor = input.toBatchTensor(150, true)
const batchTensor = input.toBatchTensor(150, true).toFloat()
const meanRgb = [122.782, 117.001, 104.298]
const normalized = normalize(batchTensor, meanRgb).div(tf.scalar(256)) as tf.Tensor4D
......@@ -57,11 +57,11 @@ export class FaceRecognitionNet extends NeuralNetwork<NetParams> {
}
public async forward(input: TNetInput): Promise<tf.Tensor2D> {
return this.forwardInput(await toNetInput(input, true))
return this.forwardInput(await toNetInput(input))
}
public async computeFaceDescriptor(input: TNetInput): Promise<Float32Array|Float32Array[]> {
const netInput = await toNetInput(input, true)
const netInput = await toNetInput(input)
const faceDescriptorTensors = tf.tidy(
() => tf.unstack(this.forwardInput(netInput))
......
src/globalApi.ts
......@@ -7,14 +7,14 @@ import { FaceDetection } from './classes/FaceDetection';
import { FaceLandmarks68 } from './classes/FaceLandmarks68';
import { FullFaceDescription } from './classes/FullFaceDescription';
import { FaceDetectionNet } from './faceDetectionNet/FaceDetectionNet';
import { FaceLandmarkNet } from './faceLandmarkNet/FaceLandmarkNet';
import { FaceLandmark68Net } from './faceLandmarkNet/FaceLandmark68Net';
import { FaceRecognitionNet } from './faceRecognitionNet/FaceRecognitionNet';
import { Mtcnn } from './mtcnn/Mtcnn';
import { MtcnnForwardParams, MtcnnResult } from './mtcnn/types';
import { TinyYolov2 } from './tinyYolov2/TinyYolov2';
export const detectionNet = new FaceDetectionNet()
export const landmarkNet = new FaceLandmarkNet()
export const landmarkNet = new FaceLandmark68Net()
export const recognitionNet = new FaceRecognitionNet()
// nets need more specific names, to avoid ambiguity in future
......
src/mtcnn/Mtcnn.ts
......@@ -32,7 +32,6 @@ export class Mtcnn extends NeuralNetwork<NetParams> {
throw new Error('Mtcnn - load model before inference')
}
const inputTensor = input.inputs[0]
const inputCanvas = input.canvases[0]
if (!inputCanvas) {
......@@ -45,14 +44,13 @@ export class Mtcnn extends NeuralNetwork<NetParams> {
const imgTensor = tf.tidy(() =>
bgrToRgbTensor(
tf.expandDims(inputTensor).toFloat() as tf.Tensor4D
tf.expandDims(tf.fromPixels(inputCanvas)).toFloat() as tf.Tensor4D
)
)
const onReturn = (results: any) => {
// dispose tensors on return
imgTensor.dispose()
input.dispose()
stats.total = Date.now() - tsTotal
return results
}
......@@ -131,7 +129,7 @@ export class Mtcnn extends NeuralNetwork<NetParams> {
): Promise<MtcnnResult[]> {
return (
await this.forwardInput(
await toNetInput(input, true, true),
await toNetInput(input),
forwardParams
)
).results
......@@ -142,7 +140,7 @@ export class Mtcnn extends NeuralNetwork<NetParams> {
forwardParams: MtcnnForwardParams = {}
): Promise<{ results: MtcnnResult[], stats: any }> {
return this.forwardInput(
await toNetInput(input, true, true),
await toNetInput(input),
forwardParams
)
}
......
src/mtcnn/MtcnnBox.ts 0 → 100644
import { Box } from 'tfjs-image-recognition-base';
export class MtcnnBox extends Box<MtcnnBox> {
constructor(left: number, top: number, right: number, bottom: number) {
super({ left, top, right, bottom }, true)
}
}
\ No newline at end of file
src/mtcnn/extractImagePatches.ts
import * as tf from '@tensorflow/tfjs-core';
import { Box, createCanvas, Dimensions, getContext2dOrThrow } from 'tfjs-image-recognition-base';
import { normalize } from './normalize';
import { BoundingBox, Dimensions, getContext2dOrThrow, createCanvas } from 'tfjs-image-recognition-base';
export async function extractImagePatches(
img: HTMLCanvasElement,
boxes: BoundingBox[],
boxes: Box[],
{ width, height }: Dimensions
): Promise<tf.Tensor4D[]> {
......
src/mtcnn/stage1.ts
......@@ -3,6 +3,7 @@ import { BoundingBox, nonMaxSuppression, Point } from 'tfjs-image-recognition-ba
import { CELL_SIZE, CELL_STRIDE } from './config';
import { getSizesForScale } from './getSizesForScale';
import { MtcnnBox } from './MtcnnBox';
import { normalize } from './normalize';
import { PNet } from './PNet';
import { PNetParams } from './types';
......@@ -45,7 +46,7 @@ function extractBoundingBoxes(
const score = scoresTensor.get(idx.y, idx.x)
const region = new BoundingBox(
const region = new MtcnnBox(
regionsTensor.get(idx.y, idx.x, 0),
regionsTensor.get(idx.y, idx.x, 1),
regionsTensor.get(idx.y, idx.x, 2),
......
src/mtcnn/stage2.ts
import * as tf from '@tensorflow/tfjs-core';
import { BoundingBox, nonMaxSuppression } from 'tfjs-image-recognition-base';
import { Box, nonMaxSuppression } from 'tfjs-image-recognition-base';
import { extractImagePatches } from './extractImagePatches';
import { MtcnnBox } from './MtcnnBox';
import { RNet } from './RNet';
import { RNetParams } from './types';
export async function stage2(
img: HTMLCanvasElement,
inputBoxes: BoundingBox[],
inputBoxes: Box[],
scoreThreshold: number,
params: RNetParams,
stats: any
......@@ -41,7 +42,7 @@ export async function stage2(
const filteredBoxes = indices.map(idx => inputBoxes[idx])
const filteredScores = indices.map(idx => scores[idx])
let finalBoxes: BoundingBox[] = []
let finalBoxes: Box[] = []
let finalScores: number[] = []
if (filteredBoxes.length > 0) {
......@@ -54,7 +55,7 @@ export async function stage2(
stats.stage2_nms = Date.now() - ts
const regions = indicesNms.map(idx =>
new BoundingBox(
new MtcnnBox(
rnetOuts[indices[idx]].regions.get(0, 0),
rnetOuts[indices[idx]].regions.get(0, 1),
rnetOuts[indices[idx]].regions.get(0, 2),
......
src/mtcnn/stage3.ts
import * as tf from '@tensorflow/tfjs-core';
import { BoundingBox, nonMaxSuppression, Point } from 'tfjs-image-recognition-base';
import { BoundingBox, Box, nonMaxSuppression, Point } from 'tfjs-image-recognition-base';
import { extractImagePatches } from './extractImagePatches';
import { MtcnnBox } from './MtcnnBox';
import { ONet } from './ONet';
import { ONetParams } from './types';
......@@ -38,7 +39,7 @@ export async function stage3(
.filter(c => c.score > scoreThreshold)
.map(({ idx }) => idx)
const filteredRegions = indices.map(idx => new BoundingBox(
const filteredRegions = indices.map(idx => new MtcnnBox(
onetOuts[idx].regions.get(0, 0),
onetOuts[idx].regions.get(0, 1),
onetOuts[idx].regions.get(0, 2),
......@@ -48,7 +49,7 @@ export async function stage3(
.map((idx, i) => inputBoxes[idx].calibrate(filteredRegions[i]))
const filteredScores = indices.map(idx => scores[idx])
let finalBoxes: BoundingBox[] = []
let finalBoxes: Box[] = []
let finalScores: number[] = []
let points: Point[][] = []
......
test/data/mtcnnFaceLandmarkPositions.json 0 → 100644
[[{"x":117.85171800851822,"y":58.91067498922348},{"x":157.70139408111572,"y":64.48519098758698},{"x":142.3133249282837,"y":88.54253697395325},{"x":110.16610914468765,"y":99.86233913898468},{"x":149.25052666664124,"y":106.37608766555786}], [{"x":260.46802616119385,"y":82.86598587036133},{"x":305.55760955810547,"y":83.54110813140869},{"x":281.4357223510742,"y":113.98349380493164},{"x":257.06039476394653,"y":125.50608730316164},{"x":306.0191822052002,"y":127.20984458923341}], [{"x":82.91613873839378,"y":292.6100924015045},{"x":133.91112035512924,"y":304.814593821764},{"x":104.43486452102661,"y":330.3951778411865},{"x":72.6984107196331,"y":342.633121073246},{"x":120.51901644468307,"y":354.2677878141403}], [{"x":278.20400857925415,"y":273.83238887786865},{"x":318.7582621574402,"y":273.39686036109924},{"x":295.54277753829956,"y":300.43398427963257},{"x":279.5109224319458,"y":311.497838973999},{"x":317.0187101364136,"y":313.05305886268616}], [{"x":489.58824399113655,"y":224.56882098317146},{"x":534.514480471611,"y":223.28146517276764},{"x":507.2082565128803,"y":250.17186474800113},{"x":493.0139665305615,"y":271.0716395378113},{"x":530.7517347931862,"y":270.4143014550209}], [{"x":606.397784024477,"y":105.43332603573799},{"x":645.2468676567078,"y":111.50095802545547},{"x":625.1735819578171,"y":133.40740483999252},{"x":598.8033188581467,"y":141.26283955574036},{"x":637.2144679427147,"y":147.32198816537857}]]
\ No newline at end of file
test/tests/dom/extractFaceTensors.test.ts
import { bufferToImage, extractFaceTensors, Rect } from '../../../src';
import { bufferToImage, extractFaceTensors, Rect, tf } from '../../../src';
describe('extractFaceTensors', () => {
let imgEl: HTMLImageElement
let imgTensor: tf.Tensor3D
beforeAll(async () => {
const img = await (await fetch('base/test/images/face1.png')).blob()
imgEl = await bufferToImage(img)
imgTensor = tf.fromPixels(await bufferToImage(img))
})
describe('extracts tensors', () => {
it('single box', async () => {
const rect = new Rect(0, 0, 50, 60)
const tensors = await extractFaceTensors(imgEl, [rect])
const tensors = await extractFaceTensors(imgTensor, [rect])
expect(tensors.length).toEqual(1)
expect(tensors[0].shape).toEqual([1, 60, 50, 3])
expect(tensors[0].shape).toEqual([60, 50, 3])
tensors[0].dispose()
})
......@@ -25,11 +25,11 @@ describe('extractFaceTensors', () => {
new Rect(0, 0, 50, 60),
new Rect(50, 50, 70, 80),
]
const tensors = await extractFaceTensors(imgEl, rects)
const tensors = await extractFaceTensors(imgTensor, rects)
expect(tensors.length).toEqual(2)
expect(tensors[0].shape).toEqual([1, 60, 50, 3])
expect(tensors[1].shape).toEqual([1, 80, 70, 3])
expect(tensors[0].shape).toEqual([60, 50, 3])
expect(tensors[1].shape).toEqual([80, 70, 3])
tensors[0].dispose()
tensors[1].dispose()
})
......@@ -40,25 +40,25 @@ describe('extractFaceTensors', () => {
it('clips upper left corner', async () => {
const rect = new Rect(-10, -10, 110, 110)
const tensors = await extractFaceTensors(imgEl, [rect])
const tensors = await extractFaceTensors(imgTensor, [rect])
expect(tensors[0].shape).toEqual([1, 100, 100, 3])
expect(tensors[0].shape).toEqual([100, 100, 3])
tensors[0].dispose()
})
it('clips bottom right corner', async () => {
const rect = new Rect(imgEl.width - 100, imgEl.height - 100, 110, 110)
const tensors = await extractFaceTensors(imgEl, [rect])
const rect = new Rect(imgTensor.shape[1] - 100, imgTensor.shape[0] - 100, 110, 110)
const tensors = await extractFaceTensors(imgTensor, [rect])
expect(tensors[0].shape).toEqual([1, 100, 100, 3])
expect(tensors[0].shape).toEqual([100, 100, 3])
tensors[0].dispose()
})
it('clips upper left and bottom right corners', async () => {
const rect = new Rect(-10, -10, imgEl.width + 20, imgEl.height + 20)
const tensors = await extractFaceTensors(imgEl, [rect])
const rect = new Rect(-10, -10, imgTensor.shape[1] + 20, imgTensor.shape[0] + 20)
const tensors = await extractFaceTensors(imgTensor, [rect])
expect(tensors[0].shape).toEqual([1, imgEl.height, imgEl.width, 3])
expect(tensors[0].shape).toEqual([imgTensor.shape[1], imgTensor.shape[0], 3])
tensors[0].dispose()
})
......
test/tests/e2e/allFacesMtcnn.test.ts
import { bufferToImage } from 'tfjs-image-recognition-base';
import { FaceLandmarks5 } from '../../../src';
import { describeWithNets, expectAllTensorsReleased } from '../../utils';
import { expectMtcnnResults } from './expectedResults';
import {
assembleExpectedFullFaceDescriptions,
describeWithNets,
expectAllTensorsReleased,
ExpectedFullFaceDescription,
} from '../../utils';
import { expectAllFacesResults, expectedMtcnnBoxes } from './expectedResults';
describe('allFacesMtcnn', () => {
let imgEl: HTMLImageElement
let facesFaceDescriptors: number[][]
let expectedFullFaceDescriptions: ExpectedFullFaceDescription[]
beforeAll(async () => {
const img = await (await fetch('base/test/images/faces.jpg')).blob()
imgEl = await bufferToImage(img)
facesFaceDescriptors = await (await fetch('base/test/data/facesFaceDescriptorsMtcnn.json')).json()
expectedFullFaceDescriptions = await assembleExpectedFullFaceDescriptions(expectedMtcnnBoxes, 'mtcnnFaceLandmarkPositions.json')
})
describeWithNets('computes full face descriptions', { withAllFacesMtcnn: true }, ({ allFacesMtcnn }) => {
......@@ -25,14 +29,13 @@ describe('allFacesMtcnn', () => {
const results = await allFacesMtcnn(imgEl, forwardParams)
expect(results.length).toEqual(6)
const mtcnnResult = results.map(res => ({
faceDetection: res.detection,
faceLandmarks: res.landmarks as FaceLandmarks5
}))
expectMtcnnResults(mtcnnResult, [0, 1, 2, 3, 4, 5], 1, 1)
results.forEach(({ descriptor }, i) => {
expect(descriptor).toEqual(new Float32Array(facesFaceDescriptors[i]))
})
const expectedScores = [1, 1, 1, 1, 0.99, 0.99]
const deltas = {
maxBoxDelta: 2,
maxLandmarksDelta: 1,
maxDescriptorDelta: 0.4
}
expectAllFacesResults(results, expectedFullFaceDescriptions, expectedScores, deltas)
})
})
......
test/tests/e2e/allFacesSsdMobilenetv1.test.ts
import * as tf from '@tensorflow/tfjs-core';
import { bufferToImage, NetInput, Point, toNetInput } from '../../../src';
import { describeWithNets, expectAllTensorsReleased, expectPointClose, expectRectClose } from '../../utils';
import { expectedSsdBoxes } from './expectedResults';
import { bufferToImage } from '../../../src';
import {
assembleExpectedFullFaceDescriptions,
describeWithNets,
expectAllTensorsReleased,
ExpectedFullFaceDescription,
} from '../../utils';
import { expectAllFacesResults, expectedSsdBoxes } from './expectedResults';
describe('allFacesSsdMobilenetv1', () => {
let imgEl: HTMLImageElement
let facesFaceLandmarkPositions: Point[][]
let facesFaceDescriptors: number[][]
let expectedFullFaceDescriptions: ExpectedFullFaceDescription[]
beforeAll(async () => {
const img = await (await fetch('base/test/images/faces.jpg')).blob()
imgEl = await bufferToImage(img)
facesFaceLandmarkPositions = await (await fetch('base/test/data/facesFaceLandmarkPositions.json')).json()
facesFaceDescriptors = await (await fetch('base/test/data/facesFaceDescriptorsSsd.json')).json()
expectedFullFaceDescriptions = await assembleExpectedFullFaceDescriptions(expectedSsdBoxes)
})
describeWithNets('computes full face descriptions', { withAllFacesSsdMobilenetv1: true }, ({ allFacesSsdMobilenetv1 }) => {
const expectedScores = [0.97, 0.88, 0.83, 0.82, 0.59, 0.52]
const maxBoxDelta = 5
const maxLandmarkPointsDelta = 1
it('scores > 0.8', async () => {
const results = await allFacesSsdMobilenetv1(imgEl, 0.8)
expect(results.length).toEqual(4)
results.forEach(({ detection, landmarks, descriptor }, i) => {
expect(detection.getImageWidth()).toEqual(imgEl.width)
expect(detection.getImageHeight()).toEqual(imgEl.height)
expect(detection.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(detection.getBox(), expectedSsdBoxes[i], maxBoxDelta)
landmarks.getPositions().forEach((pt, j) => expectPointClose(pt, facesFaceLandmarkPositions[i][j], maxLandmarkPointsDelta))
expect(descriptor).toEqual(new Float32Array(facesFaceDescriptors[i]))
})
const expectedScores = [-1, 0.81, 0.97, 0.88, 0.84, -1]
const deltas = {
maxBoxDelta: 5,
maxLandmarksDelta: 4,
maxDescriptorDelta: 0.01
}
expectAllFacesResults(results, expectedFullFaceDescriptions, expectedScores, deltas)
})
it('scores > 0.5', async () => {
const results = await allFacesSsdMobilenetv1(imgEl, 0.5)
expect(results.length).toEqual(6)
results.forEach(({ detection, landmarks, descriptor }, i) => {
expect(detection.getImageWidth()).toEqual(imgEl.width)
expect(detection.getImageHeight()).toEqual(imgEl.height)
expect(detection.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(detection.getBox(), expectedSsdBoxes[i], maxBoxDelta)
landmarks.getPositions().forEach((pt, j) => expectPointClose(pt, facesFaceLandmarkPositions[i][j], maxLandmarkPointsDelta))
expect(descriptor).toEqual(new Float32Array(facesFaceDescriptors[i]))
})
const expectedScores = [0.54, 0.81, 0.97, 0.88, 0.84, 0.61]
const deltas = {
maxBoxDelta: 5,
maxLandmarksDelta: 4,
maxDescriptorDelta: 0.01
}
expectAllFacesResults(results, expectedFullFaceDescriptions, expectedScores, deltas)
})
})
......@@ -66,8 +65,7 @@ describe('allFacesSsdMobilenetv1', () => {
const tensor = tf.fromPixels(imgEl)
await expectAllTensorsReleased(async () => {
const netInput = (new NetInput([tensor])).managed()
await allFacesSsdMobilenetv1(netInput)
await allFacesSsdMobilenetv1(tensor)
})
tensor.dispose()
......@@ -77,7 +75,7 @@ describe('allFacesSsdMobilenetv1', () => {
const tensor = tf.tidy(() => tf.fromPixels(imgEl).expandDims()) as tf.Tensor4D
await expectAllTensorsReleased(async () => {
await allFacesSsdMobilenetv1(await toNetInput(tensor, true))
await allFacesSsdMobilenetv1(tensor)
})
tensor.dispose()
......
test/tests/e2e/allFacesTinyYolov2.test.ts
import * as tf from '@tensorflow/tfjs-core';
import { TinyYolov2Types } from 'tfjs-tiny-yolov2';
import { bufferToImage, NetInput, Point, toNetInput } from '../../../src';
import { describeWithNets, expectAllTensorsReleased, expectMaxDelta, expectPointClose, expectRectClose } from '../../utils';
import { expectedTinyYolov2SeparableConvBoxes } from './expectedResults';
import { bufferToImage } from '../../../src';
import {
assembleExpectedFullFaceDescriptions,
describeWithNets,
expectAllTensorsReleased,
ExpectedFullFaceDescription,
} from '../../utils';
import { expectAllFacesResults, expectedTinyYolov2Boxes } from './expectedResults';
describe('allFacesTinyYolov2', () => {
let imgEl: HTMLImageElement
let facesFaceLandmarkPositions: Point[][]
let facesFaceDescriptors: number[][]
let expectedFullFaceDescriptions: ExpectedFullFaceDescription[]
beforeAll(async () => {
const img = await (await fetch('base/test/images/faces.jpg')).blob()
imgEl = await bufferToImage(img)
facesFaceLandmarkPositions = await (await fetch('base/test/data/facesFaceLandmarkPositions.json')).json()
facesFaceDescriptors = await (await fetch('base/test/data/facesFaceDescriptorsSsd.json')).json()
expectedFullFaceDescriptions = await assembleExpectedFullFaceDescriptions(expectedTinyYolov2Boxes)
})
describeWithNets('computes full face descriptions', { withAllFacesTinyYolov2: true }, ({ allFacesTinyYolov2 }) => {
it('TinyYolov2Types.SizeType.LG', async () => {
const expectedScores = [0.9, 0.9, 0.89, 0.85, 0.85, 0.85]
const maxBoxDelta = 5
const maxLandmarkPointsDelta = 10
const maxDescriptorDelta = 0.06
const results = await allFacesTinyYolov2(imgEl, { inputSize: TinyYolov2Types.SizeType.LG })
const detectionOrder = [0, 2, 3, 4, 1, 5]
expect(results.length).toEqual(6)
results.forEach(({ detection, landmarks, descriptor }, i) => {
expect(detection.getImageWidth()).toEqual(imgEl.width)
expect(detection.getImageHeight()).toEqual(imgEl.height)
expect(detection.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(detection.getBox(), expectedTinyYolov2SeparableConvBoxes[i], maxBoxDelta)
landmarks.getPositions().forEach((pt, j) => expectPointClose(pt, facesFaceLandmarkPositions[detectionOrder[i]][j], maxLandmarkPointsDelta))
descriptor.forEach((val, j) => expectMaxDelta(val, facesFaceDescriptors[detectionOrder[i]][j], maxDescriptorDelta))
})
const expectedScores = [0.85, 0.88, 0.9, 0.86, 0.9, 0.85]
const deltas = {
maxBoxDelta: 25,
maxLandmarksDelta: 10,
maxDescriptorDelta: 0.24
}
expectAllFacesResults(results, expectedFullFaceDescriptions, expectedScores, deltas)
})
it('TinyYolov2Types.SizeType.MD', async () => {
const expectedScores = [0.85, 0.85, 0.84, 0.83, 0.8, 0.8]
const maxBoxDelta = 17
const maxLandmarkPointsDelta = 16
const maxDescriptorDelta = 0.05
const results = await allFacesTinyYolov2(imgEl, { inputSize: TinyYolov2Types.SizeType.MD })
const boxOrder = [5, 1, 4, 3, 2, 0]
const detectionOrder = [5, 2, 1, 4, 3, 0]
expect(results.length).toEqual(6)
results.forEach(({ detection, landmarks, descriptor }, i) => {
expect(detection.getImageWidth()).toEqual(imgEl.width)
expect(detection.getImageHeight()).toEqual(imgEl.height)
expect(detection.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(detection.getBox(), expectedTinyYolov2SeparableConvBoxes[boxOrder[i]], maxBoxDelta)
landmarks.getPositions().forEach((pt, j) => expectPointClose(pt, facesFaceLandmarkPositions[detectionOrder[i]][j], maxLandmarkPointsDelta))
descriptor.forEach((val, j) => expectMaxDelta(val, facesFaceDescriptors[detectionOrder[i]][j], maxDescriptorDelta))
})
const expectedScores = [0.85, 0.8, 0.8, 0.85, 0.85, 0.82]
const deltas = {
maxBoxDelta: 34,
maxLandmarksDelta: 18,
maxDescriptorDelta: 0.2
}
expectAllFacesResults(results, expectedFullFaceDescriptions, expectedScores, deltas)
})
})
......@@ -76,8 +68,7 @@ describe('allFacesTinyYolov2', () => {
const tensor = tf.fromPixels(imgEl)
await expectAllTensorsReleased(async () => {
const netInput = (new NetInput([tensor])).managed()
await allFacesTinyYolov2(netInput)
await allFacesTinyYolov2(tensor)
})
tensor.dispose()
......@@ -87,7 +78,7 @@ describe('allFacesTinyYolov2', () => {
const tensor = tf.tidy(() => tf.fromPixels(imgEl).expandDims()) as tf.Tensor4D
await expectAllTensorsReleased(async () => {
await allFacesTinyYolov2(await toNetInput(tensor, true))
await allFacesTinyYolov2(tensor)
})
tensor.dispose()
......
test/tests/e2e/expectedResults.ts
import * as faceapi from '../../../src';
import { expectMaxDelta, expectPointClose, expectRectClose } from '../../utils';
import { Point } from '../../../src';
import { FullFaceDescription } from '../../../src/classes/FullFaceDescription';
import { euclideanDistance } from '../../../src/euclideanDistance';
import {
ExpectedFullFaceDescription,
expectMaxDelta,
expectPointClose,
expectRectClose,
sortBoxes,
sortByDistanceToOrigin,
sortFullFaceDescriptions,
} from '../../utils';
import { IPoint, IRect } from '../../../src';
import { FaceDetection } from '../../../src/classes/FaceDetection';
import { sortFaceDetections } from '../../utils';
export const expectedSsdBoxes = [
export type BoxAndLandmarksDeltas = {
maxBoxDelta: number
maxLandmarksDelta: number
}
export type AllFacesDeltas = BoxAndLandmarksDeltas & {
maxDescriptorDelta: number
}
export const expectedSsdBoxes = sortBoxes([
{ x: 48, y: 253, width: 104, height: 129 },
{ x: 260, y: 227, width: 76, height: 117 },
{ x: 466, y: 165, width: 88, height: 130 },
{ x: 234, y: 36, width: 84, height: 119 },
{ x: 577, y: 65, width: 84, height: 105 },
{ x: 84, y: 14, width: 79, height: 132 }
]
])
export const expectedMtcnnBoxes = [
{ x: 70, y: 21, width: 112, height: 112 },
{ x: 36, y: 250, width: 133, height: 132 },
{ x: 221, y: 43, width: 112, height: 111 },
{ x: 247, y: 231, width: 106, height: 107 },
{ x: 566, y: 67, width: 104, height: 104 },
{ x: 451, y: 176, width: 122, height: 122 }
]
export const expectedTinyYolov2Boxes = [
export const expectedTinyYolov2Boxes = sortBoxes([
{ x: 52, y: 263, width: 106, height: 102 },
{ x: 455, y: 191, width: 103, height: 97 },
{ x: 236, y: 57, width: 90, height: 85 },
{ x: 257, y: 243, width: 86, height: 95 },
{ x: 578, y: 76, width: 86, height: 91 },
{ x: 87, y: 30, width: 92, height: 93 }
]
])
export const expectedTinyYolov2SeparableConvBoxes = [
export const expectedTinyYolov2SeparableConvBoxes = sortBoxes([
{ x: 42, y: 257, width: 111, height: 121 },
{ x: 454, y: 175, width: 104, height: 121 },
{ x: 230, y: 45, width: 94, height: 104 },
{ x: 574, y: 62, width: 88, height: 113 },
{ x: 260, y: 233, width: 82, height: 104 },
{ x: 83, y: 24, width: 85, height: 111 }
]
])
export const expectedMtcnnFaceLandmarks = [
[new Point(117, 58), new Point(156, 63), new Point(141, 86), new Point(109, 98), new Point(147, 104)],
[new Point(82, 292), new Point(134, 304), new Point(104, 330), new Point(72, 342), new Point(120, 353)],
[new Point(261, 82), new Point(306, 83), new Point(282, 113), new Point(257, 124), new Point(306, 126)],
[new Point(277, 273), new Point(318, 273), new Point(295, 300), new Point(279, 311), new Point(316, 313)],
[new Point(607, 110), new Point(645, 115), new Point(626, 138), new Point(601, 144), new Point(639, 150)],
[new Point(489, 224), new Point(534, 223), new Point(507, 250), new Point(493, 271), new Point(530, 270)]
]
export const expectedMtcnnBoxes = sortBoxes([
{ x: 70, y: 21, width: 112, height: 112 },
{ x: 36, y: 250, width: 133, height: 132 },
{ x: 221, y: 43, width: 112, height: 111 },
{ x: 247, y: 231, width: 106, height: 107 },
{ x: 566, y: 67, width: 104, height: 104 },
{ x: 451, y: 176, width: 122, height: 122 }
])
export function expectMtcnnResults(
results: { faceDetection: faceapi.FaceDetection, faceLandmarks: faceapi.FaceLandmarks5 }[],
boxOrder: number[],
maxBoxDelta: number,
maxLandmarkPointsDelta: number
expectedMtcnnFaceLandmarks: IPoint[][],
deltas: BoxAndLandmarksDeltas
) {
results.forEach((result, i) => {
sortByDistanceToOrigin(results, res => res.faceDetection.box).forEach((result, i) => {
const { faceDetection, faceLandmarks } = result
expect(faceDetection instanceof faceapi.FaceDetection).toBe(true)
expect(faceLandmarks instanceof faceapi.FaceLandmarks5).toBe(true)
expectRectClose(faceDetection.getBox(), expectedMtcnnBoxes[boxOrder[i]], maxBoxDelta)
faceLandmarks.getPositions().forEach((pt, j) => expectPointClose(pt, expectedMtcnnFaceLandmarks[boxOrder[i]][j], maxLandmarkPointsDelta))
expectRectClose(faceDetection.getBox(), expectedMtcnnBoxes[i], deltas.maxBoxDelta)
faceLandmarks.getPositions().forEach((pt, j) => expectPointClose(pt, expectedMtcnnFaceLandmarks[i][j], deltas.maxLandmarksDelta))
expectMaxDelta(faceDetection.getScore(), 0.99, 0.01)
})
}
export function expectDetectionResults(results: FaceDetection[], allExpectedFaceDetections: IRect[], expectedScores: number[], maxBoxDelta: number) {
const expectedDetections = expectedScores
.map((score, i) => ({
score,
...allExpectedFaceDetections[i]
}))
.filter(expected => expected.score !== -1)
const sortedResults = sortFaceDetections(results)
expectedDetections.forEach((expectedDetection, i) => {
const det = sortedResults[i]
expect(det.score).toBeCloseTo(expectedDetection.score, 2)
expectRectClose(det.box, expectedDetection, maxBoxDelta)
})
}
export function expectAllFacesResults(results: FullFaceDescription[], allExpectedFullFaceDescriptions: ExpectedFullFaceDescription[], expectedScores: number[], deltas: AllFacesDeltas) {
const expectedFullFaceDescriptions = expectedScores
.map((score, i) => ({
score,
...allExpectedFullFaceDescriptions[i]
}))
.filter(expected => expected.score !== -1)
const sortedResults = sortFullFaceDescriptions(results)
expectedFullFaceDescriptions.forEach((expected, i) => {
const { detection, landmarks, descriptor } = sortedResults[i]
expect(detection.score).toBeCloseTo(expected.score, 2)
expectRectClose(detection.box, expected.detection, deltas.maxBoxDelta)
landmarks.getPositions().forEach((pt, j) => expectPointClose(pt, expected.landmarks[j], deltas.maxLandmarksDelta))
expect(euclideanDistance(descriptor, expected.descriptor)).toBeLessThan(deltas.maxDescriptorDelta)
})
}
\ No newline at end of file
test/tests/e2e/faceDetectionNet.test.ts
import * as faceapi from '../../../src';
import { describeWithNets, expectAllTensorsReleased, expectRectClose } from '../../utils';
import { expectedSsdBoxes } from './expectedResults';
import { expectedSsdBoxes, expectDetectionResults } from './expectedResults';
describe('faceDetectionNet', () => {
......@@ -13,62 +13,52 @@ describe('faceDetectionNet', () => {
describeWithNets('uncompressed weights', { withFaceDetectionNet: { quantized: false } }, ({ faceDetectionNet }) => {
const expectedScores = [0.98, 0.89, 0.82, 0.75, 0.58, 0.55]
const maxBoxDelta = 1
it('scores > 0.8', async () => {
const detections = await faceDetectionNet.locateFaces(imgEl) as faceapi.FaceDetection[]
expect(detections.length).toEqual(3)
detections.forEach((det, i) => {
expect(det.getImageWidth()).toEqual(imgEl.width)
expect(det.getImageHeight()).toEqual(imgEl.height)
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedSsdBoxes[i], maxBoxDelta)
})
const expectedScores = [-1, -1, 0.98, 0.88, 0.81, -1]
const maxBoxDelta = 3
expectDetectionResults(detections, expectedSsdBoxes, expectedScores, maxBoxDelta)
})
it('scores > 0.5', async () => {
const detections = await faceDetectionNet.locateFaces(imgEl, 0.5) as faceapi.FaceDetection[]
expect(detections.length).toEqual(6)
detections.forEach((det, i) => {
expect(det.getImageWidth()).toEqual(imgEl.width)
expect(det.getImageHeight()).toEqual(imgEl.height)
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedSsdBoxes[i], maxBoxDelta)
})
const expectedScores = [0.57, 0.74, 0.98, 0.88, 0.81, 0.58]
const maxBoxDelta = 3
expectDetectionResults(detections, expectedSsdBoxes, expectedScores, maxBoxDelta)
})
})
describeWithNets('quantized weights', { withFaceDetectionNet: { quantized: true } }, ({ faceDetectionNet }) => {
const expectedScores = [0.97, 0.88, 0.83, 0.82, 0.59, 0.52]
const maxBoxDelta = 5
it('scores > 0.8', async () => {
const detections = await faceDetectionNet.locateFaces(imgEl) as faceapi.FaceDetection[]
expect(detections.length).toEqual(4)
detections.forEach((det, i) => {
expect(det.getImageWidth()).toEqual(imgEl.width)
expect(det.getImageHeight()).toEqual(imgEl.height)
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedSsdBoxes[i], maxBoxDelta)
})
const expectedScores = [-1, 0.81, 0.97, 0.88, 0.84, -1]
const maxBoxDelta = 4
expectDetectionResults(detections, expectedSsdBoxes, expectedScores, maxBoxDelta)
})
it('scores > 0.5', async () => {
const detections = await faceDetectionNet.locateFaces(imgEl, 0.5) as faceapi.FaceDetection[]
expect(detections.length).toEqual(6)
detections.forEach((det, i) => {
expect(det.getImageWidth()).toEqual(imgEl.width)
expect(det.getImageHeight()).toEqual(imgEl.height)
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedSsdBoxes[i], maxBoxDelta)
})
const expectedScores = [0.54, 0.81, 0.97, 0.88, 0.84, 0.61]
const maxBoxDelta = 5
expectDetectionResults(detections, expectedSsdBoxes, expectedScores, maxBoxDelta)
})
})
......
test/tests/e2e/faceLandmarkNet.test.ts
......@@ -11,7 +11,7 @@ import {
toNetInput,
} from '../../../src';
import { FaceLandmarks68 } from '../../../src/classes/FaceLandmarks68';
import { FaceLandmarkNet } from '../../../src/faceLandmarkNet/FaceLandmarkNet';
import { FaceLandmark68Net } from '../../../src/faceLandmarkNet/FaceLandmark68Net';
import { describeWithNets, expectAllTensorsReleased, expectMaxDelta } from '../../utils';
function getInputDims (input: tf.Tensor | TMediaElement): Dimensions {
......@@ -54,8 +54,8 @@ describe('faceLandmarkNet', () => {
expect(result.getShift().x).toEqual(0)
expect(result.getShift().y).toEqual(0)
result.getPositions().forEach(({ x, y }, i) => {
expectMaxDelta(x, faceLandmarkPositions1[i].x, 0.1)
expectMaxDelta(y, faceLandmarkPositions1[i].y, 0.1)
expectMaxDelta(x, faceLandmarkPositions1[i].x, 1)
expectMaxDelta(y, faceLandmarkPositions1[i].y, 1)
})
})
......@@ -68,8 +68,8 @@ describe('faceLandmarkNet', () => {
expect(result.getShift().x).toEqual(0)
expect(result.getShift().y).toEqual(0)
result.getPositions().forEach(({ x, y }, i) => {
expectMaxDelta(x, faceLandmarkPositionsRect[i].x, 0.1)
expectMaxDelta(y, faceLandmarkPositionsRect[i].y, 0.1)
expectMaxDelta(x, faceLandmarkPositionsRect[i].x, 2)
expectMaxDelta(y, faceLandmarkPositionsRect[i].y, 2)
})
})
......@@ -128,8 +128,8 @@ describe('faceLandmarkNet', () => {
expect(result.getShift().x).toEqual(0)
expect(result.getShift().y).toEqual(0)
result.getPositions().forEach(({ x, y }, i) => {
expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 0.1)
expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 0.1)
expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 2)
expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 2)
})
})
})
......@@ -153,33 +153,8 @@ describe('faceLandmarkNet', () => {
expect(result.getShift().x).toEqual(0)
expect(result.getShift().y).toEqual(0)
result.getPositions().forEach(({ x, y }, i) => {
expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 0.1)
expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 0.1)
})
})
})
it('computes face landmarks for tf.Tensor4D', async () => {
const inputs = [imgEl1, imgEl2].map(el => tf.fromPixels(el))
const faceLandmarkPositions = [
faceLandmarkPositions1,
faceLandmarkPositions2,
faceLandmarkPositionsRect
]
const results = await faceLandmarkNet.detectLandmarks(tf.stack(inputs) as tf.Tensor4D) as FaceLandmarks68[]
expect(Array.isArray(results)).toBe(true)
expect(results.length).toEqual(2)
results.forEach((result, batchIdx) => {
const { width, height } = getInputDims(inputs[batchIdx])
expect(result.getImageWidth()).toEqual(width)
expect(result.getImageHeight()).toEqual(height)
expect(result.getShift().x).toEqual(0)
expect(result.getShift().y).toEqual(0)
result.getPositions().forEach(({ x, y }, i) => {
expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 0.1)
expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 0.1)
expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 1)
expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 1)
})
})
})
......@@ -203,8 +178,8 @@ describe('faceLandmarkNet', () => {
expect(result.getShift().x).toEqual(0)
expect(result.getShift().y).toEqual(0)
result.getPositions().forEach(({ x, y }, i) => {
expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 0.1)
expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 0.1)
expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 1)
expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 1)
})
})
})
......@@ -230,7 +205,7 @@ describe('faceLandmarkNet', () => {
it('disposes all param tensors', async () => {
await expectAllTensorsReleased(async () => {
const net = new FaceLandmarkNet()
const net = new FaceLandmark68Net()
await net.load('base/weights')
net.dispose()
})
......@@ -242,7 +217,7 @@ describe('faceLandmarkNet', () => {
it('single image element', async () => {
await expectAllTensorsReleased(async () => {
const netInput = (new NetInput([imgEl1])).managed()
const netInput = new NetInput([imgEl1])
const outTensor = await faceLandmarkNet.forwardInput(netInput)
outTensor.dispose()
})
......@@ -250,7 +225,7 @@ describe('faceLandmarkNet', () => {
it('multiple image elements', async () => {
await expectAllTensorsReleased(async () => {
const netInput = (new NetInput([imgEl1, imgEl1, imgEl1])).managed()
const netInput = new NetInput([imgEl1, imgEl1, imgEl1])
const outTensor = await faceLandmarkNet.forwardInput(netInput)
outTensor.dispose()
})
......@@ -260,7 +235,7 @@ describe('faceLandmarkNet', () => {
const tensor = tf.fromPixels(imgEl1)
await expectAllTensorsReleased(async () => {
const netInput = (new NetInput([tensor])).managed()
const netInput = new NetInput([tensor])
const outTensor = await faceLandmarkNet.forwardInput(netInput)
outTensor.dispose()
})
......@@ -272,7 +247,7 @@ describe('faceLandmarkNet', () => {
const tensors = [imgEl1, imgEl1, imgEl1].map(el => tf.fromPixels(el))
await expectAllTensorsReleased(async () => {
const netInput = (new NetInput(tensors)).managed()
const netInput = new NetInput(tensors)
const outTensor = await faceLandmarkNet.forwardInput(netInput)
outTensor.dispose()
})
......@@ -284,7 +259,7 @@ describe('faceLandmarkNet', () => {
const tensor = tf.tidy(() => tf.fromPixels(imgEl1).expandDims()) as tf.Tensor4D
await expectAllTensorsReleased(async () => {
const outTensor = await faceLandmarkNet.forwardInput(await toNetInput(tensor, true))
const outTensor = await faceLandmarkNet.forwardInput(await toNetInput(tensor))
outTensor.dispose()
})
......@@ -296,7 +271,7 @@ describe('faceLandmarkNet', () => {
.map(el => tf.tidy(() => tf.fromPixels(el).expandDims())) as tf.Tensor4D[]
await expectAllTensorsReleased(async () => {
const outTensor = await faceLandmarkNet.forwardInput(await toNetInput(tensors, true))
const outTensor = await faceLandmarkNet.forwardInput(await toNetInput(tensors))
outTensor.dispose()
})
......
test/tests/e2e/faceRecognitionNet.test.ts
import * as tf from '@tensorflow/tfjs-core';
import { bufferToImage, FaceRecognitionNet, NetInput, toNetInput } from '../../../src';
import { euclideanDistance } from '../../../src/euclideanDistance';
import { createFaceRecognitionNet } from '../../../src/faceRecognitionNet';
import { describeWithNets, expectAllTensorsReleased } from '../../utils';
......@@ -30,13 +31,13 @@ describe('faceRecognitionNet', () => {
it('computes face descriptor for squared input', async () => {
const result = await faceRecognitionNet.computeFaceDescriptor(imgEl1) as Float32Array
expect(result.length).toEqual(128)
expect(result).toEqual(new Float32Array(faceDescriptor1))
expect(euclideanDistance(result, faceDescriptor1)).toBeLessThan(0.1)
})
it('computes face descriptor for rectangular input', async () => {
const result = await faceRecognitionNet.computeFaceDescriptor(imgElRect) as Float32Array
expect(result.length).toEqual(128)
expect(result).toEqual(new Float32Array(faceDescriptorRect))
expect(euclideanDistance(result, faceDescriptorRect)).toBeLessThan(0.1)
})
})
......@@ -75,11 +76,11 @@ describe('faceRecognitionNet', () => {
expect(Array.isArray(results)).toBe(true)
expect(results.length).toEqual(3)
results.forEach((result, batchIdx) => {
expect(result).toEqual(new Float32Array(faceDescriptors[batchIdx]))
expect(euclideanDistance(result, faceDescriptors[batchIdx])).toBeLessThan(0.1)
})
})
it('computes face landmarks for batch of tf.Tensor3D', async () => {
it('computes face descriptors for batch of tf.Tensor3D', async () => {
const inputs = [imgEl1, imgEl2, imgElRect].map(el => tf.fromPixels(el))
const faceDescriptors = [
......@@ -92,28 +93,11 @@ describe('faceRecognitionNet', () => {
expect(Array.isArray(results)).toBe(true)
expect(results.length).toEqual(3)
results.forEach((result, batchIdx) => {
expect(result).toEqual(new Float32Array(faceDescriptors[batchIdx]))
})
})
it('computes face landmarks for tf.Tensor4D', async () => {
const inputs = [imgEl1, imgEl2].map(el => tf.fromPixels(el))
const faceDescriptors = [
faceDescriptor1,
faceDescriptor2,
faceDescriptorRect
]
const results = await faceRecognitionNet.computeFaceDescriptor(tf.stack(inputs) as tf.Tensor4D) as Float32Array[]
expect(Array.isArray(results)).toBe(true)
expect(results.length).toEqual(2)
results.forEach((result, batchIdx) => {
expect(result).toEqual(new Float32Array(faceDescriptors[batchIdx]))
expect(euclideanDistance(result, faceDescriptors[batchIdx])).toBeLessThan(0.1)
})
})
it('computes face landmarks for batch of mixed inputs', async () => {
it('computes face descriptors for batch of mixed inputs', async () => {
const inputs = [imgEl1, tf.fromPixels(imgEl2), tf.fromPixels(imgElRect)]
const faceDescriptors = [
......@@ -126,7 +110,7 @@ describe('faceRecognitionNet', () => {
expect(Array.isArray(results)).toBe(true)
expect(results.length).toEqual(3)
results.forEach((result, batchIdx) => {
expect(result).toEqual(new Float32Array(faceDescriptors[batchIdx]))
expect(euclideanDistance(result, faceDescriptors[batchIdx])).toBeLessThan(0.1)
})
})
......@@ -163,7 +147,7 @@ describe('faceRecognitionNet', () => {
it('single image element', async () => {
await expectAllTensorsReleased(async () => {
const netInput = (new NetInput([imgEl1])).managed()
const netInput = new NetInput([imgEl1])
const outTensor = await faceRecognitionNet.forwardInput(netInput)
outTensor.dispose()
})
......@@ -171,7 +155,7 @@ describe('faceRecognitionNet', () => {
it('multiple image elements', async () => {
await expectAllTensorsReleased(async () => {
const netInput = (new NetInput([imgEl1, imgEl1, imgEl1])).managed()
const netInput = new NetInput([imgEl1, imgEl1, imgEl1])
const outTensor = await faceRecognitionNet.forwardInput(netInput)
outTensor.dispose()
})
......@@ -181,7 +165,7 @@ describe('faceRecognitionNet', () => {
const tensor = tf.fromPixels(imgEl1)
await expectAllTensorsReleased(async () => {
const netInput = (new NetInput([tensor])).managed()
const netInput = new NetInput([tensor])
const outTensor = await faceRecognitionNet.forwardInput(netInput)
outTensor.dispose()
})
......@@ -193,7 +177,7 @@ describe('faceRecognitionNet', () => {
const tensors = [imgEl1, imgEl1, imgEl1].map(el => tf.fromPixels(el))
await expectAllTensorsReleased(async () => {
const netInput = (new NetInput(tensors)).managed()
const netInput = new NetInput(tensors)
const outTensor = await faceRecognitionNet.forwardInput(netInput)
outTensor.dispose()
})
......@@ -205,7 +189,7 @@ describe('faceRecognitionNet', () => {
const tensor = tf.tidy(() => tf.fromPixels(imgEl1).expandDims()) as tf.Tensor4D
await expectAllTensorsReleased(async () => {
const outTensor = await faceRecognitionNet.forwardInput(await toNetInput(tensor, true))
const outTensor = await faceRecognitionNet.forwardInput(await toNetInput(tensor))
outTensor.dispose()
})
......@@ -217,7 +201,7 @@ describe('faceRecognitionNet', () => {
.map(el => tf.tidy(() => tf.fromPixels(el).expandDims())) as tf.Tensor4D[]
await expectAllTensorsReleased(async () => {
const outTensor = await faceRecognitionNet.forwardInput(await toNetInput(tensors, true))
const outTensor = await faceRecognitionNet.forwardInput(await toNetInput(tensors))
outTensor.dispose()
})
......
test/tests/e2e/mtcnn.test.ts
import * as faceapi from '../../../src';
import { describeWithNets, expectAllTensorsReleased } from '../../utils';
import { describeWithNets, expectAllTensorsReleased, sortByDistanceToOrigin } from '../../utils';
import { expectMtcnnResults } from './expectedResults';
import { IPoint } from '../../../src';
describe('mtcnn', () => {
let imgEl: HTMLImageElement
let expectedMtcnnLandmarks: IPoint[][]
beforeAll(async () => {
const img = await (await fetch('base/test/images/faces.jpg')).blob()
imgEl = await faceapi.bufferToImage(img)
expectedMtcnnLandmarks = await (await fetch('base/test/data/mtcnnFaceLandmarkPositions.json')).json()
})
describeWithNets('uncompressed weights', { withMtcnn: { quantized: false } }, ({ mtcnn }) => {
......@@ -22,7 +25,12 @@ describe('mtcnn', () => {
const results = await mtcnn.forward(imgEl, forwardParams)
expect(results.length).toEqual(6)
expectMtcnnResults(results, [0, 1, 2, 3, 4, 5], 1, 1)
const deltas = {
maxBoxDelta: 2,
maxLandmarksDelta: 5
}
expectMtcnnResults(results, expectedMtcnnLandmarks, deltas)
})
it('minFaceSize = 80, finds all faces', async () => {
......@@ -33,7 +41,11 @@ describe('mtcnn', () => {
const results = await mtcnn.forward(imgEl, forwardParams)
expect(results.length).toEqual(6)
expectMtcnnResults(results, [0, 5, 3, 1, 2, 4], 12, 12)
const deltas = {
maxBoxDelta: 15,
maxLandmarksDelta: 13
}
expectMtcnnResults(results, expectedMtcnnLandmarks, deltas)
})
it('all optional params passed, finds all faces', async () => {
......@@ -46,7 +58,12 @@ describe('mtcnn', () => {
const results = await mtcnn.forward(imgEl, forwardParams)
expect(results.length).toEqual(6)
expectMtcnnResults(results, [5, 1, 4, 2, 3, 0], 6, 10)
const deltas = {
maxBoxDelta: 8,
maxLandmarksDelta: 7
}
expectMtcnnResults(results, expectedMtcnnLandmarks, deltas)
})
it('scale steps passed, finds all faces', async () => {
......@@ -56,7 +73,12 @@ describe('mtcnn', () => {
const results = await mtcnn.forward(imgEl, forwardParams)
expect(results.length).toEqual(6)
expectMtcnnResults(results, [5, 1, 3, 0, 2, 4], 7, 15)
const deltas = {
maxBoxDelta: 8,
maxLandmarksDelta: 10
}
expectMtcnnResults(results, expectedMtcnnLandmarks, deltas)
})
})
......
test/tests/e2e/tinyYolov2.test.ts
import { TinyYolov2Types } from 'tfjs-tiny-yolov2';
import { bufferToImage, createTinyYolov2, TinyYolov2 } from '../../../src';
import { describeWithNets, expectAllTensorsReleased, expectRectClose } from '../../utils';
import { expectedTinyYolov2Boxes } from './expectedResults';
import { describeWithNets, expectAllTensorsReleased } from '../../utils';
import { expectDetectionResults, expectedTinyYolov2Boxes } from './expectedResults';
describe('tinyYolov2', () => {
......@@ -18,43 +18,31 @@ describe('tinyYolov2', () => {
it('inputSize lg, finds all faces', async () => {
const detections = await tinyYolov2.locateFaces(imgEl, { inputSize: TinyYolov2Types.SizeType.LG })
const expectedScores = [0.86, 0.86, 0.85, 0.83, 0.81, 0.81]
const maxBoxDelta = 3
const boxOrder = [0, 1, 2, 3, 4, 5]
const expectedScores = [0.8, 0.85, 0.86, 0.83, 0.86, 0.81]
const maxBoxDelta = 4
expect(detections.length).toEqual(6)
detections.forEach((det, i) => {
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedTinyYolov2Boxes[boxOrder[i]], maxBoxDelta)
})
expectDetectionResults(detections, expectedTinyYolov2Boxes, expectedScores, maxBoxDelta)
})
it('inputSize md, finds all faces', async () => {
const detections = await tinyYolov2.locateFaces(imgEl, { inputSize: TinyYolov2Types.SizeType.MD })
const expectedScores = [0.89, 0.87, 0.83, 0.82, 0.81, 0.72]
const maxBoxDelta = 16
const boxOrder = [5, 4, 0, 2, 1, 3]
const expectedScores = [0.89, 0.81, 0.82, 0.72, 0.81, 0.86]
const maxBoxDelta = 27
expect(detections.length).toEqual(6)
detections.forEach((det, i) => {
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedTinyYolov2Boxes[boxOrder[i]], maxBoxDelta)
})
expectDetectionResults(detections, expectedTinyYolov2Boxes, expectedScores, maxBoxDelta)
})
it('inputSize custom, finds all faces', async () => {
const detections = await tinyYolov2.locateFaces(imgEl, { inputSize: 416 })
const expectedScores = [0.89, 0.87, 0.83, 0.82, 0.81, 0.72]
const maxBoxDelta = 16
const boxOrder = [5, 4, 0, 2, 1, 3]
const expectedScores = [0.89, 0.81, 0.82, 0.72, 0.81, 0.86]
const maxBoxDelta = 27
expect(detections.length).toEqual(6)
detections.forEach((det, i) => {
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedTinyYolov2Boxes[boxOrder[i]], maxBoxDelta)
})
expectDetectionResults(detections, expectedTinyYolov2Boxes, expectedScores, maxBoxDelta)
})
})
......@@ -64,43 +52,31 @@ describe('tinyYolov2', () => {
it('inputSize lg, finds all faces', async () => {
const detections = await tinyYolov2.locateFaces(imgEl, { inputSize: TinyYolov2Types.SizeType.LG })
const expectedScores = [0.86, 0.86, 0.85, 0.83, 0.81, 0.81]
const expectedScores = [0.81, 0.85, 0.86, 0.83, 0.86, 0.81]
const maxBoxDelta = 1
const boxOrder = [0, 1, 2, 3, 4, 5]
expect(detections.length).toEqual(6)
detections.forEach((det, i) => {
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedTinyYolov2Boxes[boxOrder[i]], maxBoxDelta)
})
expectDetectionResults(detections, expectedTinyYolov2Boxes, expectedScores, maxBoxDelta)
})
it('inputSize md, finds all faces', async () => {
const detections = await tinyYolov2.locateFaces(imgEl, { inputSize: TinyYolov2Types.SizeType.MD })
const expectedScores = [0.89, 0.87, 0.83, 0.83, 0.81, 0.73]
const maxBoxDelta = 14
const boxOrder = [5, 4, 2, 0, 1, 3]
const expectedScores = [0.89, 0.82, 0.82, 0.72, 0.81, 0.86]
const maxBoxDelta = 24
expect(detections.length).toEqual(6)
detections.forEach((det, i) => {
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedTinyYolov2Boxes[boxOrder[i]], maxBoxDelta)
})
expectDetectionResults(detections, expectedTinyYolov2Boxes, expectedScores, maxBoxDelta)
})
it('inputSize custom, finds all faces', async () => {
const detections = await tinyYolov2.locateFaces(imgEl, { inputSize: 416 })
const expectedScores = [0.89, 0.87, 0.83, 0.83, 0.81, 0.73]
const maxBoxDelta = 14
const boxOrder = [5, 4, 2, 0, 1, 3]
const expectedScores = [0.89, 0.82, 0.82, 0.72, 0.81, 0.86]
const maxBoxDelta = 24
expect(detections.length).toEqual(6)
detections.forEach((det, i) => {
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedTinyYolov2Boxes[boxOrder[i]], maxBoxDelta)
})
expectDetectionResults(detections, expectedTinyYolov2Boxes, expectedScores, maxBoxDelta)
})
})
......
test/tests/e2e/tinyYolov2SeparableConv.test.ts
......@@ -2,7 +2,7 @@ import { TinyYolov2Types } from 'tfjs-tiny-yolov2';
import { bufferToImage, createTinyYolov2, TinyYolov2 } from '../../../src';
import { describeWithNets, expectAllTensorsReleased, expectRectClose } from '../../utils';
import { expectedTinyYolov2SeparableConvBoxes } from './expectedResults';
import { expectedTinyYolov2SeparableConvBoxes, expectDetectionResults, expectedTinyYolov2Boxes } from './expectedResults';
describe('tinyYolov2, with separable convolutions', () => {
......@@ -18,43 +18,31 @@ describe('tinyYolov2, with separable convolutions', () => {
it('inputSize lg, finds all faces', async () => {
const detections = await tinyYolov2.locateFaces(imgEl, { inputSize: TinyYolov2Types.SizeType.LG })
const expectedScores = [0.9, 0.9, 0.89, 0.85, 0.85, 0.85]
const maxBoxDelta = 1
const boxOrder = [0, 1, 2, 3, 4, 5]
const expectedScores = [0.85, 0.88, 0.9, 0.85, 0.9, 0.85]
const maxBoxDelta = 25
expect(detections.length).toEqual(6)
detections.forEach((det, i) => {
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedTinyYolov2SeparableConvBoxes[boxOrder[i]], maxBoxDelta)
})
expectDetectionResults(detections, expectedTinyYolov2Boxes, expectedScores, maxBoxDelta)
})
it('inputSize md, finds all faces', async () => {
const detections = await tinyYolov2.locateFaces(imgEl, { inputSize: TinyYolov2Types.SizeType.MD })
const expectedScores = [0.85, 0.85, 0.84, 0.83, 0.8, 0.8]
const maxBoxDelta = 17
const boxOrder = [5, 1, 4, 3, 2, 0]
const expectedScores = [0.85, 0.8, 0.8, 0.85, 0.85, 0.83]
const maxBoxDelta = 34
expect(detections.length).toEqual(6)
detections.forEach((det, i) => {
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedTinyYolov2SeparableConvBoxes[boxOrder[i]], maxBoxDelta)
})
expectDetectionResults(detections, expectedTinyYolov2Boxes, expectedScores, maxBoxDelta)
})
it('inputSize custom, finds all faces', async () => {
const detections = await tinyYolov2.locateFaces(imgEl, { inputSize: 416 })
const expectedScores = [0.85, 0.85, 0.84, 0.83, 0.8, 0.8]
const maxBoxDelta = 17
const boxOrder = [5, 1, 4, 3, 2, 0]
const expectedScores = [0.85, 0.8, 0.8, 0.85, 0.85, 0.83]
const maxBoxDelta = 34
expect(detections.length).toEqual(6)
detections.forEach((det, i) => {
expect(det.getScore()).toBeCloseTo(expectedScores[i], 2)
expectRectClose(det.getBox(), expectedTinyYolov2SeparableConvBoxes[boxOrder[i]], maxBoxDelta)
})
expectDetectionResults(detections, expectedTinyYolov2Boxes, expectedScores, maxBoxDelta)
})
})
......
test/tests/faceLandmarkNet/FaceLandmark68NetBase.test.ts 0 → 100644
import * as tf from '@tensorflow/tfjs-core';
import { FaceLandmark68NetBase } from '../../../src/faceLandmarkNet/FaceLandmark68NetBase';
describe('FaceLandmark68NetBase', () => {
describe('postProcess', () => {
const net = new FaceLandmark68NetBase('')
describe('single batch', () => {
it('transform x coordinates for width < height', () => {
const landmarksData = Array(136).fill(0)
landmarksData[0] = 0.4
landmarksData[1] = 0.55
landmarksData[2] = 0.2
landmarksData[3] = 0.55
landmarksData[4] = 0.1
landmarksData[5] = 0.55
const out = net.postProcess(
tf.tensor2d(landmarksData, [1, 136]),
128,
[{ width: 200, height: 300 }]
).dataSync()
expect(out[0]).toBeCloseTo(0.35, 2)
expect(out[1]).toBeCloseTo(0.55, 2)
expect(out[2]).toBeCloseTo(0.05, 2)
expect(out[3]).toBeCloseTo(0.55, 2)
expect(out[4]).toBeCloseTo(-0.1, 2)
expect(out[5]).toBeCloseTo(0.55, 2)
})
it('transform y coordinates for height < width', () => {
const landmarksData = Array(136).fill(0)
landmarksData[0] = 0.55
landmarksData[1] = 0.4
landmarksData[2] = 0.55
landmarksData[3] = 0.2
landmarksData[4] = 0.55
landmarksData[5] = 0.1
const out = net.postProcess(
tf.tensor2d(landmarksData, [1, 136]),
128,
[{ width: 300, height: 200 }]
).dataSync()
expect(out[0]).toBeCloseTo(0.55, 2)
expect(out[1]).toBeCloseTo(0.35, 2)
expect(out[2]).toBeCloseTo(0.55, 2)
expect(out[3]).toBeCloseTo(0.05, 2)
expect(out[4]).toBeCloseTo(0.55, 2)
expect(out[5]).toBeCloseTo(-0.1, 2)
})
it('no transformation for height === width', () => {
const landmarksData = Array(136).fill(0)
landmarksData[0] = 0.55
landmarksData[1] = 0.4
landmarksData[2] = 0.55
landmarksData[3] = 0.2
landmarksData[4] = 0.55
landmarksData[5] = 0.1
const out = net.postProcess(
tf.tensor2d(landmarksData, [1, 136]),
128,
[{ width: 300, height: 300 }]
).dataSync()
expect(out[0]).toBeCloseTo(0.55, 2)
expect(out[1]).toBeCloseTo(0.4, 2)
expect(out[2]).toBeCloseTo(0.55, 2)
expect(out[3]).toBeCloseTo(0.2, 2)
expect(out[4]).toBeCloseTo(0.55, 2)
expect(out[5]).toBeCloseTo(0.1, 2)
})
})
describe('multiple batches', () => {
it('transform coordinates correctly for multiple batches', () => {
const landmarksData1 = Array(136).fill(0)
landmarksData1[0] = 0.4
landmarksData1[1] = 0.55
landmarksData1[2] = 0.2
landmarksData1[3] = 0.55
landmarksData1[4] = 0.1
landmarksData1[5] = 0.55
const landmarksData2 = Array(136).fill(0)
landmarksData2[0] = 0.55
landmarksData2[1] = 0.4
landmarksData2[2] = 0.55
landmarksData2[3] = 0.2
landmarksData2[4] = 0.55
landmarksData2[5] = 0.1
const out = net.postProcess(
tf.tensor2d(landmarksData1.concat(landmarksData2).concat(landmarksData1), [3, 136]),
128,
[{ width: 200, height: 300 }, { width: 300, height: 200 }, { width: 300, height: 300 }]
).dataSync()
expect(out[0]).toBeCloseTo(0.35, 2)
expect(out[1]).toBeCloseTo(0.55, 2)
expect(out[2]).toBeCloseTo(0.05, 2)
expect(out[3]).toBeCloseTo(0.55, 2)
expect(out[4]).toBeCloseTo(-0.1, 2)
expect(out[5]).toBeCloseTo(0.55, 2)
expect(out[0 + 136]).toBeCloseTo(0.55, 2)
expect(out[1 + 136]).toBeCloseTo(0.35, 2)
expect(out[2 + 136]).toBeCloseTo(0.55, 2)
expect(out[3 + 136]).toBeCloseTo(0.05, 2)
expect(out[4 + 136]).toBeCloseTo(0.55, 2)
expect(out[5 + 136]).toBeCloseTo(-0.1, 2)
expect(out[0 + (136 * 2)]).toBeCloseTo(0.4, 2)
expect(out[1 + (136 * 2)]).toBeCloseTo(0.55, 2)
expect(out[2 + (136 * 2)]).toBeCloseTo(0.2, 2)
expect(out[3 + (136 * 2)]).toBeCloseTo(0.55, 2)
expect(out[4 + (136 * 2)]).toBeCloseTo(0.1, 2)
expect(out[5 + (136 * 2)]).toBeCloseTo(0.55, 2)
})
})
})
})
test/utils.ts
......@@ -11,6 +11,9 @@ import {
TinyYolov2,
} from '../src/';
import { allFacesMtcnnFactory, allFacesSsdMobilenetv1Factory, allFacesTinyYolov2Factory } from '../src/allFacesFactory';
import { FaceDetection } from '../src/classes/FaceDetection';
import { FaceLandmarks } from '../src/classes/FaceLandmarks';
import { FullFaceDescription } from '../src/classes/FullFaceDescription';
import { allFacesMtcnnFunction, allFacesSsdMobilenetv1Function, allFacesTinyYolov2Function } from '../src/globalApi';
jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000
......@@ -25,14 +28,16 @@ export async function expectAllTensorsReleased(fn: () => any) {
expect(tf.memory().numTensors - numTensorsBefore).toEqual(0)
}
export function pointDistance(pt1: IPoint, pt2: IPoint) {
return Math.sqrt(Math.pow(pt1.x - pt2.x, 2) + Math.pow(pt1.y - pt2.y, 2))
}
export function expectPointClose(
result: IPoint,
expectedPoint: IPoint,
maxDelta: number
) {
const { x, y } = result
expectMaxDelta(x, expectedPoint.x, maxDelta)
expectMaxDelta(y, expectedPoint.y, maxDelta)
expect(pointDistance(result, expectedPoint)).toBeLessThan(maxDelta)
}
export function expectRectClose(
......@@ -40,10 +45,52 @@ export function expectRectClose(
expectedBox: IRect,
maxDelta: number
) {
const { width, height } = result
expectPointClose(result, expectedBox, maxDelta)
expectMaxDelta(width, expectedBox.width, maxDelta)
expectMaxDelta(height, expectedBox.height, maxDelta)
expectPointClose({ x: result.width, y: result.height }, { x:expectedBox.width, y: expectedBox.height }, maxDelta)
}
export function sortByDistanceToOrigin<T>(objs: T[], originGetter: (obj: T) => IPoint) {
const origin = { x: 0, y: 0 }
return objs.sort((obj1, obj2) =>
pointDistance(originGetter(obj1), origin)
- pointDistance(originGetter(obj2), origin)
)
}
export function sortBoxes(boxes: IRect[]) {
return sortByDistanceToOrigin(boxes, rect => rect)
}
export function sortFaceDetections(boxes: FaceDetection[]) {
return sortByDistanceToOrigin(boxes, det => det.box)
}
export function sortLandmarks(landmarks: FaceLandmarks[]) {
return sortByDistanceToOrigin(landmarks, l => l.getPositions()[0])
}
export function sortFullFaceDescriptions(descs: FullFaceDescription[]) {
return sortByDistanceToOrigin(descs, d => d.detection.box)
}
export type ExpectedFullFaceDescription = {
detection: IRect
landmarks: IPoint[]
descriptor: Float32Array
}
export async function assembleExpectedFullFaceDescriptions(
detections: IRect[],
landmarksFile: string = 'facesFaceLandmarkPositions.json'
): Promise<ExpectedFullFaceDescription[]> {
const landmarks = await (await fetch(`base/test/data/${landmarksFile}`)).json()
const descriptors = await (await fetch('base/test/data/facesFaceDescriptors.json')).json()
return detections.map((detection, i) => ({
detection,
landmarks: landmarks[i],
descriptor: descriptors[i]
}))
}
export type WithNetOptions = {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment