Skip to content

F
face
Commit c44a4c12 by vincent
Options

new composable tasks API + forward options

parent 9c006357
Showing with 598 additions and 375 deletions
package-lock.json
......@@ -4912,9 +4912,9 @@
}
},
"tfjs-tiny-yolov2": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/tfjs-tiny-yolov2/-/tfjs-tiny-yolov2-0.1.3.tgz",
"integrity": "sha512-oY77SnFFzOizXRYPfRcHO9LuqB+dJkXvOBZz6b04E5HrJvVkCJu/MDJBIHuMdbe2TuVmPIhcX+Yr1bqLp3gXKw==",
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/tfjs-tiny-yolov2/-/tfjs-tiny-yolov2-0.2.1.tgz",
"integrity": "sha512-HSdBu6dMyQdtueY32wSO+5IajXDrvu7MufvUBaLD0CubKbfJWM1JogsONUkvp3N948UAI2/K35p9+eEP2woXpw==",
"requires": {
"@tensorflow/tfjs-core": "0.13.2",
"tfjs-image-recognition-base": "0.1.2",
......
package.json
......@@ -25,7 +25,7 @@
"dependencies": {
"@tensorflow/tfjs-core": "^0.13.2",
"tfjs-image-recognition-base": "^0.1.2",
"tfjs-tiny-yolov2": "^0.1.3",
"tfjs-tiny-yolov2": "^0.2.1",
"tslib": "^1.9.3"
},
"devDependencies": {
......
src/allFacesFactory.ts deleted 100644 → 0
import { Point, Rect, TNetInput } from 'tfjs-image-recognition-base';
import { TinyYolov2Types } from 'tfjs-tiny-yolov2';
import { TinyYolov2 } from '.';
import { FaceDetection } from './classes/FaceDetection';
import { FaceLandmarks68 } from './classes/FaceLandmarks68';
import { FullFaceDescription } from './classes/FullFaceDescription';
import { extractFaces } from './dom';
import { FaceDetectionNet } from './faceDetectionNet/FaceDetectionNet';
import { FaceLandmark68Net } from './faceLandmarkNet/FaceLandmark68Net';
import { FaceRecognitionNet } from './faceRecognitionNet/FaceRecognitionNet';
import { Mtcnn } from './mtcnn/Mtcnn';
import { MtcnnForwardParams } from './mtcnn/types';
function computeDescriptorsFactory(
recognitionNet: FaceRecognitionNet
) {
return async function(input: TNetInput, alignedFaceBoxes: Rect[], useBatchProcessing: boolean) {
const alignedFaceCanvases = await extractFaces(input, alignedFaceBoxes)
const descriptors = useBatchProcessing
? await recognitionNet.computeFaceDescriptor(alignedFaceCanvases) as Float32Array[]
: await Promise.all(alignedFaceCanvases.map(
canvas => recognitionNet.computeFaceDescriptor(canvas)
)) as Float32Array[]
return descriptors
}
}
function allFacesFactory(
detectFaces: (input: TNetInput) => Promise<FaceDetection[]>,
landmarkNet: FaceLandmark68Net,
recognitionNet: FaceRecognitionNet
) {
const computeDescriptors = computeDescriptorsFactory(recognitionNet)
return async function(
input: TNetInput,
useBatchProcessing: boolean = false
): Promise<FullFaceDescription[]> {
const detections = await detectFaces(input)
const faceCanvases = await extractFaces(input, detections)
const faceLandmarksByFace = useBatchProcessing
? await landmarkNet.detectLandmarks(faceCanvases) as FaceLandmarks68[]
: await Promise.all(faceCanvases.map(
canvas => landmarkNet.detectLandmarks(canvas)
)) as FaceLandmarks68[]
const alignedFaceBoxes = faceLandmarksByFace.map(
(landmarks, i) => landmarks.align(detections[i].getBox())
)
const descriptors = await computeDescriptors(input, alignedFaceBoxes, useBatchProcessing)
return detections.map((detection, i) =>
new FullFaceDescription(
detection,
faceLandmarksByFace[i].shiftByPoint<FaceLandmarks68>(
new Point(detection.box.x, detection.box.y)
),
descriptors[i]
)
)
}
}
export function allFacesSsdMobilenetv1Factory(
ssdMobilenetv1: FaceDetectionNet,
landmarkNet: FaceLandmark68Net,
recognitionNet: FaceRecognitionNet
) {
return async function(
input: TNetInput,
minConfidence: number = 0.8,
useBatchProcessing: boolean = false
): Promise<FullFaceDescription[]> {
const detectFaces = (input: TNetInput) => ssdMobilenetv1.locateFaces(input, minConfidence)
const allFaces = allFacesFactory(detectFaces, landmarkNet, recognitionNet)
return allFaces(input, useBatchProcessing)
}
}
export function allFacesTinyYolov2Factory(
tinyYolov2: TinyYolov2,
landmarkNet: FaceLandmark68Net,
recognitionNet: FaceRecognitionNet
) {
return async function(
input: TNetInput,
forwardParams: TinyYolov2Types.TinyYolov2ForwardParams = {},
useBatchProcessing: boolean = false
): Promise<FullFaceDescription[]> {
const detectFaces = (input: TNetInput) => tinyYolov2.locateFaces(input, forwardParams)
const allFaces = allFacesFactory(detectFaces, landmarkNet, recognitionNet)
return allFaces(input, useBatchProcessing)
}
}
export function allFacesMtcnnFactory(
mtcnn: Mtcnn,
recognitionNet: FaceRecognitionNet
) {
const computeDescriptors = computeDescriptorsFactory(recognitionNet)
return async function(
input: TNetInput,
mtcnnForwardParams: MtcnnForwardParams = {},
useBatchProcessing: boolean = false
): Promise<FullFaceDescription[]> {
const results = await mtcnn.forward(input, mtcnnForwardParams)
const alignedFaceBoxes = results.map(
({ faceLandmarks }) => faceLandmarks.align()
)
const descriptors = await computeDescriptors(input, alignedFaceBoxes, useBatchProcessing)
return results.map(({ faceDetection, faceLandmarks }, i) =>
new FullFaceDescription(
faceDetection,
faceLandmarks,
descriptors[i]
)
)
}
}
\ No newline at end of file
src/classes/FaceDetectionWithLandmarks.ts 0 → 100644
import { FaceDetection } from './FaceDetection';
import { FaceLandmarks } from './FaceLandmarks';
export class FaceDetectionWithLandmarks {
private _detection: FaceDetection
private _relativeLandmarks: FaceLandmarks
constructor(
detection: FaceDetection,
relativeLandmarks: FaceLandmarks
) {
this._detection = detection
this._relativeLandmarks = relativeLandmarks
}
public get detection(): FaceDetection { return this._detection }
public get relativeLandmarks(): FaceLandmarks { return this._relativeLandmarks }
public get landmarks(): FaceLandmarks {
const { x, y } = this.detection.box
return this._relativeLandmarks.shift(x, y)
}
public forSize(width: number, height: number): FaceDetectionWithLandmarks {
return new FaceDetectionWithLandmarks(
this._detection.forSize(width, height),
this._relativeLandmarks.forSize(width, height)
)
}
}
\ No newline at end of file
src/classes/FullFaceDescription.ts
import { FaceDetection } from './FaceDetection';
import { FaceDetectionWithLandmarks } from './FaceDetectionWithLandmarks';
import { FaceLandmarks } from './FaceLandmarks';
export class FullFaceDescription {
constructor(
private _detection: FaceDetection,
private _landmarks: FaceLandmarks,
private _descriptor: Float32Array
) {}
public get detection(): FaceDetection {
return this._detection
}
export class FullFaceDescription extends FaceDetectionWithLandmarks {
private _descriptor: Float32Array
public get landmarks(): FaceLandmarks {
return this._landmarks
constructor(
detection: FaceDetection,
landmarks: FaceLandmarks,
descriptor: Float32Array
) {
super(detection, landmarks)
this._descriptor = descriptor
}
public get descriptor(): Float32Array {
......@@ -21,10 +19,7 @@ export class FullFaceDescription {
}
public forSize(width: number, height: number): FullFaceDescription {
return new FullFaceDescription(
this._detection.forSize(width, height),
this._landmarks.forSize(width, height),
this._descriptor
)
const { detection, landmarks } = super.forSize(width, height)
return new FullFaceDescription(detection, landmarks, this.descriptor)
}
}
\ No newline at end of file
src/faceDetectionNet/index.ts deleted 100644 → 0
import { FaceDetectionNet } from './FaceDetectionNet';
export * from './FaceDetectionNet';
export function createFaceDetectionNet(weights: Float32Array) {
const net = new FaceDetectionNet()
net.extractWeights(weights)
return net
}
export function faceDetectionNet(weights: Float32Array) {
console.warn('faceDetectionNet(weights: Float32Array) will be deprecated in future, use createFaceDetectionNet instead')
return createFaceDetectionNet(weights)
}
\ No newline at end of file
src/faceLandmarkNet/index.ts
......@@ -9,9 +9,4 @@ export function createFaceLandmarkNet(weights: Float32Array) {
const net = new FaceLandmarkNet()
net.extractWeights(weights)
return net
}
export function faceLandmarkNet(weights: Float32Array) {
console.warn('faceLandmarkNet(weights: Float32Array) will be deprecated in future, use createFaceLandmarkNet instead')
return createFaceLandmarkNet(weights)
}
\ No newline at end of file
src/faceRecognitionNet/index.ts
......@@ -6,9 +6,4 @@ export function createFaceRecognitionNet(weights: Float32Array) {
const net = new FaceRecognitionNet()
net.extractWeights(weights)
return net
}
export function faceRecognitionNet(weights: Float32Array) {
console.warn('faceRecognitionNet(weights: Float32Array) will be deprecated in future, use createFaceRecognitionNet instead')
return createFaceRecognitionNet(weights)
}
\ No newline at end of file
src/globalApi.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
import { NetInput, TNetInput } from 'tfjs-image-recognition-base';
import { TinyYolov2Types } from 'tfjs-tiny-yolov2';
import { allFacesMtcnnFactory, allFacesSsdMobilenetv1Factory, allFacesTinyYolov2Factory } from './allFacesFactory';
import { FaceDetection } from './classes/FaceDetection';
import { FaceLandmarks68 } from './classes/FaceLandmarks68';
import { FullFaceDescription } from './classes/FullFaceDescription';
import { FaceDetectionNet } from './faceDetectionNet/FaceDetectionNet';
import { FaceLandmark68Net } from './faceLandmarkNet/FaceLandmark68Net';
import { FaceLandmark68TinyNet } from './faceLandmarkNet/FaceLandmark68TinyNet';
import { FaceRecognitionNet } from './faceRecognitionNet/FaceRecognitionNet';
import { Mtcnn } from './mtcnn/Mtcnn';
import { MtcnnForwardParams, MtcnnResult } from './mtcnn/types';
import { TinyYolov2 } from './tinyYolov2/TinyYolov2';
export const detectionNet = new FaceDetectionNet()
export const landmarkNet = new FaceLandmark68Net()
export const recognitionNet = new FaceRecognitionNet()
// nets need more specific names, to avoid ambiguity in future
// when alternative net implementations are provided
export const nets = {
ssdMobilenetv1: detectionNet,
faceLandmark68Net: landmarkNet,
faceLandmark68TinyNet: new FaceLandmark68TinyNet(),
faceRecognitionNet: recognitionNet,
mtcnn: new Mtcnn(),
tinyYolov2: new TinyYolov2()
}
export function loadSsdMobilenetv1Model(url: string) {
return nets.ssdMobilenetv1.load(url)
}
export function loadFaceLandmarkModel(url: string) {
return nets.faceLandmark68Net.load(url)
}
export function loadFaceLandmarkTinyModel(url: string) {
return nets.faceLandmark68TinyNet.load(url)
}
export function loadFaceRecognitionModel(url: string) {
return nets.faceRecognitionNet.load(url)
}
export function loadMtcnnModel(url: string) {
return nets.mtcnn.load(url)
}
export function loadTinyYolov2Model(url: string) {
return nets.tinyYolov2.load(url)
}
export function loadFaceDetectionModel(url: string) {
return loadSsdMobilenetv1Model(url)
}
export function loadModels(url: string) {
console.warn('loadModels will be deprecated in future')
return Promise.all([
loadSsdMobilenetv1Model(url),
loadFaceLandmarkModel(url),
loadFaceRecognitionModel(url),
loadMtcnnModel(url),
loadTinyYolov2Model(url)
])
}
export function locateFaces(
input: TNetInput,
minConfidence?: number,
maxResults?: number
): Promise<FaceDetection[]> {
return nets.ssdMobilenetv1.locateFaces(input, minConfidence, maxResults)
}
export const ssdMobilenetv1 = locateFaces
export function detectLandmarks(
input: TNetInput
): Promise<FaceLandmarks68 | FaceLandmarks68[]> {
return nets.faceLandmark68Net.detectLandmarks(input)
}
export function detectLandmarksTiny(
input: TNetInput
): Promise<FaceLandmarks68 | FaceLandmarks68[]> {
return nets.faceLandmark68TinyNet.detectLandmarks(input)
}
export function computeFaceDescriptor(
input: TNetInput
): Promise<Float32Array | Float32Array[]> {
return nets.faceRecognitionNet.computeFaceDescriptor(input)
}
export function mtcnn(
input: TNetInput,
forwardParams: MtcnnForwardParams
): Promise<MtcnnResult[]> {
return nets.mtcnn.forward(input, forwardParams)
}
export function tinyYolov2(
input: TNetInput,
forwardParams: TinyYolov2Types.TinyYolov2ForwardParams
): Promise<FaceDetection[]> {
return nets.tinyYolov2.locateFaces(input, forwardParams)
}
export type allFacesSsdMobilenetv1Function = (
input: tf.Tensor | NetInput | TNetInput,
minConfidence?: number,
useBatchProcessing?: boolean
) => Promise<FullFaceDescription[]>
export const allFacesSsdMobilenetv1: allFacesSsdMobilenetv1Function = allFacesSsdMobilenetv1Factory(
nets.ssdMobilenetv1,
nets.faceLandmark68Net,
nets.faceRecognitionNet
)
export type allFacesTinyYolov2Function = (
input: tf.Tensor | NetInput | TNetInput,
forwardParams?: TinyYolov2Types.TinyYolov2ForwardParams,
useBatchProcessing?: boolean
) => Promise<FullFaceDescription[]>
export const allFacesTinyYolov2: allFacesTinyYolov2Function = allFacesTinyYolov2Factory(
nets.tinyYolov2,
nets.faceLandmark68Net,
nets.faceRecognitionNet
)
export type allFacesMtcnnFunction = (
input: tf.Tensor | NetInput | TNetInput,
mtcnnForwardParams?: MtcnnForwardParams,
useBatchProcessing?: boolean
) => Promise<FullFaceDescription[]>
export const allFacesMtcnn: allFacesMtcnnFunction = allFacesMtcnnFactory(
nets.mtcnn,
nets.faceRecognitionNet
)
export const allFaces = allFacesSsdMobilenetv1
src/globalApi/ComposableTask.ts 0 → 100644
export class ComposableTask<T> {
public async then(
onfulfilled: (value: T) => T | PromiseLike<T>
): Promise<T> {
return onfulfilled(await this.run())
}
public async run(): Promise<T> {
throw new Error('ComposableTask - run is not implemented')
}
}
src/globalApi/ComputeFaceDescriptorsTasks.ts 0 → 100644
import { TNetInput } from 'tfjs-image-recognition-base';
import { FaceDetectionWithLandmarks } from '../classes/FaceDetectionWithLandmarks';
import { FullFaceDescription } from '../classes/FullFaceDescription';
import { extractFaces } from '../dom';
import { ComposableTask } from './ComposableTask';
import { nets } from './nets';
export class ComputeFaceDescriptorsTaskBase<TReturn, DetectFaceLandmarksReturnType> extends ComposableTask<TReturn> {
constructor(
protected detectFaceLandmarksTask: ComposableTask<DetectFaceLandmarksReturnType> | Promise<DetectFaceLandmarksReturnType>,
protected input: TNetInput
) {
super()
}
}
export class ComputeAllFaceDescriptorsTask extends ComputeFaceDescriptorsTaskBase<FullFaceDescription[], FaceDetectionWithLandmarks[]> {
public async run(): Promise<FullFaceDescription[]> {
const facesWithLandmarks = await this.detectFaceLandmarksTask
const alignedFaceCanvases = await extractFaces(
this.input,
facesWithLandmarks.map(({ landmarks }) => landmarks.align())
)
return await Promise.all(facesWithLandmarks.map(async ({ detection, landmarks }, i) => {
const descriptor = await nets.faceRecognitionNet.computeFaceDescriptor(alignedFaceCanvases[i]) as Float32Array
return new FullFaceDescription(detection, landmarks, descriptor)
}))
}
}
export class ComputeSingleFaceDescriptorTask extends ComputeFaceDescriptorsTaskBase<FullFaceDescription | undefined, FaceDetectionWithLandmarks | undefined> {
public async run(): Promise<FullFaceDescription | undefined> {
const detectionWithLandmarks = await this.detectFaceLandmarksTask
if (!detectionWithLandmarks) {
return
}
const { detection, landmarks } = detectionWithLandmarks
const alignedFaceCanvas = (await extractFaces(this.input, [landmarks.align()]))[0]
const descriptor = await nets.faceRecognitionNet.computeFaceDescriptor(alignedFaceCanvas) as Float32Array
return new FullFaceDescription(detection, landmarks, descriptor)
}
}
\ No newline at end of file
src/globalApi/DetectFacesLandmarksTasks.ts 0 → 100644
import { TNetInput } from 'tfjs-image-recognition-base';
import { FaceDetection } from '../classes/FaceDetection';
import { FaceDetectionWithLandmarks } from '../classes/FaceDetectionWithLandmarks';
import { FaceLandmarks68 } from '../classes/FaceLandmarks68';
import { extractFaces } from '../dom';
import { FaceLandmark68Net } from '../faceLandmarkNet/FaceLandmark68Net';
import { FaceLandmark68TinyNet } from '../faceLandmarkNet/FaceLandmark68TinyNet';
import { ComposableTask } from './ComposableTask';
import { ComputeAllFaceDescriptorsTask, ComputeSingleFaceDescriptorTask } from './ComputeFaceDescriptorsTasks';
import { nets } from './nets';
export class DetectFaceLandmarksTaskBase<ReturnType, DetectFacesReturnType> extends ComposableTask<ReturnType> {
constructor(
protected detectFacesTask: ComposableTask<DetectFacesReturnType> | Promise<DetectFacesReturnType>,
protected input: TNetInput,
protected useTinyLandmarkNet: boolean
) {
super()
}
protected get landmarkNet(): FaceLandmark68Net | FaceLandmark68TinyNet {
return this.useTinyLandmarkNet
? nets.faceLandmark68TinyNet
: nets.faceLandmark68Net
}
}
export class DetectAllFaceLandmarksTask extends DetectFaceLandmarksTaskBase<FaceDetectionWithLandmarks[], FaceDetection[]> {
public async run(): Promise<FaceDetectionWithLandmarks[]> {
const detections = await this.detectFacesTask
const faceCanvases = await extractFaces(this.input, detections)
const faceLandmarksByFace = await Promise.all(faceCanvases.map(
canvas => this.landmarkNet.detectLandmarks(canvas)
)) as FaceLandmarks68[]
return detections.map((detection, i) =>
new FaceDetectionWithLandmarks(detection, faceLandmarksByFace[i])
)
}
withFaceDescriptors(): ComputeAllFaceDescriptorsTask {
return new ComputeAllFaceDescriptorsTask(this, this.input)
}
}
export class DetectSingleFaceLandmarksTask extends DetectFaceLandmarksTaskBase<FaceDetectionWithLandmarks | undefined, FaceDetection | undefined> {
public async run(): Promise<FaceDetectionWithLandmarks | undefined> {
const detection = await this.detectFacesTask
if (!detection) {
return
}
const faceCanvas = (await extractFaces(this.input, [detection]))[0]
return new FaceDetectionWithLandmarks(
detection,
await this.landmarkNet.detectLandmarks(faceCanvas) as FaceLandmarks68
)
}
withFaceDescriptor(): ComputeSingleFaceDescriptorTask {
return new ComputeSingleFaceDescriptorTask(this, this.input)
}
}
\ No newline at end of file
src/globalApi/DetectFacesTasks.ts 0 → 100644
import { TNetInput } from 'tfjs-image-recognition-base';
import { TinyYolov2Options } from 'tfjs-tiny-yolov2';
import { FaceDetection } from '../classes/FaceDetection';
import { MtcnnOptions } from '../mtcnn/MtcnnOptions';
import { SsdMobilenetv1Options } from '../ssdMobilenetv1/SsdMobilenetv1Options';
import { TinyFaceDetectorOptions } from '../tinyFaceDetector/TinyFaceDetectorOptions';
import { ComposableTask } from './ComposableTask';
import { DetectAllFaceLandmarksTask, DetectSingleFaceLandmarksTask } from './DetectFacesLandmarksTasks';
import { nets } from './nets';
import { FaceDetectionOptions } from './types';
export function detectSingleFace(
input: TNetInput,
options: FaceDetectionOptions = new SsdMobilenetv1Options()
): DetectSingleFaceTask {
return new DetectSingleFaceTask(input, options)
}
export function detectAllFaces(
input: TNetInput,
options: FaceDetectionOptions = new SsdMobilenetv1Options()
): DetectAllFacesTask {
return new DetectAllFacesTask(input, options)
}
export class DetectFacesTaskBase<TReturn> extends ComposableTask<TReturn> {
constructor(
protected input: TNetInput,
protected options: FaceDetectionOptions = new SsdMobilenetv1Options()
) {
super()
}
}
export class DetectAllFacesTask extends DetectFacesTaskBase<FaceDetection[]> {
public async run(): Promise<FaceDetection[]> {
const { input, options } = this
if (options instanceof MtcnnOptions) {
return (await nets.mtcnn.forward(input, options))
.map(result => result.faceDetection)
}
const faceDetectionFunction = options instanceof TinyFaceDetectorOptions
? (input: TNetInput) => nets.tinyFaceDetector.locateFaces(input, options)
: (
options instanceof SsdMobilenetv1Options
? (input: TNetInput) => nets.ssdMobilenetv1.locateFaces(input, options)
: (
options instanceof TinyYolov2Options
? (input: TNetInput) => nets.tinyYolov2.locateFaces(input, options)
: null
)
)
if (!faceDetectionFunction) {
throw new Error('detectFaces - expected options to be instance of TinyFaceDetectorOptions | SsdMobilenetv1Options | MtcnnOptions | TinyYolov2Options')
}
return faceDetectionFunction(input)
}
withFaceLandmarks(useTinyLandmarkNet: boolean = false): DetectAllFaceLandmarksTask {
return new DetectAllFaceLandmarksTask(this, this.input, useTinyLandmarkNet)
}
}
export class DetectSingleFaceTask extends DetectFacesTaskBase<FaceDetection | undefined> {
public async run(): Promise<FaceDetection | undefined> {
return (await new DetectAllFacesTask(this.input, this.options))
.sort((f1, f2) => f1.score - f2.score)[0]
}
withFaceLandmarks(useTinyLandmarkNet: boolean = false): DetectSingleFaceLandmarksTask {
return new DetectSingleFaceLandmarksTask(this, this.input, useTinyLandmarkNet)
}
}
\ No newline at end of file
src/globalApi/allFaces.ts 0 → 100644
import { TNetInput } from 'tfjs-image-recognition-base';
import { ITinyYolov2Options, TinyYolov2Options } from 'tfjs-tiny-yolov2';
import { FullFaceDescription } from '../classes';
import { IMtcnnOptions, MtcnnOptions } from '../mtcnn/MtcnnOptions';
import { SsdMobilenetv1Options } from '../ssdMobilenetv1';
import { detectAllFaces } from './DetectFacesTasks';
// export allFaces API for backward compatibility
export async function allFacesSsdMobilenetv1(
input: TNetInput,
minConfidence?: number
): Promise<FullFaceDescription[]> {
return await detectAllFaces(input, new SsdMobilenetv1Options(minConfidence ? { minConfidence } : {}))
.withFaceLandmarks()
.withFaceDescriptors()
}
export async function allFacesTinyYolov2(
input: TNetInput,
forwardParams: ITinyYolov2Options = {}
): Promise<FullFaceDescription[]> {
return await detectAllFaces(input, new TinyYolov2Options(forwardParams))
.withFaceLandmarks()
.withFaceDescriptors()
}
export async function allFacesMtcnn(
input: TNetInput,
forwardParams: IMtcnnOptions = {}
): Promise<FullFaceDescription[]> {
return await detectAllFaces(input, new MtcnnOptions(forwardParams))
.withFaceLandmarks()
.withFaceDescriptors()
}
export const allFaces = allFacesSsdMobilenetv1
src/globalApi/index.ts 0 → 100644
export * from './allFaces'
export * from './ComposableTask'
export * from './ComputeFaceDescriptorsTasks'
export * from './DetectFacesTasks'
export * from './DetectFacesLandmarksTasks'
export * from './nets'
export * from './types'
src/globalApi/nets.ts 0 → 100644
import { TNetInput } from 'tfjs-image-recognition-base';
import { ITinyYolov2Options } from 'tfjs-tiny-yolov2';
import { FaceDetection } from '../classes/FaceDetection';
import { FaceLandmarks68 } from '../classes/FaceLandmarks68';
import { FaceLandmark68Net } from '../faceLandmarkNet/FaceLandmark68Net';
import { FaceLandmark68TinyNet } from '../faceLandmarkNet/FaceLandmark68TinyNet';
import { FaceRecognitionNet } from '../faceRecognitionNet/FaceRecognitionNet';
import { Mtcnn } from '../mtcnn/Mtcnn';
import { MtcnnOptions } from '../mtcnn/MtcnnOptions';
import { MtcnnResult } from '../mtcnn/MtcnnResult';
import { SsdMobilenetv1 } from '../ssdMobilenetv1/SsdMobilenetv1';
import { SsdMobilenetv1Options } from '../ssdMobilenetv1/SsdMobilenetv1Options';
import { TinyFaceDetector } from '../tinyFaceDetector/TinyFaceDetector';
import { TinyFaceDetectorOptions } from '../tinyFaceDetector/TinyFaceDetectorOptions';
import { TinyYolov2 } from '../tinyYolov2/TinyYolov2';
export const nets = {
ssdMobilenetv1: new SsdMobilenetv1(),
tinyFaceDetector: new TinyFaceDetector(),
tinyYolov2: new TinyYolov2(),
mtcnn: new Mtcnn(),
faceLandmark68Net: new FaceLandmark68Net(),
faceLandmark68TinyNet: new FaceLandmark68TinyNet(),
faceRecognitionNet: new FaceRecognitionNet()
}
/**
* Attempts to detect all faces in an image using SSD Mobilenetv1 Network.
*
* @param input The input image.
* @param options (optional, default: see SsdMobilenetv1Options constructor for default parameters).
* @returns Bounding box of each face with score.
*/
export const ssdMobilenetv1 = (input: TNetInput, options: SsdMobilenetv1Options): Promise<FaceDetection[]> =>
nets.ssdMobilenetv1.locateFaces(input, options)
/**
* Attempts to detect all faces in an image using the Tiny Face Detector.
*
* @param input The input image.
* @param options (optional, default: see TinyFaceDetectorOptions constructor for default parameters).
* @returns Bounding box of each face with score.
*/
export const tinyFaceDetector = (input: TNetInput, options: TinyFaceDetectorOptions): Promise<FaceDetection[]> =>
nets.tinyFaceDetector.locateFaces(input, options)
/**
* Attempts to detect all faces in an image using the Tiny Yolov2 Network.
*
* @param input The input image.
* @param options (optional, default: see TinyYolov2Options constructor for default parameters).
* @returns Bounding box of each face with score.
*/
export const tinyYolov2 = (input: TNetInput, options: ITinyYolov2Options): Promise<FaceDetection[]> =>
nets.tinyYolov2.locateFaces(input, options)
/**
* Attempts to detect all faces in an image and the 5 point face landmarks
* of each detected face using the MTCNN Network.
*
* @param input The input image.
* @param options (optional, default: see MtcnnOptions constructor for default parameters).
* @returns Bounding box of each face with score and 5 point face landmarks.
*/
export const mtcnn = (input: TNetInput, options: MtcnnOptions): Promise<MtcnnResult[]> =>
nets.mtcnn.forward(input, options)
/**
* Detects the 68 point face landmark positions of the face shown in an image.
*
* @param inputs The face image extracted from the bounding box of a face. Can
* also be an array of input images, which will be batch processed.
* @returns 68 point face landmarks or array thereof in case of batch input.
*/
export const detectFaceLandmarks = (input: TNetInput): Promise<FaceLandmarks68 | FaceLandmarks68[]> =>
nets.faceLandmark68Net.detectLandmarks(input)
/**
* Detects the 68 point face landmark positions of the face shown in an image
* using a tinier version of the 68 point face landmark model, which is slightly
* faster at inference, but also slightly less accurate.
*
* @param inputs The face image extracted from the bounding box of a face. Can
* also be an array of input images, which will be batch processed.
* @returns 68 point face landmarks or array thereof in case of batch input.
*/
export const detectFaceLandmarksTiny = (input: TNetInput): Promise<FaceLandmarks68 | FaceLandmarks68[]> =>
nets.faceLandmark68TinyNet.detectLandmarks(input)
/**
* Computes a 128 entry vector (face descriptor / face embeddings) from the face shown in an image,
* which uniquely represents the features of that persons face. The computed face descriptor can
* be used to measure the similarity between faces, by computing the euclidean distance of two
* face descriptors.
*
* @param inputs The face image extracted from the aligned bounding box of a face. Can
* also be an array of input images, which will be batch processed.
* @returns Face descriptor with 128 entries or array thereof in case of batch input.
*/
export const computeFaceDescriptor = (input: TNetInput): Promise<Float32Array | Float32Array[]> =>
nets.faceRecognitionNet.computeFaceDescriptor(input)
export const loadSsdMobilenetv1Model = (url: string) => nets.ssdMobilenetv1.load(url)
export const loadTinyFaceDetectorModel = (url: string) => nets.tinyFaceDetector.load(url)
export const loadMtcnnModel = (url: string) => nets.mtcnn.load(url)
export const loadTinyYolov2Model = (url: string) => nets.tinyYolov2.load(url)
export const loadFaceLandmarkModel = (url: string) => nets.faceLandmark68Net.load(url)
export const loadFaceLandmarkTinyModel = (url: string) => nets.faceLandmark68TinyNet.load(url)
export const loadFaceRecognitionModel = (url: string) => nets.faceRecognitionNet.load(url)
// backward compatibility
export const loadFaceDetectionModel = loadSsdMobilenetv1Model
export const locateFaces = ssdMobilenetv1
export const detectLandmarks = detectFaceLandmarks
\ No newline at end of file
src/globalApi/types.ts 0 → 100644
import { TNetInput } from 'tfjs-image-recognition-base';
import { TinyYolov2Options } from 'tfjs-tiny-yolov2';
import { FaceDetection } from '../classes/FaceDetection';
import { MtcnnOptions } from '../mtcnn/MtcnnOptions';
import { SsdMobilenetv1Options } from '../ssdMobilenetv1/SsdMobilenetv1Options';
import { TinyFaceDetectorOptions } from '../tinyFaceDetector/TinyFaceDetectorOptions';
export type FaceDetectionOptions = TinyFaceDetectorOptions | SsdMobilenetv1Options | MtcnnOptions | TinyYolov2Options
export type FaceDetectionFunction = (input: TNetInput) => Promise<FaceDetection[]>
\ No newline at end of file
src/index.ts
......@@ -10,9 +10,10 @@ export * from './classes';
export * from './dom'
export * from './euclideanDistance';
export * from './faceDetectionNet';
export * from './faceLandmarkNet';
export * from './faceRecognitionNet';
export * from './globalApi';
export * from './mtcnn';
export * from './ssdMobilenetv1';
export * from './tinyFaceDetector';
export * from './tinyYolov2';
\ No newline at end of file
src/mtcnn/Mtcnn.ts
......@@ -6,14 +6,15 @@ import { FaceLandmarks5 } from '../classes/FaceLandmarks5';
import { bgrToRgbTensor } from './bgrToRgbTensor';
import { CELL_SIZE } from './config';
import { extractParams } from './extractParams';
import { getDefaultMtcnnForwardParams } from './getDefaultMtcnnForwardParams';
import { getSizesForScale } from './getSizesForScale';
import { loadQuantizedParams } from './loadQuantizedParams';
import { IMtcnnOptions, MtcnnOptions } from './MtcnnOptions';
import { MtcnnResult } from './MtcnnResult';
import { pyramidDown } from './pyramidDown';
import { stage1 } from './stage1';
import { stage2 } from './stage2';
import { stage3 } from './stage3';
import { MtcnnForwardParams, MtcnnResult, NetParams } from './types';
import { NetParams } from './types';
export class Mtcnn extends NeuralNetwork<NetParams> {
......@@ -23,7 +24,7 @@ export class Mtcnn extends NeuralNetwork<NetParams> {
public async forwardInput(
input: NetInput,
forwardParams: MtcnnForwardParams = {}
forwardParams: IMtcnnOptions = {}
): Promise<{ results: MtcnnResult[], stats: any }> {
const { params } = this
......@@ -63,7 +64,7 @@ export class Mtcnn extends NeuralNetwork<NetParams> {
maxNumScales,
scoreThresholds,
scaleSteps
} = Object.assign({}, getDefaultMtcnnForwardParams(), forwardParams)
} = new MtcnnOptions(forwardParams)
const scales = (scaleSteps || pyramidDown(minFaceSize, scaleFactor, [height, width]))
.filter(scale => {
......@@ -72,6 +73,15 @@ export class Mtcnn extends NeuralNetwork<NetParams> {
})
.slice(0, maxNumScales)
console.log({
minFaceSize,
scaleFactor,
maxNumScales,
scoreThresholds,
scales
})
stats.scales = scales
stats.pyramid = scales.map(scale => getSizesForScale(scale, [height, width]))
......@@ -100,8 +110,8 @@ export class Mtcnn extends NeuralNetwork<NetParams> {
const out3 = await stage3(inputCanvas, out2.boxes, scoreThresholds[2], params.onet, stats)
stats.total_stage3 = Date.now() - ts
const results = out3.boxes.map((box, idx) => ({
faceDetection: new FaceDetection(
const results = out3.boxes.map((box, idx) => new MtcnnResult(
new FaceDetection(
out3.scores[idx],
new Rect(
box.left / width,
......@@ -114,18 +124,18 @@ export class Mtcnn extends NeuralNetwork<NetParams> {
width
}
),
faceLandmarks: new FaceLandmarks5(
new FaceLandmarks5(
out3.points[idx].map(pt => pt.div(new Point(width, height))),
{ width, height }
)
}))
))
return onReturn({ results, stats })
}
public async forward(
input: TNetInput,
forwardParams: MtcnnForwardParams = {}
forwardParams: IMtcnnOptions = {}
): Promise<MtcnnResult[]> {
return (
await this.forwardInput(
......@@ -137,7 +147,7 @@ export class Mtcnn extends NeuralNetwork<NetParams> {
public async forwardWithStats(
input: TNetInput,
forwardParams: MtcnnForwardParams = {}
forwardParams: IMtcnnOptions = {}
): Promise<{ results: MtcnnResult[], stats: any }> {
return this.forwardInput(
await toNetInput(input),
......
src/mtcnn/MtcnnOptions.ts 0 → 100644
export interface IMtcnnOptions {
minFaceSize?: number
scaleFactor?: number
maxNumScales?: number
scoreThresholds?: number[]
scaleSteps?: number[]
}
export class MtcnnOptions {
protected _name: string = 'MtcnnOptions'
private _minFaceSize: number
private _scaleFactor: number
private _maxNumScales: number
private _scoreThresholds: number[]
private _scaleSteps: number[] | undefined
constructor({ minFaceSize, scaleFactor, maxNumScales, scoreThresholds, scaleSteps }: IMtcnnOptions = {}) {
this._minFaceSize = minFaceSize || 20
this._scaleFactor = scaleFactor || 0.709
this._maxNumScales = maxNumScales || 10
this._scoreThresholds = scoreThresholds || [0.6, 0.7, 0.7]
this._scaleSteps = scaleSteps
if (typeof this._minFaceSize !== 'number' || this._minFaceSize < 0) {
throw new Error(`${this._name} - expected minFaceSize to be a number > 0`)
}
if (typeof this._scaleFactor !== 'number' || this._scaleFactor <= 0 || this._scaleFactor >= 1) {
throw new Error(`${this._name} - expected scaleFactor to be a number between 0 and 1`)
}
if (typeof this._maxNumScales !== 'number' || this._maxNumScales < 0) {
throw new Error(`${this._name} - expected maxNumScales to be a number > 0`)
}
if (
!Array.isArray(this._scoreThresholds)
|| this._scoreThresholds.length !== 3
|| this._scoreThresholds.some(th => typeof th !== 'number')
) {
throw new Error(`${this._name} - expected scoreThresholds to be an array of numbers of length 3`)
}
if (
!Array.isArray(this._scaleSteps)
|| this._scaleSteps.some(th => typeof th !== 'number')
) {
throw new Error(`${this._name} - expected scaleSteps to be an array of numbers`)
}
}
get minFaceSize(): number { return this._minFaceSize }
get scaleFactor(): number { return this._scaleFactor }
get maxNumScales(): number { return this._maxNumScales }
get scoreThresholds(): number[] { return this._scoreThresholds }
get scaleSteps(): number[] | undefined { return this._scaleSteps }
}
\ No newline at end of file
src/mtcnn/MtcnnResult.ts 0 → 100644
import { FaceDetection } from '../classes/FaceDetection';
import { FaceDetectionWithLandmarks } from '../classes/FaceDetectionWithLandmarks';
import { FaceLandmarks5 } from '../classes/FaceLandmarks5';
export class MtcnnResult extends FaceDetectionWithLandmarks {
// aliases for backward compatibily
get faceDetection(): FaceDetection { return this.detection }
get faceLandmarks(): FaceLandmarks5 { return this.faceLandmarks }
}
\ No newline at end of file
src/mtcnn/getDefaultMtcnnForwardParams.ts deleted 100644 → 0
export function getDefaultMtcnnForwardParams() {
return {
minFaceSize: 20,
scaleFactor: 0.709,
maxNumScales: 10,
scoreThresholds: [0.6, 0.7, 0.7]
}
}
\ No newline at end of file
src/mtcnn/index.ts
import { Mtcnn } from './Mtcnn';
export * from './Mtcnn';
export * from './MtcnnOptions';
export function createMtcnn(weights: Float32Array) {
const net = new Mtcnn()
......
src/mtcnn/types.ts
......@@ -40,16 +40,3 @@ export type NetParams = {
rnet: RNetParams
onet: ONetParams
}
export type MtcnnResult = {
faceDetection: FaceDetection,
faceLandmarks: FaceLandmarks5
}
export type MtcnnForwardParams = {
minFaceSize?: number
scaleFactor?: number
maxNumScales?: number
scoreThresholds?: number[]
scaleSteps?: number[]
}
src/faceDetectionNet/FaceDetectionNet.ts → src/ssdMobilenetv1/SsdMobilenetv1.ts
......@@ -8,12 +8,13 @@ import { mobileNetV1 } from './mobileNetV1';
import { nonMaxSuppression } from './nonMaxSuppression';
import { outputLayer } from './outputLayer';
import { predictionLayer } from './predictionLayer';
import { NetParams } from './types';
import { NetParams, ISsdMobilenetv1Options } from './types';
import { SsdMobilenetv1Options } from './SsdMobilenetv1Options';
export class FaceDetectionNet extends NeuralNetwork<NetParams> {
export class SsdMobilenetv1 extends NeuralNetwork<NetParams> {
constructor() {
super('FaceDetectionNet')
super('SsdMobilenetv1')
}
public forwardInput(input: NetInput) {
......@@ -21,7 +22,7 @@ export class FaceDetectionNet extends NeuralNetwork<NetParams> {
const { params } = this
if (!params) {
throw new Error('FaceDetectionNet - load model before inference')
throw new Error('SsdMobilenetv1 - load model before inference')
}
return tf.tidy(() => {
......@@ -45,10 +46,11 @@ export class FaceDetectionNet extends NeuralNetwork<NetParams> {
public async locateFaces(
input: TNetInput,
minConfidence: number = 0.8,
maxResults: number = 100
options: ISsdMobilenetv1Options
): Promise<FaceDetection[]> {
const { maxResults, minConfidence } = new SsdMobilenetv1Options(options)
const netInput = await toNetInput(input)
const {
......
src/ssdMobilenetv1/SsdMobilenetv1Options.ts 0 → 100644
import { ISsdMobilenetv1Options } from './types';
export class SsdMobilenetv1Options {
protected _name: string = 'MtcnnOptions'
private _minConfidence: number
private _maxResults: number
constructor({ minConfidence, maxResults }: ISsdMobilenetv1Options = {}) {
this._minConfidence = minConfidence || 0.5
this._maxResults = maxResults || 100
if (typeof this._minConfidence !== 'number' || this._minConfidence <= 0 || this._minConfidence >= 1) {
throw new Error(`${this._name} - expected minConfidence to be a number between 0 and 1`)
}
if (typeof this._maxResults !== 'number') {
throw new Error(`${this._name} - expected maxResults to be a number`)
}
}
get minConfidence(): number { return this._minConfidence }
get maxResults(): number { return this._maxResults }
}
\ No newline at end of file
src/ssdMobilenetv1/index.ts 0 → 100644
import { SsdMobilenetv1 } from './SsdMobilenetv1';
export * from './SsdMobilenetv1';
export * from './SsdMobilenetv1Options';
export function createSsdMobilenetv1(weights: Float32Array) {
const net = new SsdMobilenetv1()
net.extractWeights(weights)
return net
}
export function createFaceDetectionNet(weights: Float32Array) {
return createSsdMobilenetv1(weights)
}
// alias for backward compatibily
export class FaceDetectionNet extends SsdMobilenetv1 {}
\ No newline at end of file
src/faceDetectionNet/types.ts → src/ssdMobilenetv1/types.ts
......@@ -71,3 +71,8 @@ export type NetParams = {
prediction_layer: PredictionLayerParams,
output_layer: OutputLayerParams
}
export interface ISsdMobilenetv1Options {
minConfidence?: number
maxResults?: number
}
\ No newline at end of file
src/tinyFaceDetector/TinyFaceDetector.ts
import { Point, TNetInput } from 'tfjs-image-recognition-base';
import { TinyYolov2 as TinyYolov2Base, TinyYolov2Types } from 'tfjs-tiny-yolov2';
import { TinyYolov2 as TinyYolov2Base, ITinyYolov2Options } from 'tfjs-tiny-yolov2';
import { FaceDetection } from '../classes';
import { BOX_ANCHORS, DEFAULT_MODEL_NAME, IOU_THRESHOLD, MEAN_RGB } from './const';
......@@ -24,7 +24,7 @@ export class TinyFaceDetector extends TinyYolov2Base {
return this.config.anchors
}
public async locateFaces(input: TNetInput, forwardParams: TinyYolov2Types.TinyYolov2ForwardParams): Promise<FaceDetection[]> {
public async locateFaces(input: TNetInput, forwardParams: ITinyYolov2Options): Promise<FaceDetection[]> {
const objectDetections = await this.detect(input, forwardParams)
return objectDetections.map(det => new FaceDetection(det.score, det.relativeBox, { width: det.imageWidth, height: det.imageHeight }))
}
......
src/tinyFaceDetector/TinyFaceDetectorOptions.ts
import { TinyYolov2Options } from '../tinyYolov2/TinyYolov2Options';
import { TinyYolov2Options } from 'tfjs-tiny-yolov2';
export class TinyFaceDetectorOptions extends TinyYolov2Options {
protected _name: string = 'TinyFaceDetectorOptions'
......
src/tinyFaceDetector/const.ts
......@@ -12,4 +12,4 @@ export const BOX_ANCHORS = [
export const MEAN_RGB: [number, number, number] = [117.001, 114.697, 97.404]
export const DEFAULT_MODEL_NAME = 'tiny_face_detection_model'
\ No newline at end of file
export const DEFAULT_MODEL_NAME = 'tiny_face_detector_model'
\ No newline at end of file
src/tinyYolov2/TinyYolov2.ts
import { Point, TNetInput } from 'tfjs-image-recognition-base';
import { TinyYolov2 as TinyYolov2Base, TinyYolov2Types } from 'tfjs-tiny-yolov2';
import { ITinyYolov2Options, TinyYolov2 as TinyYolov2Base } from 'tfjs-tiny-yolov2';
import { FaceDetection } from '../classes';
import {
......@@ -41,7 +41,7 @@ export class TinyYolov2 extends TinyYolov2Base {
return this.config.anchors
}
public async locateFaces(input: TNetInput, forwardParams: TinyYolov2Types.TinyYolov2ForwardParams): Promise<FaceDetection[]> {
public async locateFaces(input: TNetInput, forwardParams: ITinyYolov2Options): Promise<FaceDetection[]> {
const objectDetections = await this.detect(input, forwardParams)
return objectDetections.map(det => new FaceDetection(det.score, det.relativeBox, { width: det.imageWidth, height: det.imageHeight }))
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment