Skip to content

F
face
Commit adaed9e7 by vincent
Options

remove duplicated code and use tfjs-image-recognition-base

parent 5add150b
Showing with 194 additions and 2117 deletions
karma.conf.js
...@@ -25,7 +25,12 @@ module.exports = function(config) { ...@@ -25,7 +25,12 @@ module.exports = function(config) {
'**/*.ts': ['karma-typescript'] '**/*.ts': ['karma-typescript']
}, },
karmaTypescriptConfig: { karmaTypescriptConfig: {
tsconfig: 'tsconfig.test.json' tsconfig: 'tsconfig.test.json',
bundlerOptions: {
transforms: [
require("karma-typescript-es6-transform")()
]
}
}, },
browsers: ['Chrome'], browsers: ['Chrome'],
browserNoActivityTimeout: 60000, browserNoActivityTimeout: 60000,
......
package.json
...@@ -20,7 +20,8 @@ ...@@ -20,7 +20,8 @@
"author": "justadudewhohacks", "author": "justadudewhohacks",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"@tensorflow/tfjs-core": "^0.11.9" "@tensorflow/tfjs-core": "^0.12.14",
"tfjs-image-recognition-base": "git+https://github.com/justadudewhohacks/tfjs-image-recognition-base.git"
}, },
"devDependencies": { "devDependencies": {
"@types/jasmine": "^2.8.8", "@types/jasmine": "^2.8.8",
...@@ -30,6 +31,7 @@ ...@@ -30,6 +31,7 @@
"karma-chrome-launcher": "^2.2.0", "karma-chrome-launcher": "^2.2.0",
"karma-jasmine": "^1.1.2", "karma-jasmine": "^1.1.2",
"karma-typescript": "^3.0.12", "karma-typescript": "^3.0.12",
"karma-typescript-es6-transform": "^1.0.4",
"rollup": "^0.59.1", "rollup": "^0.59.1",
"rollup-plugin-commonjs": "^9.1.3", "rollup-plugin-commonjs": "^9.1.3",
"rollup-plugin-node-resolve": "^3.3.0", "rollup-plugin-node-resolve": "^3.3.0",
......
src/BoundingBox.ts deleted 100644 → 0
import { Rect } from './Rect';
import { Dimensions } from './types';
import { isDimensions } from './utils';
export class BoundingBox {
constructor(
private _left: number,
private _top: number,
private _right: number,
private _bottom: number
) {}
public get left() : number {
return this._left
}
public get top() : number {
return this._top
}
public get right() : number {
return this._right
}
public get bottom() : number {
return this._bottom
}
public get width() : number {
return this.right - this.left
}
public get height() : number {
return this.bottom - this.top
}
public get area() : number {
return this.width * this.height
}
public toSquare(): BoundingBox {
let { left, top, right, bottom } = this
const off = (Math.abs(this.width - this.height) / 2)
if (this.width < this.height) {
left -= off
right += off
}
if (this.height < this.width) {
top -= off
bottom += off
}
return new BoundingBox(left, top, right, bottom)
}
public round(): BoundingBox {
return new BoundingBox(
Math.round(this.left),
Math.round(this.top),
Math.round(this.right),
Math.round(this.bottom)
)
}
public padAtBorders(imageHeight: number, imageWidth: number) {
const w = this.width + 1
const h = this.height + 1
let dx = 1
let dy = 1
let edx = w
let edy = h
let x = this.left
let y = this.top
let ex = this.right
let ey = this.bottom
if (ex > imageWidth) {
edx = -ex + imageWidth + w
ex = imageWidth
}
if (ey > imageHeight) {
edy = -ey + imageHeight + h
ey = imageHeight
}
if (x < 1) {
edy = 2 - x
x = 1
}
if (y < 1) {
edy = 2 - y
y = 1
}
return { dy, edy, dx, edx, y, ey, x, ex, w, h }
}
public calibrate(region: BoundingBox) {
return new BoundingBox(
this.left + (region.left * this.width),
this.top + (region.top * this.height),
this.right + (region.right * this.width),
this.bottom + (region.bottom * this.height)
).toSquare().round()
}
public rescale(s: Dimensions | number) {
const scaleX = isDimensions(s) ? (s as Dimensions).width : s as number
const scaleY = isDimensions(s) ? (s as Dimensions).height : s as number
return new BoundingBox(this.left * scaleX, this.top * scaleY, this.right * scaleX, this.bottom * scaleY)
}
public toRect(): Rect {
return new Rect(this.left, this.top, this.width, this.height)
}
}
\ No newline at end of file
src/NetInput.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
import { isTensor3D, isTensor4D } from './commons/isTensor';
import { padToSquare } from './padToSquare';
import { Point } from './Point';
import { TResolvedNetInput, Dimensions } from './types';
import { createCanvasFromMedia } from './utils';
export class NetInput {
private _inputs: tf.Tensor3D[] = []
private _canvases: HTMLCanvasElement[] = []
private _isManaged: boolean = false
private _isBatchInput: boolean = false
private _inputDimensions: number[][] = []
private _paddings: Point[] = []
private _inputSize: number = 0
constructor(
inputs: tf.Tensor4D | Array<TResolvedNetInput>,
isBatchInput: boolean = false,
keepCanvases: boolean = false
) {
if (isTensor4D(inputs)) {
this._inputs = tf.unstack(inputs as tf.Tensor4D) as tf.Tensor3D[]
}
if (Array.isArray(inputs)) {
this._inputs = inputs.map((input, idx) => {
if (isTensor3D(input)) {
// TODO: make sure not to dispose original tensors passed in by the user
return tf.clone(input as tf.Tensor3D)
}
if (isTensor4D(input)) {
const shape = (input as tf.Tensor4D).shape
const batchSize = shape[0]
if (batchSize !== 1) {
throw new Error(`NetInput - tf.Tensor4D with batchSize ${batchSize} passed, but not supported in input array`)
}
return (input as tf.Tensor4D).reshape(shape.slice(1) as [number, number, number]) as tf.Tensor3D
}
const canvas = input instanceof HTMLCanvasElement ? input : createCanvasFromMedia(input as HTMLImageElement | HTMLVideoElement)
if (keepCanvases) {
this._canvases[idx] = canvas
}
return tf.fromPixels(canvas)
})
}
this._isBatchInput = this.batchSize > 1 || isBatchInput
this._inputDimensions = this._inputs.map(t => t.shape)
}
public get inputs(): tf.Tensor3D[] {
return this._inputs
}
public get canvases(): HTMLCanvasElement[] {
return this._canvases
}
public get isManaged(): boolean {
return this._isManaged
}
public get isBatchInput(): boolean {
return this._isBatchInput
}
public get batchSize(): number {
return this._inputs.length
}
public get inputDimensions(): number[][] {
return this._inputDimensions
}
public get paddings(): Point[] {
return this._paddings
}
public get inputSize(): number {
return this._inputSize
}
public get relativePaddings(): Point[] {
return Array(this.inputs.length).fill(0).map(
(_, batchIdx) => this.getRelativePaddings(batchIdx)
)
}
public get reshapedInputDimensions(): Dimensions[] {
return Array(this.inputs.length).fill(0).map(
(_, batchIdx) => this.getReshapedInputDimensions(batchIdx)
)
}
public getInputDimensions(batchIdx: number): number[] {
return this._inputDimensions[batchIdx]
}
public getInputHeight(batchIdx: number): number {
return this._inputDimensions[batchIdx][0]
}
public getInputWidth(batchIdx: number): number {
return this._inputDimensions[batchIdx][1]
}
public getPaddings(batchIdx: number): Point {
return this._paddings[batchIdx]
}
public getRelativePaddings(batchIdx: number): Point {
return new Point(
(this.getPaddings(batchIdx).x + this.getInputWidth(batchIdx)) / this.getInputWidth(batchIdx),
(this.getPaddings(batchIdx).y + this.getInputHeight(batchIdx)) / this.getInputHeight(batchIdx)
)
}
public getReshapedInputDimensions(batchIdx: number): Dimensions {
const [h, w] = [this.getInputHeight(batchIdx), this.getInputWidth(batchIdx)]
const f = this.inputSize / Math.max(h, w)
return {
height: Math.floor(h * f),
width: Math.floor(w * f)
}
}
public toBatchTensor(inputSize: number, isCenterInputs: boolean = true): tf.Tensor4D {
this._inputSize = inputSize
return tf.tidy(() => {
const inputTensors = this._inputs.map((inputTensor: tf.Tensor3D) => {
const [originalHeight, originalWidth] = inputTensor.shape
let imgTensor = inputTensor.expandDims().toFloat() as tf.Tensor4D
imgTensor = padToSquare(imgTensor, isCenterInputs)
const [heightAfterPadding, widthAfterPadding] = imgTensor.shape.slice(1)
if (heightAfterPadding !== inputSize || widthAfterPadding !== inputSize) {
imgTensor = tf.image.resizeBilinear(imgTensor, [inputSize, inputSize])
}
this._paddings.push(new Point(
widthAfterPadding - originalWidth,
heightAfterPadding - originalHeight
))
return imgTensor
})
const batchTensor = tf.stack(inputTensors).as4D(this.batchSize, inputSize, inputSize, 3)
if (this.isManaged) {
this.dispose()
}
return batchTensor
})
}
/**
* By setting the isManaged flag, all newly created tensors will be
* automatically disposed after the batch tensor has been created
*/
public managed() {
this._isManaged = true
return this
}
public dispose() {
this._inputs.forEach(t => t.dispose())
}
}
\ No newline at end of file
src/Point.ts deleted 100644 → 0
export interface IPoint {
x: number
y: number
}
export class Point implements IPoint {
public x: number
public y: number
constructor(x: number, y: number) {
this.x = x
this.y = y
}
public add(pt: IPoint): Point {
return new Point(this.x + pt.x, this.y + pt.y)
}
public sub(pt: IPoint): Point {
return new Point(this.x - pt.x, this.y - pt.y)
}
public mul(pt: IPoint): Point {
return new Point(this.x * pt.x, this.y * pt.y)
}
public div(pt: IPoint): Point {
return new Point(this.x / pt.x, this.y / pt.y)
}
public abs(): Point {
return new Point(Math.abs(this.x), Math.abs(this.y))
}
public magnitude(): number {
return Math.sqrt(Math.pow(this.x, 2) + Math.pow(this.y, 2))
}
public floor(): Point {
return new Point(Math.floor(this.x), Math.floor(this.y))
}
}
\ No newline at end of file
src/Rect.ts deleted 100644 → 0
import { BoundingBox } from './BoundingBox';
export interface IRect {
x: number
y: number
width: number
height: number
}
export class Rect implements IRect {
public x: number
public y: number
public width: number
public height: number
constructor(x: number, y: number, width: number, height: number) {
this.x = x
this.y = y
this.width = width
this.height = height
}
public get right() {
return this.x + this.width
}
public get bottom() {
return this.y + this.height
}
public toSquare(): Rect {
let { x, y, width, height } = this
const diff = Math.abs(width - height)
if (width < height) {
x -= (diff / 2)
width += diff
}
if (height < width) {
y -= (diff / 2)
height += diff
}
return new Rect(x, y, width, height)
}
public pad(padX: number, padY: number): Rect {
let { x, y, width, height } = this
return new Rect(x - (padX / 2), y - (padY / 2), width + padX, height + padY)
}
public floor(): Rect {
return new Rect(
Math.floor(this.x),
Math.floor(this.y),
Math.floor(this.width),
Math.floor(this.height)
)
}
public toBoundingBox(): BoundingBox {
return new BoundingBox(this.x, this.y, this.x + this.width, this.y + this.height)
}
public clipAtImageBorders(imgWidth: number, imgHeight: number): Rect {
const { x, y, right, bottom } = this
const clippedX = Math.max(x, 0)
const clippedY = Math.max(y, 0)
const newWidth = right - clippedX
const newHeight = bottom - clippedY
const clippedWidth = Math.min(newWidth, imgWidth - clippedX)
const clippedHeight = Math.min(newHeight, imgHeight - clippedY)
return (new Rect(clippedX, clippedY, clippedWidth, clippedHeight)).floor()
}
}
\ No newline at end of file
src/allFacesFactory.ts
import { Point, Rect, TNetInput } from 'tfjs-image-recognition-base';
import { TinyYolov2 } from '.'; import { TinyYolov2 } from '.';
import { FaceDetection } from './classes/FaceDetection';
import { FaceLandmarks68 } from './classes/FaceLandmarks68';
import { FullFaceDescription } from './classes/FullFaceDescription';
import { extractFaceTensors } from './extractFaceTensors'; import { extractFaceTensors } from './extractFaceTensors';
import { FaceDetection } from './FaceDetection';
import { FaceDetectionNet } from './faceDetectionNet/FaceDetectionNet'; import { FaceDetectionNet } from './faceDetectionNet/FaceDetectionNet';
import { FaceLandmarkNet } from './faceLandmarkNet/FaceLandmarkNet'; import { FaceLandmarkNet } from './faceLandmarkNet/FaceLandmarkNet';
import { FaceLandmarks68 } from './faceLandmarkNet/FaceLandmarks68';
import { FaceRecognitionNet } from './faceRecognitionNet/FaceRecognitionNet'; import { FaceRecognitionNet } from './faceRecognitionNet/FaceRecognitionNet';
import { FullFaceDescription } from './FullFaceDescription';
import { Mtcnn } from './mtcnn/Mtcnn'; import { Mtcnn } from './mtcnn/Mtcnn';
import { MtcnnForwardParams } from './mtcnn/types'; import { MtcnnForwardParams } from './mtcnn/types';
import { Rect } from './Rect';
import { TinyYolov2ForwardParams } from './tinyYolov2/types'; import { TinyYolov2ForwardParams } from './tinyYolov2/types';
import { TNetInput } from './types';
function computeDescriptorsFactory( function computeDescriptorsFactory(
recognitionNet: FaceRecognitionNet recognitionNet: FaceRecognitionNet
...@@ -62,7 +62,9 @@ function allFacesFactory( ...@@ -62,7 +62,9 @@ function allFacesFactory(
return detections.map((detection, i) => return detections.map((detection, i) =>
new FullFaceDescription( new FullFaceDescription(
detection, detection,
faceLandmarksByFace[i].shiftByPoint<FaceLandmarks68>(detection.getBox()), faceLandmarksByFace[i].shiftByPoint<FaceLandmarks68>(
new Point(detection.box.x, detection.box.y)
),
descriptors[i] descriptors[i]
) )
) )
......
src/FaceDetection.ts → src/classes/FaceDetection.ts
import { Rect } from './Rect'; import { Dimensions, Rect } from 'tfjs-image-recognition-base';
import { Dimensions } from './types';
export class FaceDetection { export class FaceDetection {
private _score: number private _score: number
......
src/FaceLandmarks.ts → src/classes/FaceLandmarks.ts
import { getCenterPoint } from './commons/getCenterPoint'; import { Dimensions, getCenterPoint, Point, Rect } from 'tfjs-image-recognition-base';
import { FaceDetection } from './FaceDetection'; import { FaceDetection } from './FaceDetection';
import { IPoint, Point } from './Point';
import { Rect } from './Rect';
import { Dimensions } from './types';
// face alignment constants // face alignment constants
const relX = 0.5 const relX = 0.5
...@@ -66,7 +64,7 @@ export class FaceLandmarks { ...@@ -66,7 +64,7 @@ export class FaceLandmarks {
) )
} }
public shiftByPoint<T extends FaceLandmarks>(pt: IPoint): T { public shiftByPoint<T extends FaceLandmarks>(pt: Point): T {
return this.shift(pt.x, pt.y) return this.shift(pt.x, pt.y)
} }
......
src/mtcnn/FaceLandmarks5.ts → src/classes/FaceLandmarks5.ts
import { getCenterPoint } from '../commons/getCenterPoint'; import { getCenterPoint, Point } from 'tfjs-image-recognition-base';
import { FaceLandmarks } from '../FaceLandmarks';
import { Point } from '../Point'; import { FaceLandmarks } from './FaceLandmarks';
export class FaceLandmarks5 extends FaceLandmarks { export class FaceLandmarks5 extends FaceLandmarks {
......
src/faceLandmarkNet/FaceLandmarks68.ts → src/classes/FaceLandmarks68.ts
import { getCenterPoint } from '../commons/getCenterPoint'; import { getCenterPoint, Point } from 'tfjs-image-recognition-base';
import { FaceDetection } from '../FaceDetection';
import { FaceLandmarks } from '../FaceLandmarks'; import { FaceLandmarks } from '../classes/FaceLandmarks';
import { Point } from '../Point';
import { Rect } from '../Rect';
export class FaceLandmarks68 extends FaceLandmarks { export class FaceLandmarks68 extends FaceLandmarks {
public getJawOutline(): Point[] { public getJawOutline(): Point[] {
......
src/classes/index.ts 0 → 100644
export * from './FaceDetection';
export * from './FaceLandmarks';
export * from './FaceLandmarks5';
export * from './FaceLandmarks68';
export * from './FullFaceDescription';
\ No newline at end of file
src/commons/NeuralNetwork.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
import { ParamMapping } from './types';
export class NeuralNetwork<TNetParams> {
protected _params: TNetParams | undefined = undefined
protected _paramMappings: ParamMapping[] = []
constructor(private _name: string) {}
public get params(): TNetParams | undefined {
return this._params
}
public get paramMappings(): ParamMapping[] {
return this._paramMappings
}
public getParamFromPath(paramPath: string): tf.Tensor {
const { obj, objProp } = this.traversePropertyPath(paramPath)
return obj[objProp]
}
public reassignParamFromPath(paramPath: string, tensor: tf.Tensor) {
const { obj, objProp } = this.traversePropertyPath(paramPath)
obj[objProp].dispose()
obj[objProp] = tensor
}
public getParamList() {
return this._paramMappings.map(({ paramPath }) => ({
path: paramPath,
tensor: this.getParamFromPath(paramPath)
}))
}
public getTrainableParams() {
return this.getParamList().filter(param => param.tensor instanceof tf.Variable)
}
public getFrozenParams() {
return this.getParamList().filter(param => !(param.tensor instanceof tf.Variable))
}
public variable() {
this.getFrozenParams().forEach(({ path, tensor }) => {
this.reassignParamFromPath(path, tf.variable(tensor))
})
}
public freeze() {
this.getTrainableParams().forEach(({ path, tensor }) => {
this.reassignParamFromPath(path, tf.tensor(tensor as any))
})
}
public dispose(throwOnRedispose: boolean = true) {
this.getParamList().forEach(param => {
if (throwOnRedispose && param.tensor.isDisposed) {
throw new Error(`param tensor has already been disposed for path ${param.path}`)
}
param.tensor.dispose()
})
this._params = undefined
}
public async load(weightsOrUrl: Float32Array | string | undefined): Promise<void> {
if (weightsOrUrl instanceof Float32Array) {
this.extractWeights(weightsOrUrl)
return
}
if (weightsOrUrl && typeof weightsOrUrl !== 'string') {
throw new Error(`${this._name}.load - expected model uri, or weights as Float32Array`)
}
const {
paramMappings,
params
} = await this.loadQuantizedParams(weightsOrUrl)
this._paramMappings = paramMappings
this._params = params
}
public extractWeights(weights: Float32Array) {
const {
paramMappings,
params
} = this.extractParams(weights)
this._paramMappings = paramMappings
this._params = params
}
private traversePropertyPath(paramPath: string) {
if (!this.params) {
throw new Error(`traversePropertyPath - model has no loaded params`)
}
const result = paramPath.split('/').reduce((res: { nextObj: any, obj?: any, objProp?: string }, objProp) => {
if (!res.nextObj.hasOwnProperty(objProp)) {
throw new Error(`traversePropertyPath - object does not have property ${objProp}, for path ${paramPath}`)
}
return { obj: res.nextObj, objProp, nextObj: res.nextObj[objProp] }
}, { nextObj: this.params })
const { obj, objProp } = result
if (!obj || !objProp || !(obj[objProp] instanceof tf.Tensor)) {
throw new Error(`traversePropertyPath - parameter is not a tensor, for path ${paramPath}`)
}
return { obj, objProp }
}
protected loadQuantizedParams(_: any): Promise<{ params: TNetParams, paramMappings: ParamMapping[] }> {
throw new Error(`${this._name}.loadQuantizedParams - not implemented`)
}
protected extractParams(_: any): { params: TNetParams, paramMappings: ParamMapping[] } {
throw new Error(`${this._name}.extractParams - not implemented`)
}
}
\ No newline at end of file
src/commons/disposeUnusedWeightTensors.ts deleted 100644 → 0
import { ParamMapping } from './types';
export function disposeUnusedWeightTensors(weightMap: any, paramMappings: ParamMapping[]) {
Object.keys(weightMap).forEach(path => {
if (!paramMappings.some(pm => pm.originalPath === path)) {
weightMap[path].dispose()
}
})
}
src/commons/extractConvParamsFactory.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { ExtractWeightsFunction, ParamMapping } from 'tfjs-image-recognition-base';
import { ConvParams, ExtractWeightsFunction, ParamMapping } from './types'; import { ConvParams } from './types';
export function extractConvParamsFactory( export function extractConvParamsFactory(
extractWeights: ExtractWeightsFunction, extractWeights: ExtractWeightsFunction,
......
src/commons/extractFCParamsFactory.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { ExtractWeightsFunction, ParamMapping } from 'tfjs-image-recognition-base';
import { ExtractWeightsFunction, FCParams, ParamMapping } from './types'; import { FCParams } from './types';
export function extractFCParamsFactory( export function extractFCParamsFactory(
extractWeights: ExtractWeightsFunction, extractWeights: ExtractWeightsFunction,
......
src/commons/extractWeightEntryFactory.ts deleted 100644 → 0
import { isTensor } from './isTensor';
import { ParamMapping } from './types';
export function extractWeightEntryFactory(weightMap: any, paramMappings: ParamMapping[]) {
return function<T> (originalPath: string, paramRank: number, mappedPath?: string): T {
const tensor = weightMap[originalPath]
if (!isTensor(tensor, paramRank)) {
throw new Error(`expected weightMap[${originalPath}] to be a Tensor${paramRank}D, instead have ${tensor}`)
}
paramMappings.push(
{ originalPath, paramPath: mappedPath || originalPath }
)
return tensor
}
}
src/commons/extractWeightsFactory.ts deleted 100644 → 0
export function extractWeightsFactory(weights: Float32Array) {
let remainingWeights = weights
function extractWeights(numWeights: number): Float32Array {
const ret = remainingWeights.slice(0, numWeights)
remainingWeights = remainingWeights.slice(numWeights)
return ret
}
function getRemainingWeights(): Float32Array {
return remainingWeights
}
return {
extractWeights,
getRemainingWeights
}
}
\ No newline at end of file
src/commons/getCenterPoint.ts deleted 100644 → 0
import { Point } from '../Point';
export function getCenterPoint(pts: Point[]): Point {
return pts.reduce((sum, pt) => sum.add(pt), new Point(0, 0))
.div(new Point(pts.length, pts.length))
}
\ No newline at end of file
src/commons/isMediaElement.ts deleted 100644 → 0
export function isMediaElement(input: any) {
return input instanceof HTMLImageElement
|| input instanceof HTMLVideoElement
|| input instanceof HTMLCanvasElement
}
\ No newline at end of file
src/commons/isTensor.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
export function isTensor(tensor: any, dim: number) {
return tensor instanceof tf.Tensor && tensor.shape.length === dim
}
export function isTensor1D(tensor: any) {
return isTensor(tensor, 1)
}
export function isTensor2D(tensor: any) {
return isTensor(tensor, 2)
}
export function isTensor3D(tensor: any) {
return isTensor(tensor, 3)
}
export function isTensor4D(tensor: any) {
return isTensor(tensor, 4)
}
\ No newline at end of file
src/commons/loadWeightMap.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
export function getModelUris(uri: string | undefined, defaultModelName: string) {
const defaultManifestFilename = `${defaultModelName}-weights_manifest.json`
if (!uri) {
return {
modelBaseUri: '',
manifestUri: defaultManifestFilename
}
}
if (uri === '/') {
return {
modelBaseUri: '/',
manifestUri: `/${defaultManifestFilename}`
}
}
const protocol = uri.startsWith('http://') ? 'http://' : uri.startsWith('https://') ? 'https://' : '';
uri = uri.replace(protocol, '');
const parts = uri.split('/').filter(s => s)
const manifestFile = uri.endsWith('.json')
? parts[parts.length - 1]
: defaultManifestFilename
let modelBaseUri = protocol + (uri.endsWith('.json') ? parts.slice(0, parts.length - 1) : parts).join('/')
modelBaseUri = uri.startsWith('/') ? `/${modelBaseUri}` : modelBaseUri
return {
modelBaseUri,
manifestUri: modelBaseUri === '/' ? `/${manifestFile}` : `${modelBaseUri}/${manifestFile}`
}
}
export async function loadWeightMap(
uri: string | undefined,
defaultModelName: string
): Promise<any> {
const { manifestUri, modelBaseUri } = getModelUris(uri, defaultModelName)
const manifest = await (await fetch(manifestUri)).json()
return tf.io.loadWeights(manifest, modelBaseUri)
}
\ No newline at end of file
src/commons/nonMaxSuppression.ts deleted 100644 → 0
import { BoundingBox } from '../BoundingBox';
import { iou } from '../iou';
export function nonMaxSuppression(
boxes: BoundingBox[],
scores: number[],
iouThreshold: number,
isIOU: boolean = true
): number[] {
let indicesSortedByScore = scores
.map((score, boxIndex) => ({ score, boxIndex }))
.sort((c1, c2) => c1.score - c2.score)
.map(c => c.boxIndex)
const pick: number[] = []
while(indicesSortedByScore.length > 0) {
const curr = indicesSortedByScore.pop() as number
pick.push(curr)
const indices = indicesSortedByScore
const outputs: number[] = []
for (let i = 0; i < indices.length; i++) {
const idx = indices[i]
const currBox = boxes[curr]
const idxBox = boxes[idx]
outputs.push(iou(currBox, idxBox, isIOU))
}
indicesSortedByScore = indicesSortedByScore.filter(
(_, j) => outputs[j] <= iouThreshold
)
}
return pick
}
\ No newline at end of file
src/commons/normalize.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
export function normalize(x: tf.Tensor4D, meanRgb: number[]): tf.Tensor4D {
return tf.tidy(() => {
const [r, g, b] = meanRgb
const avg_r = tf.fill([...x.shape.slice(0, 3), 1], r)
const avg_g = tf.fill([...x.shape.slice(0, 3), 1], g)
const avg_b = tf.fill([...x.shape.slice(0, 3), 1], b)
const avg_rgb = tf.concat([avg_r, avg_g, avg_b], 3)
return tf.sub(x, avg_rgb)
})
}
\ No newline at end of file
src/commons/types.ts
...@@ -8,18 +8,4 @@ export type ConvParams = { ...@@ -8,18 +8,4 @@ export type ConvParams = {
export type FCParams = { export type FCParams = {
weights: tf.Tensor2D weights: tf.Tensor2D
bias: tf.Tensor1D bias: tf.Tensor1D
}
export type ExtractWeightsFunction = (numWeights: number) => Float32Array
export type BatchReshapeInfo = {
originalWidth: number
originalHeight: number
paddingX: number
paddingY: number
}
export type ParamMapping = {
originalPath?: string
paramPath: string
} }
\ No newline at end of file
src/drawing/index.ts deleted 100644 → 0
import { FaceDetection } from '../FaceDetection';
import { FaceLandmarks68 } from '../faceLandmarkNet';
import { FaceLandmarks } from '../FaceLandmarks';
import { Point } from '../Point';
import { getContext2dOrThrow, resolveInput, round } from '../utils';
import { DrawBoxOptions, DrawLandmarksOptions, DrawOptions, DrawTextOptions } from './types';
export function getDefaultDrawOptions(): DrawOptions {
return {
color: 'blue',
lineWidth: 2,
fontSize: 20,
fontStyle: 'Georgia'
}
}
export function drawBox(
ctx: CanvasRenderingContext2D,
x: number,
y: number,
w: number,
h: number,
options: DrawBoxOptions
) {
const drawOptions = Object.assign(
getDefaultDrawOptions(),
(options || {})
)
ctx.strokeStyle = drawOptions.color
ctx.lineWidth = drawOptions.lineWidth
ctx.strokeRect(x, y, w, h)
}
export function drawText(
ctx: CanvasRenderingContext2D,
x: number,
y: number,
text: string,
options: DrawTextOptions
) {
const drawOptions = Object.assign(
getDefaultDrawOptions(),
(options || {})
)
const padText = 2 + drawOptions.lineWidth
ctx.fillStyle = drawOptions.color
ctx.font = `${drawOptions.fontSize}px ${drawOptions.fontStyle}`
ctx.fillText(text, x + padText, y + padText + (drawOptions.fontSize * 0.6))
}
export function drawDetection(
canvasArg: string | HTMLCanvasElement,
detection: FaceDetection | FaceDetection[],
options?: DrawBoxOptions & DrawTextOptions & { withScore: boolean }
) {
const canvas = resolveInput(canvasArg)
if (!(canvas instanceof HTMLCanvasElement)) {
throw new Error('drawBox - expected canvas to be of type: HTMLCanvasElement')
}
const detectionArray = Array.isArray(detection)
? detection
: [detection]
detectionArray.forEach((det) => {
const {
x,
y,
width,
height
} = det.getBox()
const drawOptions = Object.assign(
getDefaultDrawOptions(),
(options || {})
)
const { withScore } = Object.assign({ withScore: true }, (options || {}))
const ctx = getContext2dOrThrow(canvas)
drawBox(
ctx,
x,
y,
width,
height,
drawOptions
)
if (withScore) {
drawText(
ctx,
x,
y,
`${round(det.getScore())}`,
drawOptions
)
}
})
}
function drawContour(
ctx: CanvasRenderingContext2D,
points: Point[],
isClosed: boolean = false
) {
ctx.beginPath()
points.slice(1).forEach(({ x, y }, prevIdx) => {
const from = points[prevIdx]
ctx.moveTo(from.x, from.y)
ctx.lineTo(x, y)
})
if (isClosed) {
const from = points[points.length - 1]
const to = points[0]
if (!from || !to) {
return
}
ctx.moveTo(from.x, from.y)
ctx.lineTo(to.x, to.y)
}
ctx.stroke()
}
export function drawLandmarks(
canvasArg: string | HTMLCanvasElement,
faceLandmarks: FaceLandmarks | FaceLandmarks[],
options?: DrawLandmarksOptions & { drawLines: boolean }
) {
const canvas = resolveInput(canvasArg)
if (!(canvas instanceof HTMLCanvasElement)) {
throw new Error('drawLandmarks - expected canvas to be of type: HTMLCanvasElement')
}
const drawOptions = Object.assign(
getDefaultDrawOptions(),
(options || {})
)
const { drawLines } = Object.assign({ drawLines: false }, (options || {}))
const ctx = getContext2dOrThrow(canvas)
const { lineWidth, color } = drawOptions
const faceLandmarksArray = Array.isArray(faceLandmarks) ? faceLandmarks : [faceLandmarks]
faceLandmarksArray.forEach(landmarks => {
if (drawLines && landmarks instanceof FaceLandmarks68) {
ctx.strokeStyle = color
ctx.lineWidth = lineWidth
drawContour(ctx, landmarks.getJawOutline())
drawContour(ctx, landmarks.getLeftEyeBrow())
drawContour(ctx, landmarks.getRightEyeBrow())
drawContour(ctx, landmarks.getNose())
drawContour(ctx, landmarks.getLeftEye(), true)
drawContour(ctx, landmarks.getRightEye(), true)
drawContour(ctx, landmarks.getMouth(), true)
return
}
// else draw points
const ptOffset = lineWidth / 2
ctx.fillStyle = color
landmarks.getPositions().forEach(pt => ctx.fillRect(pt.x - ptOffset, pt.y - ptOffset, lineWidth, lineWidth))
})
}
\ No newline at end of file
src/drawing/types.ts deleted 100644 → 0
export type DrawBoxOptions = {
lineWidth?: number
color?: string
}
export type DrawTextOptions = {
lineWidth?: number
fontSize?: number
fontStyle?: string
color?: string
}
export type DrawLandmarksOptions = {
lineWidth?: number
color?: string
}
export type DrawOptions = {
lineWidth: number
fontSize: number
fontStyle: string
color: string
}
\ No newline at end of file
src/extractFaceTensors.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { Rect, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
import { FaceDetection } from './FaceDetection'; import { FaceDetection } from './classes/FaceDetection';
import { Rect } from './Rect';
import { toNetInput } from './toNetInput';
import { TNetInput } from './types';
/** /**
* Extracts the tensors of the image regions containing the detected faces. * Extracts the tensors of the image regions containing the detected faces.
...@@ -42,7 +40,7 @@ export async function extractFaceTensors( ...@@ -42,7 +40,7 @@ export async function extractFaceTensors(
.map(box => box.clipAtImageBorders(imgWidth, imgHeight)) .map(box => box.clipAtImageBorders(imgWidth, imgHeight))
const faceTensors = boxes.map(({ x, y, width, height }) => const faceTensors = boxes.map(({ x, y, width, height }) =>
tf.slice(imgTensor, [0, y, x, 0], [1, height, width, numChannels]) tf.slice4d(imgTensor, [0, y, x, 0], [1, height, width, numChannels])
) )
if (netInput.isManaged) { if (netInput.isManaged) {
......
src/extractFaces.ts
import { FaceDetection } from './FaceDetection'; import {
import { Rect } from './Rect'; createCanvas,
import { toNetInput } from './toNetInput'; getContext2dOrThrow,
import { TNetInput } from './types'; imageTensorToCanvas,
import { createCanvas, getContext2dOrThrow, imageTensorToCanvas } from './utils'; Rect,
TNetInput,
toNetInput,
} from 'tfjs-image-recognition-base';
import { FaceDetection } from './classes/FaceDetection';
/** /**
* Extracts the image regions containing the detected faces. * Extracts the image regions containing the detected faces.
......
src/faceDetectionNet/FaceDetectionNet.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { NetInput, NeuralNetwork, Rect, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
import { NeuralNetwork } from '../commons/NeuralNetwork'; import { FaceDetection } from '../classes/FaceDetection';
import { FaceDetection } from '../FaceDetection';
import { NetInput } from '../NetInput';
import { Rect } from '../Rect';
import { toNetInput } from '../toNetInput';
import { TNetInput } from '../types';
import { extractParams } from './extractParams'; import { extractParams } from './extractParams';
import { loadQuantizedParams } from './loadQuantizedParams'; import { loadQuantizedParams } from './loadQuantizedParams';
import { mobileNetV1 } from './mobileNetV1'; import { mobileNetV1 } from './mobileNetV1';
......
src/faceDetectionNet/extractParams.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { extractWeightsFactory, ExtractWeightsFunction, ParamMapping } from 'tfjs-image-recognition-base';
import { extractWeightsFactory } from '../commons/extractWeightsFactory'; import { ConvParams } from '../commons/types';
import { ConvParams, ExtractWeightsFunction, ParamMapping } from '../commons/types';
import { MobileNetV1, NetParams, PointwiseConvParams, PredictionLayerParams } from './types'; import { MobileNetV1, NetParams, PointwiseConvParams, PredictionLayerParams } from './types';
function extractorsFactory(extractWeights: ExtractWeightsFunction, paramMappings: ParamMapping[]) { function extractorsFactory(extractWeights: ExtractWeightsFunction, paramMappings: ParamMapping[]) {
......
src/faceDetectionNet/loadQuantizedParams.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import {
import { disposeUnusedWeightTensors } from '../commons/disposeUnusedWeightTensors'; disposeUnusedWeightTensors,
import { extractWeightEntryFactory } from '../commons/extractWeightEntryFactory'; extractWeightEntryFactory,
import { isTensor3D } from '../commons/isTensor'; isTensor3D,
import { loadWeightMap } from '../commons/loadWeightMap'; loadWeightMap,
import { ConvParams, ParamMapping } from '../commons/types'; ParamMapping,
} from 'tfjs-image-recognition-base';
import { ConvParams } from '../commons/types';
import { BoxPredictionParams, MobileNetV1, NetParams, PointwiseConvParams, PredictionLayerParams } from './types'; import { BoxPredictionParams, MobileNetV1, NetParams, PointwiseConvParams, PredictionLayerParams } from './types';
const DEFAULT_MODEL_NAME = 'ssd_mobilenetv1_model' const DEFAULT_MODEL_NAME = 'ssd_mobilenetv1_model'
......
src/faceLandmarkNet/FaceLandmarkNet.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { isEven, NetInput, NeuralNetwork, Point, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
import { FaceLandmarks68 } from '../classes/FaceLandmarks68';
import { convLayer } from '../commons/convLayer'; import { convLayer } from '../commons/convLayer';
import { NeuralNetwork } from '../commons/NeuralNetwork';
import { ConvParams } from '../commons/types'; import { ConvParams } from '../commons/types';
import { NetInput } from '../NetInput';
import { Point } from '../Point';
import { toNetInput } from '../toNetInput';
import { TNetInput } from '../types';
import { isEven } from '../utils';
import { extractParams } from './extractParams'; import { extractParams } from './extractParams';
import { FaceLandmarks68 } from './FaceLandmarks68';
import { fullyConnectedLayer } from './fullyConnectedLayer'; import { fullyConnectedLayer } from './fullyConnectedLayer';
import { loadQuantizedParams } from './loadQuantizedParams'; import { loadQuantizedParams } from './loadQuantizedParams';
import { NetParams } from './types'; import { NetParams } from './types';
......
src/faceLandmarkNet/extractParams.ts
import { extractWeightsFactory, ParamMapping } from 'tfjs-image-recognition-base';
import { extractConvParamsFactory } from '../commons/extractConvParamsFactory'; import { extractConvParamsFactory } from '../commons/extractConvParamsFactory';
import { extractFCParamsFactory } from '../commons/extractFCParamsFactory'; import { extractFCParamsFactory } from '../commons/extractFCParamsFactory';
import { extractWeightsFactory } from '../commons/extractWeightsFactory';
import { ParamMapping } from '../commons/types';
import { NetParams } from './types'; import { NetParams } from './types';
export function extractParams(weights: Float32Array): { params: NetParams, paramMappings: ParamMapping[] } { export function extractParams(weights: Float32Array): { params: NetParams, paramMappings: ParamMapping[] } {
......
src/faceLandmarkNet/index.ts
import { FaceLandmarkNet } from './FaceLandmarkNet'; import { FaceLandmarkNet } from './FaceLandmarkNet';
export * from './FaceLandmarkNet'; export * from './FaceLandmarkNet';
export * from './FaceLandmarks68';
export function createFaceLandmarkNet(weights: Float32Array) { export function createFaceLandmarkNet(weights: Float32Array) {
const net = new FaceLandmarkNet() const net = new FaceLandmarkNet()
......
src/faceLandmarkNet/loadQuantizedParams.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import {
disposeUnusedWeightTensors,
extractWeightEntryFactory,
loadWeightMap,
ParamMapping,
} from 'tfjs-image-recognition-base';
import { disposeUnusedWeightTensors } from '../commons/disposeUnusedWeightTensors'; import { ConvParams, FCParams } from '../commons/types';
import { extractWeightEntryFactory } from '../commons/extractWeightEntryFactory';
import { loadWeightMap } from '../commons/loadWeightMap';
import { ConvParams, FCParams, ParamMapping } from '../commons/types';
import { NetParams } from './types'; import { NetParams } from './types';
const DEFAULT_MODEL_NAME = 'face_landmark_68_model' const DEFAULT_MODEL_NAME = 'face_landmark_68_model'
......
src/faceRecognitionNet/FaceRecognitionNet.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { NetInput, NeuralNetwork, normalize, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
import { NeuralNetwork } from '../commons/NeuralNetwork';
import { normalize } from '../commons/normalize';
import { NetInput } from '../NetInput';
import { toNetInput } from '../toNetInput';
import { TNetInput } from '../types';
import { convDown } from './convLayer'; import { convDown } from './convLayer';
import { extractParams } from './extractParams'; import { extractParams } from './extractParams';
import { loadQuantizedParams } from './loadQuantizedParams'; import { loadQuantizedParams } from './loadQuantizedParams';
import { residual, residualDown } from './residualLayer'; import { residual, residualDown } from './residualLayer';
import { NetParams } from './types'; import { NetParams } from './types';
export class FaceRecognitionNet extends NeuralNetwork<NetParams> { export class FaceRecognitionNet extends NeuralNetwork<NetParams> {
constructor() { constructor() {
......
src/faceRecognitionNet/extractParams.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { extractWeightsFactory, ExtractWeightsFunction, isFloat, ParamMapping } from 'tfjs-image-recognition-base';
import { extractWeightsFactory } from '../commons/extractWeightsFactory'; import { ConvParams } from '../commons/types';
import { ConvParams, ExtractWeightsFunction, ParamMapping } from '../commons/types';
import { isFloat } from '../utils';
import { ConvLayerParams, NetParams, ResidualLayerParams, ScaleLayerParams } from './types'; import { ConvLayerParams, NetParams, ResidualLayerParams, ScaleLayerParams } from './types';
function extractorsFactory(extractWeights: ExtractWeightsFunction, paramMappings: ParamMapping[]) { function extractorsFactory(extractWeights: ExtractWeightsFunction, paramMappings: ParamMapping[]) {
......
src/faceRecognitionNet/loadQuantizedParams.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import {
disposeUnusedWeightTensors,
extractWeightEntryFactory,
isTensor2D,
loadWeightMap,
ParamMapping,
} from 'tfjs-image-recognition-base';
import { disposeUnusedWeightTensors } from '../commons/disposeUnusedWeightTensors';
import { extractWeightEntryFactory } from '../commons/extractWeightEntryFactory';
import { isTensor2D } from '../commons/isTensor';
import { loadWeightMap } from '../commons/loadWeightMap';
import { ParamMapping } from '../commons/types';
import { ConvLayerParams, NetParams, ResidualLayerParams, ScaleLayerParams } from './types'; import { ConvLayerParams, NetParams, ResidualLayerParams, ScaleLayerParams } from './types';
const DEFAULT_MODEL_NAME = 'face_recognition_model' const DEFAULT_MODEL_NAME = 'face_recognition_model'
......
src/globalApi.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { NetInput, TNetInput } from 'tfjs-image-recognition-base';
import { allFacesMtcnnFactory, allFacesSsdMobilenetv1Factory, allFacesTinyYolov2Factory } from './allFacesFactory'; import { allFacesMtcnnFactory, allFacesSsdMobilenetv1Factory, allFacesTinyYolov2Factory } from './allFacesFactory';
import { FaceDetection } from './FaceDetection'; import { FaceDetection } from './classes/FaceDetection';
import { FaceLandmarks68 } from './classes/FaceLandmarks68';
import { FullFaceDescription } from './classes/FullFaceDescription';
import { FaceDetectionNet } from './faceDetectionNet/FaceDetectionNet'; import { FaceDetectionNet } from './faceDetectionNet/FaceDetectionNet';
import { FaceLandmarkNet } from './faceLandmarkNet/FaceLandmarkNet'; import { FaceLandmarkNet } from './faceLandmarkNet/FaceLandmarkNet';
import { FaceLandmarks68 } from './faceLandmarkNet/FaceLandmarks68';
import { FaceRecognitionNet } from './faceRecognitionNet/FaceRecognitionNet'; import { FaceRecognitionNet } from './faceRecognitionNet/FaceRecognitionNet';
import { FullFaceDescription } from './FullFaceDescription';
import { Mtcnn } from './mtcnn/Mtcnn'; import { Mtcnn } from './mtcnn/Mtcnn';
import { MtcnnForwardParams, MtcnnResult } from './mtcnn/types'; import { MtcnnForwardParams, MtcnnResult } from './mtcnn/types';
import { NetInput } from './NetInput';
import { TinyYolov2 } from './tinyYolov2/TinyYolov2'; import { TinyYolov2 } from './tinyYolov2/TinyYolov2';
import { TinyYolov2ForwardParams } from './tinyYolov2/types'; import { TinyYolov2ForwardParams } from './tinyYolov2/types';
import { TNetInput } from './types';
export const detectionNet = new FaceDetectionNet() export const detectionNet = new FaceDetectionNet()
export const landmarkNet = new FaceLandmarkNet() export const landmarkNet = new FaceLandmarkNet()
......
src/index.ts
...@@ -4,15 +4,10 @@ export { ...@@ -4,15 +4,10 @@ export {
tf tf
} }
export * from 'tfjs-image-recognition-base';
export * from './BoundingBox'; export * from './classes';
export * from './FaceDetection';
export * from './FullFaceDescription';
export * from './NetInput';
export * from './Point';
export * from './Rect';
export * from './drawing';
export * from './euclideanDistance'; export * from './euclideanDistance';
export * from './extractFaces' export * from './extractFaces'
export * from './extractFaceTensors' export * from './extractFaceTensors'
...@@ -20,11 +15,5 @@ export * from './faceDetectionNet'; ...@@ -20,11 +15,5 @@ export * from './faceDetectionNet';
export * from './faceLandmarkNet'; export * from './faceLandmarkNet';
export * from './faceRecognitionNet'; export * from './faceRecognitionNet';
export * from './globalApi'; export * from './globalApi';
export * from './iou';
export * from './mtcnn'; export * from './mtcnn';
export * from './padToSquare'; export * from './tinyYolov2';
export * from './tinyYolov2'; \ No newline at end of file
export * from './toNetInput';
export * from './utils';
export * from './types';
\ No newline at end of file
src/iou.ts deleted 100644 → 0
import { BoundingBox } from './BoundingBox';
export function iou(box1: BoundingBox, box2: BoundingBox, isIOU: boolean = true) {
const width = Math.max(0.0, Math.min(box1.right, box2.right) - Math.max(box1.left, box2.left) + 1)
const height = Math.max(0.0, Math.min(box1.bottom, box2.bottom) - Math.max(box1.top, box2.top) + 1)
const interSection = width * height
return isIOU
? interSection / (box1.area + box2.area - interSection)
: interSection / Math.min(box1.area, box2.area)
}
\ No newline at end of file
src/mtcnn/Mtcnn.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { NetInput, NeuralNetwork, Point, Rect, TNetInput, toNetInput } from 'tfjs-image-recognition-base';
import { NeuralNetwork } from '../commons/NeuralNetwork'; import { FaceDetection } from '../classes/FaceDetection';
import { FaceDetection } from '../FaceDetection'; import { FaceLandmarks5 } from '../classes/FaceLandmarks5';
import { NetInput } from '../NetInput';
import { Point } from '../Point';
import { Rect } from '../Rect';
import { toNetInput } from '../toNetInput';
import { TNetInput } from '../types';
import { bgrToRgbTensor } from './bgrToRgbTensor'; import { bgrToRgbTensor } from './bgrToRgbTensor';
import { CELL_SIZE } from './config'; import { CELL_SIZE } from './config';
import { extractParams } from './extractParams'; import { extractParams } from './extractParams';
import { FaceLandmarks5 } from './FaceLandmarks5';
import { getDefaultMtcnnForwardParams } from './getDefaultMtcnnForwardParams'; import { getDefaultMtcnnForwardParams } from './getDefaultMtcnnForwardParams';
import { getSizesForScale } from './getSizesForScale'; import { getSizesForScale } from './getSizesForScale';
import { loadQuantizedParams } from './loadQuantizedParams'; import { loadQuantizedParams } from './loadQuantizedParams';
......
src/mtcnn/extractImagePatches.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { BoundingBox } from '../BoundingBox';
import { Dimensions } from '../types';
import { createCanvas, getContext2dOrThrow } from '../utils';
import { normalize } from './normalize'; import { normalize } from './normalize';
import { BoundingBox, Dimensions, getContext2dOrThrow, createCanvas } from 'tfjs-image-recognition-base';
export async function extractImagePatches( export async function extractImagePatches(
img: HTMLCanvasElement, img: HTMLCanvasElement,
......
src/mtcnn/extractParams.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { extractWeightsFactory, ExtractWeightsFunction, ParamMapping } from 'tfjs-image-recognition-base';
import { extractConvParamsFactory } from '../commons/extractConvParamsFactory'; import { extractConvParamsFactory } from '../commons/extractConvParamsFactory';
import { extractFCParamsFactory } from '../commons/extractFCParamsFactory'; import { extractFCParamsFactory } from '../commons/extractFCParamsFactory';
import { extractWeightsFactory } from '../commons/extractWeightsFactory'; import { NetParams, ONetParams, PNetParams, RNetParams, SharedParams } from './types';
import { ExtractWeightsFunction, ParamMapping } from '../commons/types';
import { NetParams, PNetParams, RNetParams, SharedParams, ONetParams } from './types';
function extractorsFactory(extractWeights: ExtractWeightsFunction, paramMappings: ParamMapping[]) { function extractorsFactory(extractWeights: ExtractWeightsFunction, paramMappings: ParamMapping[]) {
......
src/mtcnn/index.ts
import { Mtcnn } from './Mtcnn'; import { Mtcnn } from './Mtcnn';
export * from './Mtcnn'; export * from './Mtcnn';
export * from './FaceLandmarks5';
export function createMtcnn(weights: Float32Array) { export function createMtcnn(weights: Float32Array) {
const net = new Mtcnn() const net = new Mtcnn()
......
src/mtcnn/loadQuantizedParams.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import {
import { disposeUnusedWeightTensors } from '../commons/disposeUnusedWeightTensors'; disposeUnusedWeightTensors,
import { extractWeightEntryFactory } from '../commons/extractWeightEntryFactory'; extractWeightEntryFactory,
import { loadWeightMap } from '../commons/loadWeightMap'; loadWeightMap,
import { ConvParams, FCParams, ParamMapping } from '../commons/types'; ParamMapping,
} from 'tfjs-image-recognition-base';
import { ConvParams, FCParams } from '../commons/types';
import { NetParams, ONetParams, PNetParams, RNetParams, SharedParams } from './types'; import { NetParams, ONetParams, PNetParams, RNetParams, SharedParams } from './types';
const DEFAULT_MODEL_NAME = 'mtcnn_model' const DEFAULT_MODEL_NAME = 'mtcnn_model'
......
src/mtcnn/stage1.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { BoundingBox, nonMaxSuppression, Point } from 'tfjs-image-recognition-base';
import { BoundingBox } from '../BoundingBox';
import { nonMaxSuppression } from '../commons/nonMaxSuppression';
import { Point } from '../Point';
import { CELL_SIZE, CELL_STRIDE } from './config'; import { CELL_SIZE, CELL_STRIDE } from './config';
import { getSizesForScale } from './getSizesForScale'; import { getSizesForScale } from './getSizesForScale';
import { normalize } from './normalize'; import { normalize } from './normalize';
......
src/mtcnn/stage2.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { BoundingBox, nonMaxSuppression } from 'tfjs-image-recognition-base';
import { BoundingBox } from '../BoundingBox';
import { nonMaxSuppression } from '../commons/nonMaxSuppression';
import { extractImagePatches } from './extractImagePatches'; import { extractImagePatches } from './extractImagePatches';
import { RNet } from './RNet'; import { RNet } from './RNet';
import { RNetParams } from './types'; import { RNetParams } from './types';
......
src/mtcnn/stage3.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { BoundingBox, nonMaxSuppression, Point } from 'tfjs-image-recognition-base';
import { BoundingBox } from '../BoundingBox';
import { nonMaxSuppression } from '../commons/nonMaxSuppression';
import { Point } from '../Point';
import { extractImagePatches } from './extractImagePatches'; import { extractImagePatches } from './extractImagePatches';
import { ONet } from './ONet'; import { ONet } from './ONet';
import { ONetParams } from './types'; import { ONetParams } from './types';
......
src/mtcnn/types.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { FaceDetection } from '../classes/FaceDetection';
import { FaceLandmarks5 } from '../classes/FaceLandmarks5';
import { ConvParams, FCParams } from '../commons/types'; import { ConvParams, FCParams } from '../commons/types';
import { FaceDetection } from '../FaceDetection';
import { FaceLandmarks5 } from './FaceLandmarks5';
export type SharedParams = { export type SharedParams = {
conv1: ConvParams conv1: ConvParams
......
src/padToSquare.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
/**
* Pads the smaller dimension of an image tensor with zeros, such that width === height.
*
* @param imgTensor The image tensor.
* @param isCenterImage (optional, default: false) If true, add padding on both sides of the image, such that the image.
* @returns The padded tensor with width === height.
*/
export function padToSquare(
imgTensor: tf.Tensor4D,
isCenterImage: boolean = false
): tf.Tensor4D {
return tf.tidy(() => {
const [height, width] = imgTensor.shape.slice(1)
if (height === width) {
return imgTensor
}
const dimDiff = Math.abs(height - width)
const paddingAmount = Math.round(dimDiff * (isCenterImage ? 0.5 : 1))
const paddingAxis = height > width ? 2 : 1
const createPaddingTensor = (paddingAmount: number): tf.Tensor => {
const paddingTensorShape = imgTensor.shape.slice()
paddingTensorShape[paddingAxis] = paddingAmount
return tf.fill(paddingTensorShape, 0)
}
const paddingTensorAppend = createPaddingTensor(paddingAmount)
const remainingPaddingAmount = dimDiff - paddingTensorAppend.shape[paddingAxis]
const paddingTensorPrepend = isCenterImage && remainingPaddingAmount
? createPaddingTensor(remainingPaddingAmount)
: null
const tensorsToStack = [
paddingTensorPrepend,
imgTensor,
paddingTensorAppend
]
.filter(t => t !== null) as tf.Tensor4D[]
return tf.concat(tensorsToStack, paddingAxis)
})
}
\ No newline at end of file
src/tinyYolov2/TinyYolov2.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import {
import { BoundingBox } from '../BoundingBox'; BoundingBox,
NetInput,
NeuralNetwork,
nonMaxSuppression,
normalize,
Point,
sigmoid,
TNetInput,
toNetInput,
} from 'tfjs-image-recognition-base';
import { FaceDetection } from '../classes/FaceDetection';
import { convLayer } from '../commons/convLayer'; import { convLayer } from '../commons/convLayer';
import { NeuralNetwork } from '../commons/NeuralNetwork';
import { nonMaxSuppression } from '../commons/nonMaxSuppression';
import { normalize } from '../commons/normalize';
import { FaceDetection } from '../FaceDetection';
import { NetInput } from '../NetInput';
import { Point } from '../Point';
import { toNetInput } from '../toNetInput';
import { TNetInput } from '../types';
import { sigmoid } from '../utils';
import { BOX_ANCHORS, BOX_ANCHORS_SEPARABLE, INPUT_SIZES, IOU_THRESHOLD, MEAN_RGB, NUM_BOXES } from './config'; import { BOX_ANCHORS, BOX_ANCHORS_SEPARABLE, INPUT_SIZES, IOU_THRESHOLD, MEAN_RGB, NUM_BOXES } from './config';
import { convWithBatchNorm } from './convWithBatchNorm'; import { convWithBatchNorm } from './convWithBatchNorm';
import { extractParams } from './extractParams'; import { extractParams } from './extractParams';
...@@ -18,6 +20,7 @@ import { getDefaultParams } from './getDefaultParams'; ...@@ -18,6 +20,7 @@ import { getDefaultParams } from './getDefaultParams';
import { loadQuantizedParams } from './loadQuantizedParams'; import { loadQuantizedParams } from './loadQuantizedParams';
import { NetParams, PostProcessingParams, TinyYolov2ForwardParams } from './types'; import { NetParams, PostProcessingParams, TinyYolov2ForwardParams } from './types';
export class TinyYolov2 extends NeuralNetwork<NetParams> { export class TinyYolov2 extends NeuralNetwork<NetParams> {
private _withSeparableConvs: boolean private _withSeparableConvs: boolean
......
src/tinyYolov2/config.ts
import { Point } from '../Point'; import { Point } from 'tfjs-image-recognition-base';
export const INPUT_SIZES = { xs: 224, sm: 320, md: 416, lg: 608 } export const INPUT_SIZES = { xs: 224, sm: 320, md: 416, lg: 608 }
export const NUM_BOXES = 5 export const NUM_BOXES = 5
......
src/tinyYolov2/extractParams.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { extractWeightsFactory, ExtractWeightsFunction, ParamMapping } from 'tfjs-image-recognition-base';
import { extractConvParamsFactory } from '../commons/extractConvParamsFactory'; import { extractConvParamsFactory } from '../commons/extractConvParamsFactory';
import { extractWeightsFactory } from '../commons/extractWeightsFactory';
import { ExtractWeightsFunction, ParamMapping } from '../commons/types';
import { BatchNorm, ConvWithBatchNorm, NetParams, SeparableConvParams } from './types'; import { BatchNorm, ConvWithBatchNorm, NetParams, SeparableConvParams } from './types';
function extractorsFactory(extractWeights: ExtractWeightsFunction, paramMappings: ParamMapping[]) { function extractorsFactory(extractWeights: ExtractWeightsFunction, paramMappings: ParamMapping[]) {
......
src/tinyYolov2/loadQuantizedParams.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import {
import { disposeUnusedWeightTensors } from '../commons/disposeUnusedWeightTensors'; disposeUnusedWeightTensors,
import { extractWeightEntryFactory } from '../commons/extractWeightEntryFactory'; extractWeightEntryFactory,
import { loadWeightMap } from '../commons/loadWeightMap'; loadWeightMap,
import { ConvParams, ParamMapping } from '../commons/types'; ParamMapping,
} from 'tfjs-image-recognition-base';
import { ConvParams } from '../commons/types';
import { BatchNorm, ConvWithBatchNorm, NetParams, SeparableConvParams } from './types'; import { BatchNorm, ConvWithBatchNorm, NetParams, SeparableConvParams } from './types';
const DEFAULT_MODEL_NAME = 'tiny_yolov2_model' const DEFAULT_MODEL_NAME = 'tiny_yolov2_model'
......
src/tinyYolov2/types.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { Point } from 'tfjs-image-recognition-base';
import { ConvParams } from '../commons/types'; import { ConvParams } from '../commons/types';
import { Point } from '../Point';
export type BatchNorm = { export type BatchNorm = {
sub: tf.Tensor1D sub: tf.Tensor1D
......
src/toNetInput.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
import { isMediaElement } from './commons/isMediaElement';
import { isTensor3D, isTensor4D } from './commons/isTensor';
import { NetInput } from './NetInput';
import { TNetInput } from './types';
import { awaitMediaLoaded, resolveInput } from './utils';
/**
* Validates the input to make sure, they are valid net inputs and awaits all media elements
* to be finished loading.
*
* @param input The input, which can be a media element or an array of different media elements.
* @param manageCreatedInput If a new NetInput instance is created from the inputs, this flag
* determines, whether to set the NetInput as managed or not.
* @returns A NetInput instance, which can be passed into one of the neural networks.
*/
export async function toNetInput(
inputs: TNetInput,
manageCreatedInput: boolean = false,
keepCanvases: boolean = false
): Promise<NetInput> {
if (inputs instanceof NetInput) {
return inputs
}
const afterCreate = (netInput: NetInput) => manageCreatedInput
? netInput.managed()
: netInput
if (isTensor4D(inputs)) {
return afterCreate(new NetInput(inputs as tf.Tensor4D))
}
let inputArgArray = Array.isArray(inputs)
? inputs
: [inputs]
if (!inputArgArray.length) {
throw new Error('toNetInput - empty array passed as input')
}
const getIdxHint = (idx: number) => Array.isArray(inputs) ? ` at input index ${idx}:` : ''
const inputArray = inputArgArray.map(resolveInput)
inputArray.forEach((input, i) => {
if (!isMediaElement(input) && !isTensor3D(input) && !isTensor4D(input)) {
if (typeof inputArgArray[i] === 'string') {
throw new Error(`toNetInput -${getIdxHint(i)} string passed, but could not resolve HTMLElement for element id ${inputArgArray[i]}`)
}
throw new Error(`toNetInput -${getIdxHint(i)} expected media to be of type HTMLImageElement | HTMLVideoElement | HTMLCanvasElement | tf.Tensor3D, or to be an element id`)
}
if (isTensor4D(input)) {
// if tf.Tensor4D is passed in the input array, the batch size has to be 1
const batchSize = input.shape[0]
if (batchSize !== 1) {
throw new Error(`toNetInput -${getIdxHint(i)} tf.Tensor4D with batchSize ${batchSize} passed, but not supported in input array`)
}
}
})
// wait for all media elements being loaded
await Promise.all(
inputArray.map(input => isMediaElement(input) && awaitMediaLoaded(input))
)
return afterCreate(new NetInput(inputArray, Array.isArray(inputs), keepCanvases))
}
\ No newline at end of file
src/types.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
import { NetInput } from './NetInput';
export type TMediaElement = HTMLImageElement | HTMLVideoElement | HTMLCanvasElement
export type TResolvedNetInput = TMediaElement | tf.Tensor3D | tf.Tensor4D
export type TNetInputArg = string | TResolvedNetInput
export type TNetInput = TNetInputArg | Array<TNetInputArg> | NetInput | tf.Tensor4D
export type Dimensions = {
width: number
height: number
}
\ No newline at end of file
src/utils.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
import { isTensor4D } from './commons/isTensor';
import { Dimensions } from './types';
export function isFloat(num: number) {
return num % 1 !== 0
}
export function isEven(num: number) {
return num % 2 === 0
}
export function round(num: number) {
return Math.floor(num * 100) / 100
}
export function sigmoid(x: number) {
return 1 / (1 + Math.exp(-x))
}
export function isDimensions(obj: any): boolean {
return obj && obj.width && obj.height
}
export function resolveInput(arg: string | any) {
if (typeof arg === 'string') {
return document.getElementById(arg)
}
return arg
}
export function isLoaded(media: HTMLImageElement | HTMLVideoElement) : boolean {
return (media instanceof HTMLImageElement && media.complete)
|| (media instanceof HTMLVideoElement && media.readyState >= 3)
}
export function awaitMediaLoaded(media: HTMLImageElement | HTMLVideoElement | HTMLCanvasElement) {
return new Promise((resolve, reject) => {
if (media instanceof HTMLCanvasElement || isLoaded(media)) {
return resolve()
}
function onLoad(e: Event) {
if (!e.currentTarget) return
e.currentTarget.removeEventListener('load', onLoad)
e.currentTarget.removeEventListener('error', onError)
resolve(e)
}
function onError(e: Event) {
if (!e.currentTarget) return
e.currentTarget.removeEventListener('load', onLoad)
e.currentTarget.removeEventListener('error', onError)
reject(e)
}
media.addEventListener('load', onLoad)
media.addEventListener('error', onError)
})
}
export function getContext2dOrThrow(canvas: HTMLCanvasElement): CanvasRenderingContext2D {
const ctx = canvas.getContext('2d')
if (!ctx) {
throw new Error('canvas 2d context is null')
}
return ctx
}
export function createCanvas({ width, height }: Dimensions): HTMLCanvasElement {
const canvas = document.createElement('canvas')
canvas.width = width
canvas.height = height
return canvas
}
export function createCanvasFromMedia(media: HTMLImageElement | HTMLVideoElement, dims?: Dimensions): HTMLCanvasElement {
if (!isLoaded(media)) {
throw new Error('createCanvasFromMedia - media has not finished loading yet')
}
const { width, height } = dims || getMediaDimensions(media)
const canvas = createCanvas({ width, height })
getContext2dOrThrow(canvas).drawImage(media, 0, 0, width, height)
return canvas
}
export function getMediaDimensions(media: HTMLImageElement | HTMLVideoElement) {
if (media instanceof HTMLImageElement) {
return { width: media.naturalWidth, height: media.naturalHeight }
}
if (media instanceof HTMLVideoElement) {
return { width: media.videoWidth, height: media.videoHeight }
}
return media
}
export function bufferToImage(buf: Blob): Promise<HTMLImageElement> {
return new Promise((resolve, reject) => {
if (!(buf instanceof Blob)) {
return reject('bufferToImage - expected buf to be of type: Blob')
}
const reader = new FileReader()
reader.onload = () => {
const img = new Image()
img.onload = () => resolve(img)
img.onerror = reject
img.src = reader.result
}
reader.onerror = reject
reader.readAsDataURL(buf)
})
}
export async function imageTensorToCanvas(
imgTensor: tf.Tensor,
canvas?: HTMLCanvasElement
): Promise<HTMLCanvasElement> {
const targetCanvas = canvas || document.createElement('canvas')
const [height, width, numChannels] = imgTensor.shape.slice(isTensor4D(imgTensor) ? 1 : 0)
const imgTensor3D = tf.tidy(() => imgTensor.as3D(height, width, numChannels).toInt())
await tf.toPixels(imgTensor3D, targetCanvas)
imgTensor3D.dispose()
return targetCanvas
}
\ No newline at end of file
test/tests/NetInput.test.ts deleted 100644 → 0
import { NetInput } from '../../src/NetInput';
import { bufferToImage } from '../../src/utils';
import { expectAllTensorsReleased, tensor3D } from '../utils';
describe('NetInput', () => {
let imgEl: HTMLImageElement
beforeAll(async () => {
const img = await (await fetch('base/test/images/face1.png')).blob()
imgEl = await bufferToImage(img)
})
describe('no memory leaks', () => {
describe('constructor', () => {
it('single image element', async () => {
await expectAllTensorsReleased(() => {
const net = new NetInput([imgEl])
net.dispose()
})
})
it('multiple image elements', async () => {
await expectAllTensorsReleased(() => {
const net = new NetInput([imgEl, imgEl, imgEl])
net.dispose()
})
})
it('single tf.Tensor3D', async () => {
const tensor = tensor3D()
await expectAllTensorsReleased(() => {
const net = new NetInput([tensor])
net.dispose()
})
tensor.dispose()
})
it('multiple tf.Tensor3Ds', async () => {
const tensors = [tensor3D(), tensor3D(), tensor3D()]
await expectAllTensorsReleased(() => {
const net = new NetInput(tensors)
net.dispose()
})
tensors.forEach(t => t.dispose())
})
})
describe('toBatchTensor', () => {
it('single image element', async () => {
await expectAllTensorsReleased(() => {
const net = new NetInput([imgEl])
const batchTensor = net.toBatchTensor(100, false)
net.dispose()
batchTensor.dispose()
})
})
it('multiple image elements', async () => {
await expectAllTensorsReleased(() => {
const net = new NetInput([imgEl, imgEl, imgEl])
const batchTensor = net.toBatchTensor(100, false)
net.dispose()
batchTensor.dispose()
})
})
it('managed, single image element', async () => {
await expectAllTensorsReleased(() => {
const net = (new NetInput([imgEl])).managed()
const batchTensor = net.toBatchTensor(100, false)
batchTensor.dispose()
})
})
it('managed, multiple image elements', async () => {
await expectAllTensorsReleased(() => {
const net = (new NetInput([imgEl, imgEl, imgEl])).managed()
const batchTensor = net.toBatchTensor(100, false)
batchTensor.dispose()
})
})
})
})
})
test/tests/commons/NeuralNetwork.test.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
import { NeuralNetwork } from '../../../src/commons/NeuralNetwork';
class FakeNeuralNetwork extends NeuralNetwork<any> {
constructor(
convFilter: tf.Tensor = tf.tensor(0),
convBias: tf.Tensor = tf.tensor(0),
fcWeights: tf.Tensor = tf.tensor(0)
) {
super('FakeNeuralNetwork')
this._params = {
conv: {
filter: convFilter,
bias: convBias,
},
fc: fcWeights
}
this._paramMappings = [
{ originalPath: 'conv2d/filter', paramPath: 'conv/filter' },
{ originalPath: 'conv2d/bias', paramPath: 'conv/bias' },
{ originalPath: 'dense/weights', paramPath: 'fc' }
]
}
}
describe('NeuralNetwork', () => {
describe('getParamFromPath', () => {
it('returns correct params', () => tf.tidy(() => {
const convFilter = tf.tensor(0)
const convBias = tf.tensor(0)
const fcWeights = tf.tensor(0)
const net = new FakeNeuralNetwork(convFilter, convBias, fcWeights)
expect(net.getParamFromPath('conv/filter')).toEqual(convFilter)
expect(net.getParamFromPath('conv/bias')).toEqual(convBias)
expect(net.getParamFromPath('fc')).toEqual(fcWeights)
}))
it('throws if param is not a tensor', () => tf.tidy(() => {
const net = new FakeNeuralNetwork(null as any)
const fakePath = 'conv/filter'
expect(
() => net.getParamFromPath(fakePath)
).toThrowError(`traversePropertyPath - parameter is not a tensor, for path ${fakePath}`)
}))
it('throws if key path invalid', () => tf.tidy(() => {
const net = new FakeNeuralNetwork()
const fakePath = 'conv2d/foo'
expect(
() => net.getParamFromPath(fakePath)
).toThrowError(`traversePropertyPath - object does not have property conv2d, for path ${fakePath}`)
}))
})
describe('reassignParamFromPath', () => {
it('sets correct params', () => tf.tidy(() => {
const net = new FakeNeuralNetwork()
const convFilter = tf.tensor(0)
const convBias = tf.tensor(0)
const fcWeights = tf.tensor(0)
net.reassignParamFromPath('conv/filter', convFilter)
net.reassignParamFromPath('conv/bias', convBias)
net.reassignParamFromPath('fc', fcWeights)
expect(net.params.conv.filter).toEqual(convFilter)
expect(net.params.conv.bias).toEqual(convBias)
expect(net.params.fc).toEqual(fcWeights)
}))
it('throws if param is not a tensor', () => tf.tidy(() => {
const net = new FakeNeuralNetwork(null as any)
const fakePath = 'conv/filter'
expect(
() => net.reassignParamFromPath(fakePath, tf.tensor(0))
).toThrowError(`traversePropertyPath - parameter is not a tensor, for path ${fakePath}`)
}))
it('throws if key path invalid', () => tf.tidy(() => {
const net = new FakeNeuralNetwork()
const fakePath = 'conv2d/foo'
expect(
() => net.reassignParamFromPath(fakePath, tf.tensor(0))
).toThrowError(`traversePropertyPath - object does not have property conv2d, for path ${fakePath}`)
}))
})
describe('getParamList', () => {
it('returns param tensors with path', () => tf.tidy(() => {
const convFilter = tf.tensor(0)
const convBias = tf.tensor(0)
const fcWeights = tf.tensor(0)
const net = new FakeNeuralNetwork(convFilter, convBias, fcWeights)
const paramList = net.getParamList()
expect(paramList.length).toEqual(3)
expect(paramList[0].path).toEqual('conv/filter')
expect(paramList[1].path).toEqual('conv/bias')
expect(paramList[2].path).toEqual('fc')
expect(paramList[0].tensor).toEqual(convFilter)
expect(paramList[1].tensor).toEqual(convBias)
expect(paramList[2].tensor).toEqual(fcWeights)
}))
})
describe('getFrozenParams', () => {
it('returns all frozen params', () => tf.tidy(() => {
const convFilter = tf.tensor(0)
const convBias = tf.tensor(0)
const fcWeights = tf.variable(tf.scalar(0))
const net = new FakeNeuralNetwork(convFilter, convBias, fcWeights)
const frozenParams = net.getFrozenParams()
expect(frozenParams.length).toEqual(2)
expect(frozenParams[0].path).toEqual('conv/filter')
expect(frozenParams[1].path).toEqual('conv/bias')
expect(frozenParams[0].tensor).toEqual(convFilter)
expect(frozenParams[1].tensor).toEqual(convBias)
}))
})
describe('getTrainableParams', () => {
it('returns all trainable params', () => tf.tidy(() => {
const convFilter = tf.variable(tf.scalar(0))
const convBias = tf.variable(tf.scalar(0))
const fcWeights = tf.tensor(0)
const net = new FakeNeuralNetwork(convFilter, convBias, fcWeights)
const trainableParams = net.getTrainableParams()
expect(trainableParams.length).toEqual(2)
expect(trainableParams[0].path).toEqual('conv/filter')
expect(trainableParams[1].path).toEqual('conv/bias')
expect(trainableParams[0].tensor).toEqual(convFilter)
expect(trainableParams[1].tensor).toEqual(convBias)
}))
})
describe('dispose', () => {
it('disposes all param tensors', () => tf.tidy(() => {
const numTensors = tf.memory().numTensors
const net = new FakeNeuralNetwork()
net.dispose()
expect(net.params).toBe(undefined)
expect(tf.memory().numTensors - numTensors).toEqual(0)
}))
})
describe('variable', () => {
it('make all param tensors trainable', () => tf.tidy(() => {
const net = new FakeNeuralNetwork()
net.variable()
expect(net.params.conv.filter instanceof tf.Variable).toBe(true)
expect(net.params.conv.bias instanceof tf.Variable).toBe(true)
expect(net.params.fc instanceof tf.Variable).toBe(true)
}))
it('disposes old tensors', () => tf.tidy(() => {
const net = new FakeNeuralNetwork()
const numTensors = tf.memory().numTensors
net.variable()
expect(tf.memory().numTensors - numTensors).toEqual(0)
}))
})
describe('freeze', () => {
it('freezes all param variables', () => tf.tidy(() => {
const net = new FakeNeuralNetwork(
tf.variable(tf.scalar(0)),
tf.variable(tf.scalar(0)),
tf.variable(tf.scalar(0))
)
net.freeze()
expect(net.params.conv.filter instanceof tf.Variable).toBe(false)
expect(net.params.conv.bias instanceof tf.Variable).toBe(false)
expect(net.params.fc instanceof tf.Variable).toBe(false)
}))
it('disposes old tensors', () => tf.tidy(() => {
const net = new FakeNeuralNetwork(
tf.variable(tf.scalar(0)),
tf.variable(tf.scalar(0)),
tf.variable(tf.scalar(0))
)
const numTensors = tf.memory().numTensors
net.freeze()
expect(tf.memory().numTensors - numTensors).toEqual(0)
}))
})
})
test/tests/commons/loadWeightMap.test.ts deleted 100644 → 0
import { getModelUris } from '../../../src/commons/loadWeightMap';
const FAKE_DEFAULT_MODEL_NAME = 'fake_model_name'
describe('loadWeightMap', () => {
describe('getModelUris', () => {
it('returns uris from relative url if no argument passed', () => {
const result = getModelUris(undefined, FAKE_DEFAULT_MODEL_NAME)
expect(result.manifestUri).toEqual(`${FAKE_DEFAULT_MODEL_NAME}-weights_manifest.json`)
expect(result.modelBaseUri).toEqual('')
})
it('returns uris from relative url for empty string', () => {
const result = getModelUris('', FAKE_DEFAULT_MODEL_NAME)
expect(result.manifestUri).toEqual(`${FAKE_DEFAULT_MODEL_NAME}-weights_manifest.json`)
expect(result.modelBaseUri).toEqual('')
})
it('returns uris for top level url, leading slash preserved', () => {
const result = getModelUris('/', FAKE_DEFAULT_MODEL_NAME)
expect(result.manifestUri).toEqual(`/${FAKE_DEFAULT_MODEL_NAME}-weights_manifest.json`)
expect(result.modelBaseUri).toEqual('/')
})
it('returns uris, given url path', () => {
const uri = 'path/to/modelfiles'
const result = getModelUris(uri, FAKE_DEFAULT_MODEL_NAME)
expect(result.manifestUri).toEqual(`${uri}/${FAKE_DEFAULT_MODEL_NAME}-weights_manifest.json`)
expect(result.modelBaseUri).toEqual(uri)
})
it('returns uris, given url path, leading slash preserved', () => {
const uri = '/path/to/modelfiles'
const result = getModelUris(`/${uri}`, FAKE_DEFAULT_MODEL_NAME)
expect(result.manifestUri).toEqual(`${uri}/${FAKE_DEFAULT_MODEL_NAME}-weights_manifest.json`)
expect(result.modelBaseUri).toEqual(uri)
})
it('returns uris, given manifest uri', () => {
const uri = 'path/to/modelfiles/model-weights_manifest.json'
const result = getModelUris(uri, FAKE_DEFAULT_MODEL_NAME)
expect(result.manifestUri).toEqual(uri)
expect(result.modelBaseUri).toEqual('path/to/modelfiles')
})
it('returns uris, given manifest uri, leading slash preserved', () => {
const uri = '/path/to/modelfiles/model-weights_manifest.json'
const result = getModelUris(uri, FAKE_DEFAULT_MODEL_NAME)
expect(result.manifestUri).toEqual(uri)
expect(result.modelBaseUri).toEqual('/path/to/modelfiles')
})
it('returns correct uris, given external path', () => {
const uri = 'https://example.com/path/to/modelfiles';
const result = getModelUris(uri, FAKE_DEFAULT_MODEL_NAME)
expect(result.manifestUri).toEqual(`${uri}/${FAKE_DEFAULT_MODEL_NAME}-weights_manifest.json`)
expect(result.modelBaseUri).toEqual(uri)
})
})
})
test/tests/e2e/allFacesMtcnn.test.ts
import * as faceapi from '../../../src'; import { bufferToImage } from 'tfjs-image-recognition-base';
import { FaceLandmarks5 } from '../../../src/mtcnn/FaceLandmarks5';
import { NetInput } from '../../../src/NetInput'; import { FaceLandmarks5 } from '../../../src';
import { describeWithNets, expectAllTensorsReleased } from '../../utils'; import { describeWithNets, expectAllTensorsReleased } from '../../utils';
import { expectMtcnnResults } from './expectedResults'; import { expectMtcnnResults } from './expectedResults';
...@@ -11,7 +11,7 @@ describe('allFacesMtcnn', () => { ...@@ -11,7 +11,7 @@ describe('allFacesMtcnn', () => {
beforeAll(async () => { beforeAll(async () => {
const img = await (await fetch('base/test/images/faces.jpg')).blob() const img = await (await fetch('base/test/images/faces.jpg')).blob()
imgEl = await faceapi.bufferToImage(img) imgEl = await bufferToImage(img)
facesFaceDescriptors = await (await fetch('base/test/data/facesFaceDescriptorsMtcnn.json')).json() facesFaceDescriptors = await (await fetch('base/test/data/facesFaceDescriptorsMtcnn.json')).json()
}) })
......
test/tests/e2e/allFacesSsdMobilenetv1.test.ts
import * as faceapi from '../../../src';
import { describeWithNets, expectAllTensorsReleased, expectRectClose, expectPointClose } from '../../utils';
import { expectedSsdBoxes } from './expectedResults';
import { NetInput } from '../../../src/NetInput';
import { toNetInput } from '../../../src';
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { Point } from '../../../src/Point';
import { bufferToImage, NetInput, Point, toNetInput } from '../../../src';
import { describeWithNets, expectAllTensorsReleased, expectPointClose, expectRectClose } from '../../utils';
import { expectedSsdBoxes } from './expectedResults';
describe('allFacesSsdMobilenetv1', () => { describe('allFacesSsdMobilenetv1', () => {
...@@ -14,7 +13,7 @@ describe('allFacesSsdMobilenetv1', () => { ...@@ -14,7 +13,7 @@ describe('allFacesSsdMobilenetv1', () => {
beforeAll(async () => { beforeAll(async () => {
const img = await (await fetch('base/test/images/faces.jpg')).blob() const img = await (await fetch('base/test/images/faces.jpg')).blob()
imgEl = await faceapi.bufferToImage(img) imgEl = await bufferToImage(img)
facesFaceLandmarkPositions = await (await fetch('base/test/data/facesFaceLandmarkPositions.json')).json() facesFaceLandmarkPositions = await (await fetch('base/test/data/facesFaceLandmarkPositions.json')).json()
facesFaceDescriptors = await (await fetch('base/test/data/facesFaceDescriptorsSsd.json')).json() facesFaceDescriptors = await (await fetch('base/test/data/facesFaceDescriptorsSsd.json')).json()
}) })
......
test/tests/e2e/allFacesTinyYolov2.test.ts
import * as faceapi from '../../../src';
import { describeWithNets, expectAllTensorsReleased, expectRectClose, expectPointClose, expectMaxDelta } from '../../utils';
import { expectedTinyYolov2SeparableConvBoxes } from './expectedResults';
import { NetInput } from '../../../src/NetInput';
import { toNetInput } from '../../../src';
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { Point } from '../../../src/Point';
import { bufferToImage, NetInput, Point, toNetInput } from '../../../src';
import { SizeType } from '../../../src/tinyYolov2/types'; import { SizeType } from '../../../src/tinyYolov2/types';
import { describeWithNets, expectAllTensorsReleased, expectMaxDelta, expectPointClose, expectRectClose } from '../../utils';
import { expectedTinyYolov2SeparableConvBoxes } from './expectedResults';
describe('allFacesTinyYolov2', () => { describe('allFacesTinyYolov2', () => {
...@@ -15,7 +13,7 @@ describe('allFacesTinyYolov2', () => { ...@@ -15,7 +13,7 @@ describe('allFacesTinyYolov2', () => {
beforeAll(async () => { beforeAll(async () => {
const img = await (await fetch('base/test/images/faces.jpg')).blob() const img = await (await fetch('base/test/images/faces.jpg')).blob()
imgEl = await faceapi.bufferToImage(img) imgEl = await bufferToImage(img)
facesFaceLandmarkPositions = await (await fetch('base/test/data/facesFaceLandmarkPositions.json')).json() facesFaceLandmarkPositions = await (await fetch('base/test/data/facesFaceLandmarkPositions.json')).json()
facesFaceDescriptors = await (await fetch('base/test/data/facesFaceDescriptorsSsd.json')).json() facesFaceDescriptors = await (await fetch('base/test/data/facesFaceDescriptorsSsd.json')).json()
}) })
......
test/tests/e2e/expectedResults.ts
import * as faceapi from '../../../src'; import * as faceapi from '../../../src';
import { Point } from '../../../src/Point';
import { expectMaxDelta, expectPointClose, expectRectClose } from '../../utils'; import { expectMaxDelta, expectPointClose, expectRectClose } from '../../utils';
import { Point } from '../../../src';
export const expectedSsdBoxes = [ export const expectedSsdBoxes = [
{ x: 48, y: 253, width: 104, height: 129 }, { x: 48, y: 253, width: 104, height: 129 },
......
test/tests/e2e/faceLandmarkNet.test.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import * as faceapi from '../../../src'; import {
import { isTensor3D } from '../../../src/commons/isTensor'; bufferToImage,
import { Point } from '../../../src/Point'; createFaceLandmarkNet,
import { Dimensions, TMediaElement } from '../../../src/types'; Dimensions,
import { expectMaxDelta, expectAllTensorsReleased, describeWithNets } from '../../utils'; isTensor3D,
import { NetInput } from '../../../src/NetInput'; NetInput,
import { toNetInput } from '../../../src'; Point,
TMediaElement,
toNetInput,
} from '../../../src';
import { FaceLandmarks68 } from '../../../src/classes/FaceLandmarks68';
import { FaceLandmarkNet } from '../../../src/faceLandmarkNet/FaceLandmarkNet';
import { describeWithNets, expectAllTensorsReleased, expectMaxDelta } from '../../utils';
function getInputDims (input: tf.Tensor | TMediaElement): Dimensions { function getInputDims (input: tf.Tensor | TMediaElement): Dimensions {
if (input instanceof tf.Tensor) { if (input instanceof tf.Tensor) {
...@@ -27,11 +33,11 @@ describe('faceLandmarkNet', () => { ...@@ -27,11 +33,11 @@ describe('faceLandmarkNet', () => {
beforeAll(async () => { beforeAll(async () => {
const img1 = await (await fetch('base/test/images/face1.png')).blob() const img1 = await (await fetch('base/test/images/face1.png')).blob()
imgEl1 = await faceapi.bufferToImage(img1) imgEl1 = await bufferToImage(img1)
const img2 = await (await fetch('base/test/images/face2.png')).blob() const img2 = await (await fetch('base/test/images/face2.png')).blob()
imgEl2 = await faceapi.bufferToImage(img2) imgEl2 = await bufferToImage(img2)
const imgRect = await (await fetch('base/test/images/face_rectangular.png')).blob() const imgRect = await (await fetch('base/test/images/face_rectangular.png')).blob()
imgElRect = await faceapi.bufferToImage(imgRect) imgElRect = await bufferToImage(imgRect)
faceLandmarkPositions1 = await (await fetch('base/test/data/faceLandmarkPositions1.json')).json() faceLandmarkPositions1 = await (await fetch('base/test/data/faceLandmarkPositions1.json')).json()
faceLandmarkPositions2 = await (await fetch('base/test/data/faceLandmarkPositions2.json')).json() faceLandmarkPositions2 = await (await fetch('base/test/data/faceLandmarkPositions2.json')).json()
faceLandmarkPositionsRect = await (await fetch('base/test/data/faceLandmarkPositionsRect.json')).json() faceLandmarkPositionsRect = await (await fetch('base/test/data/faceLandmarkPositionsRect.json')).json()
...@@ -42,7 +48,7 @@ describe('faceLandmarkNet', () => { ...@@ -42,7 +48,7 @@ describe('faceLandmarkNet', () => {
it('computes face landmarks for squared input', async () => { it('computes face landmarks for squared input', async () => {
const { width, height } = imgEl1 const { width, height } = imgEl1
const result = await faceLandmarkNet.detectLandmarks(imgEl1) as faceapi.FaceLandmarks68 const result = await faceLandmarkNet.detectLandmarks(imgEl1) as FaceLandmarks68
expect(result.getImageWidth()).toEqual(width) expect(result.getImageWidth()).toEqual(width)
expect(result.getImageHeight()).toEqual(height) expect(result.getImageHeight()).toEqual(height)
expect(result.getShift().x).toEqual(0) expect(result.getShift().x).toEqual(0)
...@@ -56,7 +62,7 @@ describe('faceLandmarkNet', () => { ...@@ -56,7 +62,7 @@ describe('faceLandmarkNet', () => {
it('computes face landmarks for rectangular input', async () => { it('computes face landmarks for rectangular input', async () => {
const { width, height } = imgElRect const { width, height } = imgElRect
const result = await faceLandmarkNet.detectLandmarks(imgElRect) as faceapi.FaceLandmarks68 const result = await faceLandmarkNet.detectLandmarks(imgElRect) as FaceLandmarks68
expect(result.getImageWidth()).toEqual(width) expect(result.getImageWidth()).toEqual(width)
expect(result.getImageHeight()).toEqual(height) expect(result.getImageHeight()).toEqual(height)
expect(result.getShift().x).toEqual(0) expect(result.getShift().x).toEqual(0)
...@@ -74,7 +80,7 @@ describe('faceLandmarkNet', () => { ...@@ -74,7 +80,7 @@ describe('faceLandmarkNet', () => {
it('computes face landmarks for squared input', async () => { it('computes face landmarks for squared input', async () => {
const { width, height } = imgEl1 const { width, height } = imgEl1
const result = await faceLandmarkNet.detectLandmarks(imgEl1) as faceapi.FaceLandmarks68 const result = await faceLandmarkNet.detectLandmarks(imgEl1) as FaceLandmarks68
expect(result.getImageWidth()).toEqual(width) expect(result.getImageWidth()).toEqual(width)
expect(result.getImageHeight()).toEqual(height) expect(result.getImageHeight()).toEqual(height)
expect(result.getShift().x).toEqual(0) expect(result.getShift().x).toEqual(0)
...@@ -88,7 +94,7 @@ describe('faceLandmarkNet', () => { ...@@ -88,7 +94,7 @@ describe('faceLandmarkNet', () => {
it('computes face landmarks for rectangular input', async () => { it('computes face landmarks for rectangular input', async () => {
const { width, height } = imgElRect const { width, height } = imgElRect
const result = await faceLandmarkNet.detectLandmarks(imgElRect) as faceapi.FaceLandmarks68 const result = await faceLandmarkNet.detectLandmarks(imgElRect) as FaceLandmarks68
expect(result.getImageWidth()).toEqual(width) expect(result.getImageWidth()).toEqual(width)
expect(result.getImageHeight()).toEqual(height) expect(result.getImageHeight()).toEqual(height)
expect(result.getShift().x).toEqual(0) expect(result.getShift().x).toEqual(0)
...@@ -112,7 +118,7 @@ describe('faceLandmarkNet', () => { ...@@ -112,7 +118,7 @@ describe('faceLandmarkNet', () => {
faceLandmarkPositionsRect faceLandmarkPositionsRect
] ]
const results = await faceLandmarkNet.detectLandmarks(inputs) as faceapi.FaceLandmarks68[] const results = await faceLandmarkNet.detectLandmarks(inputs) as FaceLandmarks68[]
expect(Array.isArray(results)).toBe(true) expect(Array.isArray(results)).toBe(true)
expect(results.length).toEqual(3) expect(results.length).toEqual(3)
results.forEach((result, batchIdx) => { results.forEach((result, batchIdx) => {
...@@ -137,7 +143,7 @@ describe('faceLandmarkNet', () => { ...@@ -137,7 +143,7 @@ describe('faceLandmarkNet', () => {
faceLandmarkPositionsRect faceLandmarkPositionsRect
] ]
const results = await faceLandmarkNet.detectLandmarks(inputs) as faceapi.FaceLandmarks68[] const results = await faceLandmarkNet.detectLandmarks(inputs) as FaceLandmarks68[]
expect(Array.isArray(results)).toBe(true) expect(Array.isArray(results)).toBe(true)
expect(results.length).toEqual(3) expect(results.length).toEqual(3)
results.forEach((result, batchIdx) => { results.forEach((result, batchIdx) => {
...@@ -162,7 +168,7 @@ describe('faceLandmarkNet', () => { ...@@ -162,7 +168,7 @@ describe('faceLandmarkNet', () => {
faceLandmarkPositionsRect faceLandmarkPositionsRect
] ]
const results = await faceLandmarkNet.detectLandmarks(tf.stack(inputs) as tf.Tensor4D) as faceapi.FaceLandmarks68[] const results = await faceLandmarkNet.detectLandmarks(tf.stack(inputs) as tf.Tensor4D) as FaceLandmarks68[]
expect(Array.isArray(results)).toBe(true) expect(Array.isArray(results)).toBe(true)
expect(results.length).toEqual(2) expect(results.length).toEqual(2)
results.forEach((result, batchIdx) => { results.forEach((result, batchIdx) => {
...@@ -187,7 +193,7 @@ describe('faceLandmarkNet', () => { ...@@ -187,7 +193,7 @@ describe('faceLandmarkNet', () => {
faceLandmarkPositionsRect faceLandmarkPositionsRect
] ]
const results = await faceLandmarkNet.detectLandmarks(inputs) as faceapi.FaceLandmarks68[] const results = await faceLandmarkNet.detectLandmarks(inputs) as FaceLandmarks68[]
expect(Array.isArray(results)).toBe(true) expect(Array.isArray(results)).toBe(true)
expect(results.length).toEqual(3) expect(results.length).toEqual(3)
results.forEach((result, batchIdx) => { results.forEach((result, batchIdx) => {
...@@ -213,7 +219,7 @@ describe('faceLandmarkNet', () => { ...@@ -213,7 +219,7 @@ describe('faceLandmarkNet', () => {
await expectAllTensorsReleased(async () => { await expectAllTensorsReleased(async () => {
const res = await fetch('base/weights_uncompressed/face_landmark_68_model.weights') const res = await fetch('base/weights_uncompressed/face_landmark_68_model.weights')
const weights = new Float32Array(await res.arrayBuffer()) const weights = new Float32Array(await res.arrayBuffer())
const net = faceapi.createFaceLandmarkNet(weights) const net = createFaceLandmarkNet(weights)
net.dispose() net.dispose()
}) })
}) })
...@@ -224,7 +230,7 @@ describe('faceLandmarkNet', () => { ...@@ -224,7 +230,7 @@ describe('faceLandmarkNet', () => {
it('disposes all param tensors', async () => { it('disposes all param tensors', async () => {
await expectAllTensorsReleased(async () => { await expectAllTensorsReleased(async () => {
const net = new faceapi.FaceLandmarkNet() const net = new FaceLandmarkNet()
await net.load('base/weights') await net.load('base/weights')
net.dispose() net.dispose()
}) })
......
test/tests/e2e/faceRecognitionNet.test.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import * as faceapi from '../../../src'; import { bufferToImage, FaceRecognitionNet, NetInput, toNetInput } from '../../../src';
import { NetInput } from '../../../src/NetInput'; import { createFaceRecognitionNet } from '../../../src/faceRecognitionNet';
import { expectAllTensorsReleased, describeWithNets } from '../../utils'; import { describeWithNets, expectAllTensorsReleased } from '../../utils';
import { toNetInput } from '../../../src';
describe('faceRecognitionNet', () => { describe('faceRecognitionNet', () => {
...@@ -16,11 +15,11 @@ describe('faceRecognitionNet', () => { ...@@ -16,11 +15,11 @@ describe('faceRecognitionNet', () => {
beforeAll(async () => { beforeAll(async () => {
const img1 = await (await fetch('base/test/images/face1.png')).blob() const img1 = await (await fetch('base/test/images/face1.png')).blob()
imgEl1 = await faceapi.bufferToImage(img1) imgEl1 = await bufferToImage(img1)
const img2 = await (await fetch('base/test/images/face2.png')).blob() const img2 = await (await fetch('base/test/images/face2.png')).blob()
imgEl2 = await faceapi.bufferToImage(img2) imgEl2 = await bufferToImage(img2)
const imgRect = await (await fetch('base/test/images/face_rectangular.png')).blob() const imgRect = await (await fetch('base/test/images/face_rectangular.png')).blob()
imgElRect = await faceapi.bufferToImage(imgRect) imgElRect = await bufferToImage(imgRect)
faceDescriptor1 = await (await fetch('base/test/data/faceDescriptor1.json')).json() faceDescriptor1 = await (await fetch('base/test/data/faceDescriptor1.json')).json()
faceDescriptor2 = await (await fetch('base/test/data/faceDescriptor2.json')).json() faceDescriptor2 = await (await fetch('base/test/data/faceDescriptor2.json')).json()
faceDescriptorRect = await (await fetch('base/test/data/faceDescriptorRect.json')).json() faceDescriptorRect = await (await fetch('base/test/data/faceDescriptorRect.json')).json()
...@@ -141,7 +140,7 @@ describe('faceRecognitionNet', () => { ...@@ -141,7 +140,7 @@ describe('faceRecognitionNet', () => {
await expectAllTensorsReleased(async () => { await expectAllTensorsReleased(async () => {
const res = await fetch('base/weights_uncompressed/face_recognition_model.weights') const res = await fetch('base/weights_uncompressed/face_recognition_model.weights')
const weights = new Float32Array(await res.arrayBuffer()) const weights = new Float32Array(await res.arrayBuffer())
const net = faceapi.createFaceRecognitionNet(weights) const net = createFaceRecognitionNet(weights)
net.dispose() net.dispose()
}) })
}) })
...@@ -152,7 +151,7 @@ describe('faceRecognitionNet', () => { ...@@ -152,7 +151,7 @@ describe('faceRecognitionNet', () => {
it('disposes all param tensors', async () => { it('disposes all param tensors', async () => {
await expectAllTensorsReleased(async () => { await expectAllTensorsReleased(async () => {
const net = new faceapi.FaceRecognitionNet() const net = new FaceRecognitionNet()
await net.load('base/weights') await net.load('base/weights')
net.dispose() net.dispose()
}) })
......
test/tests/e2e/mtcnn.test.ts
...@@ -46,7 +46,7 @@ describe('mtcnn', () => { ...@@ -46,7 +46,7 @@ describe('mtcnn', () => {
const results = await mtcnn.forward(imgEl, forwardParams) const results = await mtcnn.forward(imgEl, forwardParams)
expect(results.length).toEqual(6) expect(results.length).toEqual(6)
expectMtcnnResults(results, [5, 1, 4, 3, 2, 0], 6, 10) expectMtcnnResults(results, [5, 1, 4, 2, 3, 0], 6, 10)
}) })
it('scale steps passed, finds all faces', async () => { it('scale steps passed, finds all faces', async () => {
......
test/tests/extractFaceTensors.test.ts
import { extractFaceTensors, Rect } from '../../src'; import { bufferToImage, extractFaceTensors, Rect } from '../../src';
import { bufferToImage } from '../../src/utils';
describe('extractFaceTensors', () => { describe('extractFaceTensors', () => {
......
test/tests/extractFaces.test.ts
import { extractFaces, Rect } from '../../src'; import { bufferToImage, createCanvasFromMedia, extractFaces, Rect } from '../../src';
import { bufferToImage, createCanvasFromMedia } from '../../src/utils';
describe('extractFaces', () => { describe('extractFaces', () => {
......
test/tests/padToSquare.test.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
import { padToSquare } from '../../src/padToSquare';
import { ones, zeros } from '../utils';
describe('padToSquare', () => {
describe('even size', () => {
it('is padded to square by 2 columns', () => tf.tidy(() => {
const imgTensor = tf.tensor4d(Array(24).fill(1), [1, 4, 2, 3])
const result = padToSquare(imgTensor)
expect(result.shape).toEqual([1, 4, 4, 3])
const paddedCols = tf.unstack(result, 2)
expect(paddedCols.length).toEqual(4)
expect(paddedCols[0].dataSync()).toEqual(ones(12))
expect(paddedCols[1].dataSync()).toEqual(ones(12))
expect(paddedCols[2].dataSync()).toEqual(zeros(12))
expect(paddedCols[3].dataSync()).toEqual(zeros(12))
}))
it('is padded to square by 2 columns and centered', () => tf.tidy(() => {
const imgTensor = tf.tensor4d(Array(24).fill(1), [1, 4, 2, 3])
const result = padToSquare(imgTensor, true)
expect(result.shape).toEqual([1, 4, 4, 3])
const paddedCols = tf.unstack(result, 2)
expect(paddedCols.length).toEqual(4)
expect(paddedCols[0].dataSync()).toEqual(zeros(12))
expect(paddedCols[1].dataSync()).toEqual(ones(12))
expect(paddedCols[2].dataSync()).toEqual(ones(12))
expect(paddedCols[3].dataSync()).toEqual(zeros(12))
}))
it('is padded to square by 1 column', () => tf.tidy(() => {
const imgTensor = tf.tensor4d(Array(36).fill(1), [1, 4, 3, 3])
const result = padToSquare(imgTensor)
expect(result.shape).toEqual([1, 4, 4, 3])
const paddedCols = tf.unstack(result, 2)
expect(paddedCols.length).toEqual(4)
expect(paddedCols[0].dataSync()).toEqual(ones(12))
expect(paddedCols[1].dataSync()).toEqual(ones(12))
expect(paddedCols[2].dataSync()).toEqual(ones(12))
expect(paddedCols[3].dataSync()).toEqual(zeros(12))
}))
it('is padded to square by 1 column and centered', () => tf.tidy(() => {
const imgTensor = tf.tensor4d(Array(36).fill(1), [1, 4, 3, 3])
const result = padToSquare(imgTensor, true)
expect(result.shape).toEqual([1, 4, 4, 3])
const paddedCols = tf.unstack(result, 2)
expect(paddedCols.length).toEqual(4)
expect(paddedCols[0].dataSync()).toEqual(ones(12))
expect(paddedCols[1].dataSync()).toEqual(ones(12))
expect(paddedCols[2].dataSync()).toEqual(ones(12))
expect(paddedCols[3].dataSync()).toEqual(zeros(12))
}))
})
describe('uneven size', () => {
it('is padded to square by 3 columns', () => tf.tidy(() => {
const imgTensor = tf.tensor4d(Array(30).fill(1), [1, 5, 2, 3])
const result = padToSquare(imgTensor)
expect(result.shape).toEqual([1, 5, 5, 3])
const paddedCols = tf.unstack(result, 2)
expect(paddedCols.length).toEqual(5)
expect(paddedCols[0].dataSync()).toEqual(ones(15))
expect(paddedCols[1].dataSync()).toEqual(ones(15))
expect(paddedCols[2].dataSync()).toEqual(zeros(15))
expect(paddedCols[3].dataSync()).toEqual(zeros(15))
expect(paddedCols[4].dataSync()).toEqual(zeros(15))
}))
it('is padded to square by 3 columns and centered', () => tf.tidy(() => {
const imgTensor = tf.tensor4d(Array(30).fill(1), [1, 5, 2, 3])
const result = padToSquare(imgTensor, true)
expect(result.shape).toEqual([1, 5, 5, 3])
const paddedCols = tf.unstack(result, 2)
expect(paddedCols.length).toEqual(5)
expect(paddedCols[0].dataSync()).toEqual(zeros(15))
expect(paddedCols[1].dataSync()).toEqual(ones(15))
expect(paddedCols[2].dataSync()).toEqual(ones(15))
expect(paddedCols[3].dataSync()).toEqual(zeros(15))
expect(paddedCols[4].dataSync()).toEqual(zeros(15))
}))
it('is padded to square by 1 column', () => tf.tidy(() => {
const imgTensor = tf.tensor4d(Array(60).fill(1), [1, 5, 4, 3])
const result = padToSquare(imgTensor)
expect(result.shape).toEqual([1, 5, 5, 3])
const paddedCols = tf.unstack(result, 2)
expect(paddedCols.length).toEqual(5)
expect(paddedCols[0].dataSync()).toEqual(ones(15))
expect(paddedCols[1].dataSync()).toEqual(ones(15))
expect(paddedCols[2].dataSync()).toEqual(ones(15))
expect(paddedCols[3].dataSync()).toEqual(ones(15))
expect(paddedCols[4].dataSync()).toEqual(zeros(15))
}))
it('is padded to square by 1 column and centered', () => tf.tidy(() => {
const imgTensor = tf.tensor4d(Array(60).fill(1), [1, 5, 4, 3])
const result = padToSquare(imgTensor, true)
expect(result.shape).toEqual([1, 5, 5, 3])
const paddedCols = tf.unstack(result, 2)
expect(paddedCols.length).toEqual(5)
expect(paddedCols[0].dataSync()).toEqual(ones(15))
expect(paddedCols[1].dataSync()).toEqual(ones(15))
expect(paddedCols[2].dataSync()).toEqual(ones(15))
expect(paddedCols[3].dataSync()).toEqual(ones(15))
expect(paddedCols[4].dataSync()).toEqual(zeros(15))
}))
})
})
test/tests/toNetInput.test.ts deleted 100644 → 0
import * as tf from '@tensorflow/tfjs-core';
import { NetInput } from '../../src/NetInput';
import { toNetInput } from '../../src/toNetInput';
import { bufferToImage, createCanvasFromMedia } from '../../src/utils';
import { expectAllTensorsReleased } from '../utils';
describe('toNetInput', () => {
let imgEl: HTMLImageElement, canvasEl: HTMLCanvasElement
beforeAll(async () => {
const img = await (await fetch('base/test/images/face1.png')).blob()
imgEl = await bufferToImage(img)
canvasEl = createCanvasFromMedia(imgEl)
})
describe('valid args', () => {
it('from HTMLImageElement', async () => {
const netInput = await toNetInput(imgEl, true)
expect(netInput instanceof NetInput).toBe(true)
expect(netInput.batchSize).toEqual(1)
})
it('from HTMLCanvasElement', async () => {
const netInput = await toNetInput(canvasEl, true)
expect(netInput instanceof NetInput).toBe(true)
expect(netInput.batchSize).toEqual(1)
})
it('from HTMLImageElement array', async () => {
const netInput = await toNetInput([
imgEl,
imgEl
], true)
expect(netInput instanceof NetInput).toBe(true)
expect(netInput.batchSize).toEqual(2)
})
it('from HTMLCanvasElement array', async () => {
const netInput = await toNetInput([
canvasEl,
canvasEl
], true)
expect(netInput instanceof NetInput).toBe(true)
expect(netInput.batchSize).toEqual(2)
})
it('from mixed media array', async () => {
const netInput = await toNetInput([
imgEl,
canvasEl,
canvasEl
], true)
expect(netInput instanceof NetInput).toBe(true)
expect(netInput.batchSize).toEqual(3)
})
})
describe('invalid args', () => {
it('undefined', async () => {
let errorMessage
try {
await toNetInput(undefined as any)
} catch (error) {
errorMessage = error.message;
}
expect(errorMessage).toBe('toNetInput - expected media to be of type HTMLImageElement | HTMLVideoElement | HTMLCanvasElement | tf.Tensor3D, or to be an element id')
})
it('empty array', async () => {
let errorMessage
try {
await toNetInput([])
} catch (error) {
errorMessage = error.message;
}
expect(errorMessage).toBe('toNetInput - empty array passed as input')
})
it('undefined at input index 1', async () => {
let errorMessage
try {
await toNetInput([document.createElement('img'), undefined] as any)
} catch (error) {
errorMessage = error.message;
}
expect(errorMessage).toBe('toNetInput - at input index 1: expected media to be of type HTMLImageElement | HTMLVideoElement | HTMLCanvasElement | tf.Tensor3D, or to be an element id')
})
})
describe('no memory leaks', () => {
it('single image element', async () => {
await expectAllTensorsReleased(async () => {
const netInput = await toNetInput(imgEl)
netInput.dispose()
})
})
it('multiple image elements', async () => {
await expectAllTensorsReleased(async () => {
const netInput = await toNetInput([imgEl, imgEl, imgEl])
netInput.dispose()
})
})
it('single tf.Tensor3D', async () => {
const tensor = tf.fromPixels(imgEl)
await expectAllTensorsReleased(async () => {
const netInput = await toNetInput(tensor)
netInput.dispose()
})
tensor.dispose()
})
it('multiple tf.Tensor3Ds', async () => {
const tensors = [imgEl, imgEl, imgEl].map(el => tf.fromPixels(el))
await expectAllTensorsReleased(async () => {
const netInput = await toNetInput(tensors)
netInput.dispose()
})
tensors.forEach(t => t.dispose())
})
it('single batch size 1 tf.Tensor4Ds', async () => {
const tensor = tf.tidy(() => tf.fromPixels(imgEl).expandDims()) as tf.Tensor4D
await expectAllTensorsReleased(async () => {
const netInput = await toNetInput(tensor)
netInput.dispose()
})
tensor.dispose()
})
it('multiple batch size 1 tf.Tensor4Ds', async () => {
const tensors = [imgEl, imgEl, imgEl]
.map(el => tf.tidy(() => tf.fromPixels(el).expandDims())) as tf.Tensor4D[]
await expectAllTensorsReleased(async () => {
const netInput = await toNetInput(tensors)
netInput.dispose()
})
tensors.forEach(t => t.dispose())
})
})
})
test/utils.ts
import * as tf from '@tensorflow/tfjs-core'; import * as tf from '@tensorflow/tfjs-core';
import { IRect } from '../build/Rect';
import * as faceapi from '../src/'; import * as faceapi from '../src/';
import { NeuralNetwork } from '../src/commons/NeuralNetwork';
import { IPoint } from '../src/'; import { IPoint } from '../src/';
import { allFacesMtcnnFactory, allFacesSsdMobilenetv1Factory, allFacesTinyYolov2Factory } from '../src/allFacesFactory'; import { allFacesMtcnnFactory, allFacesSsdMobilenetv1Factory, allFacesTinyYolov2Factory } from '../src/allFacesFactory';
import { allFacesMtcnnFunction, allFacesSsdMobilenetv1Function, allFacesTinyYolov2, allFacesTinyYolov2Function } from '../src/globalApi'; import { allFacesMtcnnFunction, allFacesSsdMobilenetv1Function, allFacesTinyYolov2Function } from '../src/globalApi';
export function zeros(length: number): Float32Array { import { NeuralNetwork, IRect } from 'tfjs-image-recognition-base';
return new Float32Array(length)
}
export function ones(length: number): Float32Array { jasmine.DEFAULT_TIMEOUT_INTERVAL = 60000
return new Float32Array(length).fill(1)
}
export function expectMaxDelta(val1: number, val2: number, maxDelta: number) { export function expectMaxDelta(val1: number, val2: number, maxDelta: number) {
expect(Math.abs(val1 - val2)).toBeLessThan(maxDelta) expect(Math.abs(val1 - val2)).toBeLessThan(maxDelta)
...@@ -25,10 +19,6 @@ export async function expectAllTensorsReleased(fn: () => any) { ...@@ -25,10 +19,6 @@ export async function expectAllTensorsReleased(fn: () => any) {
expect(tf.memory().numTensors - numTensorsBefore).toEqual(0) expect(tf.memory().numTensors - numTensorsBefore).toEqual(0)
} }
export function tensor3D() {
return tf.tensor3d([[[0]]])
}
export function expectPointClose( export function expectPointClose(
result: IPoint, result: IPoint,
expectedPoint: IPoint, expectedPoint: IPoint,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment