Skip to content

F
face
Unverified Commit b8d62591 by justadudewhohacks Committed by GitHub
Options

Merge pull request #33 from justadudewhohacks/facelandmarks-fix

pad face landmark net input to square and center to prevent stretching of input image
parents fd698812 4177c46e
Showing with 258 additions and 79 deletions
.gitignore
node_modules node_modules
.rpt2_cache .rpt2_cache
\ No newline at end of file .env*
tmp
\ No newline at end of file
.npmignore
...@@ -4,4 +4,5 @@ node_modules ...@@ -4,4 +4,5 @@ node_modules
examples examples
test test
proto proto
weights weights
\ No newline at end of file tools
\ No newline at end of file
build/Rect.d.ts
...@@ -10,5 +10,6 @@ export declare class Rect implements IRect { ...@@ -10,5 +10,6 @@ export declare class Rect implements IRect {
width: number; width: number;
height: number; height: number;
constructor(x: number, y: number, width: number, height: number); constructor(x: number, y: number, width: number, height: number);
toSquare(): Rect;
floor(): Rect; floor(): Rect;
} }
build/Rect.js
...@@ -7,6 +7,19 @@ var Rect = /** @class */ (function () { ...@@ -7,6 +7,19 @@ var Rect = /** @class */ (function () {
this.width = width; this.width = width;
this.height = height; this.height = height;
} }
Rect.prototype.toSquare = function () {
var _a = this, x = _a.x, y = _a.y, width = _a.width, height = _a.height;
var diff = Math.abs(width - height);
if (width < height) {
x -= (diff / 2);
width += diff;
}
if (height < width) {
y -= (diff / 2);
height += diff;
}
return new Rect(x, y, width, height);
};
Rect.prototype.floor = function () { Rect.prototype.floor = function () {
return new Rect(Math.floor(this.x), Math.floor(this.y), Math.floor(this.width), Math.floor(this.height)); return new Rect(Math.floor(this.x), Math.floor(this.y), Math.floor(this.width), Math.floor(this.height));
}; };
......
build/Rect.js.map
{"version":3,"file":"Rect.js","sourceRoot":"","sources":["../src/Rect.ts"],"names":[],"mappings":";;AAOA;IAME,cAAY,CAAS,EAAE,CAAS,EAAE,KAAa,EAAE,MAAc;QAC7D,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACV,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACV,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAEM,oBAAK,GAAZ;QACE,OAAO,IAAI,IAAI,CACb,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EACtB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CACxB,CAAA;IACH,CAAC;IACH,WAAC;AAAD,CAAC,AArBD,IAqBC;AArBY,oBAAI"} {"version":3,"file":"Rect.js","sourceRoot":"","sources":["../src/Rect.ts"],"names":[],"mappings":";;AAOA;IAME,cAAY,CAAS,EAAE,CAAS,EAAE,KAAa,EAAE,MAAc;QAC7D,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACV,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACV,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;QAClB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAEM,uBAAQ,GAAf;QACM,IAAA,SAA8B,EAA5B,QAAC,EAAE,QAAC,EAAE,gBAAK,EAAE,kBAAM,CAAS;QAClC,IAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,MAAM,CAAC,CAAA;QACrC,IAAI,KAAK,GAAG,MAAM,EAAE;YAClB,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAA;YACf,KAAK,IAAI,IAAI,CAAA;SACd;QACD,IAAI,MAAM,GAAG,KAAK,EAAE;YAClB,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAA;YACf,MAAM,IAAI,IAAI,CAAA;SACf;QACD,OAAO,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;IACtC,CAAC;IAEM,oBAAK,GAAZ;QACE,OAAO,IAAI,IAAI,CACb,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAClB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EACtB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CACxB,CAAA;IACH,CAAC;IACH,WAAC;AAAD,CAAC,AAnCD,IAmCC;AAnCY,oBAAI"}
\ No newline at end of file \ No newline at end of file
build/faceLandmarkNet/FaceLandmarkNet.d.ts
...@@ -6,5 +6,7 @@ export declare class FaceLandmarkNet { ...@@ -6,5 +6,7 @@ export declare class FaceLandmarkNet {
private _params; private _params;
load(weightsOrUrl: Float32Array | string | undefined): Promise<void>; load(weightsOrUrl: Float32Array | string | undefined): Promise<void>;
extractWeights(weights: Float32Array): void; extractWeights(weights: Float32Array): void;
forwardTensor(imgTensor: tf.Tensor4D): tf.Tensor2D;
forward(input: tf.Tensor | NetInput | TNetInput): Promise<tf.Tensor2D>;
detectLandmarks(input: tf.Tensor | NetInput | TNetInput): Promise<FaceLandmarks>; detectLandmarks(input: tf.Tensor | NetInput | TNetInput): Promise<FaceLandmarks>;
} }
build/faceLandmarkNet/FaceLandmarkNet.js
...@@ -4,8 +4,10 @@ var tslib_1 = require("tslib"); ...@@ -4,8 +4,10 @@ var tslib_1 = require("tslib");
var tf = require("@tensorflow/tfjs-core"); var tf = require("@tensorflow/tfjs-core");
var convLayer_1 = require("../commons/convLayer"); var convLayer_1 = require("../commons/convLayer");
var getImageTensor_1 = require("../commons/getImageTensor"); var getImageTensor_1 = require("../commons/getImageTensor");
var padToSquare_1 = require("../padToSquare");
var Point_1 = require("../Point"); var Point_1 = require("../Point");
var toNetInput_1 = require("../toNetInput"); var toNetInput_1 = require("../toNetInput");
var utils_1 = require("../utils");
var extractParams_1 = require("./extractParams"); var extractParams_1 = require("./extractParams");
var FaceLandmarks_1 = require("./FaceLandmarks"); var FaceLandmarks_1 = require("./FaceLandmarks");
var fullyConnectedLayer_1 = require("./fullyConnectedLayer"); var fullyConnectedLayer_1 = require("./fullyConnectedLayer");
...@@ -45,6 +47,72 @@ var FaceLandmarkNet = /** @class */ (function () { ...@@ -45,6 +47,72 @@ var FaceLandmarkNet = /** @class */ (function () {
FaceLandmarkNet.prototype.extractWeights = function (weights) { FaceLandmarkNet.prototype.extractWeights = function (weights) {
this._params = extractParams_1.extractParams(weights); this._params = extractParams_1.extractParams(weights);
}; };
FaceLandmarkNet.prototype.forwardTensor = function (imgTensor) {
var params = this._params;
if (!params) {
throw new Error('FaceLandmarkNet - load model before inference');
}
return tf.tidy(function () {
var _a = imgTensor.shape.slice(), batchSize = _a[0], height = _a[1], width = _a[2];
var x = padToSquare_1.padToSquare(imgTensor, true);
var _b = x.shape.slice(1), heightAfterPadding = _b[0], widthAfterPadding = _b[1];
// work with 128 x 128 sized face images
if (heightAfterPadding !== 128 || widthAfterPadding !== 128) {
x = tf.image.resizeBilinear(x, [128, 128]);
}
var out = conv(x, params.conv0_params);
out = maxPool(out);
out = conv(out, params.conv1_params);
out = conv(out, params.conv2_params);
out = maxPool(out);
out = conv(out, params.conv3_params);
out = conv(out, params.conv4_params);
out = maxPool(out);
out = conv(out, params.conv5_params);
out = conv(out, params.conv6_params);
out = maxPool(out, [1, 1]);
out = conv(out, params.conv7_params);
var fc0 = tf.relu(fullyConnectedLayer_1.fullyConnectedLayer(out.as2D(out.shape[0], -1), params.fc0_params));
var fc1 = fullyConnectedLayer_1.fullyConnectedLayer(fc0, params.fc1_params);
var createInterleavedTensor = function (fillX, fillY) {
return tf.stack([
tf.fill([68], fillX),
tf.fill([68], fillY)
], 1).as2D(batchSize, 136);
};
/* shift coordinates back, to undo centered padding
((x * widthAfterPadding) - shiftX) / width
((y * heightAfterPadding) - shiftY) / height
*/
var shiftX = Math.floor(Math.abs(widthAfterPadding - width) / 2);
var shiftY = Math.floor(Math.abs(heightAfterPadding - height) / 2);
var landmarkTensor = fc1
.mul(createInterleavedTensor(widthAfterPadding, heightAfterPadding))
.sub(createInterleavedTensor(shiftX, shiftY))
.div(createInterleavedTensor(width, height));
return landmarkTensor;
});
};
FaceLandmarkNet.prototype.forward = function (input) {
return tslib_1.__awaiter(this, void 0, void 0, function () {
var netInput, _a;
return tslib_1.__generator(this, function (_b) {
switch (_b.label) {
case 0:
if (!(input instanceof tf.Tensor)) return [3 /*break*/, 1];
_a = input;
return [3 /*break*/, 3];
case 1: return [4 /*yield*/, toNetInput_1.toNetInput(input)];
case 2:
_a = _b.sent();
_b.label = 3;
case 3:
netInput = _a;
return [2 /*return*/, this.forwardTensor(getImageTensor_1.getImageTensor(netInput))];
}
});
});
};
FaceLandmarkNet.prototype.detectLandmarks = function (input) { FaceLandmarkNet.prototype.detectLandmarks = function (input) {
return tslib_1.__awaiter(this, void 0, void 0, function () { return tslib_1.__awaiter(this, void 0, void 0, function () {
var _this = this; var _this = this;
...@@ -52,9 +120,6 @@ var FaceLandmarkNet = /** @class */ (function () { ...@@ -52,9 +120,6 @@ var FaceLandmarkNet = /** @class */ (function () {
return tslib_1.__generator(this, function (_d) { return tslib_1.__generator(this, function (_d) {
switch (_d.label) { switch (_d.label) {
case 0: case 0:
if (!this._params) {
throw new Error('FaceLandmarkNet - load model before inference');
}
if (!(input instanceof tf.Tensor)) return [3 /*break*/, 1]; if (!(input instanceof tf.Tensor)) return [3 /*break*/, 1];
_a = input; _a = input;
return [3 /*break*/, 3]; return [3 /*break*/, 3];
...@@ -65,37 +130,18 @@ var FaceLandmarkNet = /** @class */ (function () { ...@@ -65,37 +130,18 @@ var FaceLandmarkNet = /** @class */ (function () {
case 3: case 3:
netInput = _a; netInput = _a;
outTensor = tf.tidy(function () { outTensor = tf.tidy(function () {
var params = _this._params;
var imgTensor = getImageTensor_1.getImageTensor(netInput); var imgTensor = getImageTensor_1.getImageTensor(netInput);
var _a = imgTensor.shape.slice(1), height = _a[0], width = _a[1]; var _a = imgTensor.shape.slice(1), height = _a[0], width = _a[1];
imageDimensions = { width: width, height: height }; imageDimensions = { width: width, height: height };
// work with 128 x 128 sized face images return _this.forwardTensor(imgTensor);
if (imgTensor.shape[1] !== 128 || imgTensor.shape[2] !== 128) {
imgTensor = tf.image.resizeBilinear(imgTensor, [128, 128]);
}
var out = conv(imgTensor, params.conv0_params);
out = maxPool(out);
out = conv(out, params.conv1_params);
out = conv(out, params.conv2_params);
out = maxPool(out);
out = conv(out, params.conv3_params);
out = conv(out, params.conv4_params);
out = maxPool(out);
out = conv(out, params.conv5_params);
out = conv(out, params.conv6_params);
out = maxPool(out, [1, 1]);
out = conv(out, params.conv7_params);
var fc0 = tf.relu(fullyConnectedLayer_1.fullyConnectedLayer(out.as2D(out.shape[0], -1), params.fc0_params));
var fc1 = fullyConnectedLayer_1.fullyConnectedLayer(fc0, params.fc1_params);
return fc1;
}); });
_c = (_b = Array).from; _c = (_b = Array).from;
return [4 /*yield*/, outTensor.data()]; return [4 /*yield*/, outTensor.data()];
case 4: case 4:
faceLandmarksArray = _c.apply(_b, [_d.sent()]); faceLandmarksArray = _c.apply(_b, [_d.sent()]);
outTensor.dispose(); outTensor.dispose();
xCoords = faceLandmarksArray.filter(function (c, i) { return (i - 1) % 2; }); xCoords = faceLandmarksArray.filter(function (_, i) { return utils_1.isEven(i); });
yCoords = faceLandmarksArray.filter(function (c, i) { return i % 2; }); yCoords = faceLandmarksArray.filter(function (_, i) { return !utils_1.isEven(i); });
return [2 /*return*/, new FaceLandmarks_1.FaceLandmarks(Array(68).fill(0).map(function (_, i) { return new Point_1.Point(xCoords[i], yCoords[i]); }), imageDimensions)]; return [2 /*return*/, new FaceLandmarks_1.FaceLandmarks(Array(68).fill(0).map(function (_, i) { return new Point_1.Point(xCoords[i], yCoords[i]); }), imageDimensions)];
} }
}); });
......
build/faceLandmarkNet/FaceLandmarkNet.js.map
{"version":3,"file":"FaceLandmarkNet.js","sourceRoot":"","sources":["../../src/faceLandmarkNet/FaceLandmarkNet.ts"],"names":[],"mappings":";;;AAAA,0CAA4C;AAE5C,kDAAiD;AACjD,4DAA2D;AAG3D,kCAAiC;AACjC,4CAA2C;AAE3C,iDAAgD;AAChD,iDAAgD;AAChD,6DAA4D;AAC5D,6DAA4D;AAG5D,cAAc,CAAc,EAAE,MAAkB;IAC9C,OAAO,qBAAS,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAA;AAC5C,CAAC;AAED,iBAAiB,CAAc,EAAE,OAAkC;IAAlC,wBAAA,EAAA,WAA6B,CAAC,EAAE,CAAC,CAAC;IACjE,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;AAChD,CAAC;AAED;IAAA;IAyEA,CAAC;IArEc,8BAAI,GAAjB,UAAkB,YAA+C;;;;;;wBAC/D,IAAI,YAAY,YAAY,YAAY,EAAE;4BACxC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAAA;4BACjC,sBAAM;yBACP;wBAED,IAAI,YAAY,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE;4BACpD,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAA;yBACzF;wBACD,KAAA,IAAI,CAAA;wBAAW,qBAAM,yCAAmB,CAAC,YAAY,CAAC,EAAA;;wBAAtD,GAAK,OAAO,GAAG,SAAuC,CAAA;;;;;KACvD;IAEM,wCAAc,GAArB,UAAsB,OAAqB;QACzC,IAAI,CAAC,OAAO,GAAG,6BAAa,CAAC,OAAO,CAAC,CAAA;IACvC,CAAC;IAEY,yCAAe,GAA5B,UAA6B,KAAuC;;;;;;;wBAClE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;4BACjB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;yBACjE;6BAEgB,CAAA,KAAK,YAAY,EAAE,CAAC,MAAM,CAAA,EAA1B,wBAA0B;wBACvC,KAAA,KAAK,CAAA;;4BACL,qBAAM,uBAAU,CAAC,KAAK,CAAC,EAAA;;wBAAvB,KAAA,SAAuB,CAAA;;;wBAFrB,QAAQ,KAEa;wBAIrB,SAAS,GAAG,EAAE,CAAC,IAAI,CAAC;4BACxB,IAAM,MAAM,GAAG,KAAI,CAAC,OAAO,CAAA;4BAE3B,IAAI,SAAS,GAAG,+BAAc,CAAC,QAAQ,CAAC,CAAA;4BAClC,IAAA,6BAA0C,EAAzC,cAAM,EAAE,aAAK,CAA4B;4BAChD,eAAe,GAAG,EAAE,KAAK,OAAA,EAAE,MAAM,QAAA,EAAE,CAAA;4BAGnC,wCAAwC;4BACxC,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE;gCAC5D,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;6BAC3D;4BAED,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;4BAC9C,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;4BAClB,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;4BACpC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;4BACpC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;4BAClB,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;4BACpC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;4BACpC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;4BAClB,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;4BACpC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;4BACpC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;4BAC1B,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;4BACpC,IAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,yCAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAA;4BACvF,IAAM,GAAG,GAAG,yCAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;4BAEvD,OAAO,GAAG,CAAA;wBACZ,CAAC,CAAC,CAAA;wBAEyB,KAAA,CAAA,KAAA,KAAK,CAAA,CAAC,IAAI,CAAA;wBAAC,qBAAM,SAAS,CAAC,IAAI,EAAE,EAAA;;wBAAtD,kBAAkB,GAAG,cAAW,SAAsB,EAAC;wBAC7D,SAAS,CAAC,OAAO,EAAE,CAAA;wBAEb,OAAO,GAAG,kBAAkB,CAAC,MAAM,CAAC,UAAC,CAAC,EAAE,CAAC,IAAK,OAAA,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAX,CAAW,CAAC,CAAA;wBAC1D,OAAO,GAAG,kBAAkB,CAAC,MAAM,CAAC,UAAC,CAAC,EAAE,CAAC,IAAK,OAAA,CAAC,GAAG,CAAC,EAAL,CAAK,CAAC,CAAA;wBAE1D,sBAAO,IAAI,6BAAa,CACtB,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAC,CAAC,EAAE,CAAC,IAAK,OAAA,IAAI,aAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAjC,CAAiC,CAAC,EAClE,eAA6B,CAC9B,EAAA;;;;KACF;IACH,sBAAC;AAAD,CAAC,AAzED,IAyEC;AAzEY,0CAAe"} {"version":3,"file":"FaceLandmarkNet.js","sourceRoot":"","sources":["../../src/faceLandmarkNet/FaceLandmarkNet.ts"],"names":[],"mappings":";;;AAAA,0CAA4C;AAE5C,kDAAiD;AACjD,4DAA2D;AAG3D,8CAA6C;AAC7C,kCAAiC;AACjC,4CAA2C;AAE3C,kCAAkC;AAClC,iDAAgD;AAChD,iDAAgD;AAChD,6DAA4D;AAC5D,6DAA4D;AAG5D,cAAc,CAAc,EAAE,MAAkB;IAC9C,OAAO,qBAAS,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAA;AAC5C,CAAC;AAED,iBAAiB,CAAc,EAAE,OAAkC;IAAlC,wBAAA,EAAA,WAA6B,CAAC,EAAE,CAAC,CAAC;IACjE,OAAO,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;AAChD,CAAC;AAED;IAAA;IA+GA,CAAC;IA3Gc,8BAAI,GAAjB,UAAkB,YAA+C;;;;;;wBAC/D,IAAI,YAAY,YAAY,YAAY,EAAE;4BACxC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAAA;4BACjC,sBAAM;yBACP;wBAED,IAAI,YAAY,IAAI,OAAO,YAAY,KAAK,QAAQ,EAAE;4BACpD,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAA;yBACzF;wBACD,KAAA,IAAI,CAAA;wBAAW,qBAAM,yCAAmB,CAAC,YAAY,CAAC,EAAA;;wBAAtD,GAAK,OAAO,GAAG,SAAuC,CAAA;;;;;KACvD;IAEM,wCAAc,GAArB,UAAsB,OAAqB;QACzC,IAAI,CAAC,OAAO,GAAG,6BAAa,CAAC,OAAO,CAAC,CAAA;IACvC,CAAC;IAEM,uCAAa,GAApB,UAAqB,SAAsB;QACzC,IAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAA;QAE3B,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;SACjE;QAED,OAAO,EAAE,CAAC,IAAI,CAAC;YACP,IAAA,4BAAoD,EAAnD,iBAAS,EAAE,cAAM,EAAE,aAAK,CAA2B;YAE1D,IAAI,CAAC,GAAG,yBAAW,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;YAC9B,IAAA,qBAA0D,EAAzD,0BAAkB,EAAE,yBAAiB,CAAoB;YAEhE,wCAAwC;YACxC,IAAI,kBAAkB,KAAK,GAAG,IAAI,iBAAiB,KAAK,GAAG,EAAE;gBAC3D,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;aAC3C;YAED,IAAI,GAAG,GAAG,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;YACtC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;YAClB,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;YACpC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;YACpC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;YAClB,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;YACpC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;YACpC,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;YAClB,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;YACpC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;YACpC,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;YAC1B,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;YACpC,IAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,yCAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAA;YACvF,IAAM,GAAG,GAAG,yCAAmB,CAAC,GAAG,EAAE,MAAM,CAAC,UAAU,CAAC,CAAA;YAGvD,IAAM,uBAAuB,GAAG,UAAC,KAAa,EAAE,KAAa;gBAC7D,OAAA,EAAE,CAAC,KAAK,CAAC;oBACP,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC;oBACpB,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC;iBACrB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC;YAH1B,CAG0B,CAAA;YAG1B;;;cAGE;YACF,IAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;YAClE,IAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;YACpE,IAAM,cAAc,GAAG,GAAG;iBACvB,GAAG,CAAC,uBAAuB,CAAC,iBAAiB,EAAE,kBAAkB,CAAC,CAAC;iBACnE,GAAG,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;iBAC5C,GAAG,CAAC,uBAAuB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAA;YAE9C,OAAO,cAA6B,CAAA;QACtC,CAAC,CAAC,CAAA;IACJ,CAAC;IAEY,iCAAO,GAApB,UAAqB,KAAuC;;;;;;6BACzC,CAAA,KAAK,YAAY,EAAE,CAAC,MAAM,CAAA,EAA1B,wBAA0B;wBACvC,KAAA,KAAK,CAAA;;4BACL,qBAAM,uBAAU,CAAC,KAAK,CAAC,EAAA;;wBAAvB,KAAA,SAAuB,CAAA;;;wBAFrB,QAAQ,KAEa;wBAE3B,sBAAO,IAAI,CAAC,aAAa,CAAC,+BAAc,CAAC,QAAQ,CAAC,CAAC,EAAA;;;;KACpD;IAEY,yCAAe,GAA5B,UAA6B,KAAuC;;;;;;;6BACjD,CAAA,KAAK,YAAY,EAAE,CAAC,MAAM,CAAA,EAA1B,wBAA0B;wBACvC,KAAA,KAAK,CAAA;;4BACL,qBAAM,uBAAU,CAAC,KAAK,CAAC,EAAA;;wBAAvB,KAAA,SAAuB,CAAA;;;wBAFrB,QAAQ,KAEa;wBAIrB,SAAS,GAAG,EAAE,CAAC,IAAI,CAAC;4BACxB,IAAM,SAAS,GAAG,+BAAc,CAAC,QAAQ,CAAC,CAAA;4BAEpC,IAAA,6BAA0C,EAAzC,cAAM,EAAE,aAAK,CAA4B;4BAChD,eAAe,GAAG,EAAE,KAAK,OAAA,EAAE,MAAM,QAAA,EAAE,CAAA;4BAEnC,OAAO,KAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAA;wBACtC,CAAC,CAAC,CAAA;wBAEyB,KAAA,CAAA,KAAA,KAAK,CAAA,CAAC,IAAI,CAAA;wBAAC,qBAAM,SAAS,CAAC,IAAI,EAAE,EAAA;;wBAAtD,kBAAkB,GAAG,cAAW,SAAsB,EAAC;wBAC7D,SAAS,CAAC,OAAO,EAAE,CAAA;wBAEb,OAAO,GAAG,kBAAkB,CAAC,MAAM,CAAC,UAAC,CAAC,EAAE,CAAC,IAAK,OAAA,cAAM,CAAC,CAAC,CAAC,EAAT,CAAS,CAAC,CAAA;wBACxD,OAAO,GAAG,kBAAkB,CAAC,MAAM,CAAC,UAAC,CAAC,EAAE,CAAC,IAAK,OAAA,CAAC,cAAM,CAAC,CAAC,CAAC,EAAV,CAAU,CAAC,CAAA;wBAE/D,sBAAO,IAAI,6BAAa,CACtB,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,UAAC,CAAC,EAAE,CAAC,IAAK,OAAA,IAAI,aAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAjC,CAAiC,CAAC,EAClE,eAA6B,CAC9B,EAAA;;;;KACF;IACH,sBAAC;AAAD,CAAC,AA/GD,IA+GC;AA/GY,0CAAe"}
\ No newline at end of file \ No newline at end of file
dist/face-api.js
...@@ -510,6 +510,19 @@ ...@@ -510,6 +510,19 @@
this.width = width; this.width = width;
this.height = height; this.height = height;
} }
Rect.prototype.toSquare = function () {
var _a = this, x = _a.x, y = _a.y, width = _a.width, height = _a.height;
var diff = Math.abs(width - height);
if (width < height) {
x -= (diff / 2);
width += diff;
}
if (height < width) {
y -= (diff / 2);
height += diff;
}
return new Rect(x, y, width, height);
};
Rect.prototype.floor = function () { Rect.prototype.floor = function () {
return new Rect(Math.floor(this.x), Math.floor(this.y), Math.floor(this.width), Math.floor(this.height)); return new Rect(Math.floor(this.x), Math.floor(this.y), Math.floor(this.width), Math.floor(this.height));
}; };
...@@ -1732,6 +1745,72 @@ ...@@ -1732,6 +1745,72 @@
FaceLandmarkNet.prototype.extractWeights = function (weights) { FaceLandmarkNet.prototype.extractWeights = function (weights) {
this._params = extractParams$1(weights); this._params = extractParams$1(weights);
}; };
FaceLandmarkNet.prototype.forwardTensor = function (imgTensor) {
var params = this._params;
if (!params) {
throw new Error('FaceLandmarkNet - load model before inference');
}
return tidy(function () {
var _a = imgTensor.shape.slice(), batchSize = _a[0], height = _a[1], width = _a[2];
var x = padToSquare(imgTensor, true);
var _b = x.shape.slice(1), heightAfterPadding = _b[0], widthAfterPadding = _b[1];
// work with 128 x 128 sized face images
if (heightAfterPadding !== 128 || widthAfterPadding !== 128) {
x = image.resizeBilinear(x, [128, 128]);
}
var out = conv(x, params.conv0_params);
out = maxPool$1(out);
out = conv(out, params.conv1_params);
out = conv(out, params.conv2_params);
out = maxPool$1(out);
out = conv(out, params.conv3_params);
out = conv(out, params.conv4_params);
out = maxPool$1(out);
out = conv(out, params.conv5_params);
out = conv(out, params.conv6_params);
out = maxPool$1(out, [1, 1]);
out = conv(out, params.conv7_params);
var fc0 = relu(fullyConnectedLayer(out.as2D(out.shape[0], -1), params.fc0_params));
var fc1 = fullyConnectedLayer(fc0, params.fc1_params);
var createInterleavedTensor = function (fillX, fillY) {
return stack([
fill([68], fillX),
fill([68], fillY)
], 1).as2D(batchSize, 136);
};
/* shift coordinates back, to undo centered padding
((x * widthAfterPadding) - shiftX) / width
((y * heightAfterPadding) - shiftY) / height
*/
var shiftX = Math.floor(Math.abs(widthAfterPadding - width) / 2);
var shiftY = Math.floor(Math.abs(heightAfterPadding - height) / 2);
var landmarkTensor = fc1
.mul(createInterleavedTensor(widthAfterPadding, heightAfterPadding))
.sub(createInterleavedTensor(shiftX, shiftY))
.div(createInterleavedTensor(width, height));
return landmarkTensor;
});
};
FaceLandmarkNet.prototype.forward = function (input) {
return __awaiter$1(this, void 0, void 0, function () {
var netInput, _a;
return __generator$1(this, function (_b) {
switch (_b.label) {
case 0:
if (!(input instanceof Tensor)) return [3 /*break*/, 1];
_a = input;
return [3 /*break*/, 3];
case 1: return [4 /*yield*/, toNetInput(input)];
case 2:
_a = _b.sent();
_b.label = 3;
case 3:
netInput = _a;
return [2 /*return*/, this.forwardTensor(getImageTensor(netInput))];
}
});
});
};
FaceLandmarkNet.prototype.detectLandmarks = function (input) { FaceLandmarkNet.prototype.detectLandmarks = function (input) {
return __awaiter$1(this, void 0, void 0, function () { return __awaiter$1(this, void 0, void 0, function () {
var _this = this; var _this = this;
...@@ -1739,9 +1818,6 @@ ...@@ -1739,9 +1818,6 @@
return __generator$1(this, function (_d) { return __generator$1(this, function (_d) {
switch (_d.label) { switch (_d.label) {
case 0: case 0:
if (!this._params) {
throw new Error('FaceLandmarkNet - load model before inference');
}
if (!(input instanceof Tensor)) return [3 /*break*/, 1]; if (!(input instanceof Tensor)) return [3 /*break*/, 1];
_a = input; _a = input;
return [3 /*break*/, 3]; return [3 /*break*/, 3];
...@@ -1752,37 +1828,18 @@ ...@@ -1752,37 +1828,18 @@
case 3: case 3:
netInput = _a; netInput = _a;
outTensor = tidy(function () { outTensor = tidy(function () {
var params = _this._params;
var imgTensor = getImageTensor(netInput); var imgTensor = getImageTensor(netInput);
var _a = imgTensor.shape.slice(1), height = _a[0], width = _a[1]; var _a = imgTensor.shape.slice(1), height = _a[0], width = _a[1];
imageDimensions = { width: width, height: height }; imageDimensions = { width: width, height: height };
// work with 128 x 128 sized face images return _this.forwardTensor(imgTensor);
if (imgTensor.shape[1] !== 128 || imgTensor.shape[2] !== 128) {
imgTensor = image.resizeBilinear(imgTensor, [128, 128]);
}
var out = conv(imgTensor, params.conv0_params);
out = maxPool$1(out);
out = conv(out, params.conv1_params);
out = conv(out, params.conv2_params);
out = maxPool$1(out);
out = conv(out, params.conv3_params);
out = conv(out, params.conv4_params);
out = maxPool$1(out);
out = conv(out, params.conv5_params);
out = conv(out, params.conv6_params);
out = maxPool$1(out, [1, 1]);
out = conv(out, params.conv7_params);
var fc0 = relu(fullyConnectedLayer(out.as2D(out.shape[0], -1), params.fc0_params));
var fc1 = fullyConnectedLayer(fc0, params.fc1_params);
return fc1;
}); });
_c = (_b = Array).from; _c = (_b = Array).from;
return [4 /*yield*/, outTensor.data()]; return [4 /*yield*/, outTensor.data()];
case 4: case 4:
faceLandmarksArray = _c.apply(_b, [_d.sent()]); faceLandmarksArray = _c.apply(_b, [_d.sent()]);
outTensor.dispose(); outTensor.dispose();
xCoords = faceLandmarksArray.filter(function (c, i) { return (i - 1) % 2; }); xCoords = faceLandmarksArray.filter(function (_, i) { return isEven(i); });
yCoords = faceLandmarksArray.filter(function (c, i) { return i % 2; }); yCoords = faceLandmarksArray.filter(function (_, i) { return !isEven(i); });
return [2 /*return*/, new FaceLandmarks(Array(68).fill(0).map(function (_, i) { return new Point(xCoords[i], yCoords[i]); }), imageDimensions)]; return [2 /*return*/, new FaceLandmarks(Array(68).fill(0).map(function (_, i) { return new Point(xCoords[i], yCoords[i]); }), imageDimensions)];
} }
}); });
......
dist/face-api.js.map
This source diff could not be displayed because it is too large. You can view the blob instead.
dist/face-api.min.js
This source diff could not be displayed because it is too large. You can view the blob instead.
src/Rect.ts
...@@ -18,6 +18,20 @@ export class Rect implements IRect { ...@@ -18,6 +18,20 @@ export class Rect implements IRect {
this.height = height this.height = height
} }
public toSquare(): Rect {
let { x, y, width, height } = this
const diff = Math.abs(width - height)
if (width < height) {
x -= (diff / 2)
width += diff
}
if (height < width) {
y -= (diff / 2)
height += diff
}
return new Rect(x, y, width, height)
}
public floor(): Rect { public floor(): Rect {
return new Rect( return new Rect(
Math.floor(this.x), Math.floor(this.x),
......
src/faceLandmarkNet/FaceLandmarkNet.ts
...@@ -4,9 +4,11 @@ import { convLayer } from '../commons/convLayer'; ...@@ -4,9 +4,11 @@ import { convLayer } from '../commons/convLayer';
import { getImageTensor } from '../commons/getImageTensor'; import { getImageTensor } from '../commons/getImageTensor';
import { ConvParams } from '../commons/types'; import { ConvParams } from '../commons/types';
import { NetInput } from '../NetInput'; import { NetInput } from '../NetInput';
import { padToSquare } from '../padToSquare';
import { Point } from '../Point'; import { Point } from '../Point';
import { toNetInput } from '../toNetInput'; import { toNetInput } from '../toNetInput';
import { Dimensions, TNetInput } from '../types'; import { Dimensions, TNetInput } from '../types';
import { isEven } from '../utils';
import { extractParams } from './extractParams'; import { extractParams } from './extractParams';
import { FaceLandmarks } from './FaceLandmarks'; import { FaceLandmarks } from './FaceLandmarks';
import { fullyConnectedLayer } from './fullyConnectedLayer'; import { fullyConnectedLayer } from './fullyConnectedLayer';
...@@ -41,31 +43,25 @@ export class FaceLandmarkNet { ...@@ -41,31 +43,25 @@ export class FaceLandmarkNet {
this._params = extractParams(weights) this._params = extractParams(weights)
} }
public async detectLandmarks(input: tf.Tensor | NetInput | TNetInput) { public forwardTensor(imgTensor: tf.Tensor4D): tf.Tensor2D {
if (!this._params) { const params = this._params
if (!params) {
throw new Error('FaceLandmarkNet - load model before inference') throw new Error('FaceLandmarkNet - load model before inference')
} }
const netInput = input instanceof tf.Tensor return tf.tidy(() => {
? input const [batchSize, height, width] = imgTensor.shape.slice()
: await toNetInput(input)
let imageDimensions: Dimensions | undefined
const outTensor = tf.tidy(() => {
const params = this._params
let imgTensor = getImageTensor(netInput)
const [height, width] = imgTensor.shape.slice(1)
imageDimensions = { width, height }
let x = padToSquare(imgTensor, true)
const [heightAfterPadding, widthAfterPadding] = x.shape.slice(1)
// work with 128 x 128 sized face images // work with 128 x 128 sized face images
if (imgTensor.shape[1] !== 128 || imgTensor.shape[2] !== 128) { if (heightAfterPadding !== 128 || widthAfterPadding !== 128) {
imgTensor = tf.image.resizeBilinear(imgTensor, [128, 128]) x = tf.image.resizeBilinear(x, [128, 128])
} }
let out = conv(imgTensor, params.conv0_params) let out = conv(x, params.conv0_params)
out = maxPool(out) out = maxPool(out)
out = conv(out, params.conv1_params) out = conv(out, params.conv1_params)
out = conv(out, params.conv2_params) out = conv(out, params.conv2_params)
...@@ -80,14 +76,58 @@ export class FaceLandmarkNet { ...@@ -80,14 +76,58 @@ export class FaceLandmarkNet {
const fc0 = tf.relu(fullyConnectedLayer(out.as2D(out.shape[0], -1), params.fc0_params)) const fc0 = tf.relu(fullyConnectedLayer(out.as2D(out.shape[0], -1), params.fc0_params))
const fc1 = fullyConnectedLayer(fc0, params.fc1_params) const fc1 = fullyConnectedLayer(fc0, params.fc1_params)
return fc1
const createInterleavedTensor = (fillX: number, fillY: number) =>
tf.stack([
tf.fill([68], fillX),
tf.fill([68], fillY)
], 1).as2D(batchSize, 136)
/* shift coordinates back, to undo centered padding
((x * widthAfterPadding) - shiftX) / width
((y * heightAfterPadding) - shiftY) / height
*/
const shiftX = Math.floor(Math.abs(widthAfterPadding - width) / 2)
const shiftY = Math.floor(Math.abs(heightAfterPadding - height) / 2)
const landmarkTensor = fc1
.mul(createInterleavedTensor(widthAfterPadding, heightAfterPadding))
.sub(createInterleavedTensor(shiftX, shiftY))
.div(createInterleavedTensor(width, height))
return landmarkTensor as tf.Tensor2D
})
}
public async forward(input: tf.Tensor | NetInput | TNetInput): Promise<tf.Tensor2D> {
const netInput = input instanceof tf.Tensor
? input
: await toNetInput(input)
return this.forwardTensor(getImageTensor(netInput))
}
public async detectLandmarks(input: tf.Tensor | NetInput | TNetInput) {
const netInput = input instanceof tf.Tensor
? input
: await toNetInput(input)
let imageDimensions: Dimensions | undefined
const outTensor = tf.tidy(() => {
const imgTensor = getImageTensor(netInput)
const [height, width] = imgTensor.shape.slice(1)
imageDimensions = { width, height }
return this.forwardTensor(imgTensor)
}) })
const faceLandmarksArray = Array.from(await outTensor.data()) const faceLandmarksArray = Array.from(await outTensor.data())
outTensor.dispose() outTensor.dispose()
const xCoords = faceLandmarksArray.filter((c, i) => (i - 1) % 2) const xCoords = faceLandmarksArray.filter((_, i) => isEven(i))
const yCoords = faceLandmarksArray.filter((c, i) => i % 2) const yCoords = faceLandmarksArray.filter((_, i) => !isEven(i))
return new FaceLandmarks( return new FaceLandmarks(
Array(68).fill(0).map((_, i) => new Point(xCoords[i], yCoords[i])), Array(68).fill(0).map((_, i) => new Point(xCoords[i], yCoords[i])),
......
test/tests/e2e/faceLandmarkNet.test.ts
...@@ -31,7 +31,10 @@ describe('faceLandmarkNet', () => { ...@@ -31,7 +31,10 @@ describe('faceLandmarkNet', () => {
expect(result.getImageHeight()).toEqual(height) expect(result.getImageHeight()).toEqual(height)
expect(result.getShift().x).toEqual(0) expect(result.getShift().x).toEqual(0)
expect(result.getShift().y).toEqual(0) expect(result.getShift().y).toEqual(0)
expect(result.getPositions().map(({ x, y }) => ({ x, y }))).toEqual(faceLandmarkPositions) result.getPositions().forEach(({ x, y }, i) => {
expectMaxDelta(x, faceLandmarkPositions[i].x, 0.1)
expectMaxDelta(y, faceLandmarkPositions[i].y, 0.1)
})
}) })
}) })
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment