test cases for batch input creation + e2e face landmark net batch input tests +…

karma.conf.js

@@ -2,6 +2,7 @@ const dataFiles = [
   'test/images/*.jpg',
   'test/images/*.png',
   'test/data/*.json',
+  'test/media/*.mp4',
   'weights/**/*'
 ].map(pattern => ({
   pattern,
@@ -24,6 +25,7 @@ module.exports = function(config) {
     karmaTypescriptConfig: {
       tsconfig: 'tsconfig.test.json'
     },
-    browsers: ['Chrome']
+    browsers: ['Chrome'],
+    browserNoActivityTimeout: 60000
   })
 }

test/data/faceLandmarkPositionsRect.json

+[{"x": 1.41040606983006, "y": 39.10264679789543}, {"x": 6.364097021520138, "y": 59.51536446809769}, {"x": 9.736957371234894, "y": 77.80020213127136}, {"x": 13.817122921347618, "y": 96.05930614471436}, {"x": 20.00808236002922, "y": 110.73724734783173}, {"x": 26.37610113620758, "y": 121.9759613275528}, {"x": 33.717534959316254, "y": 132.17582309246063}, {"x": 41.72355371713638, "y": 140.95428133010864}, {"x": 50.880420446395874, "y": 146.48704254627228}, {"x": 61.8572758436203, "y": 146.95429050922394}, {"x": 72.31115305423737, "y": 142.27578461170197}, {"x": 81.12127649784088, "y": 134.44229400157928}, {"x": 90.62704622745514, "y": 123.21754229068756}, {"x": 99.09188437461853, "y": 110.66853940486908}, {"x": 106.08789837360382, "y": 92.88722932338715}, {"x": 110.42981934547424, "y": 72.83273243904114}, {"x": 110.97252583503723, "y": 51.15715354681015}, {"x": 19.203872233629227, "y": 37.71118628978729}, {"x": 30.421346753835678, "y": 37.16629707813263}, {"x": 39.97739166021347, "y": 40.400769114494324}, {"x": 49.04381215572357, "y": 45.61803185939789}, {"x": 56.723132252693176, "y": 53.29851043224335}, {"x": 56.084791123867035, "y": 46.76303869485855}, {"x": 65.71077406406403, "y": 46.11447751522064}, {"x": 75.0335134267807, "y": 46.96638810634613}, {"x": 83.45012521743774, "y": 51.2211879491806}, {"x": 90.43923103809357, "y": 59.06730401515961}, {"x": 57.634793639183044, "y": 60.69884181022644}, {"x": 56.081126153469086, "y": 72.10934686660767}, {"x": 54.33526223897934, "y": 84.50968235731125}, {"x": 52.046833634376526, "y": 99.18286514282227}, {"x": 43.99328297376633, "y": 99.4484453201294}, {"x": 49.026043593883514, "y": 104.39998996257782}, {"x": 55.01739031076431, "y": 107.4150230884552}, {"x": 59.81684720516205, "y": 105.90647208690643}, {"x": 64.0739357471466, "y": 101.74840021133423}, {"x": 27.087938010692596, "y": 55.09133046865463}, {"x": 34.27653384208679, "y": 55.21461433172226}, {"x": 42.54729217290878, "y": 57.39108908176422}, {"x": 48.454275488853455, "y": 61.82331371307373}, {"x": 41.1783966422081, "y": 61.66855639219284}, {"x": 34.177026987075806, "y": 60.50751310586929}, {"x": 59.51743584871292, "y": 62.6235288977623}, {"x": 65.59782898426056, "y": 61.08310657739639}, {"x": 73.38363683223724, "y": 63.624403953552246}, {"x": 78.78980994224548, "y": 67.99472141265869}, {"x": 72.09050583839417, "y": 68.4499539732933}, {"x": 64.50075209140778, "y": 66.9551106095314}, {"x": 34.73506963253021, "y": 112.76923489570618}, {"x": 42.499454855918884, "y": 113.63057744503021}, {"x": 49.24748706817627, "y": 114.29424142837524}, {"x": 55.50651115179062, "y": 115.7518025636673}, {"x": 62.20475721359253, "y": 116.19831931591034}, {"x": 66.55372452735901, "y": 117.66684019565582}, {"x": 71.82598805427551, "y": 118.23190891742706}, {"x": 64.1076112985611, "y": 127.00456368923187}, {"x": 58.63429069519043, "y": 129.7624831199646}, {"x": 52.65244817733765, "y": 129.39003217220306}, {"x": 47.463079154491425, "y": 127.39845669269562}, {"x": 42.28087282180786, "y": 123.10187613964081}, {"x": 38.320072412490845, "y": 114.88845479488373}, {"x": 49.383214592933655, "y": 119.21215355396271}, {"x": 55.4133198261261, "y": 120.68333745002747}, {"x": 60.75339984893799, "y": 121.00764191150665}, {"x": 67.49378943443298, "y": 118.56007528305054}, {"x": 59.50732082128525, "y": 124.34931361675262}, {"x": 54.06349813938141, "y": 124.39959263801575}, {"x": 48.627867102622986, "y": 122.32591986656189}]
\ No newline at end of file

test/tests/e2e/faceLandmarkNet.test.ts

+import * as tf from '@tensorflow/tfjs-core';
+
 import * as faceapi from '../../../src';
+import { isTensor3D } from '../../../src/commons/isTensor';
 import { FaceLandmarks } from '../../../src/faceLandmarkNet/FaceLandmarks';
 import { Point } from '../../../src/Point';
+import { Dimensions, TMediaElement } from '../../../src/types';
 import { expectMaxDelta } from '../../utils';
 
+function getInputDims (input: tf.Tensor | TMediaElement): Dimensions {
+  if (input instanceof tf.Tensor) {
+    const [height, width] = input.shape.slice(isTensor3D(input) ? 0 : 1)
+    return { width, height }
+  }
+  return input
+}
+
 describe('faceLandmarkNet', () => {
 
   let imgEl1: HTMLImageElement
   let imgEl2: HTMLImageElement
+  let imgElRect: HTMLImageElement
   let faceLandmarkPositions1: Point[]
   let faceLandmarkPositions2: Point[]
+  let faceLandmarkPositionsRect: Point[]
 
   beforeAll(async () => {
     const img1 = await (await fetch('base/test/images/face1.png')).blob()
     imgEl1 = await faceapi.bufferToImage(img1)
     const img2 = await (await fetch('base/test/images/face2.png')).blob()
     imgEl2 = await faceapi.bufferToImage(img2)
+    const imgRect = await (await fetch('base/test/images/face_rectangular.png')).blob()
+    imgElRect = await faceapi.bufferToImage(imgRect)
     faceLandmarkPositions1 = await (await fetch('base/test/data/faceLandmarkPositions1.json')).json()
     faceLandmarkPositions2 = await (await fetch('base/test/data/faceLandmarkPositions2.json')).json()
+    faceLandmarkPositionsRect = await (await fetch('base/test/data/faceLandmarkPositionsRect.json')).json()
   })
 
   describe('uncompressed weights', () => {
@@ -29,7 +46,7 @@ describe('faceLandmarkNet', () => {
       faceLandmarkNet = faceapi.faceLandmarkNet(weights)
     })
 
-    it('computes face landmarks', async () => {
+    it('computes face landmarks for squared input', async () => {
       const { width, height } = imgEl1
 
       const result = await faceLandmarkNet.detectLandmarks(imgEl1) as FaceLandmarks
@@ -43,26 +60,17 @@ describe('faceLandmarkNet', () => {
       })
     })
 
-    it('computes face landmarks for batch input', async () => {
-      const imgEls = [imgEl1, imgEl2]
+    it('computes face landmarks for rectangular input', async () => {
+      const { width, height } = imgElRect
 
-      const faceLandmarkPositions = [
-        faceLandmarkPositions1,
-        faceLandmarkPositions2
-      ]
-      const results = await faceLandmarkNet.detectLandmarks(imgEls) as FaceLandmarks[]
-      expect(Array.isArray(results)).toBe(true)
-      expect(results.length).toEqual(2)
-      results.forEach((result, batchIdx) => {
-        const { width, height } = imgEls[batchIdx]
+      const result = await faceLandmarkNet.detectLandmarks(imgElRect) as FaceLandmarks
       expect(result.getImageWidth()).toEqual(width)
       expect(result.getImageHeight()).toEqual(height)
       expect(result.getShift().x).toEqual(0)
       expect(result.getShift().y).toEqual(0)
       result.getPositions().forEach(({ x, y }, i) => {
-          expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 0.1)
-          expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 0.1)
-        })
+        expectMaxDelta(x, faceLandmarkPositionsRect[i].x, 0.1)
+        expectMaxDelta(y, faceLandmarkPositionsRect[i].y, 0.1)
       })
     })
 
@@ -91,25 +99,127 @@ describe('faceLandmarkNet', () => {
       })
     })
 
-    it('computes face landmarks for batch input', async () => {
-      const imgEls = [imgEl1, imgEl2]
+    it('computes face landmarks for rectangular input', async () => {
+      const { width, height } = imgElRect
+
+      const result = await faceLandmarkNet.detectLandmarks(imgElRect) as FaceLandmarks
+      expect(result.getImageWidth()).toEqual(width)
+      expect(result.getImageHeight()).toEqual(height)
+      expect(result.getShift().x).toEqual(0)
+      expect(result.getShift().y).toEqual(0)
+      result.getPositions().forEach(({ x, y }, i) => {
+        expectMaxDelta(x, faceLandmarkPositionsRect[i].x, 6)
+        expectMaxDelta(y, faceLandmarkPositionsRect[i].y, 6)
+      })
+    })
+
+  })
+
+
+  describe('batch inputs', () => {
+
+    let faceLandmarkNet: faceapi.FaceLandmarkNet
+
+    beforeAll(async () => {
+      const res = await fetch('base/weights/uncompressed/face_landmark_68_model.weights')
+      const weights = new Float32Array(await res.arrayBuffer())
+      faceLandmarkNet = faceapi.faceLandmarkNet(weights)
+    })
+
+    it('computes face landmarks for batch of image elements', async () => {
+      const inputs = [imgEl1, imgEl2, imgElRect]
+
+      const faceLandmarkPositions = [
+        faceLandmarkPositions1,
+        faceLandmarkPositions2,
+        faceLandmarkPositionsRect
+      ]
+      const results = await faceLandmarkNet.detectLandmarks(inputs) as FaceLandmarks[]
+      expect(Array.isArray(results)).toBe(true)
+      expect(results.length).toEqual(3)
+      results.forEach((result, batchIdx) => {
+        const { width, height } = getInputDims(inputs[batchIdx])
+        expect(result.getImageWidth()).toEqual(width)
+        expect(result.getImageHeight()).toEqual(height)
+        expect(result.getShift().x).toEqual(0)
+        expect(result.getShift().y).toEqual(0)
+        result.getPositions().forEach(({ x, y }, i) => {
+          expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 0.1)
+          expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 0.1)
+        })
+      })
+    })
+
+    it('computes face landmarks for batch of tf.Tensor3D', async () => {
+      const inputs = [imgEl1, imgEl2, imgElRect].map(el => tf.fromPixels(el))
+
+      const faceLandmarkPositions = [
+        faceLandmarkPositions1,
+        faceLandmarkPositions2,
+        faceLandmarkPositionsRect
+      ]
+      const results = await faceLandmarkNet.detectLandmarks(inputs) as FaceLandmarks[]
+      expect(Array.isArray(results)).toBe(true)
+      expect(results.length).toEqual(3)
+      results.forEach((result, batchIdx) => {
+        const { width, height } = getInputDims(inputs[batchIdx])
+        expect(result.getImageWidth()).toEqual(width)
+        expect(result.getImageHeight()).toEqual(height)
+        expect(result.getShift().x).toEqual(0)
+        expect(result.getShift().y).toEqual(0)
+        result.getPositions().forEach(({ x, y }, i) => {
+          expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 0.1)
+          expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 0.1)
+        })
+      })
+    })
+
+    it('computes face landmarks for tf.Tensor4D', async () => {
+      const inputs = [imgEl1, imgEl2].map(el => tf.fromPixels(el))
 
       const faceLandmarkPositions = [
         faceLandmarkPositions1,
-        faceLandmarkPositions2
+        faceLandmarkPositions2,
+        faceLandmarkPositionsRect
       ]
-      const results = await faceLandmarkNet.detectLandmarks(imgEls) as FaceLandmarks[]
+      const results = await faceLandmarkNet.detectLandmarks(tf.stack(inputs) as tf.Tensor4D) as FaceLandmarks[]
       expect(Array.isArray(results)).toBe(true)
       expect(results.length).toEqual(2)
       results.forEach((result, batchIdx) => {
-        const { width, height } = imgEls[batchIdx]
+        const { width, height } = getInputDims(inputs[batchIdx])
+        expect(result.getImageWidth()).toEqual(width)
+        expect(result.getImageHeight()).toEqual(height)
+        expect(result.getShift().x).toEqual(0)
+        expect(result.getShift().y).toEqual(0)
+        result.getPositions().forEach(({ x, y }, i) => {
+          expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 0.1)
+          expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 0.1)
+        })
+      })
+    })
+
+    it('computes face landmarks for batch of mixed inputs', async () => {
+      const inputs = [imgEl1, tf.fromPixels(imgEl2), tf.fromPixels(imgElRect)]
+
+      const faceLandmarkPositions = [
+        faceLandmarkPositions1,
+        faceLandmarkPositions2,
+        faceLandmarkPositionsRect
+      ]
+
+
+      const results = await faceLandmarkNet.detectLandmarks(inputs) as FaceLandmarks[]
+      expect(Array.isArray(results)).toBe(true)
+      expect(results.length).toEqual(3)
+      results.forEach((result, batchIdx) => {
+        const { width, height } = getInputDims(inputs[batchIdx])
         expect(result.getImageWidth()).toEqual(width)
         expect(result.getImageHeight()).toEqual(height)
         expect(result.getShift().x).toEqual(0)
         expect(result.getShift().y).toEqual(0)
         result.getPositions().forEach(({ x, y }, i) => {
-          expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 3)
-          expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 3)
+          expectMaxDelta(x, faceLandmarkPositions[batchIdx][i].x, 0.1)
+          expectMaxDelta(y, faceLandmarkPositions[batchIdx][i].y, 0.1)
         })
       })
     })

test/tests/toNetInput.test.ts

 import { NetInput } from '../../src/NetInput';
 import { toNetInput } from '../../src/toNetInput';
+import { bufferToImage, createCanvasFromMedia } from '../../src/utils';
+
+async function createFakeHTMLVideoElement() {
+  const videoEl = document.createElement('video')
+  videoEl.muted = true
+  videoEl.src = 'base/test/media/video.mp4'
+  await videoEl.pause()
+  await videoEl.play()
+  return videoEl
+}
 
 describe('toNetInput', () => {
 
+  let imgEl: HTMLImageElement, canvasEl: HTMLCanvasElement
+
+  beforeAll(async () => {
+    const img = await (await fetch('base/test/images/face1.png')).blob()
+    imgEl = await bufferToImage(img)
+    canvasEl = createCanvasFromMedia(imgEl)
+  })
+
   describe('valid args', () => {
 
     it('from HTMLImageElement', async () => {
-      const netInput = await toNetInput(document.createElement('img'))
+      const netInput = await toNetInput(imgEl, true)
       expect(netInput instanceof NetInput).toBe(true)
-      expect(netInput.canvases.length).toEqual(1)
+      expect(netInput.batchSize).toEqual(1)
     })
 
     it('from HTMLVideoElement', async () => {
-      const videoEl = document.createElement('video')
-      spyOnProperty(videoEl, 'readyState', 'get').and.returnValue(4)
-
-      const netInput = await toNetInput(videoEl)
+      const videoEl = await createFakeHTMLVideoElement()
+      const netInput = await toNetInput(videoEl, true)
       expect(netInput instanceof NetInput).toBe(true)
-      expect(netInput.canvases.length).toEqual(1)
+      expect(netInput.batchSize).toEqual(1)
     })
 
     it('from HTMLCanvasElement', async () => {
-      const netInput = await toNetInput(document.createElement('canvas'))
+      const netInput = await toNetInput(canvasEl, true)
       expect(netInput instanceof NetInput).toBe(true)
-      expect(netInput.canvases.length).toEqual(1)
+      expect(netInput.batchSize).toEqual(1)
     })
 
     it('from HTMLImageElement array', async () => {
       const netInput = await toNetInput([
-        document.createElement('img'),
-        document.createElement('img')
-      ])
+        imgEl,
+        imgEl
+      ], true)
       expect(netInput instanceof NetInput).toBe(true)
-      expect(netInput.canvases.length).toEqual(2)
+      expect(netInput.batchSize).toEqual(2)
     })
 
     it('from HTMLVideoElement array', async () => {
       const videoElements = [
-        document.createElement('video'),
-        document.createElement('video')
+        await createFakeHTMLVideoElement(),
+        await createFakeHTMLVideoElement()
       ]
       videoElements.forEach(videoEl =>
         spyOnProperty(videoEl, 'readyState', 'get').and.returnValue(4)
       )
 
-      const netInput = await toNetInput(videoElements)
+      const netInput = await toNetInput(videoElements, true)
       expect(netInput instanceof NetInput).toBe(true)
-      expect(netInput.canvases.length).toEqual(2)
+      expect(netInput.batchSize).toEqual(2)
     })
 
     it('from HTMLCanvasElement array', async () => {
       const netInput = await toNetInput([
-        document.createElement('canvas'),
-        document.createElement('canvas')
-      ])
+        canvasEl,
+        canvasEl
+      ], true)
       expect(netInput instanceof NetInput).toBe(true)
-      expect(netInput.canvases.length).toEqual(2)
+      expect(netInput.batchSize).toEqual(2)
     })
 
     it('from mixed media array', async () => {
-      const videoEl = document.createElement('video')
-      spyOnProperty(videoEl, 'readyState', 'get').and.returnValue(4)
-
       const netInput = await toNetInput([
-        document.createElement('img'),
-        document.createElement('canvas'),
-        videoEl
-      ])
+        imgEl,
+        canvasEl,
+        await createFakeHTMLVideoElement()
+      ], true)
       expect(netInput instanceof NetInput).toBe(true)
-      expect(netInput.canvases.length).toEqual(3)
+      expect(netInput.batchSize).toEqual(3)
     })
 
   })
@@ -81,7 +94,7 @@ describe('toNetInput', () => {
       } catch (error) {
           errorMessage = error.message;
       }
-      expect(errorMessage).toBe('toNetInput - expected media to be of type HTMLImageElement | HTMLVideoElement | HTMLCanvasElement, or to be an element id')
+      expect(errorMessage).toBe('toNetInput - expected media to be of type HTMLImageElement | HTMLVideoElement | HTMLCanvasElement | tf.Tensor3D, or to be an element id')
     })
 
     it('empty array', async () => {
@@ -101,7 +114,7 @@ describe('toNetInput', () => {
       } catch (error) {
           errorMessage = error.message;
       }
-      expect(errorMessage).toBe('toNetInput - at input index 1: expected media to be of type HTMLImageElement | HTMLVideoElement | HTMLCanvasElement, or to be an element id')
+      expect(errorMessage).toBe('toNetInput - at input index 1: expected media to be of type HTMLImageElement | HTMLVideoElement | HTMLCanvasElement | tf.Tensor3D, or to be an element id')
     })
 
   })