fixed image size overflowing in extractFaceTensors and extractFaces when…

.npmignore

@@ -2,7 +2,8 @@ node_modules
 .rpt2_cache
 
 examples
-test
 proto
 weights
+weights_uncompressed
+test
 tools
\ No newline at end of file

src/Rect.ts

@@ -18,6 +18,14 @@ export class Rect implements IRect {
     this.height = height
   }
 
+  public get right() {
+    return this.x + this.width
+  }
+
+  public get bottom() {
+    return this.y + this.height
+  }
+
   public toSquare(): Rect {
     let { x, y, width, height } = this
     const diff = Math.abs(width - height)
@@ -45,4 +53,17 @@ export class Rect implements IRect {
       Math.floor(this.height)
     )
   }
+
+  public clipAtImageBorders(imgWidth: number, imgHeight: number): Rect {
+    const { x, y, right, bottom } = this
+    const clippedX = Math.max(x, 0)
+    const clippedY = Math.max(y, 0)
+
+    const newWidth = right - clippedX
+    const newHeight = bottom - clippedY
+    const clippedWidth = Math.min(newWidth, imgWidth - clippedX)
+    const clippedHeight = Math.min(newHeight, imgHeight - clippedY)
+
+    return (new Rect(clippedX, clippedY, clippedWidth, clippedHeight)).floor()
+  }
 }
\ No newline at end of file

src/extractFaceTensors.ts

@@ -36,9 +36,11 @@ export async function extractFaceTensors(
 
     const boxes = detections.map(
       det => det instanceof FaceDetection
-        ? det.forSize(imgWidth, imgHeight).getBox().floor()
+        ? det.forSize(imgWidth, imgHeight).getBox()
         : det
     )
+      .map(box => box.clipAtImageBorders(imgWidth, imgHeight))
+
     const faceTensors = boxes.map(({ x, y, width, height }) =>
       tf.slice(imgTensor, [0, y, x, 0], [1, height, width, numChannels])
     )

src/extractFaces.ts

@@ -42,6 +42,8 @@ export async function extractFaces(
       ? det.forSize(canvas.width, canvas.height).getBox().floor()
       : det
   )
+    .map(box => box.clipAtImageBorders(canvas.width, canvas.height))
+
   return boxes.map(({ x, y, width, height }) => {
     const faceImg = createCanvas({ width, height })
     getContext2dOrThrow(faceImg)

test/tests/extractFaceTensors.test.ts

+import { extractFaceTensors, Rect } from '../../src';
+import { bufferToImage } from '../../src/utils';
+
+describe('extractFaceTensors', () => {
+
+  let imgEl: HTMLImageElement
+
+  beforeAll(async () => {
+    const img = await (await fetch('base/test/images/face1.png')).blob()
+    imgEl = await bufferToImage(img)
+  })
+
+  describe('extracts tensors', () => {
+
+    it('single box', async () => {
+      const rect = new Rect(0, 0, 50, 60)
+      const tensors = await extractFaceTensors(imgEl, [rect])
+
+      expect(tensors.length).toEqual(1)
+      expect(tensors[0].shape).toEqual([1, 60, 50, 3])
+      tensors[0].dispose()
+    })
+
+    it('multiple boxes', async () => {
+      const rects = [
+        new Rect(0, 0, 50, 60),
+        new Rect(50, 50, 70, 80),
+      ]
+      const tensors = await extractFaceTensors(imgEl, rects)
+
+      expect(tensors.length).toEqual(2)
+      expect(tensors[0].shape).toEqual([1, 60, 50, 3])
+      expect(tensors[1].shape).toEqual([1, 80, 70, 3])
+      tensors[0].dispose()
+      tensors[1].dispose()
+    })
+
+  })
+
+  describe('box out of image borders', () => {
+
+    it('clips upper left corner', async () => {
+      const rect = new Rect(-10, -10, 110, 110)
+      const tensors = await extractFaceTensors(imgEl, [rect])
+
+      expect(tensors[0].shape).toEqual([1, 100, 100, 3])
+      tensors[0].dispose()
+    })
+
+    it('clips bottom right corner', async () => {
+      const rect = new Rect(imgEl.width - 100, imgEl.height - 100, 110, 110)
+      const tensors = await extractFaceTensors(imgEl, [rect])
+
+      expect(tensors[0].shape).toEqual([1, 100, 100, 3])
+      tensors[0].dispose()
+    })
+
+    it('clips upper left and bottom right corners', async () => {
+      const rect = new Rect(-10, -10, imgEl.width + 20, imgEl.height + 20)
+      const tensors = await extractFaceTensors(imgEl, [rect])
+
+      expect(tensors[0].shape).toEqual([1, imgEl.height, imgEl.width, 3])
+      tensors[0].dispose()
+    })
+
+  })
+
+})

test/tests/extractFaces.test.ts

+import { extractFaces, Rect } from '../../src';
+import { bufferToImage, createCanvasFromMedia } from '../../src/utils';
+
+describe('extractFaces', () => {
+
+  let imgEl: HTMLImageElement, canvasEl: HTMLCanvasElement
+
+  beforeAll(async () => {
+    const img = await (await fetch('base/test/images/face1.png')).blob()
+    imgEl = await bufferToImage(img)
+    canvasEl = createCanvasFromMedia(imgEl)
+  })
+
+  describe('extracts canvases', () => {
+
+    it('HTMLImageElement, single box', async () => {
+      const rect = new Rect(0, 0, 50, 60)
+      const canvases = await extractFaces(imgEl, [rect])
+
+      expect(canvases.length).toEqual(1)
+      expect(canvases[0] instanceof HTMLCanvasElement).toBe(true)
+      expect(canvases[0].width).toEqual(50)
+      expect(canvases[0].height).toEqual(60)
+    })
+
+    it('HTMLImageElement, multiple boxes', async () => {
+      const rects = [
+        new Rect(0, 0, 50, 60),
+        new Rect(50, 50, 70, 80),
+      ]
+      const canvases = await extractFaces(imgEl, rects)
+
+      expect(canvases.length).toEqual(2)
+      expect(canvases[0] instanceof HTMLCanvasElement).toBe(true)
+      expect(canvases[0].width).toEqual(50)
+      expect(canvases[0].height).toEqual(60)
+      expect(canvases[1] instanceof HTMLCanvasElement).toBe(true)
+      expect(canvases[1].width).toEqual(70)
+      expect(canvases[1].height).toEqual(80)
+    })
+
+    it('HTMLCanvasElement, single box', async () => {
+      const rect = new Rect(0, 0, 50, 60)
+      const canvases = await extractFaces(canvasEl, [rect])
+
+      expect(canvases.length).toEqual(1)
+      expect(canvases[0] instanceof HTMLCanvasElement).toBe(true)
+      expect(canvases[0].width).toEqual(50)
+      expect(canvases[0].height).toEqual(60)
+    })
+
+    it('HTMLCanvasElement, multiple boxes', async () => {
+      const rects = [
+        new Rect(0, 0, 50, 60),
+        new Rect(50, 50, 70, 80),
+      ]
+      const canvases = await extractFaces(canvasEl, rects)
+
+      expect(canvases.length).toEqual(2)
+      expect(canvases[0] instanceof HTMLCanvasElement).toBe(true)
+      expect(canvases[0].width).toEqual(50)
+      expect(canvases[0].height).toEqual(60)
+      expect(canvases[1] instanceof HTMLCanvasElement).toBe(true)
+      expect(canvases[1].width).toEqual(70)
+      expect(canvases[1].height).toEqual(80)
+    })
+
+  })
+
+  describe('box out of image borders', () => {
+
+    it('clips upper left corner', async () => {
+      const rect = new Rect(-10, -10, 110, 110)
+      const canvases = await extractFaces(imgEl, [rect])
+
+      expect(canvases[0].width).toEqual(100)
+      expect(canvases[0].height).toEqual(100)
+    })
+
+    it('clips bottom right corner', async () => {
+      const rect = new Rect(imgEl.width - 100, imgEl.height - 100, 110, 110)
+      const canvases = await extractFaces(imgEl, [rect])
+
+      expect(canvases[0].width).toEqual(100)
+      expect(canvases[0].height).toEqual(100)
+    })
+
+    it('clips upper left and bottom right corners', async () => {
+      const rect = new Rect(-10, -10, imgEl.width + 20, imgEl.height + 20)
+      const canvases = await extractFaces(imgEl, [rect])
+
+      expect(canvases[0].width).toEqual(imgEl.width)
+      expect(canvases[0].height).toEqual(imgEl.height)
+    })
+
+  })
+
+})