better color extraction

examples/examples-browser/public/js/extract-colors.js

@@ -80,15 +80,38 @@ var extractColors = function( ctx, where, k, debug){
 
   var r, g, b;
   var colors = [];
+
+  var debugPixels = document.getElementById('ccolors').getContext('2d');
+  var debugData = debugPixels.getImageData( 0, 0, 100, 480 ),
+    dd = debugData.data;
+  var pt = (extracting*100*120*4)
+
   for( var i = 0, _i = points.length; i < _i; i++ ){
     var p = points[ i ], pointer = ( p[ 0 ] - minX + ( p[ 1 ] - minY ) * w ) * 4;
     r = data[ pointer ];
     g = data[ pointer + 1 ];
     b = data[ pointer + 2 ];
+
+    pt = ((extracting*100*120+i*2+((i/50)|0)*100)*4);
+    dd[pt++] = r; dd[pt++] = g; dd[pt++] = b;
+    dd[pt++]=255;
+
+    dd[pt++] = r; dd[pt++] = g; dd[pt++] = b;
+    dd[pt++]=255;
+    pt = ((extracting*100*120+i*2+((i/50)|0)*100)*4)+100*4;
+    dd[pt++] = r; dd[pt++] = g; dd[pt++] = b;
+    dd[pt++]=255;
+
+    dd[pt++] = r; dd[pt++] = g; dd[pt++] = b;
+    dd[pt++]=255;
+
+
+
     //data[ pointer + 2 ] = 255
     colors.push( rgbToHsv( r, g, b ) );
   }
 
+  debugPixels.putImageData( debugData, 0, 0 );
   //ctx2.putImageData( imageData, minX, minY );
 
 
@@ -105,19 +128,19 @@ var extractColors = function( ctx, where, k, debug){
     return { color: c, normalized: c.length / amount };
   } )
     .filter( function( c ){
-      return c.normalized > 1 / k;
+      return c.normalized > 1 / (k*3);
     } )
     .sort( function( a, b ){
       return a.normalized - b.normalized;
     } )
     .map( function( c ){
-      return c.color[0];//result.means.centroids.slice().sort(distanceToHSV(c.color[0]))[0];
+      return c.color.sort(colorSort)[0];//result.means.centroids.slice().sort(distanceToHSV(c.color[0]))[0];
     } );
 
   var similars = filteredResult.map(function(c, i) {
     return {color: c, i: i, similar: [i]}
   });
-  var similarColor = 0.03;
+
 
   // O^2 similarity search
   for( var i = 0, _i = similars.length; i < _i; i++ ){
@@ -129,6 +152,7 @@ var extractColors = function( ctx, where, k, debug){
       if(Math.abs(hsvDistance(c1.color, c2.color))<similarColor){
         if(c1.similar.indexOf(j) === -1){
           c1.similar.push( j );
+          c2.similar.push( i );
         }
       }
     }
@@ -165,22 +189,43 @@ var extractColors = function( ctx, where, k, debug){
     return delta <0.0000001 || delta > 0.03;
   });
 };
+var colorSort = function(a,b){return (-(a[2]-b[2]))+(-(a[1]-b[1])/2)};
+var similarColor = 0.05;
 var abs = Math.abs, min = Math.min;
 var hsvDistance = function( c1, c2 ){
-  var dh = min( abs( c2[ 0 ] - c1[ 0 ] ), 360 - abs( c2[ 0 ] - c1[ 0 ] ) ) / 180.0,
+  var dh = min( abs( c2[ 0 ] - c1[ 0 ] ), 1 - abs( c2[ 0 ] - c1[ 0 ] ) ),
     ds = abs( c2[ 1 ] - c1[ 1 ] ), dv = c2[ 2 ] - c1[ 2 ];
 
-  return dh*30 + ds / 12 + dv / 6;
+  return Math.sqrt(Math.pow(dh/2,2) + Math.pow(ds / 8, 2) + Math.pow(dv / 4,2));
 };
 
+
+function GetPixel(context, x, y)
+{
+  var p = context.getImageData(x, y, 1, 1).data;
+  var hex = "#" + ("000000" + rgbToHex(p[0], p[1], p[2])).slice(-6);
+  return hex;
+}
+
+function rgbToHex(r, g, b) {
+  if (r > 255 || g > 255 || b > 255)
+    throw "Invalid color component";
+  return ((r << 16) | (g << 8) | b).toString(16);
+}
 var extractor = function(ctx, from, k, debug) {
+  extracting++;
   return extractColors(ctx, from, k, debug)
     .map(function(m) {
-      return {color: hsvToRgb.apply(null, m.color).map(function(a){return a|0}), distance: m.distance};
+      return {color: hsvToRgb.apply(null, m.color).map(function(a){return a|0}), hsv: m.color, distance: m.distance};
     })
     .map(function(m){
+
       return {
-        html: '<span style="display: inline-block;width:32px;height:32px;background:rgb('+m.color.join(',')+');border-radius:3px;border:2px solid #000;margin:2px">'+m.distance.toFixed(2)+'</span>'
+        html: '<span style="background:rgb('+m.color.join(',')+');" class="color-preview">'+
+          'H:&nbsp;'+Math.floor(m.hsv[0]*360)+'<br>'+
+          'S:&nbsp;'+Math.round(m.hsv[1]*100)+'%<br>'+
+          'V:&nbsp;'+Math.round(m.hsv[2]*100)+'%'+
+          '</span>'
       };
     });
 };
\ No newline at end of file

examples/examples-browser/public/js/facemarks.js

@@ -45,7 +45,7 @@ var FaceMarks = function(landmarks) {
   marks.cheek.right = this.cheek(nose[3], toPoint(jaw[15]).middle(toPoint(jaw[16])), mouth[6]);
 
   marks.lips.top = this.lip(mouth[2], mouth[4], mouth[13], mouth[15], true);
-  marks.lips.bottom = this.lip(mouth[19], mouth[17], mouth[10], mouth[8], false);
+  marks.lips.bottom = this.lip(mouth[10], mouth[8], mouth[19], mouth[17], true);
 
   marks.eyes.left = this.eye(leftBrow[0], leftBrow[3], leftEye[0], leftEye[2], leftEye[3], leftEye[4]);
   marks.eyes.right = this.eye(rightBrow[4], rightBrow[1], rightEye[3], rightEye[1], rightEye[0], rightEye[5]);
@@ -136,12 +136,12 @@ FaceMarks.prototype = {
     lips.push([
       toPoint(p1).add(lipVertical.mulClone(1/3)),
       toPoint(p2).add(lipVertical.mulClone(1/3)),
-      toPoint(p3).sub(lipVertical.mulClone(1/3))
+      toPoint(p3).sub(lipVertical.mulClone(1/5))
     ]);
     lips.push([
       toPoint(p2).add(lipVertical.mulClone(1/3)).add(lipDeltaX),
-      toPoint(p4).sub(lipVertical.mulClone(1/3)).add(lipDeltaX),
-      toPoint(p3).sub(lipVertical.mulClone(1/3)).add(lipDeltaX)
+      toPoint(p4).sub(lipVertical.mulClone(1/5)).add(lipDeltaX),
+      toPoint(p3).sub(lipVertical.mulClone(1/5)).add(lipDeltaX)
     ]);
 
 

examples/examples-browser/public/js/kmean.js

@@ -65,7 +65,7 @@ KMeans.prototype = {
     var movement = true;
     while( movement ){
       iterations++;
-      if(iterations>300)
+      if(iterations>500)
         break;
       // update point-to-centroid assignments
       for( var i = 0; i < points.length; i++ ){

examples/examples-browser/views/demo.html

@@ -9,7 +9,19 @@
     <script src="js/kmean.js"></script>
     <script src="js/facemarks.js"></script>
     <script src="js/extract-colors.js"></script>
-
+    <style>
+        .color-preview {
+            display: inline-block;
+            font-family: monospace;
+            font-size: 12px;
+            width: 48px;
+            padding: 4px 8px;
+            border-radius: 3px;
+            border: 2px solid #000;
+            margin: 2px;
+            text-shadow: #fff 0 0 6px;
+        }
+    </style>
 </head>
 <body>
 
@@ -17,6 +29,7 @@
 
 <canvas id="playback" style="position: absolute;left:0"></canvas>
 <canvas id="overlay" style="position: absolute;left:0"></canvas>
+<canvas id="ccolors" style="position: absolute;left:700px" width="100" height="480"></canvas>
 
 <div id="colors"></div>
 <div>
@@ -129,15 +142,14 @@ var videoEl = inputVideo;
 
       }
 
-      const result = await faceapi.detectSingleFace( playback, options ).withFaceLandmarks();//.withFaceDescriptor();
+      let result = await faceapi.detectSingleFace( playback, options ).withFaceLandmarks();//.withFaceDescriptor();
       if(result){
 
 
 
 
         //const resizedResults = faceapi.resizeResults(result, videoEl)
-        ctx.clearRect(0,0,dims.width, dims.height)
-        faceapi.draw.drawFaceLandmarks( canvas, faceapi.resizeResults( result, dims ) );
+
 
         var toPoint = function(p) {
           return new Point(p.x, p.y)
@@ -152,6 +164,30 @@ var videoEl = inputVideo;
           leftEye = result.landmarks.getLeftEye(),
           rightEye = result.landmarks.getRightEye();
 
+        var top = toPoint(nose[6]).lerp(nose[0], 2).y;
+        if(top<0){
+          var dtop = -top/dims.height;
+
+          ctx2.drawImage(playback, dtop*dims.width/2, -top, dims.width-dtop*dims.width, dims.height+top);
+          ctx2.fillStyle = GetPixel(ctx2,4,4);
+          ctx2.fillRect(0,0,dims.width, -top);
+          result = await faceapi.detectSingleFace( playback, options ).withFaceLandmarks();//.withFaceDescriptor();
+          if(!result){
+            return setTimeout( f, 300 );
+          }
+            jaw = result.landmarks.getJawOutline(),
+            nose = result.landmarks.getNose(),
+            mouth = result.landmarks.getMouth(),
+            leftBrow = result.landmarks.getLeftEyeBrow(),
+            rightBrow = result.landmarks.getRightEyeBrow(),
+
+            leftEye = result.landmarks.getLeftEye(),
+            rightEye = result.landmarks.getRightEye();
+        }
+
+        ctx.clearRect(0,0,dims.width, dims.height);
+        faceapi.draw.drawFaceLandmarks( canvas, faceapi.resizeResults( result, dims ) );
+
         ctx.fillStyle='blue'; mouth.forEach(function(p, i) { ctx.fillText(i+'', p.x, p.y) });
         ctx.fillStyle='red'; jaw.forEach(function(p, i) { ctx.fillText(i+'', p.x, p.y) });
 
@@ -262,26 +298,26 @@ var videoEl = inputVideo;
 
 
 
+        document.getElementById('ccolors').getContext('2d').clearRect(0,0,120,480);
 
 
-
-
+        extracting = -1;
         document.getElementById('colors').innerHTML = [
           block(
             'Lips',
-            extractor(ctx2, lips, 5).map(a=>a.html).join('\n')
+            extractor(ctx2, lips, 5)
           ),
           block(
             'Blush',
-            extractor(ctx2, cheek, 5).map(a=>a.html).join('\n')
+            extractor(ctx2, cheek, 3)
           ),
           block(
             'Brows',
-            extractor(ctx2, browsSearchArea, 5).map(a=>a.html).join('\n')
+            extractor(ctx2, browsSearchArea, 2)
           ),
           block(
             'Eyes',
-            extractor(ctx2, eyes, 5, 1).map(a=>a.html).join('\n')
+            extractor(ctx2, eyes, 6, 1)
           )
         ].join('\n');
 
@@ -299,10 +335,11 @@ var videoEl = inputVideo;
       //requestAnimationFrame(f);
       setTimeout(f, 300)
     };
+  var extracting = 0;
 
+  var block = function(title, extracted) {
 
-  var block = function(title, children) {
-    return '<div class="block"><h2>'+ title +'</h2><p>'+children+'</p></div>'
+    return '<div class="block"><h2>'+ title +'</h2><p>'+extracted.map(a=>a.html).join('\n')+'</p></div>'
   };
     var f0 = async function() {